diff --git a/common/common.py b/common/common.py index 5adb6c8c53e53b562bf0a500a9e7226e312dc3bc..a1739cb89158c126be0b2d5d2d63e09860a34ef4 100644 --- a/common/common.py +++ b/common/common.py @@ -28,7 +28,7 @@ def str_to_bool(s): return s.lower() in ("yes", "true", "t", "1") -def git_repo_src(repo_url, gitee_user_name, gitee_user_pwd): +def git_repo_src(repo_url, gitee_user_name, gitee_user_pwd, dest_dir=None): """ get repo source repo_url: url of repository @@ -37,12 +37,15 @@ def git_repo_src(repo_url, gitee_user_name, gitee_user_pwd): """ repos_dir = os.getcwd() tmp = repo_url.split("//") - repo_path = os.path.join(repos_dir, tmp[1].split("/")[-1].replace(".git", "")) + if dest_dir: + repo_path = dest_dir + else: + repo_path = os.path.join(repos_dir, tmp[1].split("/")[-1].replace(".git", "")) if os.path.exists(repo_path) and os.path.isdir(repo_path): - cmd = "cd %s && git pull" % repo_path + cmd = "cd %s && git pull && cd -" % repo_path else: - cmd = "rm -rf %s && git clone --depth 1 %s//%s:%s@%s" % \ - (repo_path, tmp[0], gitee_user_name, gitee_user_pwd, tmp[1]) + cmd = "rm -rf %s && git clone --depth 1 %s//%s:%s@%s %s" % \ + (repo_path, tmp[0], gitee_user_name, gitee_user_pwd, tmp[1], repo_path) if os.system(cmd) != 0: return None if os.path.exists(repo_path): @@ -88,7 +91,6 @@ class Pexpect(object): cmd = "ssh -p %s %s@%s '%s'" % (self.port, self.user, self.ip, cmd) else: cmd = "ssh %s@%s '%s'" % (self.user, self.ip, cmd) - print(cmd) process = pexpect.spawn(cmd, timeout=timeout) self._expect(process) msg = process.readlines() @@ -105,7 +107,6 @@ class Pexpect(object): cmd = "scp -P %s %s %s@%s:%s" % (self.port, src_file, self.user, self.ip, dest_dir) else: cmd = "scp %s %s@%s:%s" % (src_file, self.user, self.ip, self.dest_dir) - print(cmd) process = pexpect.spawn(cmd) self._expect(process) msg = process.readlines() diff --git a/common/parser_config.py b/common/parser_config.py index 0532f5cf1196557276ae116fb8af39c9a50442ab..9623f4bfca64a7daa28205055b4072012a2622ac 100644 --- a/common/parser_config.py +++ b/common/parser_config.py @@ -142,7 +142,6 @@ class ParserConfigIni(object): init repos url """ repos_list = self.config.options("gitee_repository") - print(repos_list) for repo in repos_list: self.repos[repo] = self.config.get("gitee_repository", repo) diff --git a/config/config.ini b/config/config.ini index bd6f66f7bbca7bd27239c45f4e6ee90e0eab2987..c480d1437f25d04d8d0b7146523463acddfc1bc6 100644 --- a/config/config.ini +++ b/config/config.ini @@ -17,6 +17,7 @@ openEuler-20.03-LTS-Next = openEuler:20.03:LTS:Next openEuler:20.03:LTS:Next:Epo [gitee_repository] community = https://gitee.com/openeuler/community obs_meta = https://gitee.com/src-openeuler/obs_meta +obs_pkg_rpms = https://gitee.com/unsunghero/obs_pkg_rpms [obs_project_repos] # replace obs project's ":" by "-" openEuler-Mainline = openEuler:selfbuild:BaseOS/mainline_standard_aarch64 openEuler:selfbuild:BaseOS/mainline_standard_x86_64 diff --git a/core/runner.py b/core/runner.py index 01dad8ed731bf008aca57a8d4af1fcffd4ff9e61..5c4caefebc3616b19c47dc2f7442dcc5eb3f6cc4 100644 --- a/core/runner.py +++ b/core/runner.py @@ -23,6 +23,7 @@ from core.save import SaveInfo from core.project_manager import OBSPrjManager from core.gitee_to_obs import SYNCCode from core.package_manager import OBSPkgManager +from core.update_obs_repos import RPMManager class Runner(object): @@ -72,6 +73,14 @@ class Runner(object): syc = SYNCCode(**self.kwargs) syc.sync_code_to_obs() + def _update_obs_repo_rpms(self): + """ + update obs repo rpms + """ + log.debug("update repo rpms") + update_repo = RPMManager(**self.kwargs) + update_repo.update_pkgs() + def run(self): """ run main @@ -79,7 +88,9 @@ class Runner(object): """ log.debug(self.ignore_list) log.debug(self.update_enabled_flag) - if self.kwargs["repository"] == "obs_meta": + if self.kwargs["repo_rpms_update"]: + self._update_obs_repo_rpms() + elif self.kwargs["repository"] == "obs_meta": self._obs_meta_action() elif self.kwargs["repository"] not in self.ignore_list: if not self.update_enabled_flag[self.kwargs["branch"]]: diff --git a/core/update_obs_repos.py b/core/update_obs_repos.py new file mode 100755 index 0000000000000000000000000000000000000000..68cb72012a870c29d1c4078f25c68a82cb3d20bf --- /dev/null +++ b/core/update_obs_repos.py @@ -0,0 +1,250 @@ +#!/bin/evn python3 +# -*- encoding=utf8 -*- +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# Author: miao_kaibo +# Create: 2020-11-27 +# ****************************************************************************** + +""" +update obs repo rpms +""" +import os +import sys +import shutil +import time +import yaml +import threadpool +current_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) +sys.path.append(os.path.join(current_path, "..")) +from common.common import git_repo_src +from common.log_obs import log +from common.common import Pexpect +from common.parser_config import ParserConfigIni + + +class RPMManager(object): + """ + update for obs repo rpms + """ + def __init__(self, **kwargs): + """ + obs_project: obs project name + repo: obs project repo where store all packages + arch: + pkgs: + """ + self.kwargs = kwargs + par = ParserConfigIni() + self.obs_project_repo_dict = par.get_obs_repos_dict() + print(self.obs_project_repo_dict) + self.obs_project_root_path = par.get_obs_prj_root_path() + self.obs_pkg_rpms_url = par.get_repos_dict()["obs_pkg_rpms"] + self.obs_project = self.kwargs["project"] + self.rpms_to_repo_path = None + self.old_pkg_rpms = None + self.repo = self.kwargs["repo"] + self.arch = self.kwargs["arch"] + self.pkgs = self.kwargs["pkglist"] + self.old_pkg_rpms = {} + self._set_rpms_to_repo() + self.pex = Pexpect(self.kwargs["repo_server_user"], self.kwargs["repo_server_ip"], + self.kwargs["repo_server_pwd"], self.kwargs["repo_server_port"]) + log.info(self.rpms_to_repo_path) + self.obs_pkg_rpms_files_dir = None + self._download_obs_pkg_rpms_file(self.obs_pkg_rpms_url, self.kwargs["gitee_user"], \ + self.kwargs["gitee_pwd"], dest_dir = "/tmp/obs_pkg_rpms") + self.obs_pkg_rpms_file = os.path.join(self.obs_pkg_rpms_files_dir, "%s_%s.yaml" \ + % (self.obs_project.replace(":", "-"), self.arch)) + self.get_old_rpms_list_from_file(self.obs_pkg_rpms_file) + + def _set_rpms_to_repo(self): + """ + set rpms repo where new rpm go + """ + rpms_to_repo_list = self.obs_project_repo_dict[self.obs_project].split(" ") + for rpms_to_repo in rpms_to_repo_list: + if rpms_to_repo.endswith(self.arch): + self.rpms_to_repo_path = rpms_to_repo + break + + def _download_obs_pkg_rpms_file(self, url, gitee_user, gitee_pwd, dest_dir=None): + """ + download file by gitee repo + """ + self.obs_pkg_rpms_files_dir = git_repo_src(url, gitee_user, gitee_pwd, dest_dir=dest_dir) + + def get_old_rpms_list_from_file(self, file_path): + """ + get old pkg rpms dict + file_path: yaml file which store all packages's rpms + """ + if os.path.exists(file_path): + with open(file_path, "r") as f: + self.old_pkg_rpms = yaml.load(f, Loader=yaml.FullLoader) + else: + log.error("no file for get rpms") + + def get_new_rpms_by_pkg(self, pkg): + """ + get new rpms by package name + pkg: name of packages + """ + try: + cmd = "ls %s/%s/%s/%s/%s | grep 'rpm' | grep -v 'src.rpm'" \ + % (self.obs_project_root_path, self.obs_project, self.repo, self.arch, pkg) + ret = self.pex.ssh_cmd(cmd) + rpm_list = [] + for p in ret: + p = str(p, encoding = 'utf8') + if "rpm" in p: + rpm_list.append(p.replace("\r\n", "")) + except ValueError as e: + log.error(e) + except SystemError as e: + log.error(e) + except TypeError as e: + log.error(e) + + return rpm_list + + def backup_old_rpms_by_pkg(self, pkg): + """ + backup old rpms by package name + pkg: name of packages + """ + try: + if pkg in self.old_pkg_rpms and not self.old_pkg_rpms[pkg]: + return None + t = time.strftime("%Y-%m-%d-%H-%M", time.localtime()) + backup_dir = os.path.join(self.obs_project_root_path, self.rpms_to_repo_path, "backup") + pkg_bak = os.path.join(backup_dir, "%s-%s" % (pkg, t)) + cmd = "rm -r %s/%s-*; mkdir -p %s" % (backup_dir, pkg, pkg_bak) + log.debug(cmd) + ret = self.pex.ssh_cmd(cmd) + log.debug(ret) + old_rpms_list = self.old_pkg_rpms[pkg] + for f in old_rpms_list: + cmd = "mv %s/%s/%s/:full/%s %s/" % (self.obs_project_root_path, \ + self.rpms_to_repo_path, self.arch, f, pkg_bak) + log.debug(cmd) + self.pex.ssh_cmd(cmd) + except ValueError as e: + log.error(e) + return False + except SystemError as e: + log.error(e) + return False + except TypeError as e: + log.error(e) + return False + return True + + def copy_new_rpms_to_repo(self, pkg): + """ + copy new rpms by package name to repo + pkg: name of package + """ + try: + rpms_list = self.get_new_rpms_by_pkg(pkg) + log.debug(rpms_list) + self.old_pkg_rpms[pkg] = rpms_list + for r in rpms_list: + cmd = "cp %s/%s/%s/%s/%s/%s %s/%s/%s/:full/" \ + % (self.obs_project_root_path, self.obs_project, self.repo, \ + self.arch, pkg, r, self.obs_project_root_path, \ + self.rpms_to_repo_path, self.arch) + log.debug(cmd) + self.pex.ssh_cmd(cmd) + except ValueError as e: + log.error(e) + except SystemError as e: + log.error(e) + except TypeError as e: + log.error(e) + + def update_pkg(self, pkg): + """ + update one package + pkg: name of package + """ + self.backup_old_rpms_by_pkg(pkg) + self.copy_new_rpms_to_repo(pkg) + + def update_pkgs(self): + """ + update packages + """ + if not self.pkgs: + self.pkgs = list(set(os.popen("osc list %s" % self.obs_project).read().split("\n")) - set([''])) + pool = threadpool.ThreadPool(10) + requests = threadpool.makeRequests(self.update_pkg, self.pkgs) + for req in requests: + pool.putRequest(req) + pool.wait() + self.write_new_pkg_rpms_to_file() + self.update_repos_db() + + def write_new_pkg_rpms_to_file(self): + """ + write new pkg rpms to yaml and git push for storing + """ + with open(self.obs_pkg_rpms_file, "w", encoding="utf-8") as f: + yaml.dump(self.old_pkg_rpms, f) + cmd = "cd %s && git add %s && git commit -m 'update rpms' && git push" \ + % (self.obs_pkg_rpms_files_dir, self.obs_pkg_rpms_file) + os.system(cmd) + + def update_repos_db(self): + """ + update obs repos db + """ + cmd = "chown -R obsrun:obsrun %s/%s/%s/:full; obs_admin --rescan-repository %s %s %s" \ + % (self.obs_project_root_path, self.rpms_to_repo_path, self.arch, \ + self.rpms_to_repo_path.split("/")[0], self.rpms_to_repo_path.split("/")[1], self.arch) + log.debug(cmd) + self.pex.ssh_cmd(cmd) + + #def get_hdr(rpm_path): + # ts = rpm.ts() + # try: + # fdno = os.open(rpm_path, os.O_RDONLY) + # hdr = ts.hdrFromFdno(fdno) + # os.close(fdno) + # except: + # log.error("ERROR: Init rpm error!") + # return hdr + + #def get_rpm_info(self, rpm_path): + # """ + # rpm_path: rpm package's path + # """ + # cmd = "rpm -qp --info %s | grep -E 'Name|Version|Release|Architecture' | awk -F '[:]' '{print $2}'" % rpm_path + # res = os.popen(cmd).read().split("\n") + # #logger.info(res) + # return res[0].strip(), res[1].strip(), res[2].strip(), res[3].strip() + + +if __name__ == "__main__": + kw = { + "project": "openEuler:20.03:LTS", + "repo": "standard_x86_64", + "arch": "x86_64", + "repo_server_user": "root", + "repo_server_ip": "127.0.0.1", + "repo_server_pwd": "1234", + "repo_server_port": "22233", + "gitee_user": "xxxxxxxxx", + "gitee_pwd": "xxxxxxxxx", + "pkglist": ["zip", "zsh"] + } + test = RPMManager(**kw) + test.update_pkgs() diff --git a/openeuler_obs.py b/openeuler_obs.py index a293a521a7a3a9d449dfb9eb8666af7c0c516330..a12b607b844ae46039a7f339493a0038b3b4c756 100644 --- a/openeuler_obs.py +++ b/openeuler_obs.py @@ -27,33 +27,52 @@ from core.runner import Runner #ArgumentParser par = argparse.ArgumentParser() par.add_argument("-o", "--obs", default=None, - help="Local path of obs_meta repository", required=False) + help="Local path of obs_meta repository.", required=False) par.add_argument("-r", "--repository", - help="gitee repository name", required=True) + help="gitee repository name.", required=False) par.add_argument("-b", "--branch", default="master", - help="gitee repository branch name", required=False) + help="gitee repository branch name.", required=False) par.add_argument("-b2", "--branch2", default=None, - help="gitee repository branch name, cp some packages to this branch", required=False) + help="gitee repository branch name, cp some packages to this branch.", required=False) par.add_argument("-p", "--project", default=None, - help="obs project name", required=False) + help="obs project name.", required=False) par.add_argument("-p2", "--project2", default=None, - help="obs project name, cp some packages to this project", required=False) + help="obs project name, cp some packages to this project.\ + should be used with -o -b2.", required=False) par.add_argument("-ip", "--source_server_ip", default=None, - help="ip of obs source server machine", required=False) + help="ip of obs source server machine.", required=False) par.add_argument("-sport", "--source_server_port", default=None, - help="ip of obs source server machine", required=False) + help="ip of obs source server machine.", required=False) par.add_argument("-suser", "--source_server_user", default=None, - help="user of obs source server machine", required=False) + help="user of obs source server machine.", required=False) par.add_argument("-spwd", "--source_server_pwd", default=None, - help="password of obs source server machine user", required=False) + help="password of obs source server machine user.", required=False) par.add_argument("-guser", "--gitee_user", default=None, - help="user of gitee", required=False) + help="user of gitee.", required=False) par.add_argument("-gpwd", "--gitee_pwd", default=None, - help="password of gitee", required=False) + help="password of gitee.", required=False) par.add_argument("-c", "--check", default=False, - help="check obs package", required=False) + help="check obs package.", required=False) par.add_argument("--pkglist", default=None, nargs='+', - help="packages list, connect everyone by espace", required=False) + help="packages list, connect everyone by espace.", required=False) + +par.add_argument("-up", "--repo_rpms_update", default=False, + help="update obs repo rpms.\ + should be used with project repo arch rsip rsu rsup rsp gitee_user gitee_pwd, \ + and pkglist will be used if update some packges not all.", required=False) +par.add_argument("-repo", "--repo", default=None, + help="obs project repo name.", required=False) +par.add_argument("-arch", "--arch", default=None, + help="obs project arch name.", required=False) +par.add_argument("-rsip", "--repo_server_ip", default=None, + help="obs repo server machine ip.", required=False) +par.add_argument("-rsu", "--repo_server_user", default=None, + help="obs repo server user.", required=False) +par.add_argument("-rsup", "--repo_server_pwd", default=None, + help="obs repo server user password.", required=False) +par.add_argument("-rsp", "--repo_server_port", default=None, + help="obs repo server port.", required=False) + args = par.parse_args() #apply @@ -71,7 +90,16 @@ kw = { "gitee_user": args.gitee_user, "gitee_pwd": args.gitee_pwd, "check_flag": args.check, - "pkglist": args.pkglist + "pkglist": args.pkglist, + + "repo_rpms_update": args.repo_rpms_update, + "repo": args.repo, + "arch": args.arch, + "repo_server_ip": args.repo_server_ip, + "repo_server_user": args.repo_server_user, + "repo_server_port": args.repo_server_port, + "repo_server_pwd": args.repo_server_pwd, } + run = Runner(**kw) run.run()