From 4e2d2e5b4714b3718933fe33535214cb399b1bb7 Mon Sep 17 00:00:00 2001 From: miao_kaibo Date: Fri, 8 Jan 2021 15:16:03 +0800 Subject: [PATCH] add new function for checking codes between gitee and obs --- core/gitee_to_obs.py | 125 +++++++++++++++++++++++++++++++++++++++++++ core/runner.py | 15 ++++-- openeuler_obs.py | 3 ++ 3 files changed, 140 insertions(+), 3 deletions(-) diff --git a/core/gitee_to_obs.py b/core/gitee_to_obs.py index 3d35d56..a508086 100644 --- a/core/gitee_to_obs.py +++ b/core/gitee_to_obs.py @@ -16,10 +16,14 @@ """ sync code from gitee to obs """ +import re import sys import os import time import shutil +import json +import requests +import threadpool current_path = os.path.join(os.path.split(os.path.realpath(__file__))[0]) sys.path.append(os.path.join(current_path, "..")) from common.log_obs import log @@ -194,6 +198,127 @@ class SYNCCode(object): self._get_latest_gitee_pull() self._gitee_pr_to_obs(obs_project) + +class CheckCode(object): + """ + Make sure that the codes for gitee and obs are the same + """ + def __init__(self, **kwargs): + self.project = kwargs["project"] + self.branch = kwargs["branch"] + cmd = "osc list %s" % self.project + self.packages = set(os.popen(cmd).read().split("\n")) - set(['']) + self.not_same_packages = [] + + def get_gitee_spec(self, branch, package): + """ + get spec file from gitee repo + branch: branch of package in gitee + package: name of package + """ + log.info("--------------------------------------") + cmd = "curl https://gitee.com/src-openeuler/{0}/tree/{1} | grep src-openeuler > {2}".format(\ + package, branch, package) + log.info("%s - %s" % (package, cmd)) + spec_files = None + if os.system(cmd) == 0: + try: + cmd = "cat {0}| grep spec".format(package) + data = os.popen(cmd).read() + str_find = '(.*.spec)' + spec_files = re.findall(str_find, data) + log.info("%s - %s" % (package, spec_files)) + except SyntaxError as e: + log.error(e) + cmd = "rm -rf %s" % package + os.system(cmd) + if spec_files: + spec_file = spec_files[0] + cmd = "wget https://gitee.com/src-openeuler/{0}/raw/{1}/{2} -O gitee_{3}".format(\ + package, branch, spec_file, spec_file) + log.info("%s - %s" % (package, cmd)) + if os.system(cmd) != 0: + os.system(cmd) + return "gitee_" + spec_file + return None + # url = "https://gitee.com/api/v5/repos/src-openeuler/{0}/contents/%2F?ref={1}".format(package, branch) + # ret = requests.get(url) + # data = json.dumps(ret.json(), sort_keys=True, indent=4, ensure_ascii=False) + # str_find = '"download_url": "(.*.spec)",' + # url = re.findall(str_find, data)[0] + # spec_file = os.path.basename(url) + # cmd = "wget %s" % url + # if os.system(cmd) != 0: + # os.system(cmd) + # return spec_file + + def get_obs_spec(self, project, package): + """ + get spec file from obs repo + project: + package: + """ + cmd = "osc ls -e %s %s | grep '.spec'" % (project, package) + data = os.popen(cmd).read() + str_find = '(.*.spec)' + spec_files = re.findall(str_find, data) + log.info("%s - obs spec files: %s" % (package, spec_files)) + if spec_files: + for spec in spec_files: + if spec.endswith(".spec"): + spec_file = spec + cmd = "osc co %s %s %s" % (project, package, spec_file) + log.info("%s - %s" % (package, cmd)) + os.system(cmd) + return spec_file + return None + + def same_or_not(self, gitee_spec, obs_spec): + """ + check spec file between gitee spec file and obs spec file + return: False - not same; True - same + """ + if gitee_spec and obs_spec: + log.info("gitee spec:%s, obs spec:%s" % (gitee_spec, obs_spec)) + cmd = "diff %s %s" % (gitee_spec, obs_spec) + ret = os.popen(cmd).read() + cmd = "rm -rf %s %s" % (gitee_spec, obs_spec) + os.system(cmd) + if ret: + return False + else: + return True + elif not gitee_spec: + log.info("SPEC: can't not find package spec file from gitee") + return True + else: + return False + + def check(self, package): + """ + check one package + """ + gitee_spec = self.get_gitee_spec(self.branch, package) + obs_spec = self.get_obs_spec(self.project, package) + if self.same_or_not(gitee_spec, obs_spec): + log.info("codes of %s are same between gitee and obs" % package) + else: + self.not_same_packages.append(package) + log.info("codes of %s are not same between gitee and obs" % package) + + def check_all(self): + """ + check all packages + """ + if self.packages: + pool = threadpool.ThreadPool(10) + reqs = threadpool.makeRequests(self.check, self.packages) + for req in reqs: + pool.putRequest(req) + pool.wait() + log.info("codes not same between gitee and obs:%s" % self.not_same_packages) + + if __name__ == "__main__": #Now start kw = {'repository': sys.argv[1], 'branch': sys.argv[2], diff --git a/core/runner.py b/core/runner.py index 46ca6f7..546fdfd 100644 --- a/core/runner.py +++ b/core/runner.py @@ -22,6 +22,7 @@ from common.parser_config import ParserConfigIni from core.save import SaveInfo from core.project_manager import OBSPrjManager from core.gitee_to_obs import SYNCCode +from core.gitee_to_obs import CheckCode from core.package_manager import OBSPkgManager from core.update_obs_repos import RPMManager @@ -90,14 +91,22 @@ class Runner(object): update_repo = RPMManager(**self.kwargs) update_repo.update_pkgs() + def _check_codes(self): + """ + check codes between gitee and obs + """ + log.debug("check codes") + check_codes = CheckCode(**self.kwargs) + check_codes.check_all() + def run(self): """ run main return: """ - log.debug(self.ignore_list) - log.debug(self.update_enabled_flag) - if self.kwargs["repo_rpms_update"]: + if self.kwargs["check_codes"]: + self._check_codes() + elif self.kwargs["repo_rpms_update"]: self._update_obs_repo_rpms() elif self.kwargs["latest_info"]: self._save_latest_info() diff --git a/openeuler_obs.py b/openeuler_obs.py index 8f427a2..69b0298 100644 --- a/openeuler_obs.py +++ b/openeuler_obs.py @@ -79,6 +79,8 @@ par.add_argument("-latest", "--latest_info", default=False, help="store latest package info by branch to gitee repo obs_pkg_rpms, \ type is bool, default False, should be with -b -guser -gpwd.", required=False) +par.add_argument("-cc", "--check_codes", default=False, + help="check codes same or not between gitee and obs,should be with -b and -p", required=False) args = par.parse_args() #apply kw = { @@ -107,6 +109,7 @@ kw = { "repo_server_pwd": args.repo_server_pwd, "latest_info": args.latest_info, + "check_codes": args.check_codes, } run = Runner(**kw) -- Gitee