diff --git a/advisors/check_missing_file.py b/advisors/check_missing_file.py index 50e1e174334fb6f54c204154e285de53cb8629b7..ae69f5aee49212155299f51c7ad4ebb91a3027e0 100755 --- a/advisors/check_missing_file.py +++ b/advisors/check_missing_file.py @@ -16,6 +16,7 @@ This is a simple script to check if files already been submit into repository. If not, it can be used to push an issue to remind the developer. """ +import sys import argparse from datetime import datetime @@ -75,7 +76,10 @@ def get_repos_by_sig(sig): """ Get repos by sig """ - user_gitee = gitee.Gitee() + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) yaml_data = user_gitee.get_sigs() repo_list = [] @@ -91,7 +95,10 @@ def main_process(repo, push, check_file): """ Main process for command line """ - my_gitee = gitee.Gitee() + try: + my_gitee = gitee.Gitee() + except NameError: + sys.exit(1) if check_file == 'yaml': file = my_gitee.get_yaml(repo) elif check_file == 'spec': diff --git a/advisors/check_source_url.py b/advisors/check_source_url.py index fb0e13d0266db3bc717629022cbfaef939894cd1..5ad3078c454baa7821237e88cec5acf08b704fbd 100755 --- a/advisors/check_source_url.py +++ b/advisors/check_source_url.py @@ -47,7 +47,10 @@ def check_repo(repo, branch, batch_num): Check source url of multi-packages in repo like src-openeuler. batch_num is batch num of one thread """ - user_gitee = gitee.Gitee() + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) repo_info = user_gitee.get_community(repo) if not repo_info: print("WARNING: {repo}.yaml can't be found in community.".format(repo=repo)) @@ -71,11 +74,10 @@ def check_repo(repo, branch, batch_num): print("Ending check.") -def create_issue(repo, title, body): +def create_issue(user_gitee, repo, title, body): """ Create issue for repo """ - user_gitee = gitee.Gitee() created_issue = False issues = user_gitee.get_issues(repo) for issue in issues: @@ -106,7 +108,10 @@ def check_pkg(pkg, branch, check_file, lock): """ Check source url of single package """ - user_gitee = gitee.Gitee() + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) check_file.writelines("\n-----------------------Checking {}-----------------------".format( pkg)) lock.acquire() @@ -123,13 +128,13 @@ def check_pkg(pkg, branch, check_file, lock): else: title = "Source url can't be found in spec on {br}".format(br=branch) check_file.writelines("WARNING: {content}".format(content=title)) - create_issue(pkg, title, BODY_SOURCE) + create_issue(user_gitee, pkg, title, BODY_SOURCE) return False if re.search(r"%{.*?}", source): title = "Source url can't be parsed with extra macros in spec on {}.".format(branch) check_file.writelines("WARNING: {content}".format(content=title)) - create_issue(pkg, title, BODY_SOURCE) + create_issue(user_gitee, pkg, title, BODY_SOURCE) return False if source.startswith("http") or source.startswith("ftp"): @@ -148,7 +153,7 @@ def check_pkg(pkg, branch, check_file, lock): if os.path.exists(file_name): if subprocess.call(["tar -tvf {} &>/dev/null".format(file_name)], shell=True): check_file.writelines("WARNING: {content}".format(content=title)) - create_issue(pkg, title, BODY_SOURCE) + create_issue(user_gitee, pkg, title, BODY_SOURCE) result = False else: check_file.writelines("Check successfully.") @@ -156,13 +161,13 @@ def check_pkg(pkg, branch, check_file, lock): subprocess.call(["rm -rf {}".format(file_name)], shell=True) else: check_file.writelines("WARNING: {content}".format(content=title)) - create_issue(pkg, title, BODY_SOURCE) + create_issue(user_gitee, pkg, title, BODY_SOURCE) result = False return result title = "Source url is invalid in spec on {br}.".format(br=branch) check_file.writelines("WARNING: {content}".format(content=title)) - create_issue(pkg, title, BODY_SOURCE) + create_issue(user_gitee, pkg, title, BODY_SOURCE) return False diff --git a/advisors/check_version.py b/advisors/check_version.py index 65541335818478126ed3c1e47cf8e533aa6c2ee9..6f80141ea7af38947b8dbf33b0ea931865bec451 100755 --- a/advisors/check_version.py +++ b/advisors/check_version.py @@ -15,7 +15,9 @@ This is an automatic script for checking source url of package """ import os +import sys import argparse +from advisors import gitee from advisors.oa_upgradable import main_process from advisors.check_missing_file import get_repos_by_sig @@ -40,7 +42,12 @@ def main(): else: sig = 'all' - repos = get_repos_by_sig(sig) + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) + + repos = get_repos_by_sig(user_gitee, sig) total = len(repos) index = 0 upgrade_list = [] diff --git a/advisors/gitee.py b/advisors/gitee.py index 57746a967f53b019794cf71c229e9b205adf3d6e..5c246a1717e33eba26977648541de23019793558 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -22,6 +22,7 @@ import urllib import urllib.request import urllib.parse import urllib.error +import threading from datetime import datetime import yaml @@ -30,16 +31,53 @@ class Gitee(): """ Gitee is a helper class to abstract gitee.com api """ + _instance_lock = threading.Lock() + _first_init = True + + def __new__(cls): + if not hasattr(Gitee, "_instance"): + with Gitee._instance_lock: + if not hasattr(Gitee, "_instance"): + Gitee._instance = object.__new__(cls) + return Gitee._instance + def __init__(self): - self.secret = open(os.path.expanduser("~/.gitee_personal_token.json"), "r") - self.token = json.load(self.secret) + if not self._first_init: + return + self._first_init = False + with open(os.path.expanduser("~/.gitee_personal_token.json"), "r") as secret: + self.token = json.load(secret) self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW 64; rv:50.0) '\ 'Gecko/20100101 Firefox/50.0'} self.community_url = "https://gitee.com/api/v5/repos/openeuler/community/contents/" self.src_openeuler_url = "https://gitee.com/api/v5/repos/src-openeuler/{repo}/contents/" self.advisor_url = "https://gitee.com/api/v5/repos/openeuler/openEuler-Advisor/contents/" - self.time_format = "%Y-%m-%dT%H:%M:%S%z" + + self.helper_info = {} + specfile_exception_url = self.advisor_url + "advisors/helper/specfile_exceptions.yaml" + resp = self.__get_gitee_json(specfile_exception_url) + if not resp: + print("ERROR: specfile_exceptions.yaml may not exist.") + raise NameError + resp_str = base64.b64decode(resp["content"]).decode("utf-8") + self.helper_info["specfile_excepts"] = yaml.load(resp_str, Loader=yaml.Loader) + + version_exception_url = self.advisor_url + "advisors/helper/version_exceptions.yaml" + resp = self.__get_gitee_json(version_exception_url) + if not resp: + print("ERROR: version_exceptions.yaml may not exist.") + raise NameError + resp_str = base64.b64decode(resp["content"]).decode("utf-8") + self.helper_info["version_excepts"] = yaml.load(resp_str, Loader=yaml.Loader) + + upgrade_branches_url = self.advisor_url + "advisors/helper/upgrade_branches.yaml" + resp = self.__get_gitee_json(upgrade_branches_url) + if not resp: + print("ERROR: upgrade_branches.yaml may not exist.") + raise NameError + resp_str = base64.b64decode(resp["content"]).decode("utf-8") + self.helper_info["upgrade_branches"] = yaml.load(resp_str, Loader=yaml.Loader) def __post_gitee(self, url, values, headers=None): """ @@ -212,46 +250,27 @@ class Gitee(): """ Get upgrade branch info """ - upgrade_branches_url = self.advisor_url + "advisors/helper/upgrade_branches.yaml" - resp = self.__get_gitee_json(upgrade_branches_url) - if not resp: - print("ERROR: upgrade_branches.yaml may not exist.") - return None - resp_str = base64.b64decode(resp["content"]).decode("utf-8") - branches_info = yaml.load(resp_str, Loader=yaml.Loader) + branches_info = self.helper_info["upgrade_branches"] for br_info in branches_info["branches"]: if branch == br_info["name"]: return br_info print("WARNING: Don't support branch: {} in auto-upgrade.".format(branch)) - return None + return "" def get_spec_exception(self, repo): """ Get well known spec file exception """ - specfile_exception_url = self.advisor_url + "advisors/helper/specfile_exceptions.yaml" - resp = self.__get_gitee_json(specfile_exception_url) - if not resp: - print("ERROR: specfile_exceptions.yaml may not exist.") - return None - resp_str = base64.b64decode(resp["content"]).decode("utf-8") - excpt_list = yaml.load(resp_str, Loader=yaml.Loader) + excpt_list = self.helper_info["specfile_excepts"] if repo in excpt_list: return excpt_list[repo] - return None + return "" def get_version_exception(self): """ Get version recommend exceptions """ - version_exception_url = self.advisor_url + "advisors/helper/version_exceptions.yaml" - resp = self.__get_gitee_json(version_exception_url) - if not resp: - print("ERROR: version_exceptions.yaml may not exist.") - return '' - resp_str = base64.b64decode(resp["content"]).decode("utf-8") - excpt = yaml.load(resp_str, Loader=yaml.Loader) - return excpt + return self.helper_info["version_excepts"] def get_spec(self, pkg, branch="master"): """ @@ -344,11 +363,13 @@ class Gitee(): parameters["body"] = comment self.__post_gitee(issues_url, parameters) - def get_gitee_datetime(self, time_string): + @staticmethod + def get_gitee_datetime(time_string): """ Get datetime of gitee """ - result = datetime.strptime(time_string, self.time_format) + time_format = "%Y-%m-%dT%H:%M:%S%z" + result = datetime.strptime(time_string, time_format) return result.replace(tzinfo=None) diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py index 6d3c64d95ae104dd551711af786f3761dfc3d70f..cde171fa22f74e51f5bea3b3cbe91b168e6dad71 100755 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -15,6 +15,7 @@ This is a script to check upgradable information against upstream """ import os +import sys import argparse import re from pyrpm.spec import Spec, replace_macros @@ -111,7 +112,10 @@ def main_process(push, default, repo): Main process of the functionality """ print("Checking", repo) - user_gitee = gitee.Gitee() + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) spec_string = user_gitee.get_spec(repo) if not spec_string: print("WARNING: Spec of {pkg} can't be found on master".format(pkg=repo)) diff --git a/advisors/package_type.py b/advisors/package_type.py index 2010dcf67269b881d5343ef0996bb71ed794aef5..9d5f27da51e2d181cd96b48a0ea7ae08e0ec8d59 100755 --- a/advisors/package_type.py +++ b/advisors/package_type.py @@ -15,6 +15,7 @@ The class of the package_type, used to get the package type of repo. """ +import sys import subprocess import tempfile from pyrpm.spec import Spec @@ -99,7 +100,10 @@ class PackageType: :returns: library path list of package :raises: None """ - user_gitee = gitee.Gitee() + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) spec_string = user_gitee.get_spec(self.repo) if not spec_string: print("WARNING: Spec of {pkg} can't be found on master".format(pkg=self.repo)) diff --git a/advisors/review_tool.py b/advisors/review_tool.py index 6009f9d7faf03f4347faebe223ec7b49ad70bda8..c8a5b02ad66a32c58432d046bbce768372d34da8 100755 --- a/advisors/review_tool.py +++ b/advisors/review_tool.py @@ -659,8 +659,11 @@ def main(): group = params[0] repo_name = params[1] pull_id = params[2] - - user_gitee = gitee.Gitee() + + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) pull_request = user_gitee.get_pr(repo_name, pull_id, group) if not pull_request: print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist."\ diff --git a/advisors/simple_update_robot.py b/advisors/simple_update_robot.py index 5c53d141590e23990f1947675a43fcf31bccaf39..f00652a0d5a45e26c48a380e8b2a594c7ebec813 100755 --- a/advisors/simple_update_robot.py +++ b/advisors/simple_update_robot.py @@ -55,9 +55,7 @@ def get_spec_path(gt_api, pkg): """ Get specfile path in repository """ - print("pkg is : {}".format(pkg)) excpt = gt_api.get_spec_exception(pkg) - print("excpt is {}".format(excpt)) if excpt: spec_path = os.path.join(excpt["dir"], excpt["file"]) else: @@ -404,6 +402,8 @@ def auto_update_pkg(gt_api, u_pkg, u_branch, u_ver=None): pkg_ver = replace_macros(pkg_spec.version, pkg_spec) branch_info = gt_api.get_branch_info(u_branch) + if not branch_info: + sys.exit(1) if not u_ver: pkg_tags = oa_upgradable.get_ver_tags(gt_api, u_pkg) @@ -591,6 +591,8 @@ def __manual_operate(gt_api, op_args): cur_version = replace_macros(spec_file.version, spec_file) branch_info = gt_api.get_branch_info(op_args.branch) + if not branch_info: + sys.exit(1) if op_args.fork_then_clone: fork_clone_repo(gt_api, op_args.repo_pkg, op_args.branch) @@ -599,7 +601,7 @@ def __manual_operate(gt_api, op_args): if not op_args.new_version: print("Please specify the upgraded version of the {}".format(op_args.repo_pkg)) sys.exit(1) - elif not update_ver_check(op_args.repo_pkg, cur_version, op_args.new_version): + if not update_ver_check(op_args.repo_pkg, cur_version, op_args.new_version): sys.exit(1) if op_args.download: @@ -610,9 +612,9 @@ def __manual_operate(gt_api, op_args): if op_args.create_spec: create_spec(gt_api, op_args.repo_pkg, cur_version, op_args.new_version) - if op_args.build_pkg: - if not build_pkg(op_args.repo_pkg, op_args.branch, branch_info["obs_prj"]): - sys.exit(1) + if op_args.build_pkg and not build_pkg(op_args.repo_pkg, op_args.branch, + branch_info["obs_prj"]): + sys.exit(1) if op_args.check_rpm_abi: check_result = check_rpm_abi(op_args.repo_pkg) @@ -655,7 +657,10 @@ def main(): "PR and issue", action="store_true") args = pars.parse_args() - user_gitee = gitee.Gitee() + try: + user_gitee = gitee.Gitee() + except NameError: + sys.exit(1) if args.update: if args.update == "repo": diff --git a/advisors/which_archived.py b/advisors/which_archived.py index 8131f01a64d989de0c3aea47b38c7e1a659baf18..81e468edb9d2f5aa56de2d5cb3b94163ffdd12de 100755 --- a/advisors/which_archived.py +++ b/advisors/which_archived.py @@ -119,7 +119,11 @@ def get_oe_repo_dict(cwd_path, nocached): logging.debug("begin to query oe.") data = get_sigs() oe_repo_dict = {} - my_gitee = gitee.Gitee() + try: + my_gitee = gitee.Gitee() + except NameError: + sys.exit(1) + last_record_dict = {} if not nocached: last_record_dict = read_pkginfo_lasttime()