diff --git a/advisors/gitee.py b/advisors/gitee.py index 6925c2eaa62f17ea61edb2709cd41871de7fceb2..620c3913c4469a6b2d8ea8200c2ce9719b3e3dd2 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -26,12 +26,13 @@ class Gitee(object): self.headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW 64; rv:50.0) Gecko/20100101 Firefox/50.0'} self.gitee_url = "https://gitee.com/" - self.src_openeuler_url = self.gitee_url + "src-openeuler/{package}/raw/master/" + self.src_openeuler_url = self.gitee_url + "src-openeuler/{package}/raw/{branch}/" self.advisor_url = self.gitee_url + "openeuler/openEuler-Advisor/raw/master/" self.specfile_url_template = self.src_openeuler_url + "{specfile}" self.yamlfile_url_template = self.src_openeuler_url + "{package}.yaml" #self.advisor_url_template = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/upstream-info/{package}.yaml" self.advisor_url_template = self.advisor_url + "upstream-info/{package}.yaml" + self.community_url_template = self.gitee_url + "openeuler/community/raw/master/repository/{repository}.yaml" #self.specfile_exception_url = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/helper/specfile_exceptions.yaml" self.specfile_exception_url = self.advisor_url + "advisors/helper/specfile_exceptions.yaml" self.time_format = "%Y-%m-%dT%H:%M:%S%z" @@ -96,7 +97,7 @@ Yours openEuler-Advisor. def get_gitee_json(self, url): """ - get and load gitee json response + Get and load gitee json response """ headers = self.headers.copy() #headers = {} @@ -106,31 +107,29 @@ Yours openEuler-Advisor. def get_spec_exception(self): """ - get well known spec file exceptions + Get well known spec file exceptions """ resp = self.get_gitee(self.specfile_exception_url) exps = yaml.load(resp, Loader=yaml.Loader) return exps - def get_spec(self, pkg): + def get_spec(self, pkg, br="master"): """ - get openeuler spec file for specific package + Get openeuler spec file for specific package """ - specurl = self.specfile_url_template.format(package=pkg, specfile=pkg + ".spec") + specurl = self.specfile_url_template.format(branch=br, package=pkg, specfile=pkg + ".spec") exp = self.get_spec_exception() if pkg in exp: dir_name = exp[pkg]["dir"] file_name = exp[pkg]["file"] specurl = urllib.parse.urljoin(specurl, os.path.join(dir_name, file_name)) - try: resp = self.get_gitee(specurl) except urllib.error.HTTPError: resp = "" - return resp - def get_yaml(self, pkg): + def get_yaml(self, pkg, br="master"): """ get upstream yaml metadata for specific package """ @@ -140,7 +139,7 @@ Yours openEuler-Advisor. except urllib.error.HTTPError: resp = "Not found" if re.match("Not found", resp): - yamlurl = self.yamlfile_url_template.format(package=pkg) + yamlurl = self.yamlfile_url_template.format(branch=br, package=pkg) try: resp = self.get_gitee(yamlurl) except urllib.error.HTTPError: @@ -153,6 +152,17 @@ Yours openEuler-Advisor. else: return resp + def get_community(self, repo): + """ + Get yaml data from community repo + """ + yamlurl = self.community_url_template.format(repository=repo) + try: + resp = self.get_gitee(yamlurl) + except urllib.error.HTTPError: + resp = "" + return resp + def get_issues(self, pkg, prj="src-openeuler"): """ List all open issues of pkg diff --git a/advisors/helper/ver_rec_excpt.yaml b/advisors/helper/ver_rec_excpt.yaml new file mode 100644 index 0000000000000000000000000000000000000000..0e0a0d962aa9925c3a411959994fda7ea92306e7 --- /dev/null +++ b/advisors/helper/ver_rec_excpt.yaml @@ -0,0 +1,10 @@ +--- +# version recommend exception list +gegl04: + - '20001120.v002' +gimp: + - '19990910' +nss: + - '334.20030307' +glibc: + - '9000' diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py index 949f55b17227195a43c2c18e4c8db87ac43c0bf9..49d66f0ebce65177d572f43590debd173ebdef14 100755 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -12,71 +12,97 @@ import os import argparse import urllib.error + import gitee import check_upstream import version_recommend -if __name__ == "__main__": - parameters = argparse.ArgumentParser() - parameters.add_argument("-p", "--push", action="store_true", - help="Push the version bump as an issue to src-openeuler repository") - parameters.add_argument("-d", "--default", type=str, default=os.getcwd(), - help="The fallback place to look for YAML information") - parameters.add_argument("repo", type=str, - help="Repository to be checked for upstream version info") - - args = parameters.parse_args() - - gitee = gitee.Gitee() - prj_name = args.repo - spec_string = gitee.get_spec(prj_name) - if not spec_string: - print("{repo} seems to be an empty repository".format(repo=args.repo)) - sys.exit(1) - s_spec = Spec.from_string(spec_string) +def _get_rec_excpt(): + """ + Get except case of version recommend + """ + y_file = open(os.getcwd() + "/helper/ver_rec_excpt.yaml") + excpt = yaml.load(y_file, Loader=yaml.Loader) + return excpt - current_version = replace_macros(s_spec.version, s_spec) - print("Checking ", prj_name) - print("current version is ", current_version) - - try: - prj_info_string = open(os.path.join(args.default, prj_name + ".yaml")).read() - except FileNotFoundError: - prj_info_string = "" +def get_ver_tags(gt, repo, d_path=None): + """ + Get version tags of given package + """ + repo_yaml = "" + if d_path: + try: + repo_yaml = open(os.path.join(d_path, repo + ".yaml")).read() + except FileNotFoundError: + print("Failed to get YAML info from default path.") - if not prj_info_string: - print("Get YAML info from gitee") + if not repo_yaml: try: - prj_info_string = gitee.get_yaml(prj_name) + repo_yaml = gt.get_yaml(repo) except urllib.error.HTTPError: - print("Failed to get YAML info for {pkg}".format(pkg=prj_name)) - sys.exit(1) + print("Failed to get YAML info for {pkg}".format(pkg=repo)) + return None + pkg_info = yaml.load(repo_yaml, Loader=yaml.Loader) - prj_info = yaml.load(prj_info_string, Loader=yaml.Loader) - - vc_type = prj_info["version_control"] + vc_type = pkg_info["version_control"] if vc_type == "hg": - tags = check_upstream.check_hg(prj_info) + tags = check_upstream.check_hg(pkg_info) elif vc_type == "github": - tags = check_upstream.check_github(prj_info) + tags = check_upstream.check_github(pkg_info) elif vc_type == "git": - tags = check_upstream.check_git(prj_info) + tags = check_upstream.check_git(pkg_info) elif vc_type == "gitlab.gnome": - tags = check_upstream.check_gnome(prj_info) + tags = check_upstream.check_gnome(pkg_info) elif vc_type == "svn": - tags = check_upstream.check_svn(prj_info) + tags = check_upstream.check_svn(pkg_info) elif vc_type == "metacpan": - tags = check_upstream.check_metacpan(prj_info) + tags = check_upstream.check_metacpan(pkg_info) elif vc_type == "pypi": - tags = check_upstream.check_pypi(prj_info) + tags = check_upstream.check_pypi(pkg_info) else: print("Unsupport version control method {vc}".format(vc=vc_type)) + return None + + excpt_list = _get_rec_excpt() + if repo in excpt_list: + for excpt in excpt_list[repo]: + for tag in tags: + if excpt in tag: + tags.remove(tag) + return tags + + +if __name__ == "__main__": + parameters = argparse.ArgumentParser() + parameters.add_argument("-p", "--push", action="store_true", + help="Push the version bump as an issue to src-openeuler repository") + parameters.add_argument("-d", "--default", type=str, default=os.getcwd(), + help="The fallback place to look for YAML information") + parameters.add_argument("repo", type=str, + help="Repository to be checked for upstream version info") + + args = parameters.parse_args() + + user_gitee = gitee.Gitee() + spec_string = user_gitee.get_spec(args.repo) + if not spec_string: + print("{pkg}.spec can't be found on the master branch".format(pkg=args.repo)) + sys.exit(1) + + spec_file = Spec.from_string(spec_string) + cur_version = replace_macros(spec_file.version, spec_file) + + print("Checking ", args.repo) + print("current version is ", cur_version) + + pkg_tags = get_ver_tags(user_gitee, args.repo, args.default) + if pkg_tags is None: sys.exit(1) + ver_rec = version_recommend.VersionRecommend(pkg_tags, cur_version, 0) - print("known release tags :", tags) - v = version_recommend.VersionRecommend(tags, current_version, 0) - print("Latest version is ", v.latest_version) - print("Maintain version is", v.maintain_version) + print("known release tags:", pkg_tags) + print("Latest version is", ver_rec.latest_version) + print("Maintain version is", ver_rec.maintain_version) diff --git a/advisors/simple-update-robot.py b/advisors/simple-update-robot.py index 054ddd1f961bc7655956543cfcaaf3bd0efa3b3e..90b116850a2b3408c939d05c036ce1165da0a744 100755 --- a/advisors/simple-update-robot.py +++ b/advisors/simple-update-robot.py @@ -21,8 +21,11 @@ import subprocess import os.path import re import datetime + +import oa_upgradable import version_recommend + def download_source_url(spec, o_ver, n_ver): """ Download source file from Source or Source0 URL @@ -59,6 +62,52 @@ def download_upstream_url(gt, repo, n_ver): return False +def update_check(spec, o_ver, n_ver): + """ + Requirements check for upgraded package + """ + if len(spec.patches) >= 1: + print("I'm too naive to handle complicated package.") + print("This package has multiple in-house patches.") + return False + + ver_type = version_recommend.VersionType() + if(ver_type.compare(n_ver, o_ver) == 1): + return True + else: + print("Update failed >> [{pkg}: current_ver:{cur_ver}, upgraded_ver:{upd_ver}]".format( + pkg=spec.name, cur_ver=o_ver, upd_ver=n_ver)) + return False + + +def fork_clone_repo(gt, repo): + """ + Fork repo from src-openEuler to private repository, then clone it to local + """ + if not gt.fork_repo(repo): + print("The repo of {pkg} seems to have been forked.".format(pkg=repo)) + + name = gt.token["user"] + subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=name, pkg=repo)]) + os.chdir(repo) + + +def download_src(gt, spec, o_ver, n_ver): + """ + Download source code for upgraded package + """ + source_file = download_source_url(spec, o_ver, n_ver) + if source_file: + print(source_file) + else: + source_file = download_upstream_url(gt, spec.name, n_ver) + if source_file: + print(source_file) + else: + print("Failed to download the latest source code.") + return False + + def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None): """ Create new spec file for upgraded package @@ -89,55 +138,133 @@ def create_spec(repo, spec_str, o_ver, n_ver, src_fn=None): fn.write("\n") fn.close() + +def auto_update_pkg(gt, u_branch, u_pkg): + """ + Auto upgrade based on given branch for single package + """ + spec_str = gt.get_spec(u_pkg, u_branch) + if not spec_str: + print("{pkg}.spec can't be found on the {br} branch. ".format( + pkg=u_pkg, br=u_branch)) + sys.exit(1) + pkg_spec = Spec.from_string(spec_str) + pkg_ver = replace_macros(pkg_spec.version, pkg_spec) + + pkg_tags = oa_upgradable.get_ver_tags(gt, u_pkg) + if pkg_tags is None: + sys.exit(1) + ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0) + rec_up_ver = pkg_ver + if re.search("master", u_branch): + rec_up_ver = ver_rec.latest_version + elif re.search("LTS", u_branch): + rec_up_ver = ver_rec.maintain_version + else: + print("Only support master and LTS version upgrade.") + sys.exit(1) + + fork_clone_repo(gt, u_pkg) + + if not update_check(pkg_spec, pkg_ver, rec_up_ver): + sys.exit(1) + + if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver): + sys.exit(1) + + create_spec(u_pkg, spec_str, pkg_ver, rec_up_ver) + + +def auto_update_repo(gt, u_branch, u_repo): + """ + Auto upgrade based on given branch for packages in given repository + """ + repo_yaml = gt.get_community(u_repo) + if not repo_yaml: + print("{repo}.yaml in community is empty.".format(repo=u_repo)) + sys.exit(1) + + pkg_info = yaml.load(repo_yaml, Loader=yaml.loader) + pkg_list = pkg_info.get("repositories") + for pkg in pkg_list: + pkg_name = pkg.get("name") + spec_str = gt.get_spec(pkg_name, u_branch) + if not spec_str: + print("{pkg}.spec can't be found on the {br} branch. ".format( + pkg=pkg_name, br=u_branch)) + continue + pkg_spec = Spec.from_string(spec_str) + pkg_ver = replace_macros(pkg_spec.version, pkg_spec) + + pkg_tags = oa_upgradable.get_ver_tags(gt, pkg_name) + if pkg_tags is None: + continue + ver_rec = version_recommend.VersionRecommend(pkg_tags, pkg_ver, 0) + rec_up_ver = pkg_ver + if re.search("master", u_branch): + rec_up_ver = ver_rec.latest_version + elif re.search("LTS", u_branch): + rec_up_ver = ver_rec.maintain_version + else: + print("Only support master and LTS version upgrade.") + sys.exit(1) + + fork_clone_repo(gt, pkg_name) + + if not update_check(pkg_spec, pkg_ver, rec_up_ver): + continue + + if not download_src(gt, pkg_spec, pkg_ver, rec_up_ver): + continue + + create_spec(pkg_name, spec_str, pkg_ver, rec_up_ver) + + if __name__ == "__main__": pars = argparse.ArgumentParser() - pars.add_argument("pkg", type=str, help="The package to be upgraded") + pars.add_argument("repo_pkg", type=str, help="The repository or package to be upgraded") + pars.add_argument("branch", type=str, help="The branch that upgrade based") + pars.add_argument("-u", "--update", type=str, help="Auto upgrade for packages in repository or single package", + choices=["repo", "pkg"]) pars.add_argument("-n", "--new_version", type=str, help="New upstream version of package will be upgrade to") pars.add_argument("-s", "--create_spec", help="Create spec file", action="store_true") pars.add_argument("-d", "--download", help="Download upstream source code", action="store_true") - pars.add_argument("-f", "--fork", help="fork src-openeuler repo into users", action="store_true") - pars.add_argument("-c", "--clone", help="clone privatge repo to local", action="store_true") + pars.add_argument("-fc", "--fork_then_clone", help="Fork src-openeuler repo into users, then clone to local", + action="store_true") pars.add_argument("-p", "--PR", help="Create upgrade PR", action="store_true") args = pars.parse_args() + + user_gitee = gitee.Gitee() - my_gitee = gitee.Gitee() - my_version = version_recommend.VersionType() - spec_string= my_gitee.get_spec(args.pkg) - - s_spec = Spec.from_string(spec_string) - cur_ver = replace_macros(s_spec.version, s_spec) + if args.update: + if args.update == "repo": + auto_update_repo(user_gitee, args.branch, args.repo_pkg) + else: + auto_update_pkg(user_gitee, args.branch, args.repo_pkg) + else: + spec_string = user_gitee.get_spec(args.repo_pkg, args.branch) + if not spec_string: + print("{pkg}.spec can't be found on the {br} branch. ".format(pkg=args.repo_pkg, br=args.branch)) + sys.exit(1) + spec_file = Spec.from_string(spec_string) + cur_version = replace_macros(spec_file.version, spec_file) - if args.fork: - if not my_gitee.fork_repo(args.pkg): - print("The repo of {pkg} seems to have been forked.".format(pkg=args.pkg)) + if args.fork_then_clone: + fork_clone_repo(user_gitee, args.repo_pkg) - if args.clone: - user=my_gitee.token["user"] - subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=user, pkg=args.pkg)]) - os.chdir(args.pkg) + if args.download or args.create_spec: + if not args.new_version: + print("Please specify the upgraded version of the {repo}".format(repo=args.repo_pkg)) + sys.exit(1) + elif not update_check(spec_file, cur_version, args.new_version): + sys.exit(1) - if args.download: - source_file = download_source_url(s_spec, cur_ver, args.new_version) - if source_file: - print(source_file) - else: - source_file = download_upstream_url(my_gitee, args.pkg, args.new_version) - if source_file: - print(source_file) - else: - print("Failed to download the latest source code.") + if args.download: + if not download_src(user_gitee, spec_file, cur_version, args.new_version): sys.exit(1) - if args.create_spec: - if len(s_spec.patches) >= 1: - print("I'm too naive to handle complicated package.") - print("This package has multiple in-house patches.") - sys.exit(1) - if(my_version.compare(args.new_version, cur_ver) ==1): - create_spec(args.pkg, spec_string, cur_ver, args.new_version) - else: - print("Please check version of {pkg} will upgrade to, it's current version is {version}.".format( - pkg=args.pkg, version=cur_ver)) + if args.create_spec: + create_spec(args.repo_pkg, spec_string, cur_version, args.new_version) - if args.PR: - my_gitee.create_pr(my_gitee.token["user"], args.pkg) + if args.PR: + user_gitee.create_pr(user_gitee.token["user"], args.repo_pkg) diff --git a/advisors/version_recommend.py b/advisors/version_recommend.py index d4b0d7c848acac74864a2bc9f4e96efe708dbf94..d8f211225f3de22d4f222ff1cfb6d2c7a59dbefb 100755 --- a/advisors/version_recommend.py +++ b/advisors/version_recommend.py @@ -72,7 +72,7 @@ class VersionType(object): :return 0: z1 equal then z2 :raises: None """ - return self._compare(self, z1, z2) + return self._compare(z1, z2) def _compare(self, z1, z2): """ @@ -91,11 +91,15 @@ class VersionType(object): len2 = len(d2) length = min(len1, len2) for index in range(length): - if d1[index].isdigit() and d1[index].isdigit(): + if d1[index].isdigit() and d2[index].isdigit(): if int(d1[index]) > int(d2[index]): return 1 elif int(d1[index]) < int(d2[index]): return -1 + elif d1[index].isdigit(): + return 1 + elif d2[index].isdigit(): + return -1 else: if d1[index] > d2[index]: return 1 @@ -126,7 +130,7 @@ class VersionType(object): :returns: The split result :raises: None """ - for f, s in re.findall(r'([\d]+)|([^\d.]+)', x): + for f, s in re.findall(r'([\d]+)|([^\d.-]+)', x): if f: float(f) yield f @@ -1085,6 +1089,10 @@ class VersionRecommend(object): if m is None: # 版本号应该是数字开头 return False + m = re.search(r'b\d', version) + if not m is None: + return False + if 'rc' in version \ or 'RC' in version \ or 'dev' in version \