From b88e7add10a4e920e243a679aac720ebeb689131 Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 21 Sep 2020 18:26:37 +0800 Subject: [PATCH 1/2] Update and Support patch match 1.rename simple-update-robot 2.support to get origin release tag 3.support patch match commits within two release tag --- advisors/check_upstream.py | 45 +++--- advisors/match_patches.py | 128 ++++++++++++++++++ advisors/oa_upgradable.py | 20 +-- ...update-robot.py => simple_update_robot.py} | 3 +- 4 files changed, 167 insertions(+), 29 deletions(-) create mode 100644 advisors/match_patches.py rename advisors/{simple-update-robot.py => simple_update_robot.py} (99%) diff --git a/advisors/check_upstream.py b/advisors/check_upstream.py index 873c614b..c5a78cfc 100755 --- a/advisors/check_upstream.py +++ b/advisors/check_upstream.py @@ -84,7 +84,7 @@ def dirty_redirect_tricks(url, resp): if "" in cookie: cookie.remove("") return need_trick, new_url, list(cookie) -def check_hg_raw(info): +def check_hg_raw(info, clean_tag): eprint("{repo} > Using hg raw-tags".format(repo=info["src_repo"]+"/raw-tags")) resp = load_last_query_result(info) if resp == "": @@ -112,11 +112,12 @@ def check_hg_raw(info): tags = [] for l in resp.splitlines(): tags.append(l.split()[0]) - result_list = clean_tags(tags, info) + if clean_tag: + result_list = clean_tags(tags, info) return result_list -def check_hg(info): +def check_hg(info, clean_tag): eprint("{repo} > Using hg json-tags".format(repo=info["src_repo"]+"/json-tags")) resp = load_last_query_result(info) if resp == "": @@ -146,11 +147,12 @@ def check_hg(info): sort_tags = tags_json["tags"] sort_tags.sort(reverse=True, key=lambda x: x['date'][0]) result_list = [tag['tag'] for tag in sort_tags] - result_list = clean_tags(result_list, info) + if clean_tag: + result_list = clean_tags(result_list, info) return result_list -def check_metacpan(info): +def check_metacpan(info, clean_tag): resp = load_last_query_result(info) if resp == "": headers = { @@ -176,7 +178,8 @@ def check_metacpan(info): last_query["time_stamp"] = datetime.now() last_query["raw_data"] = resp info["last_query"] = last_query - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags def check_pypi(info): @@ -236,7 +239,7 @@ def __git_resp_to_tags(resp): tags.append(tag) return tags -def check_git (info): +def check_git (info, clean_tag): resp = load_last_query_result(info) if resp == "": resp = __check_git_helper(info["src_repo"]) @@ -246,11 +249,12 @@ def check_git (info): info["last_query"] = last_query tags = __git_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_github(info): +def check_github(info, clean_tag): resp = load_last_query_result(info) if info.get("query_type", "git-ls") != "git-ls": resp = "" @@ -266,10 +270,11 @@ def check_github(info): info["query_type"] = "git-ls" tags = __git_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_gnu_ftp(info): +def check_gnu_ftp(info, clean_tag): headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' } @@ -283,11 +288,12 @@ def check_gnu_ftp(info): m = re_pattern.search(l) if m: tags.append(m[1]) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_gnome(info): +def check_gnome(info, clean_tag): resp = load_last_query_result(info) src_repos = info["src_repo"].split("/") if len(src_repos) == 1: @@ -303,10 +309,11 @@ def check_gnome(info): info["last_query"] = last_query tags = __git_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_gitee(info): +def check_gitee(info, clean_tag): resp = load_last_query_result(info) repo_url = "https://gitee.com/" + info["src_repo"] if resp == "": @@ -317,10 +324,11 @@ def check_gitee(info): info["last_query"] = last_query tags = __git_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_svn(info): +def check_svn(info, clean_tag): resp = load_last_query_result(info) tag_dir = info.get("tag_dir", "tags") repo_url = info["src_repo"] + "/" + tag_dir @@ -332,7 +340,8 @@ def check_svn(info): info["last_query"] = last_query tags = __svn_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags diff --git a/advisors/match_patches.py b/advisors/match_patches.py new file mode 100644 index 00000000..2ab0a2a7 --- /dev/null +++ b/advisors/match_patches.py @@ -0,0 +1,128 @@ +#!/usr/bin/python3 + + +import os +import re +import sys +import yaml +import gitee +import argparse +import subprocess +import oa_upgradable +import simple_update_robot + + +def patches_match(pkg, branch, c_ver, u_ver): + user_gitee = gitee.Gitee() + + pkg_info = user_gitee.get_yaml(pkg) + if not pkg_info: + return None + + pkg_yaml = yaml.load(pkg_info, Loader=yaml.Loader) + + separa = pkg_yaml.get("separator") + if not separa: + separa = pkg_yaml.get("seperator") + + pkg_tags = oa_upgradable.get_ver_tags(user_gitee, pkg, clean_tag=False) + c_tag = "" + u_tag = "" + for tag in pkg_tags: + if re.search("\d[a-z]\d*", tag, re.IGNORECASE) \ + or re.search(r'rc\d*$', tag, re.IGNORECASE) \ + or re.search(r'dev\d*$', tag, re.IGNORECASE) \ + or re.search(r'beta\d*$', tag, re.IGNORECASE) \ + or re.search(r"alpha.*$", tag, re.IGNORECASE) \ + or re.search(r'pl\d*$', tag, re.IGNORECASE) \ + or re.search(r'pre\d*$', tag, re.IGNORECASE) \ + or re.search(r'bp\d*$', tag, re.IGNORECASE): + continue + elif c_ver in tag or c_ver.replace(".", separa) in tag: + c_tag = tag + elif u_ver in tag or u_ver.replace(".", separa) in tag: + u_tag = tag + if not c_tag or not u_tag: + print("WARNING: c_tag or u_tag is null.") + sys.exit(1) + + vc_type = pkg_yaml.get("version_control", None) + if vc_type is None: + print("Missing version_control in {}.yaml".format(pkg)) + return None + if vc_type == "github": + repo_url = "https://github.com/" + pkg_yaml.get("src_repo") + ".git" + elif vc_type == "git": + repo_url = pkg_yaml.get("src_repo") + elif vc_type == "gitee": + repo_url = "https://gitee.com/" + pkg_yaml.get("src_repo") + ".git" + elif vc_type == "gitlab.gnome": + src_path = pkg_yaml.get("src_repo").split("/") + if len(src_path) == 1: + repo_url = "https://gitlab.gnome.org/GNOME/" + pkg_yaml.get("src_repo") + ".git" + else: + repo_url = "https://gitlab.gnome.org/"+ pkg_yaml.get("src_repo") + ".git" + elif vc_type == "svn": + repo_url = pkg_yaml.get("src_repo") + else: + print("WARNING: Current can't support {vc} for {repo}".format(vc=vc_type, repo=pkg)) + return None + + dir_pkg = os.path.basename(repo_url).split(".")[0] + if os.path.exists(dir_pkg): + subprocess.call(["rm -rf {}".format(dir_pkg)], shell=True) + + if vc_type == "svn": + pass + else: + print("git clone {url}".format(tag=u_tag, url=repo_url)) + subprocess.call(["git clone {url}".format(tag=u_tag, url=repo_url)], shell=True) + + os.chdir(dir_pkg) + commit_str = subprocess.check_output(["git log -p {t1}...{t2}".format( + t1=u_tag, t2=c_tag)], shell=True).decode('ISO-8859-1').strip('\n') + os.chdir(os.pardir) + subprocess.call(["rm -rf {}".format(dir_pkg)], shell=True) + + if not commit_str: + print("WARNING: No commits between {t1} and {t2}.".format(t1=u_tag, t2=c_tag)) + return None + + if os.path.exists(pkg): + subprocess.call(["rm -rf {}".format(pkg)], shell=True) + + simple_update_robot.fork_clone_repo(user_gitee, pkg, branch) + patch_match = {} + for fn in os.listdir(pkg): + if not fn.endswith(".patch"): + continue + f = open("{path}/{file_name}".format(path=pkg, file_name=fn), "r") + index_match = [] + part_matched = False + for line in f.readlines(): + if re.match("index", line): + index_head = line[0:13] + if index_head in commit_str: + index_match.append(line.strip('\n')) + else: + part_matched = True + + if len(index_match): + if part_matched: + patch_match[fn] = index_match + else: + patch_match[fn]= None + + subprocess.call(["rm -rf {}".format(pkg)], shell=True) + return patch_match + + +if __name__ == "__main__": + pars = argparse.ArgumentParser() + pars.add_argument("pkg", type=str, help="The package to be matched.") + pars.add_argument("branch", type=str, help="Branch to be matched.") + pars.add_argument("cur_ver", type=str, help="Current version of package.") + pars.add_argument("up_ver", type=str, help="Upgrade version of package.") + args = pars.parse_args() + + patches_match(args.pkg, args.branch, args.cur_ver, args.up_ver) diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py index 172be752..61ca7082 100755 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -24,7 +24,7 @@ def _filter_except(excpts, sources): return sources -def get_ver_tags(gt, repo, cwd_path=None): +def get_ver_tags(gt, repo, clean_tag=True, cwd_path=None): """ Get version tags of given package """ @@ -47,25 +47,25 @@ def get_ver_tags(gt, repo, cwd_path=None): print("Missing version_control in YAML file") return None if vc_type == "hg": - tags = check_upstream.check_hg(pkg_info) + tags = check_upstream.check_hg(pkg_info, clean_tag) elif vc_type == "hg-raw": - tags = check_upstream.check_hg_raw(pkg_info) + tags = check_upstream.check_hg_raw(pkg_info, clean_tag) elif vc_type == "github": - tags = check_upstream.check_github(pkg_info) + tags = check_upstream.check_github(pkg_info, clean_tag) elif vc_type == "git": - tags = check_upstream.check_git(pkg_info) + tags = check_upstream.check_git(pkg_info, clean_tag) elif vc_type == "gitlab.gnome": - tags = check_upstream.check_gnome(pkg_info) + tags = check_upstream.check_gnome(pkg_info, clean_tag) elif vc_type == "svn": - tags = check_upstream.check_svn(pkg_info) + tags = check_upstream.check_svn(pkg_info, clean_tag) elif vc_type == "metacpan": - tags = check_upstream.check_metacpan(pkg_info) + tags = check_upstream.check_metacpan(pkg_info, clean_tag) elif vc_type == "pypi": tags = check_upstream.check_pypi(pkg_info) elif vc_type == "gitee": - tags = check_upstream.check_gitee(pkg_info) + tags = check_upstream.check_gitee(pkg_info, clean_tag) elif vc_type == "gnu-ftp": - tags = check_upstream.check_gnu_ftp(pkg_info) + tags = check_upstream.check_gnu_ftp(pkg_info, clean_tag) else: print("Unsupport version control method {vc}".format(vc=vc_type)) return None diff --git a/advisors/simple-update-robot.py b/advisors/simple_update_robot.py similarity index 99% rename from advisors/simple-update-robot.py rename to advisors/simple_update_robot.py index 35e95b6a..38e3cc35 100755 --- a/advisors/simple-update-robot.py +++ b/advisors/simple_update_robot.py @@ -97,7 +97,8 @@ def fork_clone_repo(gt, repo, br): name = gt.token["user"] while True: - subprocess.call(["rm", "-rf", "{pkg}".format(pkg=repo)]) + if os.path.exists(repo): + subprocess.call(["rm", "-rf", "{pkg}".format(pkg=repo)]) subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=name, pkg=repo)]) os.chdir(repo) if subprocess.call(["git", "checkout", "{branch}".format(branch=br)]): -- Gitee From d5a189ba770d6d7d39bbbce09015372ab342318a Mon Sep 17 00:00:00 2001 From: Leo Fang Date: Mon, 21 Sep 2020 19:11:17 +0800 Subject: [PATCH 2/2] Optimization and fix 1.optimization for dir delete and code struct 2.fix errors detected by pylint3 Signed-off-by: Leo Fang --- advisors/check_upstream.py | 26 ++++---- advisors/match_patches.py | 109 ++++++++++++++++++++++++-------- advisors/oa_upgradable.py | 2 +- advisors/simple_update_robot.py | 3 +- 4 files changed, 99 insertions(+), 41 deletions(-) diff --git a/advisors/check_upstream.py b/advisors/check_upstream.py index 5edf00f9..0b7a3dfe 100755 --- a/advisors/check_upstream.py +++ b/advisors/check_upstream.py @@ -99,7 +99,7 @@ def dirty_redirect_tricks(url, resp): cookie.remove("") return need_trick, new_url, list(cookie) -def check_hg_raw(info, clean_tag): +def check_hg_raw(info, clean_tag=True): """ Check hg version info via raw-tags """ @@ -135,7 +135,7 @@ def check_hg_raw(info, clean_tag): return result_list -def check_hg(info, clean_tag): +def check_hg(info, clean_tag=True): """ Check hg version info via json """ @@ -173,7 +173,7 @@ def check_hg(info, clean_tag): return result_list -def check_metacpan(info, clean_tag): +def check_metacpan(info, clean_tag=True): """ Check perl module version info via metacpan api """ @@ -213,7 +213,7 @@ def check_metacpan(info, clean_tag): return tags -def check_pypi(info, clean_tag): +def check_pypi(info, clean_tag=True): """ Check python module version info via pypi api """ @@ -232,10 +232,12 @@ def check_pypi(info, clean_tag): if not tags: eprint("{repo} > No Response or JSON parse failed".format(repo=info["src_repo"])) sys.exit(1) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_rubygem(info): +def check_rubygem(info, clean_tag=True): """ Check ruby module version info via rubygem api """ @@ -254,6 +256,8 @@ def check_rubygem(info): if not tags: eprint("{repo} > No Response or JSON parse failed".format(repo=info["src_repo"])) sys.exit(1) + if clean_tag: + tags = clean_tags(tags, info) return tags def __check_subprocess(cmd_list): @@ -312,7 +316,7 @@ def __git_resp_to_tags(resp): tags.append(tag) return tags -def check_git (info, clean_tag): +def check_git(info, clean_tag=True): """ Check version info via git command """ @@ -330,7 +334,7 @@ def check_git (info, clean_tag): return tags -def check_github(info, clean_tag): +def check_github(info, clean_tag=True): """ Check version info via github api """ @@ -353,7 +357,7 @@ def check_github(info, clean_tag): tags = clean_tags(tags, info) return tags -def check_gnu_ftp(info, clean_tag): +def check_gnu_ftp(info, clean_tag=True): """ Check version info via compare ftp release tar file for gnu """ @@ -394,7 +398,7 @@ def check_ftp(info): tags = clean_tags(tags, info) return tags -def check_gnome(info, clean_tag): +def check_gnome(info, clean_tag=True): """ Check version info via gitlab.gnome.org api """ @@ -417,7 +421,7 @@ def check_gnome(info, clean_tag): tags = clean_tags(tags, info) return tags -def check_gitee(info, clean_tag): +def check_gitee(info, clean_tag=True): """ Check version info via gitee """ @@ -435,7 +439,7 @@ def check_gitee(info, clean_tag): tags = clean_tags(tags, info) return tags -def check_svn(info, clean_tag): +def check_svn(info, clean_tag=True): """ Check version info via svn """ diff --git a/advisors/match_patches.py b/advisors/match_patches.py index 2ab0a2a7..016cfa82 100644 --- a/advisors/match_patches.py +++ b/advisors/match_patches.py @@ -1,18 +1,71 @@ #!/usr/bin/python3 - +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# +# ******************************************************************************/ +""" +This module supply the function for patch matching +""" import os import re -import sys import yaml import gitee +import shutil import argparse import subprocess import oa_upgradable import simple_update_robot +def _get_github_url(pkg_info): + """Get github repo url of package""" + url = "https://github.com/" + pkg_info.get("src_repo") + ".git" + return url + + +def _get_git_url(pkg_info): + """Get git repo url of package""" + url = pkg_info.get("src_repo") + return url + + +def _get_gitee_url(pkg_info): + """Get gitee repo url of package""" + url = "https://github.com/" + pkg_info.get("src_repo") + ".git" + return url + + +def _get_gitlab_url(pkg_info): + """Get gitlab repo url of package""" + src_path = pkg_info.get("src_repo").split("/") + if len(src_path) == 1: + url = "https://gitlab.gnome.org/GNOME/" + pkg_info.get("src_repo") + ".git" + else: + url = "https://gitlab.gnome.org/" + pkg_info.get("src_repo") + ".git" + return url + + +def _get_svn_url(pkg_info): + """Get svn repo url of package""" + url = pkg_info.get("src_repo") + return url + + def patches_match(pkg, branch, c_ver, u_ver): + """ + Patches match function for pkg on src-openeuler + when patch all matched, then return 'patch_name': None + when patch part matched, then return 'patch_name': matched index number + """ user_gitee = gitee.Gitee() pkg_info = user_gitee.get_yaml(pkg) @@ -23,13 +76,14 @@ def patches_match(pkg, branch, c_ver, u_ver): separa = pkg_yaml.get("separator") if not separa: - separa = pkg_yaml.get("seperator") + print("WARNING: Key word: separator can't be found in {}.yaml".format(pkg)) + return None pkg_tags = oa_upgradable.get_ver_tags(user_gitee, pkg, clean_tag=False) c_tag = "" u_tag = "" for tag in pkg_tags: - if re.search("\d[a-z]\d*", tag, re.IGNORECASE) \ + if re.search(r'\d[a-z]\d*', tag, re.IGNORECASE) \ or re.search(r'rc\d*$', tag, re.IGNORECASE) \ or re.search(r'dev\d*$', tag, re.IGNORECASE) \ or re.search(r'beta\d*$', tag, re.IGNORECASE) \ @@ -43,53 +97,52 @@ def patches_match(pkg, branch, c_ver, u_ver): elif u_ver in tag or u_ver.replace(".", separa) in tag: u_tag = tag if not c_tag or not u_tag: - print("WARNING: c_tag or u_tag is null.") - sys.exit(1) + print("WARNING: current version tag or upgrade version tag can't be found in repository.") + return None vc_type = pkg_yaml.get("version_control", None) if vc_type is None: print("Missing version_control in {}.yaml".format(pkg)) return None - if vc_type == "github": - repo_url = "https://github.com/" + pkg_yaml.get("src_repo") + ".git" - elif vc_type == "git": - repo_url = pkg_yaml.get("src_repo") - elif vc_type == "gitee": - repo_url = "https://gitee.com/" + pkg_yaml.get("src_repo") + ".git" - elif vc_type == "gitlab.gnome": - src_path = pkg_yaml.get("src_repo").split("/") - if len(src_path) == 1: - repo_url = "https://gitlab.gnome.org/GNOME/" + pkg_yaml.get("src_repo") + ".git" - else: - repo_url = "https://gitlab.gnome.org/"+ pkg_yaml.get("src_repo") + ".git" - elif vc_type == "svn": - repo_url = pkg_yaml.get("src_repo") + + switcher = { + "github":_get_github_url, + "git":_get_git_url, + "gitee":_get_gitee_url, + "gitlab.gnome":_get_gitlab_url, + "svn":_get_svn_url + } + + get_repo_url = switcher.get(vc_type, None) + if get_repo_url: + repo_url = get_repo_url(pkg_yaml) else: print("WARNING: Current can't support {vc} for {repo}".format(vc=vc_type, repo=pkg)) return None dir_pkg = os.path.basename(repo_url).split(".")[0] if os.path.exists(dir_pkg): - subprocess.call(["rm -rf {}".format(dir_pkg)], shell=True) + shutil.rmtree(dir_pkg, ignore_errors=True) if vc_type == "svn": - pass + print("WARNING: Now can't support patch match in svn type.") + return None else: - print("git clone {url}".format(tag=u_tag, url=repo_url)) - subprocess.call(["git clone {url}".format(tag=u_tag, url=repo_url)], shell=True) + print("git clone {url}".format(url=repo_url)) + subprocess.call(["git clone {url}".format(url=repo_url)], shell=True) os.chdir(dir_pkg) commit_str = subprocess.check_output(["git log -p {t1}...{t2}".format( t1=u_tag, t2=c_tag)], shell=True).decode('ISO-8859-1').strip('\n') os.chdir(os.pardir) - subprocess.call(["rm -rf {}".format(dir_pkg)], shell=True) + shutil.rmtree(dir_pkg, ignore_errors=True) if not commit_str: print("WARNING: No commits between {t1} and {t2}.".format(t1=u_tag, t2=c_tag)) return None if os.path.exists(pkg): - subprocess.call(["rm -rf {}".format(pkg)], shell=True) + shutil.rmtree(pkg, ignore_errors=True) simple_update_robot.fork_clone_repo(user_gitee, pkg, branch) patch_match = {} @@ -111,9 +164,9 @@ def patches_match(pkg, branch, c_ver, u_ver): if part_matched: patch_match[fn] = index_match else: - patch_match[fn]= None + patch_match[fn] = None - subprocess.call(["rm -rf {}".format(pkg)], shell=True) + shutil.rmtree(pkg, ignore_errors=True) return patch_match diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py index 22462ba4..e4fbb603 100755 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -24,7 +24,7 @@ def _filter_except(excpts, sources): return sources -def get_ver_tags(gt, repo, clean_tag=True, cwd_path=None): +def get_ver_tags(my_gitee, repo, clean_tag=True, cwd_path=None): """ Get version tags of given package """ diff --git a/advisors/simple_update_robot.py b/advisors/simple_update_robot.py index 38e3cc35..12a80fd8 100755 --- a/advisors/simple_update_robot.py +++ b/advisors/simple_update_robot.py @@ -22,6 +22,7 @@ import os.path import re import time import datetime +import shutil import oa_upgradable import version_recommend @@ -98,7 +99,7 @@ def fork_clone_repo(gt, repo, br): name = gt.token["user"] while True: if os.path.exists(repo): - subprocess.call(["rm", "-rf", "{pkg}".format(pkg=repo)]) + shutil.rmtree(repo, ignore_errors=True) subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=name, pkg=repo)]) os.chdir(repo) if subprocess.call(["git", "checkout", "{branch}".format(branch=br)]): -- Gitee