diff --git a/advisors/check_upstream.py b/advisors/check_upstream.py index dc74d2d7c717dedcf4b4b05936777eed8561b76f..0b7a3dfec3c21b335ee91d92c93c8b27c426031c 100755 --- a/advisors/check_upstream.py +++ b/advisors/check_upstream.py @@ -99,7 +99,7 @@ def dirty_redirect_tricks(url, resp): cookie.remove("") return need_trick, new_url, list(cookie) -def check_hg_raw(info): +def check_hg_raw(info, clean_tag=True): """ Check hg version info via raw-tags """ @@ -130,11 +130,12 @@ def check_hg_raw(info): tags = [] for line in resp.splitlines(): tags.append(line.split()[0]) - result_list = clean_tags(tags, info) + if clean_tag: + result_list = clean_tags(tags, info) return result_list -def check_hg(info): +def check_hg(info, clean_tag=True): """ Check hg version info via json """ @@ -167,11 +168,12 @@ def check_hg(info): sort_tags = tags_json["tags"] sort_tags.sort(reverse=True, key=lambda x: x['date'][0]) result_list = [tag['tag'] for tag in sort_tags] - result_list = clean_tags(result_list, info) + if clean_tag: + result_list = clean_tags(result_list, info) return result_list -def check_metacpan(info): +def check_metacpan(info, clean_tag=True): """ Check perl module version info via metacpan api """ @@ -206,11 +208,12 @@ def check_metacpan(info): last_query["time_stamp"] = datetime.now() last_query["raw_data"] = resp info["last_query"] = last_query - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_pypi(info): +def check_pypi(info, clean_tag=True): """ Check python module version info via pypi api """ @@ -229,10 +232,12 @@ def check_pypi(info): if not tags: eprint("{repo} > No Response or JSON parse failed".format(repo=info["src_repo"])) sys.exit(1) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_rubygem(info): +def check_rubygem(info, clean_tag=True): """ Check ruby module version info via rubygem api """ @@ -251,6 +256,8 @@ def check_rubygem(info): if not tags: eprint("{repo} > No Response or JSON parse failed".format(repo=info["src_repo"])) sys.exit(1) + if clean_tag: + tags = clean_tags(tags, info) return tags def __check_subprocess(cmd_list): @@ -309,7 +316,7 @@ def __git_resp_to_tags(resp): tags.append(tag) return tags -def check_git(info): +def check_git(info, clean_tag=True): """ Check version info via git command """ @@ -322,11 +329,12 @@ def check_git(info): info["last_query"] = last_query tags = __git_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_github(info): +def check_github(info, clean_tag=True): """ Check version info via github api """ @@ -345,10 +353,11 @@ def check_github(info): info["query_type"] = "git-ls" tags = __git_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_gnu_ftp(info): +def check_gnu_ftp(info, clean_tag=True): """ Check version info via compare ftp release tar file for gnu """ @@ -365,7 +374,8 @@ def check_gnu_ftp(info): result = re_pattern.search(line) if result: tags.append(result[1]) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags def check_ftp(info): @@ -388,7 +398,7 @@ def check_ftp(info): tags = clean_tags(tags, info) return tags -def check_gnome(info): +def check_gnome(info, clean_tag=True): """ Check version info via gitlab.gnome.org api """ @@ -407,10 +417,11 @@ def check_gnome(info): info["last_query"] = last_query tags = __git_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_gitee(info): +def check_gitee(info, clean_tag=True): """ Check version info via gitee """ @@ -424,10 +435,11 @@ def check_gitee(info): info["last_query"] = last_query tags = __git_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags -def check_svn(info): +def check_svn(info, clean_tag=True): """ Check version info via svn """ @@ -442,7 +454,8 @@ def check_svn(info): info["last_query"] = last_query tags = __svn_resp_to_tags(resp) - tags = clean_tags(tags, info) + if clean_tag: + tags = clean_tags(tags, info) return tags diff --git a/advisors/match_patches.py b/advisors/match_patches.py new file mode 100644 index 0000000000000000000000000000000000000000..016cfa82c5d420cd550022eafb70035317702dff --- /dev/null +++ b/advisors/match_patches.py @@ -0,0 +1,181 @@ +#!/usr/bin/python3 +#****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# +# ******************************************************************************/ +""" +This module supply the function for patch matching +""" + +import os +import re +import yaml +import gitee +import shutil +import argparse +import subprocess +import oa_upgradable +import simple_update_robot + + +def _get_github_url(pkg_info): + """Get github repo url of package""" + url = "https://github.com/" + pkg_info.get("src_repo") + ".git" + return url + + +def _get_git_url(pkg_info): + """Get git repo url of package""" + url = pkg_info.get("src_repo") + return url + + +def _get_gitee_url(pkg_info): + """Get gitee repo url of package""" + url = "https://github.com/" + pkg_info.get("src_repo") + ".git" + return url + + +def _get_gitlab_url(pkg_info): + """Get gitlab repo url of package""" + src_path = pkg_info.get("src_repo").split("/") + if len(src_path) == 1: + url = "https://gitlab.gnome.org/GNOME/" + pkg_info.get("src_repo") + ".git" + else: + url = "https://gitlab.gnome.org/" + pkg_info.get("src_repo") + ".git" + return url + + +def _get_svn_url(pkg_info): + """Get svn repo url of package""" + url = pkg_info.get("src_repo") + return url + + +def patches_match(pkg, branch, c_ver, u_ver): + """ + Patches match function for pkg on src-openeuler + when patch all matched, then return 'patch_name': None + when patch part matched, then return 'patch_name': matched index number + """ + user_gitee = gitee.Gitee() + + pkg_info = user_gitee.get_yaml(pkg) + if not pkg_info: + return None + + pkg_yaml = yaml.load(pkg_info, Loader=yaml.Loader) + + separa = pkg_yaml.get("separator") + if not separa: + print("WARNING: Key word: separator can't be found in {}.yaml".format(pkg)) + return None + + pkg_tags = oa_upgradable.get_ver_tags(user_gitee, pkg, clean_tag=False) + c_tag = "" + u_tag = "" + for tag in pkg_tags: + if re.search(r'\d[a-z]\d*', tag, re.IGNORECASE) \ + or re.search(r'rc\d*$', tag, re.IGNORECASE) \ + or re.search(r'dev\d*$', tag, re.IGNORECASE) \ + or re.search(r'beta\d*$', tag, re.IGNORECASE) \ + or re.search(r"alpha.*$", tag, re.IGNORECASE) \ + or re.search(r'pl\d*$', tag, re.IGNORECASE) \ + or re.search(r'pre\d*$', tag, re.IGNORECASE) \ + or re.search(r'bp\d*$', tag, re.IGNORECASE): + continue + elif c_ver in tag or c_ver.replace(".", separa) in tag: + c_tag = tag + elif u_ver in tag or u_ver.replace(".", separa) in tag: + u_tag = tag + if not c_tag or not u_tag: + print("WARNING: current version tag or upgrade version tag can't be found in repository.") + return None + + vc_type = pkg_yaml.get("version_control", None) + if vc_type is None: + print("Missing version_control in {}.yaml".format(pkg)) + return None + + switcher = { + "github":_get_github_url, + "git":_get_git_url, + "gitee":_get_gitee_url, + "gitlab.gnome":_get_gitlab_url, + "svn":_get_svn_url + } + + get_repo_url = switcher.get(vc_type, None) + if get_repo_url: + repo_url = get_repo_url(pkg_yaml) + else: + print("WARNING: Current can't support {vc} for {repo}".format(vc=vc_type, repo=pkg)) + return None + + dir_pkg = os.path.basename(repo_url).split(".")[0] + if os.path.exists(dir_pkg): + shutil.rmtree(dir_pkg, ignore_errors=True) + + if vc_type == "svn": + print("WARNING: Now can't support patch match in svn type.") + return None + else: + print("git clone {url}".format(url=repo_url)) + subprocess.call(["git clone {url}".format(url=repo_url)], shell=True) + + os.chdir(dir_pkg) + commit_str = subprocess.check_output(["git log -p {t1}...{t2}".format( + t1=u_tag, t2=c_tag)], shell=True).decode('ISO-8859-1').strip('\n') + os.chdir(os.pardir) + shutil.rmtree(dir_pkg, ignore_errors=True) + + if not commit_str: + print("WARNING: No commits between {t1} and {t2}.".format(t1=u_tag, t2=c_tag)) + return None + + if os.path.exists(pkg): + shutil.rmtree(pkg, ignore_errors=True) + + simple_update_robot.fork_clone_repo(user_gitee, pkg, branch) + patch_match = {} + for fn in os.listdir(pkg): + if not fn.endswith(".patch"): + continue + f = open("{path}/{file_name}".format(path=pkg, file_name=fn), "r") + index_match = [] + part_matched = False + for line in f.readlines(): + if re.match("index", line): + index_head = line[0:13] + if index_head in commit_str: + index_match.append(line.strip('\n')) + else: + part_matched = True + + if len(index_match): + if part_matched: + patch_match[fn] = index_match + else: + patch_match[fn] = None + + shutil.rmtree(pkg, ignore_errors=True) + return patch_match + + +if __name__ == "__main__": + pars = argparse.ArgumentParser() + pars.add_argument("pkg", type=str, help="The package to be matched.") + pars.add_argument("branch", type=str, help="Branch to be matched.") + pars.add_argument("cur_ver", type=str, help="Current version of package.") + pars.add_argument("up_ver", type=str, help="Upgrade version of package.") + args = pars.parse_args() + + patches_match(args.pkg, args.branch, args.cur_ver, args.up_ver) diff --git a/advisors/oa_upgradable.py b/advisors/oa_upgradable.py index f0219def4a38c1463139196b1037648b49bd04be..e4fbb603d386e73788746f49317de8c56b025c76 100755 --- a/advisors/oa_upgradable.py +++ b/advisors/oa_upgradable.py @@ -24,7 +24,7 @@ def _filter_except(excpts, sources): return sources -def get_ver_tags(my_gitee, repo, cwd_path=None): +def get_ver_tags(my_gitee, repo, clean_tag=True, cwd_path=None): """ Get version tags of given package """ @@ -65,7 +65,7 @@ def get_ver_tags(my_gitee, repo, cwd_path=None): check_method = switcher.get(vc_type, None) if check_method: - tags = check_method(pkg_info) + tags = check_method(pkg_info, clean_tag) else: print("Unsupport version control method {vc}".format(vc=vc_type)) return None diff --git a/advisors/simple-update-robot.py b/advisors/simple_update_robot.py similarity index 99% rename from advisors/simple-update-robot.py rename to advisors/simple_update_robot.py index 35e95b6a78f71f026c949c386b3eb5eede1f4982..12a80fd8b85f681defa911d403a5497e06a9bcc2 100755 --- a/advisors/simple-update-robot.py +++ b/advisors/simple_update_robot.py @@ -22,6 +22,7 @@ import os.path import re import time import datetime +import shutil import oa_upgradable import version_recommend @@ -97,7 +98,8 @@ def fork_clone_repo(gt, repo, br): name = gt.token["user"] while True: - subprocess.call(["rm", "-rf", "{pkg}".format(pkg=repo)]) + if os.path.exists(repo): + shutil.rmtree(repo, ignore_errors=True) subprocess.call(["git", "clone", "git@gitee.com:{user}/{pkg}".format(user=name, pkg=repo)]) os.chdir(repo) if subprocess.call(["git", "checkout", "{branch}".format(branch=br)]):