diff --git a/advisors/check_repeated_repo.py b/advisors/check_repeated_repo.py new file mode 100755 index 0000000000000000000000000000000000000000..40267f7126ff0c39e8c8c93606698cbf1ec3da93 --- /dev/null +++ b/advisors/check_repeated_repo.py @@ -0,0 +1,83 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# +# ******************************************************************************/ +""" +This is an automatic script for checking repo is repeated or not +""" +import argparse +import os +import sys + +import yaml +import yaml2url + + +def get_url(repo_file): + """ + Get url of given package + """ + try: + repo_yaml = open(repo_file) + except FileNotFoundError: + print("WARNING: {} can't be found in local path.".format(repo_file)) + return None + + if repo_yaml: + pkg_info = yaml.load(repo_yaml, Loader=yaml.Loader) + else: + return None + + if not pkg_info: + print("WARNING: load {} yaml fail".format(repo_file)) + return None + + return yaml2url.yaml2url(pkg_info) + + +def check_repo(directory): + """ + Check repeate repo + """ + url_dict = {} + for root, _, files in os.walk(directory): + for file in files: + file_path = os.path.join(root, file) + url = get_url(file_path) + if url: + if url in url_dict.keys(): + print("WARNING: {file1} is repeat with {file2}, url is {url}" \ + .format(file1=url_dict[url], file2=file, url=url)) + else: + url_dict[url] = file + else: + continue + + +def main(): + """ + Main entrance of the functionality + """ + pars = argparse.ArgumentParser() + pars.add_argument("-d", "--default", type=str, default=None, + help="The fallback place to look for YAML information") + args = pars.parse_args() + + if args.default: + check_repo(args.default) + else: + print("WARNING: Please specify the direction (-d ../upstream-info/) to be checkd.") + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/advisors/check_upstream.py b/advisors/check_upstream.py index 0343cac5059bb151db4eea4adf35f4a9bcf8e8bb..36d8476761f0a8d1c7bb6374b2fe1541283639b7 100755 --- a/advisors/check_upstream.py +++ b/advisors/check_upstream.py @@ -12,9 +12,8 @@ from datetime import datetime #import yaml import json from urllib.parse import urljoin - import requests - +import yaml2url TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z" @@ -109,7 +108,7 @@ def check_hg_raw(info, clean_tag=True): headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' } - url = urljoin(info["src_repo"] + "/", "raw-tags") + url = yaml2url.yaml2url(info) resp = requests.get(url, headers=headers) resp = resp.text need_trick, url, cookies = dirty_redirect_tricks(url, resp) @@ -145,7 +144,7 @@ def check_hg(info, clean_tag=True): headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' } - url = urljoin(info["src_repo"] + "/", "json-tags") + url = yaml2url.yaml2url(info) resp = requests.get(url, headers=headers) resp = resp.text need_trick, url, cookies = dirty_redirect_tricks(url, resp) @@ -182,7 +181,7 @@ def check_metacpan(info, clean_tag=True): headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' } - url = urljoin("https://metacpan.org/release/", info["src_repo"]) + url = yaml2url.yaml2url(info) resp = requests.get(url, headers=headers) resp = resp.text @@ -223,7 +222,7 @@ def check_pypi(info, clean_tag=True): headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' } - url = urljoin("https://pypi.org/pypi/", info["src_repo"] + "/json") + url = yaml2url.yaml2url(info) resp = requests.get(url, headers=headers) data = resp.json() @@ -247,7 +246,7 @@ def check_rubygem(info, clean_tag=True): headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' } - url = urljoin("https://rubygems.org/api/v1/versions/", info["src_repo"] + ".json") + url = yaml2url.yaml2url(info) resp = requests.get(url, headers=headers) data = resp.json() @@ -322,7 +321,8 @@ def check_git(info, clean_tag=True): """ resp = load_last_query_result(info) if resp == "": - resp = __check_git_helper(info["src_repo"]) + url = yaml2url.yaml2url(info) + resp = __check_git_helper(url) last_query = {} last_query["time_stamp"] = datetime.now() last_query["raw_data"] = resp @@ -342,7 +342,7 @@ def check_github(info, clean_tag=True): if info.get("query_type", "git-ls") != "git-ls": resp = "" - repo_url = "https://github.com/" + info["src_repo"] + ".git" + repo_url = yaml2url.yaml2url(info) if resp == "": resp = __check_git_helper(repo_url) @@ -364,7 +364,7 @@ def check_gnu_ftp(info, clean_tag=True): headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' } - url = urljoin("https://ftp.gnu.org/gnu/", info["src_repo"] + "/") + url = yaml2url.yaml2url(info) eprint("{repo} > List ftp directory".format(repo=url)) resp = requests.get(url, headers=headers) resp = resp.text @@ -385,7 +385,7 @@ def check_ftp(info, clean_tag=True): headers = { 'User-Agent' : 'Mozilla/5.0 (X11; Linux x86_64)' } - url = urljoin('ftp', info["src_repo"] + "/") + url = yaml2url.yaml2url(info) eprint("{repo} > List ftp directory".format(repo=url)) resp = requests.get(url, headers=headers) resp = resp.text @@ -404,11 +404,7 @@ def check_gnome(info, clean_tag=True): Check version info via gitlab.gnome.org api """ resp = load_last_query_result(info) - src_repos = info["src_repo"].split("/") - if len(src_repos) == 1: - repo_url = "https://gitlab.gnome.org/GNOME/" + info["src_repo"] + ".git" - else: - repo_url = "https://gitlab.gnome.org/" + info["src_repo"] + ".git" + repo_url = yaml2url.yaml2url(info) if resp == "": resp = __check_git_helper(repo_url) @@ -427,7 +423,7 @@ def check_gitee(info, clean_tag=True): Check version info via gitee """ resp = load_last_query_result(info) - repo_url = "https://gitee.com/" + info["src_repo"] + repo_url = yaml2url.yaml2url(info) if resp == "": resp = __check_git_helper(repo_url) last_query = {} @@ -445,8 +441,7 @@ def check_svn(info, clean_tag=True): Check version info via svn """ resp = load_last_query_result(info) - tag_dir = info.get("tag_dir", "tags") - repo_url = info["src_repo"] + "/" + tag_dir + repo_url = yaml2url.yaml2url(info) if resp == "": resp = __check_svn_helper(repo_url) last_query = {} diff --git a/advisors/yaml2url.py b/advisors/yaml2url.py new file mode 100755 index 0000000000000000000000000000000000000000..e886d3def6d474914ff7580c72ec925f08a7e7c1 --- /dev/null +++ b/advisors/yaml2url.py @@ -0,0 +1,161 @@ +# !/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# +# ******************************************************************************/ +""" +This is an script for get url from repo name +""" + +from urllib.parse import urljoin + + +def __get_hg_url(pkg_info): + """ + Get hg repo url of package + """ + url = urljoin(pkg_info["src_repo"] + "/", "raw-tags") + return url + + +def __get_hg_raw_url(pkg_info): + """ + Get hg raw repo url of package + """ + url = urljoin(pkg_info["src_repo"] + "/", "json-tags") + return url + + +def __get_github_url(pkg_info): + """ + Get github repo url of package + """ + url = "https://github.com/" + pkg_info["src_repo"] + ".git" + return url + + +def __get_gnome_url(pkg_info): + """ + Get gnome repo url of package + """ + src_repos = pkg_info["src_repo"].split("/") + if len(src_repos) == 1: + url = "https://gitlab.gnome.org/GNOME/" + pkg_info["src_repo"] + ".git" + else: + url = "https://gitlab.gnome.org/" + pkg_info["src_repo"] + ".git" + return url + + +def __get_git_url(pkg_info): + """ + Get git repo url of package + """ + url = pkg_info["src_repo"] + return url + + +def __get_svn_url(pkg_info): + """ + Get svn repo url of package + """ + tag_dir = pkg_info.get("tag_dir", "tags") + url = pkg_info["src_repo"] + "/" + tag_dir + return url + + +def __get_metacpan_url(pkg_info): + """ + Get metacpan repo url of package + """ + url = urljoin("https://metacpan.org/release/", pkg_info["src_repo"]) + return url + + +def _get_rubygem_url(pkg_info): + """ + Get rubygem repo url of package + """ + url = urljoin("https://rubygems.org/api/v1/versions/", pkg_info["src_repo"] + ".json") + return url + + +def __get_gitee_url(pkg_info): + """ + Get gitee repo url of package + """ + url = "https://gitee.com/" + pkg_info["src_repo"] + return url + + +def __get_gnu_ftp_url(pkg_info): + """ + Get gnu ftp repo url of package + """ + url = urljoin("https://ftp.gnu.org/gnu/", pkg_info["src_repo"] + "/") + return url + + +def __get_ftp_url(pkg_info): + """ + Get ftp repo url of package + """ + url = urljoin('ftp', pkg_info["src_repo"] + "/") + return url + + +def __get_pypi_url(pkg_info): + """ + Get pypi repo url of package + """ + url = urljoin("https://pypi.org/pypi/", pkg_info["src_repo"] + "/json") + return url + + +def __get_rubygem_url(pkg_info): + """ + Get rubygem repo url of package + """ + url = urljoin("https://rubygems.org/api/v1/versions/", pkg_info["src_repo"] + ".json") + return url + + +def yaml2url(pkg_info): + """ + Get url from yaml + """ + vc_type = pkg_info.get("version_control", None) + if vc_type is None: + print("Missing version_control in YAML file") + return None + + switcher = { + "hg": __get_hg_url, + "hg-raw": __get_hg_raw_url, + "github": __get_github_url, + "git": __get_git_url, + "gitlab.gnome": __get_gnome_url, + "svn": __get_svn_url, + "metacpan": __get_metacpan_url, + "pypi": __get_pypi_url, + "rubygem": __get_rubygem_url, + "gitee": __get_gitee_url, + "gnu-ftp": __get_gnu_ftp_url, + "ftp": __get_ftp_url + } + + get_url_method = switcher.get(vc_type, None) + if get_url_method: + url = get_url_method(pkg_info) + else: + print("Unsupport version control method {vc}".format(vc=vc_type)) + return None + + return url diff --git a/upstream-info/libapr1.yaml b/upstream-info/libapr1.yaml index da6ac579ff7deef330f63d717880216661291c4f..af2be64ac8f76ca5c1e825fd169d475a2d84e382 100644 --- a/upstream-info/libapr1.yaml +++ b/upstream-info/libapr1.yaml @@ -1,5 +1,5 @@ --- version_control: svn -src_rpeo: https://svn.apache.org/repos/asf/apr/apr +src_repo: https://svn.apache.org/repos/asf/apr/apr tag_prefix: "^" separator: "." diff --git a/upstream-info/libtcnative-1-0.yaml b/upstream-info/libtcnative-1-0.yaml index dc4b5a6d55eaa026aa6b3844da76ac35f7cf93bc..7709142e19f66d947875328855be80043bdfcc75 100644 --- a/upstream-info/libtcnative-1-0.yaml +++ b/upstream-info/libtcnative-1-0.yaml @@ -1,5 +1,5 @@ --- version_control: github -src_rpeo: apache/tomcat-native +src_repo: apache/tomcat-native tag_prefix: "^" separator: "." diff --git a/upstream-info/xmlto.yaml b/upstream-info/xmlto.yaml index 1f1817f5258ddd1d623faf9b0ec924fec9e6c36c..4eb92f550a2ad05ef1c887f8c857e76acd4f7b70 100644 --- a/upstream-info/xmlto.yaml +++ b/upstream-info/xmlto.yaml @@ -1,5 +1,5 @@ --- version_control: git -asrc_repo: https://pagure.io/xmlto.git +src_repo: https://pagure.io/xmlto.git tag_prefix: separator: