From fce8e7fffd0e6c9d6138fabd419cd0f1c31ac650 Mon Sep 17 00:00:00 2001 From: liuqi <469227928@qq.com> Date: Wed, 12 Jan 2022 17:51:36 +0800 Subject: [PATCH] add feature branch for new community review tool --- advisors/review_tool.py | 251 +++++++++++++++++++++------------------- 1 file changed, 130 insertions(+), 121 deletions(-) diff --git a/advisors/review_tool.py b/advisors/review_tool.py index 058261be..20093a55 100755 --- a/advisors/review_tool.py +++ b/advisors/review_tool.py @@ -19,7 +19,6 @@ import sys import argparse import subprocess import shutil -import urllib import yaml from advisors import gitee @@ -45,7 +44,6 @@ categorizer = {'PRSubmissionSPEC': 'PR提交规范', 'Compatibility': '兼容性', 'PackageSubmission': '制品仓要求', 'customization': '定制项'} -SIGS_URL = "https://gitee.com/openeuler/community/raw/master/sig/sigs.yaml" headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW 64; rv:23.0) Gecko/20100101 Firefox/23.0'} __NUMBER = 0 @@ -125,7 +123,7 @@ def check_spec_change(branch, keyword): return False -def load_checklist(local, user_gitee): +def load_checklist(local): """ @Desc: load checklist @Notice: this function must be called before prepare_env(), @@ -158,27 +156,54 @@ def check_repository_changes(): """ check if src-openeuler.yaml has been changed """ - lst_files = subprocess.getoutput("git diff remotes/origin/master.. \ - repository/src-openeuler.yaml | grep '^+- name' | awk '{print $NF}'") - return bool(lst_files.splitlines()) + pr_diff = subprocess.getoutput("git show | grep '^diff --git'").splitlines() + diff_files = [diff_file.split(' ')[-1].split('/', 1)[1].strip() for diff_file in pr_diff] + lst_files = [] + for diff_file in diff_files: + if diff_file.startswith('sig') and diff_file.endswith('.yaml'): + if len(diff_file.split('/')) == 5 and diff_file.split('/')[2] in ['openeuler', 'src-openeuler']: + lst_files.append(diff_file) + return bool(lst_files) -def load_sigs(sigs_file=""): +def load_sigs(branch=None): """ - Load sigs yaml + Construct sigs """ - if sigs_file: - try: - file_descriptor = open(sigs_file, 'r', encoding='utf-8') - except IOError as error: - print("Error: open file {} failed", sigs_file, error) - return None - sigs = yaml.load(file_descriptor.read(), Loader=yaml.Loader) - else: - req = urllib.request.Request(url=SIGS_URL, headers=headers) - res = urllib.request.urlopen(req) - sigs = yaml.load(res.read().decode("utf-8"), Loader=yaml.Loader) - return sigs['sigs'] + if branch and branch != 'working_pr_{}'.format(pull_id): + subprocess.call('git checkout {}'.format(branch), shell=True) + openeuler_repos = [] + src_openeuler_repos = [] + sigs = [] + for i in os.listdir('sig'): + if i in ['README.md', 'sig-template']: + continue + if i not in [x['name'] for x in sigs]: + sigs.append({'name': i, 'repositories': []}) + if 'openeuler' in os.listdir(os.path.join('sig', i)): + for filesdir, _, repos in os.walk(os.path.join('sig', i, 'openeuler')): + for repo in repos: + with open(os.path.join(filesdir, repo)) as f: + config_info = yaml.load(f.read(), Loader=yaml.Loader) + openeuler_repos.append(config_info) + for sig in sigs: + if sig['name'] == i: + repositories = sig['repositories'] + repositories.append(os.path.join('openeuler', repo.split('.yaml')[0])) + if 'src-openeuler' in os.listdir(os.path.join('sig', i)): + for filesdir, _, src_repos in os.walk(os.path.join('sig', i, 'src-openeuler')): + for src_repo in src_repos: + with open(os.path.join(filesdir, src_repo), 'r') as f: + src_config_info = yaml.load(f.read(), Loader=yaml.Loader) + src_openeuler_repos.append(src_config_info) + for sig in sigs: + if sig['name'] == i: + repositories = sig['repositories'] + repositories.append(os.path.join('src-openeuler', src_repo.split('.yaml')[0])) + cur_branch = subprocess.getoutput("git branch | grep \\*").split(' ')[-1] + if cur_branch != 'working_pr_{}'.format(pull_id): + subprocess.call('git checkout working_pr_{}'.format(pull_id), shell=True) + return sigs, openeuler_repos, src_openeuler_repos def get_repo_sig_ownership(repo, sigs): @@ -191,27 +216,12 @@ def get_repo_sig_ownership(repo, sigs): return "" -def load_repositories(repos_file): - """ - Load repository yaml - """ - if repos_file: - try: - file_descriptor = open(repos_file, 'r', encoding='utf-8') - except IOError as error: - print("Error: open file {} failed", repos_file, error) - return None - repos = yaml.load(file_descriptor.read(), Loader=yaml.Loader) - return repos['repositories'] - return None - - -def is_exist_protected_branch_exclude_master(repo_name, repos): +def is_exist_protected_branch_exclude_master(repoName, repos): """ check there exist other protected branches exclude master """ for repo in repos: - if repo_name == repo['name']: + if repoName == repo['name']: if len(repo['branches']) == 1: return False return True @@ -243,22 +253,21 @@ def get_repo_changes(): """ dlt_repos = [] add_repos = [] - - dlt_lines = subprocess.getoutput( - "git diff remotes/origin/master.. sig/sigs.yaml | grep '^-[ ][ ]-' | awk '{print $NF}'") - for dlt_line in dlt_lines.splitlines(): - if dlt_line.startswith("openeuler") or dlt_line.startswith("src-openeuler"): - dlt_repos.append(dlt_line.strip()) - - add_lines = subprocess.getoutput( - "git diff remotes/origin/master.. sig/sigs.yaml | grep '^+[ ][ ]-' | awk '{print $NF}'") - for add_line in add_lines.splitlines(): - if add_line.startswith("openeuler") or add_line.startswith("src-openeuler"): - add_repos.append(add_line.strip()) - repo_changes = {} - cur_sigs = load_sigs() - tobe_sigs = load_sigs("sig/sigs.yaml") + cur_repos = [] + tobe_repos = [] + cur_sigs, cur_openeuler_repos, cur_src_openeuler_repos = load_sigs() + tobe_sigs, tobe_openeuler_repos, tobe_src_openeuler_repos = load_sigs(base_branch) + cur_repos.extend(cur_openeuler_repos) + cur_repos.extend(cur_src_openeuler_repos) + tobe_repos.extend(tobe_openeuler_repos) + tobe_repos.extend(tobe_src_openeuler_repos) + for cur_repo in cur_repos: + if cur_repo not in tobe_repos: + add_repos.append(cur_repo) + for tobe_repo in tobe_repos: + if tobe_repo not in cur_repos: + dlt_repos.append(tobe_repo) for dlt_repo in dlt_repos: if dlt_repo in add_repos: cur_sig = get_repo_sig_ownership(dlt_repo, cur_sigs) @@ -281,8 +290,7 @@ def check_repository_ownership_changes(info): review_body = "" rls_mgmt_owners = load_sig_owners("sig-release-management") - oe_mgmt_repos = load_repositories("repository/openeuler.yaml") - src_oe_mgmt_repos = load_repositories("repository/src-openeuler.yaml") + _, oe_mgmt_repos, src_oe_mgmt_repos = load_sigs() repo_changes = get_repo_changes() for sig_changes, repos in repo_changes.items(): sig1_owners = load_sig_owners(sig_changes[0]) @@ -297,8 +305,8 @@ def check_repository_ownership_changes(info): else: print("ERROR: repo:%s error" % repo) mgmt_repos = None - repo_name = repo.split('/')[1] - if is_exist_protected_branch_exclude_master(repo_name, mgmt_repos): + name = repo.split('/')[1] + if is_exist_protected_branch_exclude_master(name, mgmt_repos): repos_need_lgtm.append(repo) item = join_check_item(categorizer['customization'], @@ -321,13 +329,20 @@ def check_branch_add(info): review_body = "" need_mgmt_lgtm = False - add_lines = subprocess.getoutput( - "git diff remotes/origin/master.. \ - repository/src-openeuler.yaml | grep '^+[ ][ ]-' | awk '{print $NF}'") - for add_line in add_lines.splitlines(): - if add_line.strip() != "master": - need_mgmt_lgtm = True - break + pr_diffs = [diff_file.splitlines() for diff_file in subprocess.getoutput('git show').split('diff --git')[1:]] + for diff_file in pr_diffs: + diff_file_name = diff_file[0].split(' ')[-1].split('/', 1)[1] + if not (diff_file_name.endswith('.yaml') and + len(diff_file_name.split('/')) == 5 and + diff_file_name.split('/')[0] == 'sig' and + diff_file_name.split('/')[2] in ['openeuler', 'src-openeuler']): + continue + for diff_line in diff_file: + if diff_line.startswith('+- name: '): + add_branch = diff_line.split('+- name: ')[-1].strip() + if add_branch != 'master': + need_mgmt_lgtm = True + break if need_mgmt_lgtm: owners = load_sig_owners("sig-release-management") item = join_check_item(categorizer['customization'], info['claim'], info['explain']) @@ -451,6 +466,7 @@ def basic_review(cklist, branch): review_body += item return review_body + def src_openeuler_review(cklist, branch): """ Review items for src-openeuler repos @@ -465,6 +481,7 @@ def src_openeuler_review(cklist, branch): review_body += item return review_body + def community_maintainer_change_review(cstm_item, sigs): """ maintainer changed review body @@ -520,7 +537,7 @@ def community_review(custom_items): return review_body -def review(checklist, pull_request, repo_name, branch, group): +def review(): """ Return check list of this PR """ @@ -532,10 +549,10 @@ def review(checklist, pull_request, repo_name, branch, group): na=RRVIEW_STATUS['na'], question=RRVIEW_STATUS['question'], ongoing=RRVIEW_STATUS['ongoing']) - review_body += basic_review(checklist, branch) + review_body += basic_review(checklist, base_branch) if group == "src-openeuler": - review_body += src_openeuler_review(checklist, branch) + review_body += src_openeuler_review(checklist, base_branch) custom_items = checklist['customization'].get(repo_name, None) if custom_items: @@ -560,24 +577,24 @@ def check_pr_url(url): return None -def extract_params(args): +def extract_params(): """ check and extract parameters we need """ if args.url and len(args.url) > 0: res = check_pr_url(args.url) if res: - group = res.group(2) - repo_name = res.group(3) - pull_id = res.group(4) - return (group, repo_name, pull_id) + owner = res.group(2) + repo = res.group(3) + pull_number = res.group(4) + return owner, repo, pull_number print("ERROR: URL is wrong, please check!") return () if args.repo and args.pull and len(args.repo) > 0 and len(args.pull) > 0: - group = args.repo.split('/')[0] - repo_name = args.repo.split('/')[1] - pull_id = args.pull - return (group, repo_name, pull_id) + owner = args.repo.split('/')[0] + repo = args.repo.split('/')[1] + pull_number = args.pull + return owner, repo, pull_number print("WARNING: please specify the URL of PR or repository name and PR's ID.\ \nDetails use -h/--help option.") return () @@ -588,7 +605,7 @@ def args_parser(): arguments parser """ pars = argparse.ArgumentParser() - pars.add_argument("-q", "--quiet", action='store_true', default=False, \ + pars.add_argument("-q", "--quiet", action='store_true', default=False, help="No log print") pars.add_argument("-n", "--repo", type=str, help="Repository name that include group") pars.add_argument("-p", "--pull", type=str, help="Number ID of Pull Request") @@ -604,7 +621,7 @@ def args_parser(): return pars.parse_args() -def local_repo_name(group, repo_name, pull_id): +def local_repo_name(): """ combine name to avoid name conflit """ @@ -636,20 +653,20 @@ def exec_cmd(cmd, retry_times=0): return subp.returncode -def prepare_env(work_dir, reuse, pr_tuple, branch): +def prepare_env(reuse, pr_tuple): """ prepare local reposity base and PR branch Notice: this will change work directory, action related to obtain path need do before this. """ - group = pr_tuple[0] - repo_name = pr_tuple[1] - pull_id = pr_tuple[2] + owner = pr_tuple[0] + repo = pr_tuple[1] + pull_number = pr_tuple[2] if not os.path.exists(work_dir): os.makedirs(work_dir) - repo = group + "/" + repo_name + repo = owner + "/" + repo gitee_url = "https://gitee.com/{repo}.git".format(repo=repo) - local_path = os.path.join(work_dir, local_repo_name(group, repo_name, pull_id)) + local_path = os.path.join(work_dir, local_repo_name()) if os.path.exists(local_path) and not reuse: print("WARNING: %s already exist, delete it." % local_path) shutil.rmtree(local_path) @@ -658,35 +675,35 @@ def prepare_env(work_dir, reuse, pr_tuple, branch): print("Failed to git clone {}".format(gitee_url)) return 1 os.chdir(local_path) - if exec_cmd(["git", "checkout", branch]) != 0: - print("Failed to checkout %s branch" % branch) + if exec_cmd(["git", "checkout", base_branch]) != 0: + print("Failed to checkout %s branch" % base_branch) return 1 if exec_cmd(["git", "pull"]) != 0: - print("Failed to update to latest commit in %s branch" % branch) + print("Failed to update to latest commit in %s branch" % base_branch) return 1 - lines = subprocess.getoutput("git branch | grep pr_{n}".format(n=pull_id)) + lines = subprocess.getoutput("git branch | grep pr_{n}".format(n=pull_number)) for br_name in lines.splitlines(): exec_cmd(["git", "branch", "-D", br_name.strip()]) - if exec_cmd(["git", "fetch", gitee_url, "pull/{n}/head:pr_{n}".format(n=pull_id)]) != 0: - print("Failed to fetch PR:{n}".format(n=pull_id)) + if exec_cmd(["git", "fetch", gitee_url, "pull/{n}/head:pr_{n}".format(n=pull_number)]) != 0: + print("Failed to fetch PR:{n}".format(n=pull_number)) return 1 - if exec_cmd(["git", "checkout", "-b", "working_pr_{n}".format(n=pull_id)]) != 0: - print("Failed to create working branch working_pr_{n}".format(n=pull_id)) + if exec_cmd(["git", "checkout", "-b", "working_pr_{n}".format(n=pull_number)]) != 0: + print("Failed to create working branch working_pr_{n}".format(n=pull_number)) return 1 - if exec_cmd(["git", "merge", "--no-edit", "pr_{n}".format(n=pull_id)], 3) != 0: - print("Failed to merge PR:{n} to branch:{base}".format(n=pull_id, base=branch)) + if exec_cmd(["git", "merge", "--no-edit", "pr_{n}".format(n=pull_number)], 3) != 0: + print("Failed to merge PR:{n} to branch:{base}".format(n=pull_number, base=base_branch)) return 1 return 0 -def cleanup_env(work_dir, group, repo_name, pull_id): +def cleanup_env(): """ Clean up environment, e.g. temporary directory """ - shutil.rmtree(os.path.join(work_dir, local_repo_name(group, repo_name, pull_id))) + shutil.rmtree(os.path.join(work_dir, local_repo_name())) -def find_review_comment(user_gitee, group, repo_name, pull_id): +def find_review_comment(): """ Find the review comment for PR """ @@ -698,14 +715,14 @@ def find_review_comment(user_gitee, group, repo_name, pull_id): return None -def edit_review_status(edit, user_gitee, group, repo_name, pull_id): +def edit_review_status(edit): """ Edit review status """ status_num_dicts = decode_edit_content(edit) if not status_num_dicts: return 1 - comment = find_review_comment(user_gitee, group, repo_name, pull_id) + comment = find_review_comment() if not comment: print("ERROR: can not find review list") return 1 @@ -721,7 +738,7 @@ def edit_review_status(edit, user_gitee, group, repo_name, pull_id): items[head_len + num]) else: for num, status in status_num_dicts.items(): - if int(num) >= 0 and int(num) < len(items[head_len:]): + if 0 <= int(num) < len(items[head_len:]): items[head_len + num] = re.sub(match_str, RRVIEW_STATUS[status], items[head_len + num]) @@ -766,18 +783,15 @@ def decode_edit_content(edit): return dicts -def main(): - """ - Main entrance of the functionality - """ +if __name__ == "__main__": args = args_parser() if args.quiet: sys.stdout = open('/dev/null', 'w') sys.stderr = sys.stdout work_dir = os.path.realpath(args.workdir) - params = extract_params(args) + params = extract_params() if not params: - return 1 + sys.exit(1) group = params[0] repo_name = params[1] pull_id = params[2] @@ -787,28 +801,23 @@ def main(): sys.exit(1) pull_request = user_gitee.get_pr(repo_name, pull_id, group) if not pull_request: - print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." \ - % (pull_id, group, repo_name)) - return 1 + print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo_name)) + sys.exit(1) if args.edit: - if edit_review_status(args.edit, user_gitee, group, repo_name, pull_id) != 0: - return 1 + if edit_review_status(args.edit) != 0: + sys.exit(1) else: - checklist = load_checklist(args.local, user_gitee) + checklist = load_checklist(args.local) if not checklist: - return 1 - branch = pull_request['base']['label'] - ret = prepare_env(work_dir, args.reuse, params, branch) + sys.exit(1) + base_branch = pull_request['base']['label'] + ret = prepare_env(args.reuse, params) if ret != 0: user_gitee.create_pr_comment(repo_name, pull_id, FAILURE_COMMENT, group) - return 1 - review_comment = review(checklist, pull_request, repo_name, branch, group) + sys.exit(1) + review_comment = review() user_gitee.create_pr_comment(repo_name, pull_id, review_comment, group) if args.clean: - cleanup_env(work_dir, group, repo_name, pull_id) + cleanup_env() print("push review list finish.") - return 0 - -if __name__ == "__main__": - sys.exit(main()) -- Gitee