From d5f8d5b85aaa6fb0af99fa6a57b98a50692d330b Mon Sep 17 00:00:00 2001 From: licihua Date: Sun, 20 Jun 2021 17:23:53 +0800 Subject: [PATCH] fix open file exception --- advisors/review_tool.py | 180 ++++++++++++++++++++++------------------ 1 file changed, 100 insertions(+), 80 deletions(-) diff --git a/advisors/review_tool.py b/advisors/review_tool.py index 1e59b063..0d6605f6 100755 --- a/advisors/review_tool.py +++ b/advisors/review_tool.py @@ -1,5 +1,5 @@ #!/usr/bin/python3 -#****************************************************************************** +# ****************************************************************************** # Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. # licensed under the Mulan PSL v2. # You can use this software according to the terms and conditions of the Mulan PSL v2. @@ -38,23 +38,23 @@ You can use "/review status[go/nogo/na/question/ongoing]:999" if you need update CHECKLIST = "helper/reviewer_checklist.yaml" -categorizer = {'PRSubmissionSPEC':'PR提交规范', - 'CleanCode':'Clean Code', - 'OpenSourceCompliance':'开源合规性', - 'SecurityPrivacy':'安全及隐私', - 'Compatibility':'兼容性', - 'customization':'定制项'} +categorizer = {'PRSubmissionSPEC': 'PR提交规范', + 'CleanCode': 'Clean Code', + 'OpenSourceCompliance': '开源合规性', + 'SecurityPrivacy': '安全及隐私', + 'Compatibility': '兼容性', + 'customization': '定制项'} SIGS_URL = "https://gitee.com/openeuler/community/raw/master/sig/sigs.yaml" -headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW 64; rv:23.0) Gecko/20100101 Firefox/23.0'} +headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW 64; rv:23.0) Gecko/20100101 Firefox/23.0'} __NUMBER = 0 RRVIEW_STATUS = { - 'go':'[🟢]', - 'nogo':'[🔴]', - 'na':'[◯]', - 'question':'[🟡]', - 'ongoing':'[🔵]' - } + 'go': '[🟢]', + 'nogo': '[🔴]', + 'na': '[◯]', + 'question': '[🟡]', + 'ongoing': '[🔵]' +} FLAG_EDIT_ALL = 999 @@ -62,12 +62,13 @@ PR_CONFLICT_COMMENT = "Conflict exists in PR.Please resolve conflict before revi FAILURE_COMMENT = """ Failed to create review list.You can try to rebuild using "/review retrigger".:confused:""" + def check_new_code(branch): """ Check if new code file has been introduced """ lst_files = subprocess.getoutput( - "git diff --name-only --diff-filter=A remotes/origin/{}..".format(branch)) + "git diff --name-only --diff-filter=A remotes/origin/{}..".format(branch)) return bool(lst_files.splitlines()) @@ -102,12 +103,12 @@ def check_spec_change(branch, keyword): check if value of keyword changed in spec """ modify_files = subprocess.getoutput( - "git diff --name-only --diff-filter=M remotes/origin/{}..".format(branch)) + "git diff --name-only --diff-filter=M remotes/origin/{}..".format(branch)) for item in modify_files.splitlines(): if item.endswith(".spec"): lines = subprocess.getoutput( - "git diff remotes/origin/{0}.. {1} \ - | grep '^[+-]{2}:'".format(branch, item, keyword)) + "git diff remotes/origin/{0}.. {1} \ + | grep '^[+-]{2}:'".format(branch, item, keyword)) lines_list = lines.splitlines() if len(lines_list) != 2: break @@ -134,8 +135,8 @@ def load_checklist(local, user_gitee): cur_dir = os.path.dirname(os.path.realpath(__file__)) chklist_path = os.path.join(cur_dir, CHECKLIST) try: - with open(chklist_path, 'r', encoding = 'utf-8') as file_descriptor: - return yaml.load(file_descriptor.read(), Loader = yaml.Loader) + with open(chklist_path, 'r', encoding='utf-8') as file_descriptor: + return yaml.load(file_descriptor.read(), Loader=yaml.Loader) except OSError as reason: print("Load yaml failed!" + str(reason)) return None @@ -166,7 +167,11 @@ def load_sigs(sigs_file=""): Load sigs yaml """ if sigs_file: - file_descriptor = open(sigs_file, 'r', encoding = 'utf-8') + try: + file_descriptor = open(sigs_file, 'r', encoding='utf-8') + except IOError as error: + print("Error: open file {} failed", sigs_file, error) + return None sigs = yaml.load(file_descriptor.read(), Loader=yaml.Loader) else: req = urllib.request.Request(url=SIGS_URL, headers=headers) @@ -190,7 +195,11 @@ def load_repositories(repos_file): Load repository yaml """ if repos_file: - file_descriptor = open(repos_file, 'r', encoding = 'utf-8') + try: + file_descriptor = open(repos_file, 'r', encoding='utf-8') + except IOError as error: + print("Error: open file {} failed", repos_file, error) + return None repos = yaml.load(file_descriptor.read(), Loader=yaml.Loader) return repos['repositories'] return None @@ -214,12 +223,16 @@ def load_sig_owners(sig_name): """ owners = [] owners_file = "sig/{}/OWNERS".format(sig_name) - with open(owners_file, 'r') as file_descriptor: - lines = file_descriptor.readlines() - for line in lines: - if line.strip().startswith('-'): - owner = line.replace('- ', '@').strip() - owners.append(owner) + try: + with open(owners_file, 'r') as file_descriptor: + lines = file_descriptor.readlines() + for line in lines: + if line.strip().startswith('-'): + owner = line.replace('- ', '@').strip() + owners.append(owner) + except IOError as error: + print("Error: 没有找到文件或读取文件失败 {}.", owners_file, error) + return None return owners @@ -231,13 +244,13 @@ def get_repo_changes(): add_repos = [] dlt_lines = subprocess.getoutput( - "git diff remotes/origin/master.. sig/sigs.yaml | grep '^-[ ][ ]-' | awk '{print $NF}'") + "git diff remotes/origin/master.. sig/sigs.yaml | grep '^-[ ][ ]-' | awk '{print $NF}'") for dlt_line in dlt_lines.splitlines(): if dlt_line.startswith("openeuler") or dlt_line.startswith("src-openeuler"): dlt_repos.append(dlt_line.strip()) add_lines = subprocess.getoutput( - "git diff remotes/origin/master.. sig/sigs.yaml | grep '^+[ ][ ]-' | awk '{print $NF}'") + "git diff remotes/origin/master.. sig/sigs.yaml | grep '^+[ ][ ]-' | awk '{print $NF}'") for add_line in add_lines.splitlines(): if add_line.startswith("openeuler") or add_line.startswith("src-openeuler"): add_repos.append(add_line.strip()) @@ -288,14 +301,14 @@ def check_repository_ownership_changes(info): repos_need_lgtm.append(repo) item = join_check_item(categorizer['customization'], - info['claim'], info['explain']) + info['claim'], info['explain']) review_body += item.format(repos=" ".join(repos), sig1=sig_changes[0], sig2=sig_changes[1], - owners1=" ".join(sig1_owners), owners2=" ".join(sig2_owners)) + owners1=" ".join(sig1_owners), owners2=" ".join(sig2_owners)) if repos_need_lgtm: item = join_check_item(categorizer['customization'], - info['to_recycle']['claim'], info['to_recycle']['explain']) + info['to_recycle']['claim'], info['to_recycle']['explain']) review_body += item.format(repos=" ".join(repos_need_lgtm), sig1=sig_changes[0], - sig2=sig_changes[1], owners=" ".join(rls_mgmt_owners)) + sig2=sig_changes[1], owners=" ".join(rls_mgmt_owners)) repos_need_lgtm.clear() return review_body @@ -308,8 +321,8 @@ def check_branch_add(info): need_mgmt_lgtm = False add_lines = subprocess.getoutput( - "git diff remotes/origin/master.. \ - repository/src-openeuler.yaml | grep '^+[ ][ ]-' | awk '{print $NF}'") + "git diff remotes/origin/master.. \ + repository/src-openeuler.yaml | grep '^+[ ][ ]-' | awk '{print $NF}'") for add_line in add_lines.splitlines(): if add_line.strip() != "master": need_mgmt_lgtm = True @@ -330,7 +343,7 @@ def check_repository_mgmt_changes(sigs, info): ret_code, lst_files = subprocess.getstatusoutput(info['cmd']) if ret_code != 0: chk = join_check_item(categorizer['customization'], - info['failed']['claim'], info['failed']['explain']) + info['failed']['claim'], info['failed']['explain']) return chk for item in lst_files.splitlines(): if item.startswith("SUGGESTION: This PR needs to be reviewed"): @@ -343,17 +356,17 @@ def check_repository_mgmt_changes(sigs, info): owners = result.group(2) if sig not in sigs: chk = join_check_item(categorizer['customization'], - info['lgtm-chk']['claim'], info['lgtm-chk']['explain']) + info['lgtm-chk']['claim'], info['lgtm-chk']['explain']) review_body += chk.format(sig=sig, owners=owners) else: result = re.match("WARNING! deleting (.*)", item) if result: chk = join_check_item(categorizer['customization'], - info['dlt-chk']['claim'], info['dlt-chk']['explain']) + info['dlt-chk']['claim'], info['dlt-chk']['explain']) review_body += chk.format(repo=result.group(1)) if need_additional_review: review_body += join_check_item(categorizer['customization'], - info['success']['claim'], info['success']['explain']) + info['success']['claim'], info['success']['explain']) return review_body @@ -419,7 +432,7 @@ def basic_review(cklist, branch): if not langs: continue item = join_check_item(categorizer[key1], - value2['claim'], value2['explain']) + value2['claim'], value2['explain']) item = item.format(lang="/".join(langs), checker="/".join(checkers)) review_body += item continue @@ -433,7 +446,7 @@ def basic_review(cklist, branch): if branch == "master" or not check_spec_change(branch, "Version"): continue item = join_check_item(categorizer[key1], - value2['claim'], value2['explain']) + value2['claim'], value2['explain']) review_body += item return review_body @@ -445,12 +458,12 @@ def community_maintainer_change_review(cstm_item, sigs): review_body = "" if cstm_item['name'] == "maintainer-add-explain": item = join_check_item(categorizer['customization'], - cstm_item['claim'], cstm_item['explain']) + cstm_item['claim'], cstm_item['explain']) review_body += item elif cstm_item['name'] == "maintainer-change-lgtm": for sig in sigs: item = join_check_item(categorizer['customization'], - cstm_item['claim'], cstm_item['explain']) + cstm_item['claim'], cstm_item['explain']) review_body += item.format(sig=sig, owners=sigs[sig]) return review_body @@ -475,13 +488,13 @@ def community_review(custom_items): if sig in sigs: continue item = join_check_item(categorizer['customization'], - cstm_item['claim'], cstm_item['explain']) + cstm_item['claim'], cstm_item['explain']) review_body += item.format(sig=sig, owners=info_sigs[sig]) elif cstm_item['condition'] == 'repo-introduce': if not repo_change: continue item = join_check_item(categorizer['customization'], - cstm_item['claim'], cstm_item['explain']) + cstm_item['claim'], cstm_item['explain']) review_body += item elif cstm_item['condition'] == 'sanity_check': add_review = check_repository_mgmt_changes(info_sigs, cstm_item) @@ -501,10 +514,10 @@ def review(checklist, pull_request, repo_name, branch): return PR_CONFLICT_COMMENT.format(owner=pull_request['user']['login']) review_body = CHK_TABLE_HEADER.format(go=RRVIEW_STATUS['go'], - nogo=RRVIEW_STATUS['nogo'], - na=RRVIEW_STATUS['na'], - question=RRVIEW_STATUS['question'], - ongoing=RRVIEW_STATUS['ongoing']) + nogo=RRVIEW_STATUS['nogo'], + na=RRVIEW_STATUS['na'], + question=RRVIEW_STATUS['question'], + ongoing=RRVIEW_STATUS['ongoing']) review_body += basic_review(checklist, branch) custom_items = checklist['customization'].get(repo_name, None) if custom_items: @@ -513,7 +526,7 @@ def review(checklist, pull_request, repo_name, branch): else: for cstm_item in custom_items: item = join_check_item(categorizer['customization'], - cstm_item['claim'], cstm_item['explain']) + cstm_item['claim'], cstm_item['explain']) review_body += item return review_body @@ -524,7 +537,7 @@ def check_pr_url(url): """ if url: pattern = re.compile(r'https://gitee.com/(open_euler/dashboard/projects/)?' - + r'(openeuler|src-openeuler)/([A-Za-z0-9-_]*)/pulls/(\d+$)') + + r'(openeuler|src-openeuler)/([A-Za-z0-9-_]*)/pulls/(\d+$)') return pattern.match(url) return None @@ -557,27 +570,29 @@ def args_parser(): arguments parser """ pars = argparse.ArgumentParser() - pars.add_argument("-q", "--quiet", action = 'store_true', default = False, \ - help="No log print") + pars.add_argument("-q", "--quiet", action='store_true', default=False, \ + help="No log print") pars.add_argument("-n", "--repo", type=str, help="Repository name that include group") pars.add_argument("-p", "--pull", type=str, help="Number ID of Pull Request") pars.add_argument("-u", "--url", type=str, help="URL of Pull Request") pars.add_argument("-r", "--reuse", help="Reuse current local git dirctory", action="store_true") pars.add_argument("-w", "--workdir", type=str, default=os.getcwd(), - help="Work directory.Default is current directory.") + help="Work directory.Default is current directory.") pars.add_argument("-e", "--edit", type=str, - help="Edit items format.Format: status1:number_list1 status2:number_list2 ...") + help="Edit items format.Format: status1:number_list1 status2:number_list2 ...") pars.add_argument("-c", "--clean", help="Clean environment", action="store_true") pars.add_argument("-l", "--local", help="Using local checklist", action="store_true") return pars.parse_args() + def local_repo_name(group, repo_name, pull_id): """ combine name to avoid name conflit """ return "{}_{}_{}".format(group, repo_name, pull_id) + def exec_cmd(cmd, retry_times=0): """ wrapper for Popen @@ -585,23 +600,24 @@ def exec_cmd(cmd, retry_times=0): @retry_times: retry times if cmd execute failed """ subp = subprocess.run(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding="utf-8", - check=False) + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + check=False) if subp.returncode != 0 and retry_times > 0: - for i in range(1,retry_times+1): - print("cmd:%s execute failed,retry:%d" % (cmd,i)) + for i in range(1, retry_times + 1): + print("cmd:%s execute failed,retry:%d" % (cmd, i)) subp = subprocess.run(cmd, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - encoding="utf-8", - check=False) + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + encoding="utf-8", + check=False) if subp.returncode == 0: break print(subp.stdout) return subp.returncode + def prepare_env(work_dir, reuse, pr_tuple, branch): """ prepare local reposity base and PR branch @@ -639,17 +655,19 @@ def prepare_env(work_dir, reuse, pr_tuple, branch): if exec_cmd(["git", "checkout", "-b", "working_pr_{n}".format(n=pull_id)]) != 0: print("Failed to create working branch working_pr_{n}".format(n=pull_id)) return 1 - if exec_cmd(["git", "merge", "--no-edit", "pr_{n}".format(n=pull_id)],3) != 0: - print("Failed to merge PR:{n} to branch:{base}".format(n=pull_id,base=branch)) + if exec_cmd(["git", "merge", "--no-edit", "pr_{n}".format(n=pull_id)], 3) != 0: + print("Failed to merge PR:{n} to branch:{base}".format(n=pull_id, base=branch)) return 1 return 0 + def cleanup_env(work_dir, group, repo_name, pull_id): """ Clean up environment, e.g. temporary directory """ shutil.rmtree(os.path.join(work_dir, local_repo_name(group, repo_name, pull_id))) + def find_review_comment(user_gitee, group, repo_name, pull_id): """ Find the review comment for PR @@ -661,7 +679,8 @@ def find_review_comment(user_gitee, group, repo_name, pull_id): return comment return None -def edit_review_status(edit, user_gitee, group ,repo_name, pull_id): + +def edit_review_status(edit, user_gitee, group, repo_name, pull_id): """ Edit review status """ @@ -679,15 +698,15 @@ def edit_review_status(edit, user_gitee, group ,repo_name, pull_id): if len(status_num_dicts) == 1 and FLAG_EDIT_ALL in status_num_dicts.keys(): need_edit = True for num in range(len(items[head_len:])): - items[head_len+num] = re.sub(match_str, - RRVIEW_STATUS[status_num_dicts[FLAG_EDIT_ALL]], - items[head_len+num]) + items[head_len + num] = re.sub(match_str, + RRVIEW_STATUS[status_num_dicts[FLAG_EDIT_ALL]], + items[head_len + num]) else: for num, status in status_num_dicts.items(): - if int(num) >=0 and int(num) < len(items[head_len:]): - items[head_len+num] = re.sub(match_str, - RRVIEW_STATUS[status], - items[head_len+num]) + if int(num) >= 0 and int(num) < len(items[head_len:]): + items[head_len + num] = re.sub(match_str, + RRVIEW_STATUS[status], + items[head_len + num]) need_edit = True if need_edit: new_body = "".join(items) @@ -710,7 +729,7 @@ def decode_edit_content(edit): nums = sect.split(":")[1] for num in nums.split(","): if not num.isdigit(): - res = re.match("^([0-9]{1,3})-([0-9]{0,3})$",num) + res = re.match("^([0-9]{1,3})-([0-9]{0,3})$", num) if res: start = int(res.group(1)) end = FLAG_EDIT_ALL @@ -719,7 +738,7 @@ def decode_edit_content(edit): if start >= end: print("ERROR: input format error,start number must greater than end.") return {} - for i in range(start, end+1): + for i in range(start, end + 1): dicts[i] = status else: print("ERROR: input format error or contain invalid character.") @@ -750,8 +769,8 @@ def main(): sys.exit(1) pull_request = user_gitee.get_pr(repo_name, pull_id, group) if not pull_request: - print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist."\ - % (pull_id, group, repo_name)) + print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." \ + % (pull_id, group, repo_name)) return 1 if args.edit: if edit_review_status(args.edit, user_gitee, group, repo_name, pull_id) != 0: @@ -772,5 +791,6 @@ def main(): print("push review list finish.") return 0 + if __name__ == "__main__": sys.exit(main()) -- Gitee