From a3838339846373b878b274427b7628cf469c444b Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Tue, 29 Oct 2024 10:55:15 +0800 Subject: [PATCH 1/5] refactor code --- advisors/oe_review.py | 88 ++++++++++++++++++++++++++++++++----------- 1 file changed, 67 insertions(+), 21 deletions(-) diff --git a/advisors/oe_review.py b/advisors/oe_review.py index 468059ba..1908d9ce 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -154,9 +154,9 @@ def args_parser(): pars.add_argument("-u", "--url", type=str, help="URL of Pull Request") pars.add_argument("-s", "--sig", type=str, default="TC", help="When active_user is set, review all PRs in specified SIG") pars.add_argument("-m", "--model", type=str, help="Model of selection to generate review") - pars.add_argument("-e", "--editor", type=str, default="nvim", + pars.add_argument("-e", "--editor", type=str, default="neovide", help="Editor of choice to edit content, default to nvim") - + pars.add_argument("-o", "--editor-option", type=str, default="--no-fork", help="Commandline option for editor") return pars.parse_args() def edit_content(text, editor): @@ -164,7 +164,7 @@ def edit_content(text, editor): with os.fdopen(fd, 'w') as tmp: tmp.write(text) tmp.flush() - subprocess.call([editor, path]) + subprocess.call([editor["editor"], editor["editor-option"], path]) text_new = open(path).read() return text_new @@ -233,6 +233,15 @@ def sort_pr(user_gitee): NEED_REVIEW_PRS.put(None) print("sort pr exits") +def ai_review_impl(user_gitee, repo, pull_id, group): + pr_diff = user_gitee.get_diff(repo, pull_id, group) + if not pr_diff: + print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo)) + return "", "", "" + review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) + review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT) + return pr_diff, review, review_rating + def ai_review(user_gitee): while True: item = NEED_REVIEW_PRS.get() @@ -240,13 +249,12 @@ def ai_review(user_gitee): if not item: break pr_info = item["pr_info"] - pr_diff = user_gitee.get_diff(pr_info['repo'], pr_info['number'], pr_info['owner']) - if not pr_diff: - print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pr_info['number'], pr_info['owner'], pr_info['repo'])) - continue - review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) - review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT) + + pr_diff, review, review_rating = ai_review_impl(user_gitee, pr_info['repo'], pr_info['id'], pr_info['group']) + if pr_diff == "": + continue + manual_review_pr = {} manual_review_pr['pr_info'] = pr_info manual_review_pr['pull_request'] = item['pull_request'] @@ -257,30 +265,46 @@ def ai_review(user_gitee): MANUAL_REVIEW_PRS.put(None) print("ai review exits") +def manually_review_impl(user_gitee, pr_info, pull_request, review, review_rating, pr_diff, editor): + review_content = "" + review_content += "!{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) + review_content += "This PR has following labels:\n" + for label in pull_request["labels"]: + review_content += f"{label['name']} " + target_branch = pull_request["base"]["ref"] + review_content += "\nThis PR is submitted to {branch}\n".format(branch=target_branch) + history_comment = "" + sync_comment = "" + comments = user_gitee.get_pr_comments_all(pr_info['owner'], pr_info['repo'], pr_info['number']) + for comment in comments: + if comment['user']['name'] == "openeuler-ci-bot": + continue + elif comment['user']['name'] == "openeuler-sync-bot": + sync_comment += comment["body"] + "\n" + else: + history_comment += comment["body"] + "\n" + review_content += "\n# Branch Status\n" + sync_comment + review_content += "\n# History\n" + history_comment + review_comment_raw = edit_content(review_content + '\n\n' + review + '\n\n' + review_rating + '\n\n' + pr_diff, editor) + return review_comment_raw + def manually_review(user_gitee, editor): while True: item = MANUAL_REVIEW_PRS.get() #print("manually review works") if not item: break - review_content = "" pull_request = item['pull_request'] - pr_info = item["pr_info"] + pr_info = item['pr_info'] review = item['review'] review_rating = item['review_rating'] pr_diff = item['pr_diff'] - review_content += "!{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) - - review_content += "This PR has following labels:\n" - for label in pull_request["labels"]: - review_content += f"{label['name']} " - - review_comment_raw = edit_content(review_content + '\n\n' + review + '\n\n' + review_rating + '\n\n' + pr_diff, editor) + review_comment_raw = manually_review_impl(user_gitee, pr_info, pull_request, review, review_rating, pr_diff, editor) submitting_pr = {} submitting_pr['review_comment'] = review_comment_raw - submitting_pr['pr_info'] = item['pr_info'] + submitting_pr['pr_info'] = pr_info submitting_pr['pull_request'] = pull_request SUBMITTING_PRS.put(submitting_pr) @@ -318,6 +342,24 @@ def submmit_review(user_gitee): submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action, suggest_reason) print("submit review exits") +def review_pr_new(user_gitee, repo_name, pull_id, group, editor): + """ + New Implementation of Review Pull Request, reuse code from threading implementation + """ + pr_info = {} + pr_info["repo"] = repo_name + pr_info['number'] = pull_id + pr_info['owner'] = group + + pull_request = user_gitee.get_pr(repo_name, pull_id, group) + + suggest_action, suggest_reason = easy_classify(pull_request) + pr_diff, review, review_rating = ai_review_impl(user_gitee, repo_name, pull_id, group) + review_comment = manually_review_impl(user_gitee, pr_info, pull_request, review, review_rating, pr_diff, editor) + submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action, suggest_reason) + + print("Finish Review") + def review_pr(user_gitee, repo_name, pull_id, group, editor): """ Review Pull Request @@ -457,11 +499,15 @@ def main(): except NameError: sys.exit(1) + editor = {} + editor["editor"] = args.editor + editor["editor-option"] = args.editor_option + if args.active_user: generate_pending_prs_thread = threading.Thread(target=generate_pending_prs, args=(user_gitee, args.sig)) sort_pr_thread = threading.Thread(target=sort_pr, args=(user_gitee,)) ai_review_thread = threading.Thread(target=ai_review, args=(user_gitee,)) - manually_review_thread = threading.Thread(target=manually_review, args=(user_gitee, args.editor)) + manually_review_thread = threading.Thread(target=manually_review, args=(user_gitee, editor)) submmit_review_thread = threading.Thread(target=submmit_review, args=(user_gitee,)) generate_pending_prs_thread.start() @@ -486,7 +532,7 @@ def main(): group = params[0] repo_name = params[1] pull_id = params[2] - review_pr(user_gitee, repo_name, pull_id, group, args.editor) + review_pr_new(user_gitee, repo_name, pull_id, group, editor) return 0 -- Gitee From df7f0bac1cf6ced30559cf32b18645b39eb7ce81 Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Fri, 1 Nov 2024 17:59:14 +0800 Subject: [PATCH 2/5] further improve batch reviewing --- advisors/gitee.py | 40 ++++++++++----------- advisors/oe_review.py | 83 ++++++++++++++++++++++++++++++------------- 2 files changed, 78 insertions(+), 45 deletions(-) diff --git a/advisors/gitee.py b/advisors/gitee.py index 12b5152a..53f92b30 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -266,6 +266,21 @@ class Gitee(): print("get diff failed: %d, %s" % (error.code, error.reason)) return None + def get_sig_info(self, sig): + """ + Get sig-info.yaml of specific SIG + """ + url_template = "https://gitee.com/openeuler/community/raw/master/sig/{sig}/sig-info.yaml" + url = url_template.format(sig=sig) + req = urllib.request.Request(url=url, headers=self.headers) + try: + result = urllib.request.urlopen(req) + return result.read().decode("utf-8") + except urllib.error.HTTPError as error: + print("get sig-info.yaml failed: %s" % (url)) + print("get sig-info.yaml failed: %d, %s" %(error.code, error.reason)) + return None + def __get_gitee_json(self, url): """ Get and load gitee json response @@ -378,26 +393,9 @@ class Gitee(): """ Get openEuler repos by SIG """ - repo_list = [] - sigs = self.get_sigs() - if sig not in sigs.keys(): - return repo_list - - openeuler_sha = self.__get_community_sha(sigs[sig], 'openeuler') - if not openeuler_sha: - return '' - - initials_tree = self.__get_community_tree(openeuler_sha) - for initials_dir in initials_tree: - openeuler_repo_tree = self.__get_community_tree(initials_dir['sha']) - for my_repo_dir in openeuler_repo_tree: - repo_name = my_repo_dir['path'] - repo_name = repo_name[:-5] - repo_list.append(repo_name) - - return repo_list + return self.get_repos_by_sig(sig, "openeuler") - def get_repos_by_sig(self, sig): + def get_repos_by_sig(self, sig, openeuler="src-openeuler"): """ Get repos list by sig """ @@ -406,7 +404,7 @@ class Gitee(): return '' repo_list = [] - openeuler_sha = self.__get_community_sha(sigs[sig], 'src-openeuler') + openeuler_sha = self.__get_community_sha(sigs[sig], openeuler) if not openeuler_sha: return '' @@ -419,7 +417,7 @@ class Gitee(): repo_list.append(repo_name) return repo_list - + def get_community(self, repo): """ diff --git a/advisors/oe_review.py b/advisors/oe_review.py index 1908d9ce..e7a9beac 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -152,7 +152,7 @@ def args_parser(): pars.add_argument("-n", "--repo", type=str, help="Repository name that include group") pars.add_argument("-p", "--pull", type=str, help="Number ID of Pull Request") pars.add_argument("-u", "--url", type=str, help="URL of Pull Request") - pars.add_argument("-s", "--sig", type=str, default="TC", help="When active_user is set, review all PRs in specified SIG") + pars.add_argument("-s", "--sig", type=str, default="", help="When active_user is set, review all PRs in specified SIG") pars.add_argument("-m", "--model", type=str, help="Model of selection to generate review") pars.add_argument("-e", "--editor", type=str, default="neovide", help="Editor of choice to edit content, default to nvim") @@ -250,7 +250,7 @@ def ai_review(user_gitee): break pr_info = item["pr_info"] - pr_diff, review, review_rating = ai_review_impl(user_gitee, pr_info['repo'], pr_info['id'], pr_info['group']) + pr_diff, review, review_rating = ai_review_impl(user_gitee, pr_info['repo'], pr_info['number'], pr_info['owner']) if pr_diff == "": continue @@ -282,6 +282,7 @@ def manually_review_impl(user_gitee, pr_info, pull_request, review, review_ratin elif comment['user']['name'] == "openeuler-sync-bot": sync_comment += comment["body"] + "\n" else: + history_comment += comment["user"]["name"] + ":\n" history_comment += comment["body"] + "\n" review_content += "\n# Branch Status\n" + sync_comment review_content += "\n# History\n" + history_comment @@ -314,6 +315,10 @@ def manually_review(user_gitee, editor): def submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action="", suggest_reason=""): result = " is handled and review is published." + if review_comment == "": + print("!{number}: {title} is ignored".format(number=pr_info["number"], title=pull_request["title"])) + return + user_gitee.create_pr_comment(pr_info['repo'], pr_info['number'], review_comment, pr_info['owner']) if suggest_action == "/close": @@ -454,7 +459,7 @@ def review_repo(user_gitee, owner, repo): result = f'{owner}/{repo}'.format(owner=owner, repo=repo) try: PRs = user_gitee.list_pr(repo, owner) - except urllib.URLErro as e: + except urllib.URLerror as e: print(e) print(f'Failed to get PRs in {owner}/{repo}'.format(owner=owner, repo=repo)) if not PRs: @@ -484,6 +489,50 @@ def generate_pending_prs(user_gitee, sig): print("DONE PENDING GENERATE") return 0 +def get_responsible_sigs(user_gitee): + """ + Get responsible sigs from config file + """ + sigs = user_gitee.get_sigs() + result = [] + for sig in sigs: + sig_info_str = user_gitee.get_sig_info(sig) + if sig_info_str == None: + continue + sig_info = yaml.load(sig_info_str, Loader=yaml.FullLoader) + for maintainer in sig_info["maintainers"]: + if maintainer["gitee_id"].lower() == user_gitee.token['user'].lower(): + result.append((sig_info["name"])) + return result + +def review_sig(user_gitee, sig, editor): + """ + Review sig + 1. Generate pending PRs for sig + 2. Close or accept PR for easy ones + 3. Generate AI Comments for sophisitcated PR + 4. Manually edit comment + 5. Submit PR review + """ + + print("Reviewing sig: {}".format(sig)) + generate_pending_prs_thread = threading.Thread(target=generate_pending_prs, args=(user_gitee, sig)) + sort_pr_thread = threading.Thread(target=sort_pr, args=(user_gitee,)) + ai_review_thread = threading.Thread(target=ai_review, args=(user_gitee,)) + manually_review_thread = threading.Thread(target=manually_review, args=(user_gitee, editor)) + submmit_review_thread = threading.Thread(target=submmit_review, args=(user_gitee,)) + + generate_pending_prs_thread.start() + sort_pr_thread.start() + ai_review_thread.start() + manually_review_thread.start() + submmit_review_thread.start() + + generate_pending_prs_thread.join() + sort_pr_thread.join() + ai_review_thread.join() + manually_review_thread.join() + submmit_review_thread.join() def main(): """ @@ -504,27 +553,13 @@ def main(): editor["editor-option"] = args.editor_option if args.active_user: - generate_pending_prs_thread = threading.Thread(target=generate_pending_prs, args=(user_gitee, args.sig)) - sort_pr_thread = threading.Thread(target=sort_pr, args=(user_gitee,)) - ai_review_thread = threading.Thread(target=ai_review, args=(user_gitee,)) - manually_review_thread = threading.Thread(target=manually_review, args=(user_gitee, editor)) - submmit_review_thread = threading.Thread(target=submmit_review, args=(user_gitee,)) - - generate_pending_prs_thread.start() - sort_pr_thread.start() - ai_review_thread.start() - manually_review_thread.start() - submmit_review_thread.start() - - generate_pending_prs_thread.join() - sort_pr_thread.join() - ai_review_thread.join() - manually_review_thread.join() - submmit_review_thread.join() - - # 获取当前用户所有担任 maintainer 的 sig，以及所有担任 committer 的 代码仓 - # 此处应考虑建立本地缓存，保留PR的标签，最近更新，如果和本地缓存相比没有更新就忽略，否则更新 review - # 然后根据不同的模型生成 review + if args.sig == "": + sigs = get_responsible_sigs(user_gitee) + for sig in sigs: + review_sig(user_gitee, sig, editor) + else: + review_sig(user_gitee, args.sig, editor) + else: params = extract_params(args) if not params: -- Gitee From 72956fdceb552aa66c26c582a4e31cc6d804e8bb Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Sat, 16 Nov 2024 17:21:44 +0800 Subject: [PATCH 3/5] fine tune for better interaction --- advisors/gitee.py | 3 +++ advisors/oe_review.py | 44 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 43 insertions(+), 4 deletions(-) diff --git a/advisors/gitee.py b/advisors/gitee.py index 53f92b30..6a94cfb0 100755 --- a/advisors/gitee.py +++ b/advisors/gitee.py @@ -242,6 +242,9 @@ class Gitee(): print("get_gitee failed to access: %s" % (url)) print("get_gitee failed: %d, %s" % (error.code, error.reason)) return None + except urllib.error.URLError as error: + print("get_gitee failed to access: %s" % (url)) + print("get_gitee failed: %s" % (error.reason)) def get_pr(self, repo, num, owner="src-openeuler"): """ diff --git a/advisors/oe_review.py b/advisors/oe_review.py index e7a9beac..ca4a67fb 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -21,6 +21,7 @@ import subprocess import shutil import tempfile import urllib +import urllib.error import urllib.request import urllib.parse import yaml @@ -33,7 +34,9 @@ from advisors import gitee OE_REVIEW_PR_PROMPT=""" You are a code reviewer of a openEuler Pull Request, providing feedback on the code changes below. As a code reviewer, your task is: + - Above all, you need to decide "/close" the PR, or "/lgtm" and "/approve" the PR. - Review the code changes (diffs) in the patch and provide feedback. + - If changelog is updated, it should describe visible changes to end-users or developers, not simply say "upgrade to blahblah". - If there are any bugs, highlight them. - Does the code do what it says in the commit messages? - Do not highlight minor issues and nitpicks. @@ -265,6 +268,14 @@ def ai_review(user_gitee): MANUAL_REVIEW_PRS.put(None) print("ai review exits") +def clean_advisor_comment(comment): + """ + replace icon in advisor comment + """ + comment = comment.replace('[🔵]', 'ongoing').replace('[🟡]', 'question') + comment = comment.replace('[◯]', 'NA').replace('[🔴]', 'nogo').replace('[🟢]', 'GO') + return comment + def manually_review_impl(user_gitee, pr_info, pull_request, review, review_rating, pr_diff, editor): review_content = "" review_content += "!{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) @@ -275,10 +286,14 @@ def manually_review_impl(user_gitee, pr_info, pull_request, review, review_ratin review_content += "\nThis PR is submitted to {branch}\n".format(branch=target_branch) history_comment = "" sync_comment = "" + advisor_comment = "" comments = user_gitee.get_pr_comments_all(pr_info['owner'], pr_info['repo'], pr_info['number']) for comment in comments: if comment['user']['name'] == "openeuler-ci-bot": - continue + + if comment['body'].startswith("\n**以下为 openEuler-Advisor"): + advisor_comment = comment['body'] + elif comment['user']['name'] == "openeuler-sync-bot": sync_comment += comment["body"] + "\n" else: @@ -286,7 +301,9 @@ def manually_review_impl(user_gitee, pr_info, pull_request, review, review_ratin history_comment += comment["body"] + "\n" review_content += "\n# Branch Status\n" + sync_comment review_content += "\n# History\n" + history_comment - review_comment_raw = edit_content(review_content + '\n\n' + review + '\n\n' + review_rating + '\n\n' + pr_diff, editor) + + review_content += "\n# Advisor\n" + clean_advisor_comment(advisor_comment) + review_comment_raw = edit_content(review_content + '\n\n# ReviewBot\n\n' + review + '\n\n# ReviewRating\n\n' + review_rating + '\n\n' + pr_diff, editor) return review_comment_raw def manually_review(user_gitee, editor): @@ -319,7 +336,24 @@ def submit_review_impl(user_gitee, pr_info, pull_request, review_comment, sugges print("!{number}: {title} is ignored".format(number=pr_info["number"], title=pull_request["title"])) return - user_gitee.create_pr_comment(pr_info['repo'], pr_info['number'], review_comment, pr_info['owner']) + review_to_submit = "" + for line in review_comment.split("\n"): + if line == "====": + if review_to_submit == "": + continue + try: + user_gitee.create_pr_comment(pr_info['repo'], pr_info['number'], review_to_submit, pr_info['owner']) + except http.client.RemoteDisconnected as e: + print("Failed to sumit review comment: {error}".format(error=e)) + review_to_submit = "" + else: + review_to_submit += line + "\n" + else: + try: + user_gitee.create_pr_comment(pr_info['repo'], pr_info['number'], review_to_submit, pr_info['owner']) + except http.client.RemoteDisconnected as e: + print("Failed to sumit review comment: {error}".format(error=e)) + if suggest_action == "/close": result = " is closed due to {reason}.".format(reason=suggest_reason) @@ -459,7 +493,7 @@ def review_repo(user_gitee, owner, repo): result = f'{owner}/{repo}'.format(owner=owner, repo=repo) try: PRs = user_gitee.list_pr(repo, owner) - except urllib.URLerror as e: + except urllib.error.URLError as e: print(e) print(f'Failed to get PRs in {owner}/{repo}'.format(owner=owner, repo=repo)) if not PRs: @@ -496,6 +530,8 @@ def get_responsible_sigs(user_gitee): sigs = user_gitee.get_sigs() result = [] for sig in sigs: + if sig == "sig-minzuchess" or sig == "README.md": + continue sig_info_str = user_gitee.get_sig_info(sig) if sig_info_str == None: continue -- Gitee From 1b2738619373c99afbb81bbaf0b81630767b76ce Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Wed, 27 Nov 2024 21:59:36 +0800 Subject: [PATCH 4/5] add option for no_ai --- advisors/oe_review.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/advisors/oe_review.py b/advisors/oe_review.py index ca4a67fb..3c165071 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -159,6 +159,7 @@ def args_parser(): pars.add_argument("-m", "--model", type=str, help="Model of selection to generate review") pars.add_argument("-e", "--editor", type=str, default="neovide", help="Editor of choice to edit content, default to nvim") + pars.add_argument("-b", "--no_ai", action='store_true', default=False, help="No AI to generate review") pars.add_argument("-o", "--editor-option", type=str, default="--no-fork", help="Commandline option for editor") return pars.parse_args() @@ -236,16 +237,18 @@ def sort_pr(user_gitee): NEED_REVIEW_PRS.put(None) print("sort pr exits") -def ai_review_impl(user_gitee, repo, pull_id, group): +def ai_review_impl(user_gitee, repo, pull_id, group, ai_flag = True): pr_diff = user_gitee.get_diff(repo, pull_id, group) if not pr_diff: print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo)) return "", "", "" + if not ai_flag: + return pr_diff, "", "" review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT) return pr_diff, review, review_rating -def ai_review(user_gitee): +def ai_review(user_gitee, ai_flag = True): while True: item = NEED_REVIEW_PRS.get() #print("ai review works") @@ -253,7 +256,7 @@ def ai_review(user_gitee): break pr_info = item["pr_info"] - pr_diff, review, review_rating = ai_review_impl(user_gitee, pr_info['repo'], pr_info['number'], pr_info['owner']) + pr_diff, review, review_rating = ai_review_impl(user_gitee, pr_info['repo'], pr_info['number'], pr_info['owner'], ai_flag) if pr_diff == "": continue @@ -381,7 +384,7 @@ def submmit_review(user_gitee): submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action, suggest_reason) print("submit review exits") -def review_pr_new(user_gitee, repo_name, pull_id, group, editor): +def review_pr_new(user_gitee, repo_name, pull_id, group, editor, ai_flag): """ New Implementation of Review Pull Request, reuse code from threading implementation """ @@ -393,7 +396,7 @@ def review_pr_new(user_gitee, repo_name, pull_id, group, editor): pull_request = user_gitee.get_pr(repo_name, pull_id, group) suggest_action, suggest_reason = easy_classify(pull_request) - pr_diff, review, review_rating = ai_review_impl(user_gitee, repo_name, pull_id, group) + pr_diff, review, review_rating = ai_review_impl(user_gitee, repo_name, pull_id, group, ai_flag) review_comment = manually_review_impl(user_gitee, pr_info, pull_request, review, review_rating, pr_diff, editor) submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action, suggest_reason) @@ -541,7 +544,7 @@ def get_responsible_sigs(user_gitee): result.append((sig_info["name"])) return result -def review_sig(user_gitee, sig, editor): +def review_sig(user_gitee, sig, editor, ai_flag): """ Review sig 1. Generate pending PRs for sig @@ -554,7 +557,7 @@ def review_sig(user_gitee, sig, editor): print("Reviewing sig: {}".format(sig)) generate_pending_prs_thread = threading.Thread(target=generate_pending_prs, args=(user_gitee, sig)) sort_pr_thread = threading.Thread(target=sort_pr, args=(user_gitee,)) - ai_review_thread = threading.Thread(target=ai_review, args=(user_gitee,)) + ai_review_thread = threading.Thread(target=ai_review, args=(user_gitee, ai_flag)) manually_review_thread = threading.Thread(target=manually_review, args=(user_gitee, editor)) submmit_review_thread = threading.Thread(target=submmit_review, args=(user_gitee,)) @@ -592,9 +595,9 @@ def main(): if args.sig == "": sigs = get_responsible_sigs(user_gitee) for sig in sigs: - review_sig(user_gitee, sig, editor) + review_sig(user_gitee, sig, editor, not args.no_ai) else: - review_sig(user_gitee, args.sig, editor) + review_sig(user_gitee, args.sig, editor, not args.no_ai) else: params = extract_params(args) @@ -603,7 +606,8 @@ def main(): group = params[0] repo_name = params[1] pull_id = params[2] - review_pr_new(user_gitee, repo_name, pull_id, group, editor) + print(args.no_ai) + review_pr_new(user_gitee, repo_name, pull_id, group, editor, not args.no_ai) return 0 -- Gitee From 20429b1bd16c25a506b24d67b032b9edc5fbd841 Mon Sep 17 00:00:00 2001 From: Shinwell Hu Date: Wed, 18 Dec 2024 14:34:10 +0800 Subject: [PATCH 5/5] refactor code of oe_review and user_activity --- advisors/oe_review.py | 151 ++++++++++++-------------------------- advisors/user_activity.py | 4 +- command/oe_review | 2 +- 3 files changed, 49 insertions(+), 108 deletions(-) diff --git a/advisors/oe_review.py b/advisors/oe_review.py index 3c165071..583811e1 100755 --- a/advisors/oe_review.py +++ b/advisors/oe_review.py @@ -27,6 +27,7 @@ import urllib.parse import yaml import json import requests +import configparser from openai import OpenAI from advisors import gitee @@ -95,7 +96,8 @@ MANUAL_REVIEW_PRS = ThreadSafeQueue() # PRs that are being submitted SUBMITTING_PRS = ThreadSafeQueue() -def generate_review_from_ollama(pr_content, prompt, model="llama3.1:8b"): +#def generate_review_from_ollama(pr_content, prompt, model="llama3.1:8b"): +def generate_review_from_ollama(pr_content, prompt, model): base_url = "http://localhost:11434/api" json_resp = [] resp = None @@ -157,12 +159,24 @@ def args_parser(): pars.add_argument("-u", "--url", type=str, help="URL of Pull Request") pars.add_argument("-s", "--sig", type=str, default="", help="When active_user is set, review all PRs in specified SIG") pars.add_argument("-m", "--model", type=str, help="Model of selection to generate review") - pars.add_argument("-e", "--editor", type=str, default="neovide", - help="Editor of choice to edit content, default to nvim") + pars.add_argument("-e", "--editor", type=str, help="Editor of choice to edit content, default to nvim") pars.add_argument("-b", "--no_ai", action='store_true', default=False, help="No AI to generate review") - pars.add_argument("-o", "--editor-option", type=str, default="--no-fork", help="Commandline option for editor") + pars.add_argument("-o", "--editor-option", type=str, help="Commandline option for editor") return pars.parse_args() - + +def load_config(): + """ + Load config from config file + """ + cf = configparser.ConfigParser() + cf_path = os.path.expanduser("~/.config/openEuler-Advisor/config.ini") + if os.path.exists(cf_path): + cf.read(cf_path) + return cf + else: + print("ERROR: no such file:"+cf_path) + return None + def edit_content(text, editor): fd, path = tempfile.mkstemp(suffix=".tmp", prefix="oe_review") with os.fdopen(fd, 'w') as tmp: @@ -237,18 +251,18 @@ def sort_pr(user_gitee): NEED_REVIEW_PRS.put(None) print("sort pr exits") -def ai_review_impl(user_gitee, repo, pull_id, group, ai_flag = True): +def ai_review_impl(user_gitee, repo, pull_id, group, ai_flag, ai_model): pr_diff = user_gitee.get_diff(repo, pull_id, group) if not pr_diff: print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo)) return "", "", "" if not ai_flag: return pr_diff, "", "" - review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) - review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT) + review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT, ai_model) + review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT, ai_model) return pr_diff, review, review_rating -def ai_review(user_gitee, ai_flag = True): +def ai_review(user_gitee, ai_flag, ai_model): while True: item = NEED_REVIEW_PRS.get() #print("ai review works") @@ -256,7 +270,7 @@ def ai_review(user_gitee, ai_flag = True): break pr_info = item["pr_info"] - pr_diff, review, review_rating = ai_review_impl(user_gitee, pr_info['repo'], pr_info['number'], pr_info['owner'], ai_flag) + pr_diff, review, review_rating = ai_review_impl(user_gitee, pr_info['repo'], pr_info['number'], pr_info['owner'], ai_flag, ai_model) if pr_diff == "": continue @@ -384,7 +398,7 @@ def submmit_review(user_gitee): submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action, suggest_reason) print("submit review exits") -def review_pr_new(user_gitee, repo_name, pull_id, group, editor, ai_flag): +def review_pr_new(user_gitee, repo_name, pull_id, group, editor, ai_flag, ai_model): """ New Implementation of Review Pull Request, reuse code from threading implementation """ @@ -396,99 +410,12 @@ def review_pr_new(user_gitee, repo_name, pull_id, group, editor, ai_flag): pull_request = user_gitee.get_pr(repo_name, pull_id, group) suggest_action, suggest_reason = easy_classify(pull_request) - pr_diff, review, review_rating = ai_review_impl(user_gitee, repo_name, pull_id, group, ai_flag) + pr_diff, review, review_rating = ai_review_impl(user_gitee, repo_name, pull_id, group, ai_flag, ai_model) review_comment = manually_review_impl(user_gitee, pr_info, pull_request, review, review_rating, pr_diff, editor) submit_review_impl(user_gitee, pr_info, pull_request, review_comment, suggest_action, suggest_reason) print("Finish Review") -def review_pr(user_gitee, repo_name, pull_id, group, editor): - """ - Review Pull Request - """ - review_content = "" - pull_request = user_gitee.get_pr(repo_name, pull_id, group) - - suggest_action = "" - - result = "" - - review_content += "!{number}: {title}\n# {body}\n".format(number=pull_request["number"], title=pull_request["title"], body=pull_request["body"]) - if not pull_request["mergeable"]: - review_content += "This PR is not mergeable!\n" - suggest_action = "/close" - suggest_reason = "存在冲突" - - if pull_request["title"].startswith("[sync] PR-") and pull_request["user"]["login"]=="openeuler-sync-bot": - sync_pr = True - - review_content += "This PR has following labels:\n" - for label in pull_request["labels"]: - if label['name'] == "ci_failed": - suggest_action = "/close" - suggest_reason = "CI 失败" - elif label['name'] == 'openeuler-cla/no': - suggest_action = "/check-cla" - suggest_reason = "CLA 未签署" - elif label['name'] == 'ci_processing': - suggest_action = "暂不处理" - suggest_reason = "等待 CI 处理结果" - elif label['name'] == 'kind/wait_for_update': - suggest_action = "暂不处理" - suggest_reason = "等待提交人更新" - elif label['name'] == 'wait_confirm': - suggest_action = "暂不处理" - suggest_reason = "等待相关开发者确认" - elif label['name'] == 'ci_successful': - if sync_pr == True: - suggest_action = "/lgtm\n/approve" - suggest_reason = "分支同步 PR，构建成功，默认合入。" - else: - pass - review_content += f"{label['name']} " - review_content += "\n\n" - - pr_diff = user_gitee.get_diff(repo_name, pull_id, group) - if not pr_diff: - print("Failed to get PR:%s of repository:%s/%s, make sure the PR is exist." % (pull_id, group, repo_name)) - return 1 - - if suggest_action == "/close": - review = "/close" - review_comment_raw = review + "\n" + suggest_reason - elif suggest_action == "/check-cla": - review = "/check-cla" - review_comment_raw = review + "\n" + suggest_reason - elif suggest_action == "暂不处理": - review = "暂不处理" - review_comment_raw = review + "\n" + suggest_reason - elif suggest_action == "/lgtm\n/approve": - review = "/lgtm\n/approve" - review_comment_raw = review + "\n" + suggest_reason - else: - review = generate_review_from_ollama(pr_diff, OE_REVIEW_PR_PROMPT) - review_rating = generate_review_from_ollama(pr_diff, OE_REVIEW_RATING_PROMPT) - review_comment_raw = edit_content(review_content + '\n\n' + review + '\n\n' + review_rating + '\n\n' + pr_diff, editor) - - review_comment = "" - # remove lines start with hash - for l in review_comment_raw: - if l.startswith("#"): - continue - else: - review_comment += l - - user_gitee.create_pr_comment(repo_name, pull_id, review_comment, group) - if suggest_action == "/close": - result = " is closed due to {reason}.".format(reason=suggest_reason) - elif suggest_action == "暂不处理": - result = " is skipped due to {reason}.".format(reason=suggest_reason) - elif suggest_action == "/lgtm\n/approve": - result = " is approved due to {reason}.".format(reason=suggest_reason) - else: - result = " is handled and review is published." - return result - def review_repo(user_gitee, owner, repo): """" Get PRs in give repo, or doing nothing if no PR @@ -544,7 +471,7 @@ def get_responsible_sigs(user_gitee): result.append((sig_info["name"])) return result -def review_sig(user_gitee, sig, editor, ai_flag): +def review_sig(user_gitee, sig, editor, ai_flag, ai_model): """ Review sig 1. Generate pending PRs for sig @@ -557,7 +484,7 @@ def review_sig(user_gitee, sig, editor, ai_flag): print("Reviewing sig: {}".format(sig)) generate_pending_prs_thread = threading.Thread(target=generate_pending_prs, args=(user_gitee, sig)) sort_pr_thread = threading.Thread(target=sort_pr, args=(user_gitee,)) - ai_review_thread = threading.Thread(target=ai_review, args=(user_gitee, ai_flag)) + ai_review_thread = threading.Thread(target=ai_review, args=(user_gitee, ai_flag, ai_model)) manually_review_thread = threading.Thread(target=manually_review, args=(user_gitee, editor)) submmit_review_thread = threading.Thread(target=submmit_review, args=(user_gitee,)) @@ -578,6 +505,8 @@ def main(): Main entrance of the functionality """ args = args_parser() + cf = load_config() + if args.quiet: sys.stdout = open('/dev/null', 'w') sys.stderr = sys.stdout @@ -587,17 +516,27 @@ def main(): except NameError: sys.exit(1) + editor = {} - editor["editor"] = args.editor - editor["editor-option"] = args.editor_option + # command line overrides config file + editor["editor"] = cf.get('editor', 'command') + if args.editor: + editor["editor"] = args.editor + editor["editor-option"] = cf.get('editor', 'option') + if args.editor_option: + editor["editor-option"] = args.editor_option + + ai_model = cf.get('model', 'name') + if args.model: + ai_model = args.model if args.active_user: if args.sig == "": sigs = get_responsible_sigs(user_gitee) for sig in sigs: - review_sig(user_gitee, sig, editor, not args.no_ai) + review_sig(user_gitee, sig, editor, not args.no_ai, ai_model) else: - review_sig(user_gitee, args.sig, editor, not args.no_ai) + review_sig(user_gitee, args.sig, editor, not args.no_ai, ai_model) else: params = extract_params(args) @@ -607,7 +546,7 @@ def main(): repo_name = params[1] pull_id = params[2] print(args.no_ai) - review_pr_new(user_gitee, repo_name, pull_id, group, editor, not args.no_ai) + review_pr_new(user_gitee, repo_name, pull_id, group, editor, not args.no_ai, ai_model) return 0 diff --git a/advisors/user_activity.py b/advisors/user_activity.py index 83159ff2..09ee57d7 100755 --- a/advisors/user_activity.py +++ b/advisors/user_activity.py @@ -1,4 +1,4 @@ -#!/usr/bin/python3 +#!python3 """ This is a command line tool to collect activities for maintainers of given SIG """ @@ -81,6 +81,8 @@ class Advisor: continue if ignore_memberevent and event['type'] == 'MemberEvent': continue + if event['type'] == 'FollowEvent': + continue # ignore followevent for now repo_name = event['repo'].get('full_name', "") if repo_name == "": print("ERROR: " + str(event['id'])) diff --git a/command/oe_review b/command/oe_review index 47a6a219..ef2572a4 100755 --- a/command/oe_review +++ b/command/oe_review @@ -1,4 +1,4 @@ -#!/usr/bin/env python3.10 +#!/usr/bin/env python3 import sys import signal -- Gitee