diff --git a/cve-agency-manager/cve_tracking/src/cve/apply.py b/cve-agency-manager/cve_tracking/src/cve/apply.py index fede6084986c42e3aeea87c76a9251ef80d6e68f..9bd93d949cf7e3bd68dc2fa6266ffc54f882bee2 100644 --- a/cve-agency-manager/cve_tracking/src/cve/apply.py +++ b/cve-agency-manager/cve_tracking/src/cve/apply.py @@ -26,6 +26,7 @@ def apply_patch(patches_path, branches): :param branches: branch of source code :return: apply result """ + print('[INFO] Start to try apply patches') shell_file = os.path.join(os.path.dirname(CURRENT_PATCH), "shell/start.sh") try: branch_str = "master" @@ -40,27 +41,13 @@ def apply_patch(patches_path, branches): shell=False, ) result = output.decode("utf-8") - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as e: + print(f'[ERROR] Failed to execute the application patch script: {str(e)}') result = "All branch apply failed" - # _clear_env(shell_file) return result -def _clear_env(shell_file): - """ - Clear tmp path and file - :param shell_file: shell - :return: None - """ - try: - subprocess.run([shell_file, CLEAR], stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - shell=False) - except subprocess.CalledProcessError: - return - - def row(func): """ Combine the rows of the table @@ -101,7 +88,7 @@ class AutoComment: def __init__(self, header=None) -> None: self._gitee = Gitee() self._line_feed = "\n" - self._header = header or ["Patchs URL", "Branch", "Apply Result"] + self._header = header or ["Patches URL", "Branch", "Apply Result"] @row def _row(self, cols, spans=None, header=False): @@ -131,13 +118,13 @@ class AutoComment: """Comments on PR""" comments = self._gitee.get_issue_comments(repo=repo, number=number, owner=owner) if comments: - for comment in comments: - if all([header in comment.get("body", "") for header in self._header]): - logger.info( - "Issue %s in repository %s already has a fixed comment content ." - % (number, repo) - ) - return + if all([header in str(comments) for header in self._header]): + print('[WARNING] There are already relevant comments') + logger.info( + "Issue %s in repository %s already has a fixed comment content ." + % (number, repo) + ) + return if table: body = self._create_table(header=self._header, body=body) body += self._line_feed * 2 + "> 说明：补丁链接和应用结果仅供初步排查参考，实际应用请人工再次确认。" diff --git a/cve-agency-manager/cve_tracking/src/cve/cli.py b/cve-agency-manager/cve_tracking/src/cve/cli.py index 7e74f413cf37c455e46a54ca11ca1ca2e111a9a3..13fb976683a3970a0086a180c10d0255b79d2cf3 100644 --- a/cve-agency-manager/cve_tracking/src/cve/cli.py +++ b/cve-agency-manager/cve_tracking/src/cve/cli.py @@ -84,6 +84,7 @@ def _run(args, option_parser=ArgumentParser): parser_args = parser.parse_args(args=args) parser_args.func(parser_args) except Exception as e: + print(f'[ERROR] Failed to execute A script: {str(e)}') return 1 else: return 0 diff --git a/cve-agency-manager/cve_tracking/src/cve/engine.py b/cve-agency-manager/cve_tracking/src/cve/engine.py index c9fdbeec0650fd044126dd7d78e273bc2d3d5a13..3946a96835710dec7b15ae0961bd64ef4b714522 100644 --- a/cve-agency-manager/cve_tracking/src/cve/engine.py +++ b/cve-agency-manager/cve_tracking/src/cve/engine.py @@ -45,6 +45,7 @@ class Cardiac: self._request_fingerprint = RequestRepeat() def _consume(self, cve, obj): + print(f'[INFO] Crawling {str(obj.__name__)} platform') if not issubclass(obj, Crawl): raise RuntimeError("") @@ -316,6 +317,7 @@ class CrawlerProcess: return Thread(target=self.engine.save_pipe) def _extract_info(self, args): + print('[INFO] Start to extract info') file_handle = FileHandle( folder=args.o, branch=args.branch, @@ -327,7 +329,7 @@ class CrawlerProcess: apply_result = None if args.cmd: apply_result = apply_patch(args.o, args.branch) - + print(f'[INFO] Apply patch result is {str(apply_result)}') file_handle.format_text(apply_result=apply_result) if args.issue: path = os.path.join(args.o, args.name + "-" + args.cve, RECORD_FILE) @@ -363,6 +365,7 @@ class CrawlerProcess: """ Entrance to program execution """ + print('[INFO] Start to run cve_tracking') # Clears all files in the specified path if os.path.exists(args.o): shutil.rmtree(args.o) diff --git a/cve-agency-manager/cve_tracking/src/cve/gitee.py b/cve-agency-manager/cve_tracking/src/cve/gitee.py index 68aa6b921b1f0b43651795e51c870df7235a10a7..44298911997304930f3690175c664eb7a7cd14dc 100644 --- a/cve-agency-manager/cve_tracking/src/cve/gitee.py +++ b/cve-agency-manager/cve_tracking/src/cve/gitee.py @@ -13,17 +13,15 @@ """ This is a helper script for working with gitee.com """ -import sys -import os import json -import base64 +import os import urllib -import urllib.request -import urllib.parse import urllib.error -from datetime import datetime +import urllib.parse +import urllib.request + from fake_useragent import UserAgent -import yaml + from .logger import logger from .settings import GITEE_AUTH @@ -64,44 +62,11 @@ class Gitee: ) return False - def fork_repo(self, repo, owner="src-openeuler"): - """ - Fork repository in gitee - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/forks" - url = url_template.format(owner=owner, repo=repo) - values = {} - values["access_token"] = self.token["access_token"] - return self.post_gitee(url, values) - - def create_issue( - self, - repo, - version="", - branch="master", - owner="src-openeuler", - title=None, - body=None, - ): - """ - Create issue in gitee - """ - title = title or "Upgrade {pkg} to {ver} in {br}".format( - pkg=repo, ver=version, br=branch - ) - body = ( - body - or """This issue is automatically created by openEuler-Advisor. - Please check the correspond PR is accepted before close it. - Thanks. - Yours openEuler-Advisor.""" - ) - return self._post_issue(repo, title, body, owner=owner) - def create_issue_comment(self, repo, owner, number, body): """ create issue comment """ + print('[INFO] Start to add issue comment') url_template = ( "https://gitee.com/api/v5/repos/{owner}/{repo}/issues/{number}/comments" ) @@ -111,67 +76,26 @@ class Gitee: values["body"] = body return self.post_gitee(url, values) - def get_reviewers(self, repo, owner="src-openeuler"): - """ - Get reviewers of pkg - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{pkg}/collaborators" - url = url_template.format(owner=owner, pkg=repo) - return self.get_gitee_dict(url) - - def create_pr( - self, - repo, - version="", - issue=None, - title=None, - branch="master", - owner="src-openeuler", - body=None, - ): + def get_gitee_json(self, url): """ - Create PR in gitee + Get and load gitee json response """ - # assignees = "" - # reviewer_info = self.get_reviewers(repo) - # if reviewer_info: - # reviewer_list = json.loads(reviewer_info) - # assignees = ",".join(reviewer["login"] - # for reviewer in reviewer_list) - url_template = "https://gitee.com/api/v5/repos/{owner}/{pkg}/pulls" - url = url_template.format(owner=owner, pkg=repo) - values = {} - values["access_token"] = self.token["access_token"] - values["title"] = title or "Upgrade {pkg} to {ver}".format( - pkg=repo, ver=version - ) - values["head"] = "{hd}:{br}".format(hd=self.token["user"], br=branch) - values["base"] = branch - # values["assignees"] = assignees - if issue: - values["issue"] = issue - values["body"] = ( - body - or """This is a automatically created PR by openEuler-Advisor. - Please be noted that it's not throughly tested. - Review carefully before accept this PR. - Thanks. - Yours openEuler-Advisor.""" - ) - return self.post_gitee(url, values) + json_resp = [] + headers = self.headers.copy() + headers["Content-Type"] = "application/json;charset=UTF-8" + resp = self.get_gitee(url, headers) + if resp: + json_resp = json.loads(resp) + return json_resp - def create_pr_comment(self, repo, number, body, owner="src-openeuler"): + def get_issue_comments(self, repo, number, owner="src-openeuler"): """ - Post comment to the given specific PR + Get comments of specific issue """ - url_template = ( - "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls/{number}/comments" - ) - url = url_template.format(owner=owner, repo=repo, number=number) - values = {} - values["access_token"] = self.token["access_token"] - values["body"] = body - return self.post_gitee(url, values) + issues_url = "https://gitee.com/api/v5/repos/{owner}/{repo}/issues/{number}/comments?" \ + "access_token={token}&page=1&per_page=50&order=asc".format(repo=repo, owner=owner, number=number, + token=self.token["access_token"]) + return self.get_gitee_json(issues_url) def get_gitee(self, url, headers=None): """ @@ -194,276 +118,3 @@ class Gitee: logger.warning("reuqest url: %s messages: %s " % (url, str(e.reason))) return None - - def get_pr(self, repo, num, owner="src-openeuler"): - """ - Get detailed information of the given specific PR - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls/{number}" - url = url_template.format(owner=owner, repo=repo, number=num) - return self.get_gitee_json(url) - - def get_gitee_json(self, url): - """ - Get and load gitee json response - """ - json_resp = [] - headers = self.headers.copy() - headers["Content-Type"] = "application/json;charset=UTF-8" - resp = self.get_gitee(url, headers) - if resp: - json_resp = json.loads(resp) - return json_resp - - def get_branch_info(self, branch): - """ - Get upgrade branch info - """ - upgrade_branches_url = ( - self.advisor_url + "advisors/helper/upgrade_branches.yaml" - ) - resp = self.get_gitee(upgrade_branches_url) - if not resp: - print("ERROR: upgrade_branches.yaml may not exist.") - sys.exit(1) - branches_info = yaml.load(resp, Loader=yaml.Loader) - for br_info in branches_info["branches"]: - if branch == br_info["name"]: - return br_info - print("WARNING: Don't support branch: {} in auto-upgrade.".format(branch)) - sys.exit(1) - - def get_spec_exception(self, repo): - """ - Get well known spec file exception - """ - specfile_exception_url = ( - self.advisor_url + "advisors/helper/specfile_exceptions.yaml" - ) - resp = self.get_gitee(specfile_exception_url) - if not resp: - print("ERROR: specfile_exceptions.yaml may not exist.") - sys.exit(1) - excpt_list = yaml.load(resp, Loader=yaml.Loader) - if repo in excpt_list: - return excpt_list[repo] - return None - - def get_version_exception(self): - """ - Get version recommend exceptions - """ - version_exception_url = ( - self.advisor_url + "advisors/helper/version_exceptions.yaml" - ) - resp = self.get_gitee(version_exception_url) - if not resp: - print("ERROR: version_exceptions.yaml may not exist.") - sys.exit(1) - excpt = yaml.load(resp, Loader=yaml.Loader) - return excpt - - def get_spec(self, pkg, branch="master"): - """ - Get openeuler spec file for specific package - """ - specurl = self.src_openeuler_url + "{repo}.spec" - specurl = specurl.format(repo=pkg, br=branch) - excpt = self.get_spec_exception(pkg) - if excpt: - specurl = urllib.parse.urljoin( - specurl, os.path.join(excpt["dir"], excpt["file"]) - ) - resp = self.get_gitee(specurl) - return resp - - def get_yaml(self, pkg): - """ - Get upstream yaml metadata for specific package - """ - yamlurl = self.advisor_url + "upstream-info/{}.yaml".format(pkg) - resp = self.get_gitee(yamlurl) - if not resp: - yamlurl = self.src_openeuler_url + "{repo}.yaml" - yamlurl = yamlurl.format(repo=pkg, br="master") - resp = self.get_gitee(yamlurl) - if not resp: - print( - "WARNING: {}.yaml can't be found in upstream-info and repo.".format( - pkg - ) - ) - return resp - - def get_community(self, repo): - """ - Get yaml data from community repo - """ - yamlurl = ( - "https://gitee.com/api/v5/repos/openeuler/community/contents/" - "repository/{repo}.yaml".format(repo=repo) - ) - resp = self.get_gitee_json(yamlurl) - resp_str = base64.b64decode(resp["content"]) - return resp_str - - def get_issues(self, pkg, prj="src-openeuler"): - """ - List all open issues of pkg - """ - issues_url = "https://gitee.com/api/v5/repos/{prj}/{pkg}/issues?".format( - prj=prj, pkg=pkg - ) - parameters = "state=open&sort=created&direction=desc&page=1&per_page=20" - return self.get_gitee_json(issues_url + parameters) - - def get_issue_comments(self, repo, number, owner="src-openeuler"): - """ - Get comments of specific issue - """ - issues_url = "https://gitee.com/api/v5/repos/{owner}/{repo}/issues/{number}/comments?page=1&per_page=50&order=asc".format( - repo=repo, owner=owner, number=number - ) - return self.get_gitee_json(issues_url) - - def get_issue_merged_branches(self, issue_id, repo, owner="src-openeuler"): - """ - Get merged branches of specific issue - """ - issues_url = "https://gitee.com/api/v5/repos/{owner}/issues/{number}/pull_requests?".format( - owner=owner, number=issue_id - ) - - param = "&repo={}".format(repo) - return self.get_gitee_dict(issues_url, param) - - def _post_issue(self, pkg, title, body, owner="src-openeuler"): - """ - Post new issue - """ - issues_url = "https://gitee.com/api/v5/repos/{owner}/issues".format( - owner=owner) - parameters = {} - parameters["access_token"] = self.token["access_token"] - parameters["repo"] = pkg - parameters["title"] = title - parameters["body"] = body - return self.post_gitee(issues_url, parameters) - - def post_issue_comment(self, pkg, number, comment, prj="src-openeuler"): - """ - Post comment of issue - """ - issues_url = ( - "https://gitee.com/api/v5/repos/{prj}/{pkg}/issues/{number}/" - "comments".format(prj=prj, pkg=pkg, number=number) - ) - parameters = {} - parameters["access_token"] = self.token["access_token"] - parameters["body"] = comment - self.post_gitee(issues_url, parameters) - - def get_gitee_datetime(self, time_string): - """ - Get datetime of gitee - """ - result = datetime.strptime(time_string, self.time_format) - return result.replace(tzinfo=None) - - def get_gitee_dict(self, url, param=""): - url += param - token_param = "access_token={}".format(self.token["access_token"]) - separator = "?" - if param: - separator = "&" - url += separator + token_param - return self.get_gitee_json(url) - - def get_contributors(self, repo, owner): - """ - Get contributors of owner/repo - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/contributors" - url = url_template.format(owner=owner, repo=repo) - return self.get_gitee_dict(url) - - def get_branches(self, repo, owner): - """ - Get branches of owner/repo - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/branches" - url = url_template.format(owner=owner, repo=repo) - return self.get_gitee_dict(url) - - def get_commits( - self, repo, owner, sha="", author="", since="", until="", page=1, per_page=20 - ): - """ - Get commits of owner/repo - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/commits" - url = url_template.format(owner=owner, repo=repo) - param = "?page={}&per_page={}".format(page, per_page) - if sha: - param += "&sha={}".format(sha) - if author: - param += "&author={}".format(author) - if since: - param += "&since={}".format(since) - if until: - param += "&until={}".format(until) - return self.get_gitee_dict(url, param) - - def get_one_commit(self, repo, sha, owner): - """ - Get one commit of owner/repo - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/commits/{sha}" - url = url_template.format(owner=owner, repo=repo, sha=sha) - return self.get_gitee_dict(url) - - def get_pr_list( - self, repo, owner, head="", base="", state="all", page=1, per_page=20 - ): - """ - Get PR list of owner/repo - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls" - url = url_template.format(owner=owner, repo=repo) - param = "?state={}&sort=created&direction=desc&page={}&per_page={}".format( - state, page, per_page - ) - if head: - param += "&head={}".format(head) - if base: - param += "&base={}".format(base) - return self.get_gitee_dict(url, param) - - def get_pr_comments(self, repo, number, owner, page=1, per_page=20): - """ - Get PR comments of owner/repo - """ - url_template = ( - "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls/{number}/comments" - ) - url = url_template.format(owner=owner, repo=repo, number=number) - param = "?page={}&per_page={}".format(page, per_page) - return self.get_gitee_dict(url, param) - - def get_repos(self, org, repo_type, page=1, per_page=20): - """ - Get repos of org - """ - url_template = "https://gitee.com/api/v5/orgs/{org}/repos" - url = url_template.format(org=org) - param = "?type={}&page={}&per_page={}".format( - repo_type, page, per_page) - return self.get_gitee_dict(url, param) - - def get_repo(self, repo, owner): - """ - Get repo - """ - url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}" - url = url_template.format(owner=owner, repo=repo) - return self.get_gitee(url) diff --git a/cve-agency-manager/cve_tracking/src/cve/pipe.py b/cve-agency-manager/cve_tracking/src/cve/pipe.py index 872acfb379edd217bdc62719cfbe904322fa5ee1..f36e45267181644e6b78b4ea78ce758dfa83658a 100644 --- a/cve-agency-manager/cve_tracking/src/cve/pipe.py +++ b/cve-agency-manager/cve_tracking/src/cve/pipe.py @@ -246,11 +246,11 @@ class FileHandle: apply_result = lines[index:] break if self._apply in apply_result: - apply_result = [branch + ":" + self._apply for branch in self.branches] + apply_result = [branch + ":" + "apply failed" for branch in self.branches] return { "urls": list(set(patchs)), - "apply_result": dict([tuple(apply.split(":")) for apply in apply_result]), + "apply_result": dict([tuple(apply.split(":")[1:]) for apply in apply_result]), } diff --git a/cve-agency-manager/cve_tracking/src/cve/settings.py b/cve-agency-manager/cve_tracking/src/cve/settings.py index a9468cd2d2bd69b06f8c3b66bffb1d92557e0cda..49934d9296e78722c4549f1191c430deffd39bf1 100644 --- a/cve-agency-manager/cve_tracking/src/cve/settings.py +++ b/cve-agency-manager/cve_tracking/src/cve/settings.py @@ -35,4 +35,4 @@ MAX_QUEUE = 100 RECORD_FILE = "repair-verdict.txt" # Code cloud related API authorization operations, You need to set a global environment variable on the server, where you specify the name of the environment variable -GITEE_AUTH = {"token": "gitee_token", "account": "gitee_account"} +GITEE_AUTH = {"token": "TRACK_GITEE_TOKEN", "account": "gitee_account"} diff --git a/cve-agency-manager/cve_tracking/src/shell/add_patch.sh b/cve-agency-manager/cve_tracking/src/shell/add_patch.sh index a0658e75c56c53ea4c3d844b51c158cd681ae1d9..c631307568492448a02799ad33ce3f3f182dc0b3 100644 --- a/cve-agency-manager/cve_tracking/src/shell/add_patch.sh +++ b/cve-agency-manager/cve_tracking/src/shell/add_patch.sh @@ -70,11 +70,11 @@ function pre_env_build() { function git_clone() { echo "[INFO] Start to git clone ${rpm_name}" - git_status=$(rpm -qa git) - if [[ -z ${git_status} ]]; then - yum install git -y - git_status=$(rpm -qa git) - if [[ -z ${git_status} ]]; then + git_status=$(dpkg -s git 2>&1) + if [[ ${git_status} =~ "is not installed" ]]; then + apt install git -y + git_status=$(dpkg -s git 2>&1) + if [[ ${git_status} =~ "is not installed" ]]; then echo "[ERROR] Git install failed" exit 1 fi @@ -142,11 +142,11 @@ function mv_source_file() { function rpm_build() { echo "[INFO] Start to rpmbuild" - rpmbuild_status=$(rpm -qa rpm-build) - if [[ -z ${rpmbuild_status} ]]; then - yum install rpm-build -y >/dev/null 2>&1 - rpmbuild_status=$(rpm -qa rpm-build) - if [[ -z ${rpmbuild_status} ]]; then + rpmbuild_status=$(dpkg -s rpm 2>&1) + if [[ ${rpmbuild_status} =~ "is not installed" ]]; then + apt install rpm -y >/dev/null 2>&1 + rpmbuild_status=$(dpkg -s rpm 2>&1) + if [[ ${rpmbuild_status} =~ "is not installed" ]]; then echo "[ERROR] Install rpm-build failed" exit 1 fi @@ -159,7 +159,7 @@ function rpm_build() { requires_rpms=$(grep -r "is needed by" ./rpmbuild.log | awk -F " " '{print $1}') echo "${requires_rpms}" >./requires_rpms.log for rpm in ${requires_rpms[@]}; do - yum install ${rpm} -y >/dev/null 2>&1 + apt install ${rpm} -y >/dev/null 2>&1 if [[ $? -eq 0 ]]; then echo "[INFO] Successfully install dependent package ${rpm}" else diff --git a/cve-agency-manager/cve_tracking/src/shell/start.sh b/cve-agency-manager/cve_tracking/src/shell/start.sh index 2209374ac92a8711f34fbaab83616bd2fe55c41a..7da3df10add4cf965ab5a36aab1560d2e7bbf21b 100644 --- a/cve-agency-manager/cve_tracking/src/shell/start.sh +++ b/cve-agency-manager/cve_tracking/src/shell/start.sh @@ -25,11 +25,11 @@ function check() { } function install_git() { - git_status=$(rpm -qa git) - if [[ -z ${git_status} ]]; then - yum install git -y - git_status=$(rpm -qa git) - if [[ -z ${git_status} ]]; then + git_status=$(dpkg -s git 2>&1) + if [[ ${git_status} =~ "is not installed" ]]; then + apt install git -y + git_status=$(dpkg -s git 2>&1) + if [[ ${git_status} =~ "is not installed" ]]; then echo "[ERROR] Git install failed" exit 1 fi