diff --git a/cve-agency-manager/cve_tracking/.gitignore b/cve-agency-manager/cve_tracking/.gitignore index 2027f52cd9ebda8e294616082ebbf3cec258f56a..11614af2870733183efe883810764d8708bddf8f 100644 --- a/cve-agency-manager/cve_tracking/.gitignore +++ b/cve-agency-manager/cve_tracking/.gitignore @@ -1,4 +1,115 @@ -.DS_Store -*/.DS_Store -*.pyc -*.vscode \ No newline at end of file +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ diff --git a/cve-agency-manager/cve_tracking/README.md b/cve-agency-manager/cve_tracking/README.md index c9903c9370c6a3cee156489fbde6856ce83904a1..45d962b3a2b9634e06951249eae032abf782c621 100644 --- a/cve-agency-manager/cve_tracking/README.md +++ b/cve-agency-manager/cve_tracking/README.md @@ -1,23 +1,110 @@ # cve_tracking #### 介绍 -cve补丁自动获取工具 + +cve 补丁自动获取工具，该工具将会根据 cve 和 rpm 包自动在上游社区查找补丁并反馈结果，同时也可以下载查找到的补丁以及验证补丁的可用性。 #### 软件架构 -软件架构说明 +python 可执行代码 #### 安装教程 -1. xxxx -2. xxxx -3. xxxx +1. 下载代码 + + ```shell + git clone https://gitee.com/openeuler/cve-manager.git + ``` + +2. 进入工具执行目录 + + ```shell + cd xxx(上述代码下载目录)/cve-manager/cve-agency-manager/cve_tracking + ``` + +3. 在 cve-tracking.yaml 的 authentication 中设置 gitee（gitee 的私人令牌）和 gitlab（gitlab 的私人令牌，默认设置了一个临时令牌，可临时使用），github（github 的私人令牌）可不设置。 + +4. 安装依赖包 + + ``` + pip3 install -r requirements.txt + ``` + +5. 根据使用说明执行工具 #### 使用说明 -1. xxxx -2. xxxx -3. xxxx +1. 补丁查找及评论 issue + + ```shell + python3 main.py comment -c cve_num -r rpm_name -i issue_num + ``` + + > 参数说明： + > + > -c cve 的编号 + > + > -r rpm 包名称 + > + > -i 需要评论的 issue 编号 + > + > 注意：默认仓库为 src-openeuler，如果要更改，请修改 main.py 同目录下 constant.py 中的 DEFAULT_OWNER 的值。 + +2. 补丁查找及下载(验证) + + ``` + python3 main.py download -c cve_num -r rpm_name [-f patch_save_path] [-s source_path] [-p] [-b branch] + ``` + + > 参数说明： + > + > -c cve 的编号 + > + > -r rpm 包名称 + > + > -f 补丁文件的下载目录，不设置默认为/opt/cve_tracking/patches + > + > -s 源码包下载路径，不设置默认为/opt/cve_tracking/source_code + > + > -b 源码包所在的 gitee 的 src-openeuler 仓库的分支，默认为 master + > + > -p 是否进行补丁应用，默认为不应用，若需要应用，添加该参数。 + +3. 补丁验证 + + ``` + python3 main.py packing -r rpm_name -f patch_save_path -s source_path -b branch [-nd] + ``` + + > 参数说明： + > + > -r rpm 包名称 + > + > -f 补丁文件路径（指定到补丁所在的文件夹即可） + > + > -s 源码包路径，如果无需下载指定为本地源码包的路径；如果需要下载指定为需要下载源码包的路径即可 + > + > -b 源码包所在 gitee 中 src-openeuler 仓库的分支，不设置默认为 master + > + > -nd 是否需要下载源代码，默认为需要下载，若无需下载添加该参数 + +4. 意见反馈 + + 若工具没有查找到补丁且人工查找到补丁信息，可通过该功能进行反馈，将会在配置的平台中提交 issue。将发现新 patch 的平台信息上报至指定的仓库，便于 CVE 修复人员完善工具的查找。指定的仓库可以在 conf.ini 配置文件的【FEEDBACK】中进行配置 + + ```shell + python3 main.py feedback -c cve_num -p cve_platform -u patch_url -i issue_warehouse + ``` + + > 参数说明： + > + > -c CVE 的编号 + > + > -p 查找 CVE 所在的平台，可按照平台名称简写 + > + > -u 修复 CVE 的补丁链接 + > + > -i 管理 issue 的仓库，只能是（gitee、gitlab、github）中任选其一 #### 参与贡献 @@ -26,10 +113,9 @@ cve补丁自动获取工具 3. 提交代码 4. 新建 Pull Request - #### 特技 -1. 使用 Readme\_XXX.md 来支持不同的语言，例如 Readme\_en.md, Readme\_zh.md +1. 使用 Readme_XXX.md 来支持不同的语言，例如 Readme_en.md, Readme_zh.md 2. Gitee 官方博客 [blog.gitee.com](https://blog.gitee.com) 3. 你可以 [https://gitee.com/explore](https://gitee.com/explore) 这个地址来了解 Gitee 上的优秀开源项目 4. [GVP](https://gitee.com/gvp) 全称是 Gitee 最有价值开源项目，是综合评定出的优秀开源项目 diff --git a/cve-agency-manager/cve_tracking/cli/__init__.py b/cve-agency-manager/cve_tracking/cli/__init__.py index cefefd513232a95053d521436dcea1949b32d5f5..41ef326fdecdf0c1e0a9812ed47acd15b8fd5d74 100644 --- a/cve-agency-manager/cve_tracking/cli/__init__.py +++ b/cve-agency-manager/cve_tracking/cli/__init__.py @@ -12,10 +12,12 @@ # ******************************************************************************/ from .base import CveTrackingCommand from .comment import CommentCommand +from .feedback import FeedbackCommand from .save import SaveCommand from .verify import VerifyCommand __all__ = ("CveTrackingCommand", "CommentCommand", "SaveCommand", - "VerifyCommand") + "VerifyCommand", + "FeedbackCommand") diff --git a/cve-agency-manager/cve_tracking/cli/comment.py b/cve-agency-manager/cve_tracking/cli/comment.py index b254b79296f0924e8e21b667eb5989e5222a330f..b45186377c580cd21e2b634f98cf5afd0776ca5e 100644 --- a/cve-agency-manager/cve_tracking/cli/comment.py +++ b/cve-agency-manager/cve_tracking/cli/comment.py @@ -11,7 +11,7 @@ # See the Mulan PSL v2 for more details. # ******************************************************************************/ from cli.base import CveTrackingCommand -from constant import Constant +from conf import CONFIG from core.comment.issue_comment import issue_comment from core.crawler.patch import Patch from exception import InputError @@ -26,10 +26,20 @@ class CommentCommand(CveTrackingCommand): def __init__(self): super(CommentCommand, self).__init__() - self.parser = CveTrackingCommand.sub_parse.add_parser('comment', help='Add a comment to the issue on gitee') + self.parser = CveTrackingCommand.sub_parse.add_parser( + "comment", help="Add a comment to the issue on gitee" + ) self._add_common_param() - self.parser.add_argument('-i', '--issue', metavar='issue', type=str, required=True, action='store', - default=None, help='CVE related issue number') + self.parser.add_argument( + "-i", + "--issue", + metavar="issue", + type=str, + required=True, + action="store", + default=None, + help="CVE related issue number", + ) @staticmethod async def run_command(args): @@ -38,9 +48,9 @@ class CommentCommand(CveTrackingCommand): :param args: command line params :return: None """ - logger.info('Start to perform search and comment functions') + logger.info("Start to perform search and comment functions") if not all([args.cve_num, args.rpm_name, args.issue]): - raise InputError(msg='Comment command parameters: cve_num/rpm_name/issue') + raise InputError(msg="Comment command parameters: cve_num/rpm_name/issue") # find patch detail patch_obj = Patch(cve_num=args.cve_num, rpm_name=args.rpm_name) @@ -48,6 +58,6 @@ class CommentCommand(CveTrackingCommand): # comment cve issue gitee = Gitee() - gitee.set_attr(owner=Constant.DEFAULT_OWNER, repo=args.rpm_name) + gitee.set_attr(owner=CONFIG.DEFAULT_OWNER, repo=args.rpm_name) await issue_comment(patch_details=patch_details, number=args.issue, gitee=gitee) - logger.info('End to perform search and comment functions') + logger.info("End to perform search and comment functions") diff --git a/cve-agency-manager/cve_tracking/cli/feedback.py b/cve-agency-manager/cve_tracking/cli/feedback.py new file mode 100644 index 0000000000000000000000000000000000000000..1bd42ff5dd67946ef5c1269a6aa712fe9c458652 --- /dev/null +++ b/cve-agency-manager/cve_tracking/cli/feedback.py @@ -0,0 +1,112 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from cli import CveTrackingCommand +from conf import CONFIG +from constant import Constant +from core.feedback.issue_mode import IssueMode +from exception import InputError +from logger import logger + + +class FeedbackCommand(CveTrackingCommand): + """ + Cve tool feedback function command line + """ + + def __init__(self): + super(FeedbackCommand, self).__init__() + self.parser = CveTrackingCommand.sub_parse.add_parser( + name="feedback", + help="Submit an issue to the designated warehouse, and the feedback tool will find the result", + ) + self._add_params() + + def _add_params(self): + """ + Add input parameters required for feedback function + :return: None + """ + self.parser.add_argument( + "-c", + "--cve_num", + metavar="cve", + type=str, + required=True, + action="store", + default=None, + help="cve number", + ) + self.parser.add_argument( + "-p", + "--cve_platform", + type=str, + metavar="cve_platform", + required=True, + action="store", + default=None, + help="Correct cve reference URL", + ) + self.parser.add_argument( + "-u", + "--patch_url", + type=str, + metavar="patch_url", + required=True, + action="store", + default=None, + help="Correct patch url", + ) + self.parser.add_argument( + "-i", + "--issue_platform", + type=str, + metavar="issue_platform", + required=False, + action="store", + choices=Constant.CODE_PLATFORM, + default=CONFIG.FEEDBACK_PLATFORM, + help="Issue submission platform, default is gitee", + ) + + @staticmethod + async def run_command(args): + """ + Feedback function execution entrance + :param args: input params + :return: None + """ + logger.info( + f"Start add issue about cve:{args.cve_num} on platform:{args.issue_platform}" + ) + if not all( + [args.cve_num, args.cve_platform, args.patch_url, args.issue_platform] + ): + raise InputError(msg="Feedback command params, please check and try again.") + + issue_mode = IssueMode( + cve_num=args.cve_num, + cve_platform=args.cve_platform, + patch_url=args.patch_url, + issue_platform=args.issue_platform, + ) + result = await issue_mode.create_feedback_issue() + + if result: + logger.info( + f"Success to add issue about cve:{args.cve_num} on platform:{args.issue_platform}" + ) + else: + logger.error( + f"Failed to add issue about cve:{args.cve_num} on platform:{args.issue_platform}, " + f"content is cve_platform:{args.cve_platform}, patch_url:{args.patch_url}" + ) diff --git a/cve-agency-manager/cve_tracking/conf/__init__.py b/cve-agency-manager/cve_tracking/conf/__init__.py index 221fcf9d05ce88ef5aa55bbd0a2723eb4e4eb98f..683c506a89390bb5cd51fb2ad98d1e29fde7a4f3 100644 --- a/cve-agency-manager/cve_tracking/conf/__init__.py +++ b/cve-agency-manager/cve_tracking/conf/__init__.py @@ -10,61 +10,10 @@ # PURPOSE. # See the Mulan PSL v2 for more details. # ******************************************************************************/ -import configparser -import os - -from conf.setting import DefaultConfig -from exception import ConfigNotFoundError - -CONFIG_FILE = os.path.join(os.path.dirname( - os.path.abspath(os.path.dirname(__file__))), 'config.ini') - - -class Config: - """ - Configuration item load and get value - """ - - def __init__(self): - self._setattr_default() - self._setattr_customize() - - def _setattr_default(self): - """ - Set default configuration items and their values - :return: self - """ - for attr in dir(DefaultConfig): - if not str(attr).startswith('_'): - setattr(self, attr, getattr(DefaultConfig, attr)) - - def _setattr_customize(self): - """ - Set customize configuration items and their values - :return: self - """ - if not os.path.exists(CONFIG_FILE): - raise ConfigNotFoundError(CONFIG_FILE) - - conf_parser = configparser.ConfigParser() - conf_parser.read(CONFIG_FILE) - for section in conf_parser.sections(): - for key_value in conf_parser.items(section): - if key_value[1]: - setattr(self, key_value[0].upper(), key_value[1]) - - def __getattr__(self, item): - """ - When the configuration cannot be found, reload the configuration file - :param item: config key - :return: value - """ - self._setattr_default() - self._setattr_customize() - try: - return self.__dict__[item] - except KeyError: - return None +from .setting import Config, YamlConfiguration CONFIG = Config() +settings = YamlConfiguration() + +__all__ = ("CONFIG", "settings") diff --git a/cve-agency-manager/cve_tracking/conf/setting.py b/cve-agency-manager/cve_tracking/conf/setting.py index 233b05432fe0a2c99d4016e8ff2a0b4007e437a7..dcb5660551dfacda8a65567357945a5525906484 100644 --- a/cve-agency-manager/cve_tracking/conf/setting.py +++ b/cve-agency-manager/cve_tracking/conf/setting.py @@ -10,6 +10,11 @@ # PURPOSE. # See the Mulan PSL v2 for more details. # ******************************************************************************/ +import configparser +import os +import yaml +from exception import ConfigNotFoundError + class DefaultConfig: """ @@ -38,3 +43,118 @@ class DefaultConfig: # Source code package and patch file storage path, # used to download gitee source code package and package verification. SOURCE_CODE_PATH = "/opt/cve_tracking/source_code" + + # ********** + # Feedback issue related configuration + # ********** + + FEEDBACK_PLATFORM = "github" + # Feedback to the owner of the issue + FEEDBACK_ISSUE_OWNER = "liheavy" + # Feedback to the repo of the issue + FEEDBACK_ISSUE_REPO = "cve_tracking" + + DEFAULT_OWNER = "src-openeuler" + + +class Config: + """ + Configuration item load and get value + """ + + config_file = os.path.join( + os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "config.ini" + ) + + def __init__(self): + self._setattr_default() + self._setattr_customize() + + def _setattr_default(self): + """ + Set default configuration items and their values + :return: self + """ + for attr in dir(DefaultConfig): + if not str(attr).startswith("_"): + setattr(self, attr, getattr(DefaultConfig, attr)) + + def _setattr_customize(self): + """ + Set customize configuration items and their values + :return: self + """ + if not os.path.exists(self.config_file): + raise ConfigNotFoundError(self.config_file) + + conf_parser = configparser.ConfigParser() + conf_parser.read(self.config_file) + for section in conf_parser.sections(): + for key_value in conf_parser.items(section): + if key_value[1]: + setattr(self, key_value[0].upper(), key_value[1]) + + def __getattr__(self, item): + """ + When the configuration cannot be found, reload the configuration file + :param item: config key + :return: value + """ + self._setattr_default() + self._setattr_customize() + try: + return self.__dict__[item] + except KeyError: + return None + + +class YamlConfiguration: + yaml = os.path.join( + os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "cve-tracking.yaml" + ) + + def __init__(self) -> None: + if not os.path.exists(self.yaml): + raise FileNotFoundError("file not found: %s" % self.yaml) + self.__dict__.update(self._parse_yaml()) + + def _parse_yaml(self): + with open(self.yaml, "r", encoding="utf-8") as file: + try: + configs = yaml.load(file.read(), Loader=yaml.FullLoader) + except yaml.YAMLError as error: + raise ValueError( + "The format of the yaml configuration " + "file is wrong please check and try again:{0}".format(error) + ) + else: + return configs + + def __getattr__(self, name): + if name not in self.__dict__: + return None + + return self.__dict__[name] + + def get_platform(self, name=None): + if name is None: + return self.platform + platform = filter(lambda x: x["name"] == name, self.platform) + if platform: + return list(platform)[-1] + + def get_regex(self, label=None): + if label is None: + regulars = [reg for reg in self.regex] + else: + regulars = filter(lambda x: x["label"] == label, self.regex) + return [regx for reg in regulars for regx in reg["regular"]] + + def token(self, name): + token = filter(lambda x: x["name"] == name, self.authentication) + if token: + return list(token)[-1]["token"] + + @property + def configuration(self): + return self.__dict__ diff --git a/cve-agency-manager/cve_tracking/config.ini b/cve-agency-manager/cve_tracking/config.ini index 9bdd816e6253a662b6c06d4eb17a93b00a55797d..f4b3f755aa24715e620fce02b75ba1533f1d7b76 100644 --- a/cve-agency-manager/cve_tracking/config.ini +++ b/cve-agency-manager/cve_tracking/config.ini @@ -23,21 +23,6 @@ LOGGER_BUFFER = 5242880 ; Number of log files saved LOGGER_FILE_COUNT = 20 -;Gitee related configuration -[GITEE] -;Token used to access gitee api -GITEE_TOKEN = - -;Github related configuration -[GITHUB] -;Token used to access github api -GITHUB_TOKEN = - -;Gitlab related configuration -[GITLAB] -;Token used to access gitlab api -GITLAB_TOKEN = glpat-24DGxkuTA8nTR1YQsXvZ - ;Path related configuration [PATH] ;Patch file download and save path. @@ -45,4 +30,16 @@ PATCH_SAVE_PATH = /opt/cve_tracking/patches ;Source code package and patch file storage path, used to download gitee source code package and package verification. SOURCE_CODE_PATH = /opt/cve_tracking/source_code +;Feedback issue related configuration +[FEEDBACK] +;Feedback issue platform,Currently supports "gitee" "github" "gitlab",default is "gitee" +;Please configure the token of the corresponding platform synchronously in cve-tracking.yaml +FEEDBACK_PLATFORM = github +;Feedback to the owner of the issue +FEEDBACK_ISSUE_OWNER = liheavy +;Feedback to the repo of the issue +FEEDBACK_ISSUE_REPO = cve_tracking + +[WAREHOUSE] +DEFAULT_OWNER = src-openeuler \ No newline at end of file diff --git a/cve-agency-manager/cve_tracking/constant.py b/cve-agency-manager/cve_tracking/constant.py index 44c2635b221f9b1fdd3be49693a8cd14ca2fd7e5..1bcd29c71c151722ac951a0fa62380db0a3c95dd 100644 --- a/cve-agency-manager/cve_tracking/constant.py +++ b/cve-agency-manager/cve_tracking/constant.py @@ -11,35 +11,12 @@ # See the Mulan PSL v2 for more details. # ******************************************************************************/ + class Constant: """ Constant class """ - """ - CVE reference URL related - """ - # CVE reference URL that supports query - CVE_PLATFORM = ['Debian', 'Ubuntu', 'Bugzilla', 'Nvd', 'Suse'] - # CVE reference URL - NVD - NVD_BASE_URL = "https://nvd.nist.gov/vuln/detail/{cve_num}" - # CVE reference URL - Bugzilla - BUGZILLA_BASE_URL = "https://bugzilla.redhat.com/rest/bug/{cve_num}/comment" - # CVE reference URL - Ubuntu - UBUNTU_BASE_URL = "https://ubuntu.com/security/{cve_num}" - # CVE reference URL - Debian - DEBIAN_BASE_URL = "https://security-tracker.debian.org/tracker/{cve_num}" - # CVE reference URL - Suse - SUSE_BASE_URL = "https://bugzilla.suse.com/show_bug.cgi?id={cve_num}" - """ - Patch related - """ - # commit format - COMMIT_REGEX = r"http[s]?://(?:[-\w.\/;?])+(?:/rev|/ci|/commit[s]?)/(?:\?id=)?[0-9a-z]{8,40}" - GIT_COMMIT_REGEX = r"http[s]?://(?:[-\w.\/;=&?])+a=commit(?:[\w;&?])+h=[0-9a-z]{8,40}" - # pr format - PR_REGEX = r"http[s]?://(?:[-\w.\/;?])+(?:pull[s]?|merge_requests)/[1-9][0-9]*" - # issue format - ISSUE_REGEX = r"http[s]?://(?:[-\w.\/;?])+issues/[0-9A-Z]+" + # gitlab host format GITLAB_HOST_REGEX = r"http[s]?://gitlab(?:[-\w.?])+" """ @@ -49,8 +26,8 @@ class Constant: GITHUB = "github" # gitlab GITLAB = "gitlab" - # default owner is src_openEuler - DEFAULT_OWNER = "src-openeuler" + # Supported code hosting platforms + CODE_PLATFORM = ["gitee", "github", "gitlab"] """ Other diff --git a/cve-agency-manager/cve_tracking/core/comment/issue_comment.py b/cve-agency-manager/cve_tracking/core/comment/issue_comment.py index 9836c2837545c496ef6b08dc1620d843cceacd8d..1a4e2c59e33ffd54f7b62ef9901c6069a8feec6d 100644 --- a/cve-agency-manager/cve_tracking/core/comment/issue_comment.py +++ b/cve-agency-manager/cve_tracking/core/comment/issue_comment.py @@ -34,14 +34,14 @@ class Table: def _rowspan(num, platform): return """{}""".format(str(num), platform) - def span_content(self, pr, commits): + def span_content(self, pr, pr_status, commits): """ Splicing comment content :param pr: pull request :param commits: commits :return: content spliced """ - contents = self._td(pr) + self._td(",".join(commits)) + contents = self._td(pr) + self._td(pr_status) + self._td("
".join(commits)) return contents def _table_name(self): @@ -49,10 +49,17 @@ class Table: Splicing header :return: header """ - table_names = ["参考网址", "关联pr", "补丁链接"] + table_names = ["参考网址", "关联pr", "状态", "补丁链接"] table_names = " ".join(map(self._th, table_names)) return self._tr(table_names) + @staticmethod + def _row_span(detail): + number = 0 + for issues in detail: + number += len(issues["issue"].get("prs", list())) + return number + def table(self, lst): """ Splicing table @@ -64,20 +71,26 @@ class Table: platform_index = 0 platform = platforms.get("platform") details = platforms.get("details") - rowspan_content = self._rowspan(len(details), platform) + rowspan_content = self._rowspan(self._row_span(details), platform) + if not details: - td_cont = self._tr(self._td(platform) + self._td("") * 2) - contents += td_cont - - for pr, commits in details.items(): - content = self.span_content(pr, commits) - if platform_index == 0: - rowspan_content = rowspan_content + content - span_content = self._tr(rowspan_content) - platform_index += 1 - else: - span_content = self._tr(content) - contents += span_content + none_tr = self._tr(self._td(platform) + self._td("") * 3) + contents += none_tr + + for issue_info in details: + for pr in issue_info["issue"].get("prs", list()): + content = self.span_content( + pr["url"], + pr["status"], + pr.get("commits", []), + ) + if platform_index == 0: + rowspan_content = rowspan_content + content + span_content = self._tr(rowspan_content) + platform_index += 1 + else: + span_content = self._tr(content) + contents += span_content return self._table(self._table_name() + contents) @@ -93,7 +106,7 @@ async def issue_comment(patch_details, number, gitee: Gitee): table = Table().table(patch_details) instruction = "\n" * 2 + "> 说明： 抱歉，当前工具暂未找到推荐补丁，请人工查找或者之后再尝试。" for cve_info in patch_details: - if cve_info.get('details'): + if cve_info.get("details"): instruction = "\n" * 2 + "> 说明：补丁链接仅供初步排查参考，实际可用性请人工再次确认。" break diff --git a/cve-agency-manager/cve_tracking/core/crawler/patch.py b/cve-agency-manager/cve_tracking/core/crawler/patch.py index 06120b97b032cf4496c4eb0c75b156af7e553a67..97c44ffc27da8c19f73360112a68e3b685eb96c3 100644 --- a/cve-agency-manager/cve_tracking/core/crawler/patch.py +++ b/cve-agency-manager/cve_tracking/core/crawler/patch.py @@ -11,7 +11,6 @@ # See the Mulan PSL v2 for more details. # ******************************************************************************/ import re - import asyncio from collections import namedtuple @@ -20,6 +19,7 @@ from core.platform import CvePlatform from logger import logger from util.github_api import Github from util.gitlab_api import Gitlab +from conf import settings class Patch: @@ -33,10 +33,19 @@ class Patch: self.patch_info_list = list() self.patch_detail_list = list() self.issue_pr_dict = dict() + self.pr_status_dict = dict() self.github_api = Github() self.gitlab_api = Gitlab() - self._SummaryInfo = namedtuple('SummaryInfo', ['pr_list', 'issues_list']) - self._PatchDetail = namedtuple('PatchDetail', ['platform', 'details']) + self._SummaryInfo = namedtuple("SummaryInfo", ["pr_list", "issues_list"]) + self._PatchDetail = namedtuple("PatchDetail", ["platform", "details"]) + + @property + def issue_relation_info(self): + return {"issue": {"url": None, "prs": []}} + + @property + def pr_relation_info(self): + return {"url": None, "status": None, "commits": []} async def find_patches_detail(self): """ @@ -47,26 +56,35 @@ class Patch: # commit,pr,issue summary structure,convenient for deduplication and concurrent search summary_info = self._SummaryInfo(pr_list=list(), issues_list=list()) for patch_info in self.patch_info_list: - summary_info.pr_list.extend(patch_info.get('pr', [])) - summary_info.issues_list.extend(patch_info.get('issue', [])) + summary_info.pr_list.extend(patch_info.get("pr", [])) + summary_info.issues_list.extend(patch_info.get("issue", [])) # Find issue linked pr and add it to the pr list obtained earlier if summary_info.issues_list: - issue_task_list = [asyncio.create_task(self._get_issue_link_pr(issue)) for issue in - list(set(summary_info.issues_list))] + issue_task_list = await self._create_async_task( + self._get_issue_link_pr, summary_info.issues_list + ) issue_done_task, _ = await asyncio.wait(issue_task_list) for task in issue_done_task: summary_info.pr_list.extend(task.result()) - # Find pr contain commits if summary_info.pr_list: - pr_task_list = [asyncio.create_task(self._get_pr_contain_commits(pr)) for pr in - list(set(summary_info.pr_list))] + # Query pr status + pr_status_task_list = await self._create_async_task( + self._get_pr_status, summary_info.pr_list + ) + await asyncio.wait(pr_status_task_list) + # Find pr contain commits + pr_task_list = await self._create_async_task( + self._get_pr_contain_commits, summary_info.pr_list + ) await asyncio.wait(pr_task_list) # Generate the final information that needs to be output self._convert_path_detail() - logger.info(f'Find the patch information of cve "{self.cve_num}" as: \n{self.patch_detail_list}') + logger.info( + f'Find the patch information of cve "{self.cve_num}" as: \n{self.patch_detail_list}' + ) return self.patch_detail_list async def _find_patches_info(self): @@ -74,12 +92,37 @@ class Patch: Implementation method of preliminary search for CVE patch information :return: self.patch_info_list """ - logger.info(f'Start to obtain patches info of {self.cve_num} for {self.rpm_name}') - cve_platforms = CvePlatform.__subclasses__() - crawler_task = [asyncio.create_task(cve_platform(self.cve_num).crawling_patch()) - for cve_platform in cve_platforms if cve_platform.__name__ in Constant.CVE_PLATFORM] + logger.info( + f"Start to obtain patches info of {self.cve_num} for {self.rpm_name}" + ) + crawler_task = [ + asyncio.create_task( + CvePlatform( + cve_platform, + self.cve_num, + cve_platform["url"], + cve_platform["format"], + ).crawling_patch() + ) + for cve_platform in settings.get_platform() + ] done_task, _ = await asyncio.wait(crawler_task) - self.patch_info_list.extend([task.result() for task in done_task if task.result()]) + self.patch_info_list.extend( + [task.result() for task in done_task if task.result()] + ) + + @staticmethod + async def _create_async_task(method, issue_pr_list): + """ + Create asynchronous tasks + :param method: Method to be executed + :param issue_pr_list: issue or pr list + :return: task list + """ + task_list = [] + for issue_or_pr in list(set(issue_pr_list)): + task_list.append(asyncio.create_task(method(issue_or_pr))) + return task_list async def _get_issue_link_pr(self, issue): """ @@ -92,49 +135,94 @@ class Patch: pr_list = await self.github_api.issue_relevance_pull(issue_url=issue) elif Constant.GITLAB in issue: try: - issue_info = issue.split('/') + issue_info = issue.split("/") self.gitlab_api.set_attr(owner=issue_info[-5], repo=issue_info[-4]) self._set_gitlab_host(issue) issue_num = issue_info[-1] - response = await self.gitlab_api.issue_relevance_pull(issue_id=issue_num) - pr_list = self._convert_api_response(response, url_key='web_url') + response = await self.gitlab_api.issue_relevance_pull( + issue_id=issue_num + ) + pr_list = self._convert_api_response(response, url_key="web_url") except IndexError: - logger.warning(f'The issue link: {issue} does not conform to the standard format') + logger.warning( + f"The issue link: {issue} does not conform to the standard format" + ) else: - logger.warning(f'Issue {issue} failed to match the relevant code platform') + logger.warning(f"Issue {issue} failed to match the relevant code platform") # Record the relationship between issue and pr to facilitate subsequent patch information processing - logger.info(f'According to issue {issue} get pr: {str(pr_list)}') + logger.info(f"According to issue {issue} get pr: {str(pr_list)}") self.issue_pr_dict[issue] = pr_list return pr_list + async def _get_pr_status(self, pr): + """ + Query pr status + :param pr: pr + :return: self.pr_status_dict + """ + pr_number = self._set_owner_repo_by_pr(pr) + if not pr_number: + logger.info("This is not effective pr:%s" % pr) + return + if Constant.GITHUB in pr: + response = await self.github_api.get_pull_request(pr_number) + if response: + self.pr_status_dict[pr] = response.get("state") + + elif Constant.GITLAB in pr: + self._set_gitlab_host(pr) + response = await self.gitlab_api.get_single_mr(pr_number) + if response: + self.pr_status_dict[pr] = response.get("merge_status") + async def _get_pr_contain_commits(self, pr): """ Get the commits contained in pr and record them :param pr: pull requests :return: self.issue_pr_dict """ - pr_info = pr.split('/') - pr_number = pr_info[-1] - commits_list = [] + pr_number = self._set_owner_repo_by_pr(pr) + if pr_number: + commits_list = [] + if Constant.GITHUB in pr: + response = await self.github_api.get_pull_commits(pr_number) + commits_list = self._convert_api_response(response, "html_url") + elif Constant.GITLAB in pr: + self._set_gitlab_host(pr) + response = await self.gitlab_api.get_mr_context_commits( + merge_request_iid=pr_number + ) + commits_list = self._convert_api_response(response, "web_url") + + logger.info(f"According to pull {pr} get commits: {str(commits_list)}") + + self.issue_pr_dict[pr] = commits_list + + def _set_owner_repo_by_pr(self, pr): + """ + Set the owner and repo required by the api through the pr link + :param pr: pull request + :return: pr number + """ + pr_info = pr.split("/") try: + pr_number = pr_info[-1] if Constant.GITHUB in pr: self.github_api.set_attr(owner=pr_info[-4], repo=pr_info[-3]) - response = await self.github_api.get_pull_commits(pr_number) - commits_list = self._convert_api_response(response, 'html_url') elif Constant.GITLAB in pr: self.gitlab_api.set_attr(owner=pr_info[-5], repo=pr_info[-4]) - self._set_gitlab_host(pr) - response = await self.gitlab_api.get_mr_context_commits(merge_request_iid=pr_number) - commits_list = self._convert_api_response(response, 'web_url') else: - logger.warning(f'Pr {pr} failed to match the relevant code platform') + logger.warning(f"Pr {pr} failed to match the relevant code platform") + return None except IndexError: - logger.warning(f'The pull link: {pr} does not conform to the standard format') - logger.info(f'According to pull {pr} get commits: {str(commits_list)}') + logger.warning( + f"The pull link: {pr} does not conform to the standard format" + ) + return None - self.issue_pr_dict[pr] = commits_list + return pr_number @staticmethod def _convert_api_response(response, url_key): @@ -146,7 +234,9 @@ class Patch: """ pr_commits_list = list() if response: - pr_commits_list.extend([commit_info.get(url_key) for commit_info in response]) + pr_commits_list.extend( + [commit_info.get(url_key) for commit_info in response] + ) return pr_commits_list def _set_gitlab_host(self, issue_pr): @@ -159,22 +249,49 @@ class Patch: _host = _host_match.group() self.gitlab_api.host = _host + def _set_pr_relation_info(self, pr): + pr_relation_info = self.pr_relation_info + pr_relation_info["url"] = pr + pr_relation_info["commits"] = self.issue_pr_dict.get(pr, []) + pr_relation_info["status"] = self.pr_status_dict.get(pr) + return pr_relation_info + def _convert_path_detail(self): """ Process data and generate patch details :return: self.patch_detail_list """ for patch_info in self.patch_info_list: - patch_detail = self._PatchDetail(platform=patch_info.get('platform'), details=dict()) + patch_detail = self._PatchDetail( + platform=patch_info.get("platform"), details=list() + ) + issue_relation_info = self.issue_relation_info + # First, If there is commits information, pr and issue information will be ignored - if patch_info.get('commits'): - patch_detail.details[None] = patch_info.get('commits') + if patch_info.get("commits"): + pr_relation_info = self.pr_relation_info + pr_relation_info["commits"] = patch_info.get("commits") + issue_relation_info["issue"]["prs"].append(pr_relation_info) + # Second, processing pr information - for pr in patch_info.get('pr'): - patch_detail.details[pr] = self.issue_pr_dict.get(pr, []) + for pr in patch_info.get("pr"): + issue_relation_info["issue"]["prs"].append( + self._set_pr_relation_info(pr) + ) + + # None Issue associated patch + if issue_relation_info["issue"]["prs"]: + patch_detail.details.append(issue_relation_info) + # Third, processing issue information - for issue in patch_info.get('issue'): + for issue in patch_info.get("issue"): + issue_relation_info = self.issue_relation_info + issue_relation_info["issue"]["url"] = issue for pr in self.issue_pr_dict.get(issue, []): - patch_detail.details[pr] = self.issue_pr_dict.get(pr, []) + issue_relation_info["issue"]["prs"].append( + self._set_pr_relation_info(pr) + ) + + patch_detail.details.append(issue_relation_info) self.patch_detail_list.append(dict(patch_detail._asdict())) diff --git a/cve-agency-manager/cve_tracking/core/download/save.py b/cve-agency-manager/cve_tracking/core/download/save.py index 3abdab537a0ff139c1c9068e4ee33efd4525e786..302c9bc4832f68a7aa0250535c7e61b1d7bfbef1 100644 --- a/cve-agency-manager/cve_tracking/core/download/save.py +++ b/cve-agency-manager/cve_tracking/core/download/save.py @@ -26,10 +26,12 @@ def _file(cve, file): """ file = os.path.join(file, cve) os.makedirs(file, exist_ok=True) - file_name = "fix-cve-" + \ - str(sum([os.path.isfile(list_index) - for list_index in os.listdir(file)]) + 1) + ".patch" - return file_name + file_name = ( + "fix-cve-" + + str(sum([os.path.isfile(list_index) for list_index in os.listdir(file)]) + 1) + + ".patch" + ) + return os.path.join(file, file_name) def save_patch(patch_details, cve, path=CONFIG.PATCH_SAVE_PATH): @@ -42,8 +44,9 @@ def save_patch(patch_details, cve, path=CONFIG.PATCH_SAVE_PATH): """ commits = set() for patch in patch_details: - for _, commit in patch.get("details", dict()).items(): - commits.update(commit) + for issue_info in patch.get("details", list()): + for pr in issue_info["issue"].get("prs", list()): + commits.update(pr.get("commits", [])) # download patch for url in commits: diff --git a/cve-agency-manager/cve_tracking/core/platform/suse.py b/cve-agency-manager/cve_tracking/core/feedback/__init__.py similarity index 70% rename from cve-agency-manager/cve_tracking/core/platform/suse.py rename to cve-agency-manager/cve_tracking/core/feedback/__init__.py index f5d6a5e3777bd54863ec1062266e45ead7279cb2..3e26da5218904e6406ea988ff828312ffb3f9fe2 100644 --- a/cve-agency-manager/cve_tracking/core/platform/suse.py +++ b/cve-agency-manager/cve_tracking/core/feedback/__init__.py @@ -10,14 +10,3 @@ # PURPOSE. # See the Mulan PSL v2 for more details. # ******************************************************************************/ -from constant import Constant -from core.platform import CvePlatform - - -class Suse(CvePlatform): - """ - cve refer to the URL suse to find the patch implementation class - """ - - def __init__(self, cve_num, base_url=Constant.SUSE_BASE_URL): - super(Suse, self).__init__(cve_num, base_url) diff --git a/cve-agency-manager/cve_tracking/core/platform/bugzila.py b/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py similarity index 33% rename from cve-agency-manager/cve_tracking/core/platform/bugzila.py rename to cve-agency-manager/cve_tracking/core/feedback/issue_mode.py index 81852a2427deca60a1a764789849e3b6f7194edd..ec2250d065d5082eb85ba97e0f8e9299f450a669 100644 --- a/cve-agency-manager/cve_tracking/core/platform/bugzila.py +++ b/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py @@ -10,33 +10,43 @@ # PURPOSE. # See the Mulan PSL v2 for more details. # ******************************************************************************/ -import json -from json import JSONDecodeError - -from constant import Constant -from core.platform.cve_platform import CvePlatform +from conf import CONFIG from logger import logger +from util.gitee_api import Gitee +from util.github_api import Github +from util.gitlab_api import Gitlab -class Bugzilla(CvePlatform): +class IssueMode: """ - cve refer to the URL bugzilla to find the patch implementation class + Feedback by creating an issue """ - def __init__(self, cve_num, base_url=Constant.BUGZILLA_BASE_URL): - super(Bugzilla, self).__init__(cve_num, base_url) + def __init__(self, cve_num, cve_platform, patch_url, issue_platform): + self.cve_num = cve_num + self.cve_platform = cve_platform + self.patch_url = patch_url + self.issue_platform = issue_platform + self._support_issue_platform = {'gitee': Gitee, + 'github': Github, + 'gitlab': Gitlab} - @staticmethod - def format_text(text): + async def create_feedback_issue(self): """ - Rewrite the formatted web page to get the content method - :param text: content of web page - :return: formatted content + Create an issue about tool feedback + :return: True or False """ - try: - text_dict = json.loads(text) - return json.dumps(text_dict, indent=4) - except JSONDecodeError as e: - logger.error(f'The format of the content obtained by bugzilla website is incorrect, ' - f'content is {text}, message is {e.msg}') - return None + platform_api_class = self._support_issue_platform.get(self.issue_platform) + if platform_api_class is None: + logger.error(f'Issue hosting platform: {self.issue_platform} does not support') + return False + if not all([CONFIG.FEEDBACK_ISSUE_OWNER, CONFIG.FEEDBACK_ISSUE_REPO]): + logger.error(f'"FEEDBACK_ISSUE_OWNER" or "FEEDBACK_ISSUE_REPO" no setting, Please set in file "config.ini"') + return False + + title = f"CVE: {self.cve_num} patch information" + description = f"Cve reference URL: {self.cve_platform}\nPatch url: {self.patch_url}" + platform_api = platform_api_class() + platform_api.set_attr(CONFIG.FEEDBACK_ISSUE_OWNER, CONFIG.FEEDBACK_ISSUE_REPO) + response = await platform_api.create_issue(title=title, body=description) + return response is not None diff --git a/cve-agency-manager/cve_tracking/core/platform/__init__.py b/cve-agency-manager/cve_tracking/core/platform/__init__.py index 4299ebe45353d6248097191b3c9f1bea8377dc1c..d7d26770ced7f7dbc9adb8fa2c600ecfc8c44d04 100644 --- a/cve-agency-manager/cve_tracking/core/platform/__init__.py +++ b/cve-agency-manager/cve_tracking/core/platform/__init__.py @@ -11,15 +11,5 @@ # See the Mulan PSL v2 for more details. # ******************************************************************************/ from .cve_platform import CvePlatform -from .nvd import Nvd -from .bugzila import Bugzilla -from .debian import Debian -from .suse import Suse -from .ubuntu import Ubuntu -__all__ = ("CvePlatform", - "Nvd", - "Bugzilla", - "Debian", - "Ubuntu", - "Suse") +__all__ = ("CvePlatform",) diff --git a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py index bc3202a3bc422f2cc95d40bb6407869cb841e7f7..37d23c10fce2e4e59b5ec10f074a36c02e8f37fb 100644 --- a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py +++ b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py @@ -10,6 +10,8 @@ # PURPOSE. # See the Mulan PSL v2 for more details. # ******************************************************************************/ +import json +from json import JSONDecodeError import re from collections import namedtuple from functools import wraps @@ -20,6 +22,7 @@ from constant import Constant from exception import RequestError from logger import logger from request import http +from conf import settings def match(regexes, component): @@ -33,7 +36,7 @@ def match(regexes, component): def _inner_decorator(func): @wraps(func) def _inner_match(crawler_text, patch_info): - if patch_info.get('commits') or patch_info.get('pr'): + if patch_info.get("commits") or patch_info.get("pr"): return func(crawler_text, patch_info) match_list = list() for regex in regexes: @@ -41,7 +44,7 @@ def match(regexes, component): try: patch_info[component].extend(list(set(match_list))) except KeyError: - logger.error(f'Patch info does not exist in field {component}') + logger.error(f"Patch info does not exist in field {component}") return func(crawler_text, patch_info) return _inner_match @@ -54,10 +57,18 @@ class CvePlatform: The parent class of cve reference URL """ - def __init__(self, cve_num=None, base_url=None): + def __init__( + self, + platform, + cve_num=None, + base_url=None, + format_text="text", + ): + self._platform = platform self.cve_num = cve_num self.base_url = base_url - self._Patch = namedtuple('Patch', ['platform', 'commits', 'pr', 'issue']) + self._format = format_text + self._Patch = namedtuple("Patch", ["platform", "commits", "pr", "issue"]) @property def crawler_url(self): @@ -75,32 +86,74 @@ class CvePlatform: """ return self._Patch(platform=self.crawler_url, commits=[], pr=[], issue=[]) + async def _rule_redirct(self, response): + """ + Page multi layer jump data parsing + :param response: http response data + :return: response data + """ + for redirct_rule in self._platform.get("redirct", []): + format_text = self.format_text(response.text) + target_val = list( + set( + re.findall( + pattern=redirct_rule.get("regex", ""), string=format_text or "" + ) + ) + ) + try: + url = redirct_rule["prefix"] + target_val[-1] + except IndexError: + url = redirct_rule["prefix"] + response = await self._method(redirct_rule)( + url, data=redirct_rule.get("body") + ) + + return response + + @staticmethod + def _method(rule): + method = http.get if rule.get("method", "get") == "get" else http.post + return method + async def crawling_patch(self): """ Crawl patch information from the cve reference website :return: patch info """ try: - _response = await http.get(self.crawler_url) + _response = await self._method(self._platform)( + self.crawler_url, data=self._platform.get("body") + ) + if "redirct" in self._platform: + _response = await self._rule_redirct(response=_response) except RequestError: return None if _response.error or not _response.text: - logger.error(f'Failed to access URL {self.crawler_url}, detail: {_response.error}') + logger.error( + f"Failed to access URL {self.crawler_url}, detail: {_response.error}" + ) return None - formatted_text = self.format_text(_response.text) + formatted_text = ( + self.format_text(_response.text) + if self._format == "text" + else self.json(_response.text) + ) if formatted_text is None: - logger.error(f'Failed to format string {_response.text}') + logger.error(f"Failed to format string {_response.text}") return None - patch_info_dict = self.match_patch(formatted_text, dict(self.patch_info._asdict())) + patch_info_dict = self.match_patch( + formatted_text, dict(self.patch_info._asdict()) + ) return patch_info_dict @staticmethod - @match([Constant.COMMIT_REGEX, Constant.GIT_COMMIT_REGEX], 'commits') - @match([Constant.PR_REGEX], 'pr') - @match([Constant.ISSUE_REGEX], 'issue') + @match(settings.get_regex(label="commit"), "commits") + @match(settings.get_regex(label="pr"), "pr") + @match(settings.get_regex(label="issue"), "issue") def match_patch(text, patch_info): """ Matching patch related links,Use decorator to find @@ -113,7 +166,7 @@ class CvePlatform: if isinstance(value, list): patch_info[key] = list(set(value)) - logger.info(f'Find patch: {patch_info}') + logger.info(f"Find patch: {patch_info}") return patch_info @staticmethod @@ -122,5 +175,24 @@ class CvePlatform: Format the content obtained from the URL :return: Formatted content """ - html_text = BeautifulSoup(text, 'html.parser') + if not text: + return None + html_text = BeautifulSoup(text, "html.parser") return html_text.prettify() + + @staticmethod + def json(text): + """ + Rewrite the formatted web page to get the content method + :param text: content of web page + :return: formatted content + """ + try: + text_dict = json.loads(text) + return json.dumps(text_dict, indent=4) + except JSONDecodeError as e: + logger.error( + f"The format of the content obtained by bugzilla website is incorrect, " + f"content is {text}, message is {e.msg}" + ) + return None diff --git a/cve-agency-manager/cve_tracking/core/platform/debian.py b/cve-agency-manager/cve_tracking/core/platform/debian.py deleted file mode 100644 index a50d2807475a65bc45868eae91dad6f06d1db2b2..0000000000000000000000000000000000000000 --- a/cve-agency-manager/cve_tracking/core/platform/debian.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python3 -# ****************************************************************************** -# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. -# licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# ******************************************************************************/ -from constant import Constant -from core.platform.cve_platform import CvePlatform - - -class Debian(CvePlatform): - """ - cve refer to the URL debian to find the patch implementation class - """ - - def __init__(self, cve_num, base_url=Constant.DEBIAN_BASE_URL): - super(Debian, self).__init__(cve_num, base_url) diff --git a/cve-agency-manager/cve_tracking/core/platform/nvd.py b/cve-agency-manager/cve_tracking/core/platform/nvd.py deleted file mode 100644 index 98a5efc26ec19663a5b4d918a691cd4d4bf681d7..0000000000000000000000000000000000000000 --- a/cve-agency-manager/cve_tracking/core/platform/nvd.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python3 -# ****************************************************************************** -# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. -# licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# ******************************************************************************/ -from constant import Constant -from core.platform.cve_platform import CvePlatform - - -class Nvd(CvePlatform): - """ - cve refer to the URL nvd to find the patch implementation class - """ - - def __init__(self, cve_num, base_url=Constant.NVD_BASE_URL): - super(Nvd, self).__init__(cve_num, base_url) diff --git a/cve-agency-manager/cve_tracking/core/platform/ubuntu.py b/cve-agency-manager/cve_tracking/core/platform/ubuntu.py deleted file mode 100644 index d3446bb3bd10af80e30465eb2a1b1b67cf42fa50..0000000000000000000000000000000000000000 --- a/cve-agency-manager/cve_tracking/core/platform/ubuntu.py +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/python3 -# ****************************************************************************** -# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. -# licensed under the Mulan PSL v2. -# You can use this software according to the terms and conditions of the Mulan PSL v2. -# You may obtain a copy of Mulan PSL v2 at: -# http://license.coscl.org.cn/MulanPSL2 -# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR -# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR -# PURPOSE. -# See the Mulan PSL v2 for more details. -# ******************************************************************************/ -from constant import Constant -from core.platform.cve_platform import CvePlatform - - -class Ubuntu(CvePlatform): - """ - cve refer to the URL ubuntu to find the patch implementation class - """ - - def __init__(self, cve_num, base_url=Constant.UBUNTU_BASE_URL): - super(Ubuntu, self).__init__(cve_num, base_url) diff --git a/cve-agency-manager/cve_tracking/core/verification/packing.sh b/cve-agency-manager/cve_tracking/core/verification/packing.sh index 9b8f8ee9a540546e6a4914e10cba46b35ba0af3c..8ab7db464c3137f4c8dafd12c1bc3ddbdf2ed956 100644 --- a/cve-agency-manager/cve_tracking/core/verification/packing.sh +++ b/cve-agency-manager/cve_tracking/core/verification/packing.sh @@ -103,7 +103,7 @@ function update_spec() { fi # add %patch last_patch_apply=$(grep "%patch.* " ${spec_file} | sed -n '$p') - if [[ -n ${last_patch_apply} ]];then + if [[ -n ${last_patch_apply} ]]; then last_patch_apply_row=$(grep -n "${last_patch_apply}" ${spec_file} | awk -F ':' '{print $1}') last_patch_apply_num=$(echo ${last_patch_apply} | awk -F ' ' '{print $1}' | awk -F 'patch' '{print $2}') ignore_level_num=$(echo ${last_patch_apply} | awk -F ' ' '{print $2}') @@ -135,6 +135,8 @@ function mv_source_file() { function rpm_build() { echo "[INFO] Start to rpmbuild" install_rpm rpm-build rpm + apt-get build-dep ${rpm_name} >/dev/null 2>&1 + dnf builddep ${root_build_path}/SPECS/${spec_file} >/dev/null 2>&1 rpmbuild -bp ${root_build_path}/SPECS/${spec_file} >./result.log 2>&1 if [[ $? -eq 0 ]]; then echo "[INFO] build success !!!" diff --git a/cve-agency-manager/cve_tracking/cve-tracking.yaml b/cve-agency-manager/cve_tracking/cve-tracking.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a456929ea95a18bcc6d3c8d211a1adb1337fea83 --- /dev/null +++ b/cve-agency-manager/cve_tracking/cve-tracking.yaml @@ -0,0 +1,71 @@ +# 匹配cve补丁信息的正则表达式 +regex: + # 匹配内容的类型的标签，当前只支持 "commit"/"pr"/"issue" + - label: commit + # 匹配特定类型内容的正则表达式，可以设置多个匹配规则 + regular: + - http[s]?://(?:[-\w.\/;?])+(?:/rev|/ci|/commit[s]?)/(?:\?id=)?[0-9a-z]{8,40} + - http[s]?://(?:[-\w.\/;=&?])+a=commit(?:[\w;&?])+h=[0-9a-z]{8,40} + - label: pr + regular: + - http[s]?://(?:[-\w.\/;?])+(?:pull[s]?|merge_requests)/[1-9][0-9]* + - label: issue + regular: + - http[s]?://(?:[-\w.\/;?])+issues/[0-9A-Z]+ + +# 查找cve漏洞修复的平台，例如 Debian、Bugzilla、Nvd等 +platform: + # The name of the platform, similar to a label + # 平台的名称(必配项) + - name: Cnnvd + # Request address to find CVE information,{cve_num} is a placeholder for string substitution in Python + # 查找cve详情信息的请求地址，“{cve_num}”是python中字符串替换的占位符(必配项) + url: http://cnnvd.org.cn/web/vulnerability/queryLds.tag?qcvCnnvdid={cve_num} + # 发送请求的方式，默认为get请求，还可以指定为post请求，当请求方式为get时，此处可不配置 + method: get + # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式，当请求方式为get时，此处可不配置 + body: + # 解析响应体的方式，当前只支持“text”或“json”(必配项) + format: text + # 当请求无法直接获取cve信息时（根据页面中特定链接多次跳转），可以指定页面跳转的方式来获取进一步的信息 + redirct: + # 跳转地址的前缀，一般情况下为跳转页面的域名，如果正则表达式匹配到的跳转地址中有完整的域名+路径，此配置项可以不填写 + - prefix: http://cnnvd.org.cn + # 匹配页面跳转地址的正则表达式，如果匹配多个值，则获取最后一个匹配项 + regex: /web/xxk/ldxqById\.tag\?CNNVD=CNNVD[0-9-]+ + # 送请求的方法，默认为get请求，还可以指定为post请求，当请求为get时，此处可不配置 + method: get + # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式 + body: + + - name: Debian + url: https://security-tracker.debian.org/tracker/{cve_num} + format: text + + - name: Ubuntn + url: https://ubuntu.com/security/{cve_num} + format: text + + - name: Bugzilla + url: https://bugzilla.redhat.com/rest/bug/{cve_num}/comment + format: json + + - name: Nvd + url: https://nvd.nist.gov/vuln/detail/{cve_num} + format: text + + - name: Suse + url: https://bugzilla.suse.com/show_bug.cgi?id={cve_num} + format: text + +# Private token for API access +# api访问时的私人令牌，当前只支持"gitee"/"github"/"gitlab" +authentication: + # api名称，此值不可更改 + - name: gitee + # 访问api的私人令牌，可以手动获取后更改 + token: + - name: github + token: + - name: gitlab + token: glpat-24DGxkuTA8nTR1YQsXvZ diff --git "a/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" index 013dbb02e3643987a1b7b6623d12b921ea7f486b..914f2f5c7f0b26791e348c799635ef08cc6a405d 100644 --- "a/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -23,15 +23,22 @@ | python3-retrying | requests请求重试依赖程序库 | openEuler已集成 | | python3-concurrent-log-handler | python日志程序库，包括日志记录，日志转储 | openEuler已集成 | | python3-pyyaml | python处理yaml文件程序库 | openEuler已集成 | +| python3-asyncio | python异步执行io操作程序库 | openEuler已集成 | +| python3-bs4 | python处理html格式程序库 | openEuler已集成 | +| python3-aiohttp | python异步请求web资源操作程序库 | openEuler已集成 | +| python3-wget | python下载web资源程序库 | openEuler已集成 | +| python3-fake_useragent | python请求api设置代理程序库 | openEuler已集成 | +| python3-lxml | python解析xml程序库 | openEuler已集成 | ### 1.4、特性需求 -| 需求编号 | 需求描述 | 特性描述 | -| -------- | --------- | ----------------------------------------------- | -| 1 | 补丁查找 | 通过cve参考网址和查找规则进行补丁链接匹配查找。 | -| 2 | issue评论 | 将查找到的补丁链接添加到相关issue的评论中。 | -| 3 | 补丁下载 | 通过查找到的补丁链接，将补丁文件下载保存。 | -| 4 | 补丁验证 | 下载补丁文件后，进行补丁打包验证。 | +| 需求编号 | 需求描述 | 特性描述 | +| -------- | --------- | ------------------------------------------------------------ | +| 1 | 补丁查找 | 通过cve参考网址和查找规则进行补丁链接匹配查找。 | +| 2 | issue评论 | 将查找到的补丁链接添加到相关issue的评论中。 | +| 3 | 补丁下载 | 通过查找到的补丁链接，将补丁文件下载保存。 | +| 4 | 补丁验证 | 下载补丁文件后，进行补丁打包验证。 | +| 5 | 补丁反馈 | 若工具为找到补丁，通过命令向指定仓库提交issue，方便数据统计和后续工具优化。 | @@ -53,34 +60,35 @@ > 评论格式 > -> | 参考网址 | 关联pr | 补丁链接 | -> | --------------------------------------------------------- | ---------------------------------------------- | ------------------------------------------------------------ | -> | https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2020-1735 | https://github.com/ansible/ansible/pull/68780/ | https://github.com/ansible/ansible/pull/68780/commits/5292482553dc409081f7f4368398358cbf9f8672，https://github.com/ansible/ansible/pull/68780/commits/5292482553dc409081f7f4368398358cbf9f8673 | -> | https://ubuntu.com/security/CVE-2019-10156 | | https://github.com/ansible/ansible/pull/57188/commits/8254c266f962d5febe46396d5083bb9c1da74840，https://github.com/ansible/ansible/pull/57188/commits/fbda0028750a17a032d83dad9d1fb284f9ea68a4 | +> | 参考网址 | 关联pr | 状态 | 补丁链接 | +> | --------------------------------------------------------- | ---------------------------------------------- | ----- | ------------------------------------------------------------ | +> | https://bugzilla.redhat.com/show_bug.cgi?id=CVE-2020-1735 | https://github.com/ansible/ansible/pull/68780/ | close | https://github.com/ansible/ansible/pull/68780/commits/5292482553dc409081f7f4368398358cbf9f8672，https://github.com/ansible/ansible/pull/68780/commits/5292482553dc409081f7f4368398358cbf9f8673 | +> | https://ubuntu.com/security/CVE-2019-10156 | | | https://github.com/ansible/ansible/pull/57188/commits/8254c266f962d5febe46396d5083bb9c1da74840，https://github.com/ansible/ansible/pull/57188/commits/fbda0028750a17a032d83dad9d1fb284f9ea68a4 | -- 本地使用：下载工具后，通过python调用，该方式支持下载补丁和验证补丁的操作。 +​ 反馈功能：如果工具未找到补丁信息，但是人工可以找到，可以通过在评论中添加一下内容进行反馈： + +```shell +/feedback "cve参考网址" "补丁url" +``` + +​ 该操作将在https://gitee.com/liheavy/cve_tracking仓库中提交相关issue。可通过config.ini中的"FEEDBACK"中的配置来更改提交issue所在的仓库。 + +- 本地使用：下载工具后，通过python调用，该方式支持下载补丁和验证补丁的操作，也支持补丁反馈功能，具体操作见readme指导文档。 ### 3.2、用例视图 -![](\images\用例图.png) +![](images\用例图.png) > 说明： > -> - 社区维护人员是该工具的主要使用者，用于快速查找补丁，及时修复cve。 -> - 社区开发人员可以使用补丁验证模块，快速打包验证。 +> - 社区维护人员是该工具的主要使用者，用于快速查找补丁，及时修复cve，当工具未找到相关补丁时可使用反馈功能进行反馈。 +> - 社区开发人员可以使用补丁验证模块，快速打包验证，可根据反馈信息不断提升工具的查找率和正确率。 > - 版本经理主要是利用该工具，能更好保障版本的顺利发布。 ### 3.3、逻辑视图 ![](images\逻辑视图.png) -> 说明： -> -> - 补丁查找模块：通过输入cve编号和软件包信息，在cve参考网址中查找相关内容，然后通过过滤查找补丁链接。 -> - 评论模块：向cve的issue提交补丁链接相关内容。 -> - 下载模块：根据找到的补丁链接，进行补丁文件下载保存。 -> - 验证模块：根据下载的补丁文件和源代码进行打包验证。 - ### 3.4、代码结构 ![](images\代码模块.png) @@ -91,35 +99,127 @@ ### 3.6、开发视图 -![](images\代码结构.png) +![](images/开发视图.png) + + ### 3.7、质量设计 ##### 3.7.1、性能规格 -| 规格项 | 规格指标 | -| -------- | ------------------------------------------------------------ | -| 提交评论 | 由于补丁平台的网络访问可能存在不稳定性，将重试三次，总时间限制为5min | -| 补丁下载 | 由于补丁平台的网络访问可能存在不稳定性以及补丁大小区别，将重试三次，总时间限制为10min | -| 补丁验证 | 由软件包编译流程和编译环境决定，无时间限制。 | +| 规格项 | 功能点 | 规格指标 | +| ------ | ---------------------- | ------------------------------------------------------------ | +| 效率 | 补丁下载 | 由于补丁平台的网络访问可能存在不稳定性以及补丁大小区别，将重试三次，每次重试2min，总时间限制为10min | +| 效率 | 提交评论
工具反馈 | 由于补丁平台的网络访问可能存在不稳定性，将重试三次，每次重试时间为1min，总时间限制为3min | +| 效率 | 补丁验证 | 由软件包编译流程和编译环境决定，无时间限制。 | -**性能提升策略：** +**补丁查找性能提升策略：** ![](images\性能策略.png) ##### 3.7.2、可靠性设计 -结果展示多个平台的相关内容，使结果更具参考性。 +| 影响因素 | 使用场景 | 可靠性说明 | +| ---------- | :--------- | ------------------------------------------------------------ | +| 高并发 | 在线、本地 | 基于当前主要使用场景为在线使用，每次执行只输入单个cve，所以多个请求也只会顺序执行，资源控制由操作系统调度。 | +| 进程异常 | 在线、本地 | 当前工具为python调用方式，当系统故障或者进程异常退出，执行中断，后续执行不会受影响。 | +| 结果正确性 | 在线、本地 | 工具采用多个cve参考平台共同查找的方式，平台信息各自展示，可通过交叉对比提升结果准确性。同时提供反馈功能，不断提升工具准确性。 | ##### 3.7.3、安全性设计 -| 功能点 | 涉及安全项 | 解决策略 | -| ---------------- | -------------------- | ------------------------------------------------------------ | -| 上游社区查找补丁 | 上游社区用户名，密码 | 不需要登录的将直接访问，必须登录的将支持用户在配置文件配置临时用户名密码。 | -| issue评论 | gitee用户名，密码 | 主要使用场景为和cve_manager集成，将使用机器人账号密码。本地执行支持用户配置临时全局环境变量。 | -| 补丁验证 | 软件包安装，编译 | 必须使用root权限的操作，需用户保证拥有对应权限，不涉及用户名密码保存。 | +| 功能点 | 涉及安全项 | 解决策略 | +| ---------------- | ---------------- | ------------------------------------------------------------ | +| 上游社区信息获取 | 上游社区token | 不需要登录的将直接访问，必须登录的将支持用户在配置文件配置临时token。 | +| issue评论 | gitee私人令牌 | 主要使用场景为和cve_manager集成，将使用机器人令牌。本地执行支持用户配置临时全局环境变量。 | +| 补丁验证 | 软件包安装，编译 | 必须使用root权限的操作，需用户保证拥有对应权限，不涉及用户名密码保存。 | +| 补丁反馈 | 提交issue | 无需用户名密码，需要token，token可设置环境变量和写入配置文件，由用户自己保存。 | + +### 3.8、扩展性设计 + +#### 3.8.1、查找平台扩展 + +工具采用yaml配置查找平台及对应的查找规则，支持动态扩展，开发者只要按照定义的yaml配置文件（cve-tracking.yaml）规则和说明配置对应平台信息即可，当yaml配置项不满足时可调整yaml解析类进行扩展。 + +```yaml +# 匹配cve补丁信息的正则表达式 +regex: + # 匹配内容的类型的标签，当前只支持 "commit"/"pr"/"issue" + - label: commit + # 匹配特定类型内容的正则表达式，可以设置多个匹配规则 + regular: + - http[s]?://(?:[-\w.\/;?])+(?:/rev|/ci|/commit[s]?)/(?:\?id=)?[0-9a-z]{8,40} + - http[s]?://(?:[-\w.\/;=&?])+a=commit(?:[\w;&?])+h=[0-9a-z]{8,40} + - label: pr + regular: + - http[s]?://(?:[-\w.\/;?])+(?:pull[s]?|merge_requests)/[1-9][0-9]* + - label: issue + regular: + - http[s]?://(?:[-\w.\/;?])+issues/[0-9A-Z]+ + +# 查找cve漏洞修复的平台，例如 Debian、Bugzilla、Nvd等 +platform: + # 平台的名称(必配项) + - name: Cnnvd + # 查找cve详情信息的请求地址，“{cve_num}”是python中字符串替换的占位符(必配项) + url: http://cnnvd.org.cn/web/vulnerability/queryLds.tag?qcvCnnvdid={cve_num} + # 发送请求的方式，默认为get请求，还可以指定为post请求，当请求方式为get时，此处可不配置 + method: get + # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式，当请求方式为get时，此处可不配置 + body: + # 解析响应体的方式，当前只支持“text”或“json”(必配项) + format: text + # 当请求无法直接获取cve信息时（根据页面中特定链接多次跳转），可以指定页面跳转的方式来获取进一步的信息 + redirct: + # 跳转地址的前缀，一般情况下为跳转页面的域名，如果正则表达式匹配到的跳转地址中有完整的域名+路径，此配置项可以不填写 + - prefix: http://cnnvd.org.cn + # 匹配页面跳转地址的正则表达式，如果匹配多个值，则获取最后一个匹配项 + regex: /web/xxk/ldxqById\.tag\?CNNVD=CNNVD[0-9-]+ + # 送请求的方法，默认为get请求，还可以指定为post请求，当请求为get时，此处可不配置 + method: get + # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式 + body: + + - name: Debian + url: https://security-tracker.debian.org/tracker/{cve_num} + format: text + + - name: Ubuntn + url: https://ubuntu.com/security/{cve_num} + format: text + + - name: Bugzilla + url: https://bugzilla.redhat.com/rest/bug/{cve_num}/comment + format: json + + - name: Nvd + url: https://nvd.nist.gov/vuln/detail/{cve_num} + format: text + + - name: Suse + url: https://bugzilla.suse.com/show_bug.cgi?id={cve_num} + format: text + +# api访问时的私人令牌，当前只支持"gitee"/"github"/"gitlab" +authentication: + # api名称，此值不可更改 + - name: gitee + # 访问api的私人令牌，可以手动获取后更改 + token: 9c2290c567bd16064219cbbc0f61c8dc + - name: github + token: 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 + - name: gitlab + token: glpat-24DGxkuTA8nTR1YQsXvZ + -### 3.8、特性清单 +``` + +#### 3.8.2、功能扩展 + +工具采用工厂模式设计模式，当命令行和功能模块扩展时，只需增量添加对应的类即可。 + +![](images\扩展设计.png) + +### 3.9、特性清单 **AR：补丁查找** @@ -147,9 +247,9 @@ | -------: | ------------------------------------------------------------ | ------ | | 补丁验证 | 根据下载的补丁文件，以及软件包的gitee地址，下载源代码，修改spec文件，进行软件包打包验证。 | 0.5k | -### 3.9、接口清单 +### 3.10、接口清单 -#### 3.9.1、查找补丁信息 find_patches_info() +#### 3.10.1、查找补丁信息 find_patches_info() > 说明：在各个cve参考网址爬取cve的补丁信息，先初步记录参考网址上列出的网址。 @@ -178,7 +278,7 @@ patch_info: } ``` -#### 3.9.2、查找补丁链接 find_patches_detail() +#### 3.10.2、查找补丁链接 find_patches_detail() > 说明：对find_patches_info查询的信息做进一步处理，获取补丁链接。 @@ -220,23 +320,39 @@ patch_details： { "plantform": "nvd_url", "details": { - "pr_url1":["commit11","commit12"], - "pr_url2":["commit21","commit22"], - None:["commit31","commit32"] + "issue1":{ + "url": "issue1_url", + "status": "close", + "prs":[ + { + "url": "pr1_url", + "status": "open", + "commits": ["commit1", "commit2"] + } + ] + } } }, { "plantform": "buzila_url", "details": { - "pr_url1":["commit11","commit12"], - "pr_url2":["commit21","commit22"], - None:["commit31","commit32"] + "issue1":{ + "url": "", + "status": "", + "prs":[ + { + "url": "", + "status": "", + "commits": ["commit1", "commit2"] + } + ] + } } } ] ``` -#### 3.9.3、评论issue add_comment() +#### 3.10.3、评论issue add_comment() > 说明：根据patch_details获取的补丁详细信息，拼接issue评论内容，添加issue评论。 @@ -254,7 +370,7 @@ patch_details： | code | str | 调用gitee api添加issue评论返回的状态码 | | text | str | 调用gitee api添加issue评论返回的信息 | -#### 3.9.4、下载保存 save_patch() +#### 3.10.4、下载保存 save_patch() > 说明：根据find_patches_detail获取到的补丁信息，进行补丁文件下载。 @@ -270,7 +386,7 @@ patch_details： > 无返回值，下载信息记录日志 -#### 3.9.5、打包验证 packing_source() +#### 3.10.5、打包验证 packing_source() > 说明：根据save_patch保存的补丁文件，以及软件包信息，从gitee拉取源代码，修改spec，进行打包测试。 @@ -284,4 +400,19 @@ patch_details： 返回值： -> 无返回值，编译过程记录文件，编译结果记录日志。 \ No newline at end of file +> 无返回值，编译过程记录文件，编译结果记录日志。 + +#### 3.10.6、补丁信息反馈 create_feedback_issue() + +> 说明：反馈cve参考网址和补丁url信息，到指定仓库下提交issue + +入参： + +| 名称 | 类型 | 说明 | +| -------------- | ---- | -------------------------------------- | +| cve_num | str | cve编号 | +| cve_platform | str | cve参考网址 | +| patch_url | str | 补丁url链接 | +| issue_platform | str | 提交issue的平台 | +| *owner | str | 提交issue所在的owner，在配置文件中配置 | +| *repo | str | 提交issue所在的repo, 在配置文件中配置 | \ No newline at end of file diff --git "a/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\347\273\223\346\236\204.png" "b/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\347\273\223\346\236\204.png" index 38310dfce4981d997ef1f3ad221740798e71d757..ed8b9633710875acbf53e360c137567f4ea2d830 100644 Binary files "a/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\347\273\223\346\236\204.png" and "b/cve-agency-manager/cve_tracking/doc/images/\344\273\243\347\240\201\347\273\223\346\236\204.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" "b/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" new file mode 100644 index 0000000000000000000000000000000000000000..a1da01a4592a933bece141ed77d370be3b86eaa4 Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" "b/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" new file mode 100644 index 0000000000000000000000000000000000000000..57922ae14b4161e6be7d2cb2b8b5a833f4f73b4d Binary files /dev/null and "b/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\347\224\250\344\276\213\345\233\276.png" "b/cve-agency-manager/cve_tracking/doc/images/\347\224\250\344\276\213\345\233\276.png" index 5bddf5b9257e2a46d2773be414e5ef57a2bf8660..53cacf2c001132d41489e19b355def525c6f26fc 100644 Binary files "a/cve-agency-manager/cve_tracking/doc/images/\347\224\250\344\276\213\345\233\276.png" and "b/cve-agency-manager/cve_tracking/doc/images/\347\224\250\344\276\213\345\233\276.png" differ diff --git "a/cve-agency-manager/cve_tracking/doc/images/\351\200\273\350\276\221\350\247\206\345\233\276.png" "b/cve-agency-manager/cve_tracking/doc/images/\351\200\273\350\276\221\350\247\206\345\233\276.png" index 8cfbd43c05be01b61406abe44fbf519a9e4f4eb9..6edec76befbac3f31b17c8931fbd5b0f211f4998 100644 Binary files "a/cve-agency-manager/cve_tracking/doc/images/\351\200\273\350\276\221\350\247\206\345\233\276.png" and "b/cve-agency-manager/cve_tracking/doc/images/\351\200\273\350\276\221\350\247\206\345\233\276.png" differ diff --git a/cve-agency-manager/cve_tracking/requirements.txt b/cve-agency-manager/cve_tracking/requirements.txt index 8386667d5b9a6ae86a2efe0ef3e1f46fba84f6ef..c1697f7e8de4b843ef1e29900b5f455dd6e8918b 100644 --- a/cve-agency-manager/cve_tracking/requirements.txt +++ b/cve-agency-manager/cve_tracking/requirements.txt @@ -6,4 +6,5 @@ wget fake_useragent retrying concurrent_log_handler -lxml \ No newline at end of file +lxml +pyyaml \ No newline at end of file diff --git a/cve-agency-manager/cve_tracking/util/__init__.py b/cve-agency-manager/cve_tracking/util/__init__.py index b6c2caed55ed482594c023ed1eac25210359a0fa..5a7ebcdc275716a52600da9e60838866db1f34a9 100644 --- a/cve-agency-manager/cve_tracking/util/__init__.py +++ b/cve-agency-manager/cve_tracking/util/__init__.py @@ -17,9 +17,14 @@ from logger import logger class Api: - def __init__(self, repo=None, owner=None) -> None: + def __init__(self, repo=None, owner=None, token=None) -> None: self._repo = repo self._owner = owner + self.token = token + + def _set_token(self): + parameters = {"access_token": self.token} if self.token else {} + return parameters @staticmethod async def _post(url, values, **kwargs): @@ -35,6 +40,8 @@ class Api: if not response.success: logger.warning("reuqest url: %s status code: %s error info: %s" % (url, response.status_code, response.error)) + return None + return response.json def set_attr(self, owner, repo): diff --git a/cve-agency-manager/cve_tracking/util/gitee_api.py b/cve-agency-manager/cve_tracking/util/gitee_api.py index 52608ae9739f4b6e4d912633f7a4c886453591c1..24aa3151256d5411f2799a31fbe2f666c59486ca 100644 --- a/cve-agency-manager/cve_tracking/util/gitee_api.py +++ b/cve-agency-manager/cve_tracking/util/gitee_api.py @@ -15,7 +15,7 @@ This is a helper script for working with gitee.com """ import os -from conf import CONFIG +from conf import settings from exception import ConfigNotFoundError from logger import logger from . import Api @@ -25,71 +25,42 @@ class Gitee(Api): """ Gitee is a helper class to abstract gitee.com api """ - token = os.getenv("TRACK_GITEE_TOKEN", CONFIG.GITEE_TOKEN) - time_format = "%Y-%m-%dT%H:%M:%S%z" + host = "https://gitee.com/api/v5/repos" def __init__(self): + super(Gitee, self).__init__() + self.token = os.getenv("TRACK_GITEE_TOKEN", settings.token(name="gitee")) if not self.token: raise ConfigNotFoundError( - "Please set the 'TRACK_GITEE_TOKEN' environment variables of gitee.") - self.src_openeuler_url = "https://gitee.com/src-openeuler/{repo}/raw/{br}/" - self.advisor_url = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/" - super(Gitee, self).__init__() + "Please set the 'TRACK_GITEE_TOKEN' environment variables of gitee." + ) - def _set_token(self): - parameters = {"access_token": self.token} - return parameters - - async def _post_issue(self, title, body): + async def create_issue(self, title, body): """ - Post new issue + Create issue in gitee """ - issues_url = f"https://gitee.com/api/v5/repos/{self._owner}/issues" + issues_url = f"{self.host}/{self._owner}/issues" parameters = self._set_token() parameters["repo"] = self._repo parameters["title"] = title parameters["body"] = body return await self._post(issues_url, parameters) - async def create_issue( - self, - version="", - branch="master", - title=None, - body=None, - ): - """ - Create issue in gitee - """ - title = title or f"Upgrade {self._repo} to {version} in {branch}" - body = ( - body - or """This issue is automatically created by openEuler-Advisor. - Please check the correspond PR is accepted before close it. - Thanks. - Yours openEuler-Advisor.""" - ) - return await self._post_issue(title, body) - async def create_issue_comment(self, number, body): """ create issue comment """ - url = ( - f"https://gitee.com/api/v5/repos/{self._owner}/{self._repo}/issues/{number}/comments" - ) + url = f"{self.host}/{self._owner}/{self._repo}/issues/{number}/comments" values = self._set_token() values["body"] = body - logger.info(f'Add comment to issue url: {url}') + logger.info(f"Add comment to issue url: {url}") return await self._post(url, values) async def create_pr_comment(self, number, body): """ Post comment to the given specific PR """ - url = ( - f"https://gitee.com/api/v5/repos/{self._owner}/{self._repo}/pulls/{number}/comments" - ) + url = f"{self.host}/{self._owner}/{self._repo}/pulls/{number}/comments" values = self._set_token() values["body"] = body return self._post(url, values) diff --git a/cve-agency-manager/cve_tracking/util/github_api.py b/cve-agency-manager/cve_tracking/util/github_api.py index 312ca95c05aca8605283c564eae192d7d3b2b215..b607a3c3b85c745403894ab66b46dbce8d3027ae 100644 --- a/cve-agency-manager/cve_tracking/util/github_api.py +++ b/cve-agency-manager/cve_tracking/util/github_api.py @@ -10,24 +10,29 @@ # PURPOSE. # See the Mulan PSL v2 for more details. # ******************************************************************************/ +import os + from lxml import etree -from exception import RequestError -from logger import logger +from conf import settings from . import Api class Github(Api): - host = "https://api.github.com" + host = "https://api.github.com/repos" def __init__(self) -> None: super(Github, self).__init__() + self.token = os.getenv("GITHUB_TOKEN", settings.token(name="github")) @property def _headers(self): - return { - "accept": "application/vnd.github.v3+json", + headers = { + "Accept": "application/vnd.github.v3+json", } + if self.token: + headers["Authorization"] = f"token {self.token}" + return headers async def get_pull_commits(self, pull_number, page=100, curpage=1): """ @@ -37,8 +42,17 @@ class Github(Api): :param curpage: current page number :return: response """ - url = f"{self.host}/repos/{self._owner}/{self._repo}/pulls/{pull_number}/commits" - return await self._get(url, params=dict(per_page=page, page=curpage), headers=self._headers) + url = f"{self.host}/{self._owner}/{self._repo}/pulls/{pull_number}/commits" + return await self._get( + url, params=dict(per_page=page, page=curpage), headers=self._headers + ) + + async def get_pull_request(self, pull_number): + """ + Get a pull request + """ + url = f"{self.host}/{self._owner}/{self._repo}/pulls/{pull_number}" + return await self._get(url, headers=self._headers) async def get_issue(self, issue_number): """ @@ -46,7 +60,7 @@ class Github(Api): :param issue_number: issue num :return: response """ - url = f"{self.host}/repos/{self._owner}/{self._repo}/issues/{issue_number}" + url = f"{self.host}/{self._owner}/{self._repo}/issues/{issue_number}" return await self._get(url, headers=self._headers) async def get_issue_comments(self, issue_number): @@ -55,16 +69,24 @@ class Github(Api): :param issue_number: issue num :return: response """ - url = f"{self.host}/repos/{self._owner}/{self._repo}/issues/{issue_number}/comments" + url = f"{self.host}/{self._owner}/{self._repo}/issues/{issue_number}/comments" + return await self._get(url, headers=self._headers) + async def create_issue(self, title, body): + url = f"{self.host}/{self._owner}/{self._repo}/issues" + params = {"title": title, "body": body} + return await self._post( + url=url, values=None, json=params, headers=self._headers + ) + async def check_pull_merged(self, pull_number): """ Check pr status :param pull_number: pull number :return: response """ - url = f"{self.host}/repos/{self._owner}/{self._repo}/pulls/{pull_number}/merge" + url = f"{self.host}/{self._owner}/{self._repo}/pulls/{pull_number}/merge" response = await self._get(url, headers=self._headers) if response.status_code == 204: return True @@ -89,19 +111,23 @@ class Github(Api): if issue_url: response = await self._get(url=issue_url, text=True) else: - response = await self._get(url=f"https://github.com/{self._owner}/{self._repo}/issues/{issue_number}", - text=True) + response = await self._get( + url=f"https://github.com/{self._owner}/{self._repo}/issues/{issue_number}", + text=True, + ) if not response: return [] html = etree.HTML(response) link = html.xpath( "//div[contains(@class,'discussion-sidebar-item') and contains(@class,'js-discussion-sidebar-item') " - "and last() ]/form/div[contains(@class,'css-truncate')]") + "and last() ]/form/div[contains(@class,'css-truncate')]" + ) if link: return self._pull(pulls=link[-1].xpath("./a/@href")) # Search for operation logs pulls = html.xpath( - "//div[@class='TimelineItem']/div[@class='TimelineItem-body']//div[contains(@class,'flex-auto')]/a/@href") + "//div[@class='TimelineItem']/div[@class='TimelineItem-body']//div[contains(@class,'flex-auto')]/a/@href" + ) return self._pull(pulls=pulls) diff --git a/cve-agency-manager/cve_tracking/util/gitlab_api.py b/cve-agency-manager/cve_tracking/util/gitlab_api.py index 97b985fd2d2bfe5f4c9f8ed3186c1820dad3a17d..a127e8a63741bc46079e6a965818bf00e4b9ad03 100644 --- a/cve-agency-manager/cve_tracking/util/gitlab_api.py +++ b/cve-agency-manager/cve_tracking/util/gitlab_api.py @@ -13,23 +13,23 @@ import os from exception import ConfigNotFoundError -from conf import CONFIG +from conf import CONFIG, settings from . import Api class Gitlab(Api): - host = "https://gitlab.com" + host = "https://gitlab.com/api/v4/projects" def __init__(self) -> None: - if not CONFIG.GITLAB_TOKEN and not os.getenv("GITLAB-TOKEN", None): - raise ConfigNotFoundError("Please set the 'GITLAB-TOKEN' environment variable for gitlab.") + if not settings.token(name="gitlab") and not os.getenv("GITLAB-TOKEN", None): + raise ConfigNotFoundError( + "Please set the 'GITLAB-TOKEN' environment variable for gitlab." + ) super(Gitlab, self).__init__() @property def _headers(self): - return { - "PRIVATE-TOKEN": os.getenv("GITLAB-TOKEN", CONFIG.GITLAB_TOKEN) - } + return {"PRIVATE-TOKEN": os.getenv("GITLAB-TOKEN", settings.token('gitlab'))} async def get_commit_comments(self, commit_sha): """ @@ -37,7 +37,7 @@ class Gitlab(Api): :param commit_sha: The commit hash or name of a repository branch or tag """ - url = f"{self.host}/api/v4/projects/{self._owner}%2F{self._repo}/repository/commits/{commit_sha}/comments" + url = f"{self.host}/{self._owner}%2F{self._repo}/repository/commits/{commit_sha}/comments" return await self._get(url, headers=self._headers) async def get_single_mr(self, merge_request_iid): @@ -46,7 +46,7 @@ class Gitlab(Api): :param merge_request_iid: The internal ID of the merge request. """ - url = f"{self.host}/api/v4/projects/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}" + url = f"{self.host}/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}" return await self._get(url, headers=self._headers) async def get_mr_comments(self, merge_request_iid): @@ -56,7 +56,7 @@ class Gitlab(Api): :param merge_request_iid: The internal ID of the merge request. """ - url = f"{self.host}/api/v4/projects/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}/notes" + url = f"{self.host}/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}/notes" return await self._get(url, headers=self._headers) async def get_mr_context_commits(self, merge_request_iid): @@ -66,7 +66,7 @@ class Gitlab(Api): :param merge_request_iid: The internal ID of the merge request. """ - url = f"{self.host}/api/v4/projects/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}/commits" + url = f"{self.host}/{self._owner}%2F{self._repo}/merge_requests/{merge_request_iid}/commits" return await self._get(url, headers=self._headers) @@ -76,5 +76,17 @@ class Gitlab(Api): :param issue_id: The internal ID of a project issue """ - url = f"{self.host}/api/v4/projects/{self._owner}%2F{self._repo}/issues/{issue_id}/closed_by" + url = f"{self.host}/{self._owner}%2F{self._repo}/issues/{issue_id}/closed_by" return await self._get(url, headers=self._headers) + + async def create_issue(self, title, body): + """ + Creates a new project issue. + :param title: issue title + :param body: issue body + :return: + """ + url = f"{self.host}/{self._owner}%2F{self._repo}/issues" + params = {'title': title, + 'description': body} + return await self._post(url, headers=self._headers, values=params)