diff --git a/cve-agency-manager/cve_tracking/.gitignore b/cve-agency-manager/cve_tracking/.gitignore
index 11614af2870733183efe883810764d8708bddf8f..5f9908e1f652c7e6c28a3367d75a223a19cff326 100644
--- a/cve-agency-manager/cve_tracking/.gitignore
+++ b/cve-agency-manager/cve_tracking/.gitignore
@@ -8,6 +8,7 @@ __pycache__/
# Distribution / packaging
.Python
+.xlsx
build/
develop-eggs/
dist/
@@ -95,6 +96,7 @@ venv/
ENV/
env.bak/
venv.bak/
+.vscode/
# Spyder project settings
.spyderproject
diff --git a/cve-agency-manager/cve_tracking/README.md b/cve-agency-manager/cve_tracking/README.md
index 45d962b3a2b9634e06951249eae032abf782c621..adeaaf3a3fb0f8e987ce2dd28a63da729431a499 100644
--- a/cve-agency-manager/cve_tracking/README.md
+++ b/cve-agency-manager/cve_tracking/README.md
@@ -48,7 +48,7 @@ python 可执行代码
>
> -i 需要评论的 issue 编号
>
- > 注意：默认仓库为 src-openeuler，如果要更改，请修改 main.py 同目录下 constant.py 中的 DEFAULT_OWNER 的值。
+ > 注意：请修改 main.py 同目录下 cve-tracking.yaml 中的 # 访问api的私人令牌 token 的值与 config.ini 中的 DEFAULT_OWNER 的值
2. 补丁查找及下载(验证)
@@ -88,7 +88,32 @@ python 可执行代码
>
> -nd 是否需要下载源代码，默认为需要下载，若无需下载添加该参数
-4. 意见反馈
+4. 查找下载验证及评论
+
+ ```shell
+ python3 main.py findp_adaptive_patch -c cve_num -r rpm_name -f patch_save_path -s source_path -p -b branch [-nd] -i issue_num
+ ```
+
+
+ > 参数说明：
+ >
+ > -c cve 的编号
+ >
+ > -r rpm 包名称
+ >
+ > -f 补丁文件的下载目录，不设置默认为/opt/cve_tracking/patches
+ >
+ > -s 源码包下载路径，不设置默认为/opt/cve_tracking/source_code
+ >
+ > -b 源码包所在的 gitee 的 src-openeuler 仓库的分支，默认为 master
+ >
+ > -p 是否进行补丁应用，默认为不应用，若需要应用，添加该参数。
+ >
+ > -i 需要评论的 issue 编号
+ >
+ > 注意：请修改 main.py 同目录下 cve-tracking.yaml 中的 # 访问api的私人令牌 token 的值与 config.ini 中的 DEFAULT_OWNER FEEDBACK_ISSUE_OWNER FEEDBACK_ISSUE_REPO 的值
+
+5. 意见反馈
若工具没有查找到补丁且人工查找到补丁信息，可通过该功能进行反馈，将会在配置的平台中提交 issue。将发现新 patch 的平台信息上报至指定的仓库，便于 CVE 修复人员完善工具的查找。指定的仓库可以在 conf.ini 配置文件的【FEEDBACK】中进行配置
diff --git a/cve-agency-manager/cve_tracking/cli/__init__.py b/cve-agency-manager/cve_tracking/cli/__init__.py
index 41ef326fdecdf0c1e0a9812ed47acd15b8fd5d74..9242bb9b27e6b504256e953351b0264de3966bf9 100644
--- a/cve-agency-manager/cve_tracking/cli/__init__.py
+++ b/cve-agency-manager/cve_tracking/cli/__init__.py
@@ -15,9 +15,11 @@ from .comment import CommentCommand
from .feedback import FeedbackCommand
from .save import SaveCommand
from .verify import VerifyCommand
+from .findp_adaptive_patch import FindAdaptivePatch
__all__ = ("CveTrackingCommand",
"CommentCommand",
"SaveCommand",
"VerifyCommand",
- "FeedbackCommand")
+ "FeedbackCommand",
+ "FindAdaptivePatch")
diff --git a/cve-agency-manager/cve_tracking/cli/comment.py b/cve-agency-manager/cve_tracking/cli/comment.py
index b45186377c580cd21e2b634f98cf5afd0776ca5e..5f9e22b43ca2c32183fd96f789851d3641fd4568 100644
--- a/cve-agency-manager/cve_tracking/cli/comment.py
+++ b/cve-agency-manager/cve_tracking/cli/comment.py
@@ -50,7 +50,8 @@ class CommentCommand(CveTrackingCommand):
"""
logger.info("Start to perform search and comment functions")
if not all([args.cve_num, args.rpm_name, args.issue]):
- raise InputError(msg="Comment command parameters: cve_num/rpm_name/issue")
+ raise InputError(
+ msg="Comment command parameters: cve_num/rpm_name/issue")
# find patch detail
patch_obj = Patch(cve_num=args.cve_num, rpm_name=args.rpm_name)
@@ -59,5 +60,5 @@ class CommentCommand(CveTrackingCommand):
# comment cve issue
gitee = Gitee()
gitee.set_attr(owner=CONFIG.DEFAULT_OWNER, repo=args.rpm_name)
- await issue_comment(patch_details=patch_details, number=args.issue, gitee=gitee)
+ await issue_comment(patch_details=patch_details, number=args.issue, gitee=gitee, packing=False)
logger.info("End to perform search and comment functions")
diff --git a/cve-agency-manager/cve_tracking/cli/findp_adaptive_patch.py b/cve-agency-manager/cve_tracking/cli/findp_adaptive_patch.py
new file mode 100644
index 0000000000000000000000000000000000000000..ccfa4dfdae995dfdd4e7807f68440d934921b311
--- /dev/null
+++ b/cve-agency-manager/cve_tracking/cli/findp_adaptive_patch.py
@@ -0,0 +1,106 @@
+#!/usr/bin/python3
+# ******************************************************************************
+# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved.
+# licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ******************************************************************************/
+from cli.base import CveTrackingCommand
+from conf import CONFIG
+from core.crawler.patch import Patch
+from core.download import save
+from core.comment.issue_comment import issue_comment
+from exception import InputError
+from logger import logger
+from util.gitee_api import Gitee
+
+
+class FindAdaptivePatch(CveTrackingCommand):
+ """
+ Execute the command line to find the patch and save the patch file
+ """
+
+ def __init__(self):
+ super(FindAdaptivePatch, self).__init__()
+ self.parser = CveTrackingCommand.sub_parse.add_parser('findp_adaptive_patch',
+ help="Search for patches, download patch files, verify and comment on patches")
+ self._add_common_param()
+ self._add_params()
+
+ def _add_params(self):
+ """
+ Add the unique parameters of the command
+ :return: None
+ """
+ self.parser.add_argument('-f', '--patch_path', metavar='patch_patch', type=str, action='store', required=False,
+ default=CONFIG.PATCH_SAVE_PATH,
+ help='Path to save the patch file, the default is"/opt/cve_tracking/patches"')
+ self.parser.add_argument('-s', '--source_path', metavar='source_patch', type=str, action='store',
+ required=False, default=CONFIG.SOURCE_CODE_PATH,
+ help='Source download path, the default is: "/opt/cve_tracking/source_code"')
+ self.parser.add_argument('-p', '--packing', action='store_true', required=False, default=False,
+ help='Whether to package the verification patch')
+ self.parser.add_argument('-b', '--branch', metavar='branch', type=str, action='store', required=False,
+ default='master', help='Branch of rpm, default is "master"')
+ self.parser.add_argument(
+ "-i"
+ ,
+ "--issue",
+ metavar="issue",
+ type=str,
+ required=True,
+ action="store",
+ default=None,
+ help="CVE related issue number",
+ )
+ @staticmethod
+ async def run_command(args):
+ """
+ Search patch, save file function execution entry
+ :param args: command line params
+ :return: None
+ """
+ logger.info(f'Start to execute the download command, cve: {args.cve_num}, '
+ f'rpm: {args.rpm_name}, patch save path: {args.patch_path}')
+ if not all([args.rpm_name, args.cve_num, args.patch_path]):
+ raise InputError('Download command parameter error, please confirm "rpm_name"/cve_num/patch_path"')
+ if args.packing and not all([args.source_path, args.branch]):
+ raise InputError('Download command parameter error, please confirm "source_path/branch"')
+
+ # find patch info
+ patch = Patch(cve_num=args.cve_num, rpm_name=args.rpm_name)
+ patch_detail_info = await patch.find_patches_detail()
+ # download patch file
+
+ #TODO 写函数优化
+ patch_url_state = save.save_patch(patch_details=patch_detail_info, cve=args.cve_num, path=args.patch_path,args=args)
+ for patch in patch_detail_info:
+ for issue_info in patch.get("details", list()):
+ patch_url_state = _issue_info(issue_info,patch_url_state)
+
+
+ # comment cve issue
+ gitee = Gitee()
+ gitee.set_attr(owner=CONFIG.DEFAULT_OWNER, repo=args.rpm_name)
+ await issue_comment(patch_details=patch_detail_info, number=args.issue, gitee=gitee, packing=True)
+ logger.info("End to perform search and comment and download and packing functions")
+
+def _issue_info(issue_info,patch_url_state):
+ for prs in issue_info["issue"].get("prs", list()):
+ for url in prs.get("commits", []):
+ patch_url_state = _patch_url_state(prs,url,patch_url_state)
+ return patch_url_state
+
+def _patch_url_state(prs,url,patch_url_state):
+ for key, value in patch_url_state.items():
+ if key == url:
+ if not prs.get("application"):
+ prs["application"] = patch_url_state[key]
+ else:
+ prs["application"] = prs["application"] + "
" + patch_url_state[key]
+ return patch_url_state
diff --git a/cve-agency-manager/cve_tracking/cli/save.py b/cve-agency-manager/cve_tracking/cli/save.py
index 2d9d5b8bc50c307705be1f1dfd661fc7d1ae5ad2..d27ba3cf6bb622b2bb8134aa2a5100f6c7b9fbca 100644
--- a/cve-agency-manager/cve_tracking/cli/save.py
+++ b/cve-agency-manager/cve_tracking/cli/save.py
@@ -14,7 +14,6 @@ from cli.base import CveTrackingCommand
from conf import CONFIG
from core.crawler.patch import Patch
from core.download import save
-from core.verification.apply import PathApply
from exception import InputError
from logger import logger
@@ -45,7 +44,7 @@ class SaveCommand(CveTrackingCommand):
self.parser.add_argument('-p', '--packing', action='store_true', required=False, default=False,
help='Whether to package the verification patch')
self.parser.add_argument('-b', '--branch', metavar='branch', type=str, action='store', required=False,
- default='master', help='Branch of rpm, default is "master"')
+ default='master', help='Branch of rpm, default is "master"')
@staticmethod
async def run_command(args):
@@ -64,12 +63,8 @@ class SaveCommand(CveTrackingCommand):
# find patch info
patch = Patch(cve_num=args.cve_num, rpm_name=args.rpm_name)
patch_detail_info = await patch.find_patches_detail()
- # download patch file
- save.save_patch(patch_details=patch_detail_info, cve=args.cve_num, path=args.patch_path)
- logger.info(f'End to execute the save file, patch save path:{args.patch_path}')
- # apply patch
- if args.packing:
- patch_apply = PathApply(rpm_name=args.rpm_name, patch_path=args.patch_path,
- source_path=args.source_path, branch_rpm=args.branch)
- patch_apply.packing_source()
+ # download patch file and apply patch
+ patch_url_state = save.save_patch(patch_details=patch_detail_info, cve=args.cve_num, path=args.patch_path,args=args)
+ logger.info(f'End to execute the save file, patch save path:{args.patch_path}')
+ logger.info(f'Result is {patch_url_state}')
diff --git a/cve-agency-manager/cve_tracking/cli/verify.py b/cve-agency-manager/cve_tracking/cli/verify.py
index ace288d8adce701b7790a1ca0321c1e8e330995a..2bd20a80c7cdfc5235a080e7d1a1db734fdd070d 100644
--- a/cve-agency-manager/cve_tracking/cli/verify.py
+++ b/cve-agency-manager/cve_tracking/cli/verify.py
@@ -13,7 +13,7 @@
from cli.base import CveTrackingCommand
from conf import CONFIG
from core.verification.apply import PathApply
-
+from logger import logger
class VerifyCommand(CveTrackingCommand):
"""
@@ -24,17 +24,17 @@ class VerifyCommand(CveTrackingCommand):
super(VerifyCommand, self).__init__()
self.parser = CveTrackingCommand.sub_parse.add_parser('packing', help='Test patch file application results')
self._add_params()
+ self._add_common_param()
+
def _add_params(self):
"""
Add the unique parameters of the command
:return: None
"""
- self.parser.add_argument('-r', '--rpm_name', metavar='rpm_name', type=str, required=True, default=None,
- help='Source rpm name')
self.parser.add_argument('-f', '--patch_path', metavar='patch_path', type=str, action='store', required=True,
default=None, help='Path of patch file')
- self.parser.add_argument('-s', '--source_path', metavar='source_path', type=str, action='store', required=False,
+ self.parser.add_argument('-s', '--source_path', metavar='source_path', type=str, action='store',required=False,
default=CONFIG.SOURCE_CODE_PATH,
help='Source download path, the default is: "/opt/cve_tracking/source_code"')
self.parser.add_argument('-b', '--branch', metavar='branch', type=str, action='store', required=False,
@@ -52,5 +52,7 @@ class VerifyCommand(CveTrackingCommand):
"""
branch = "no" if args.no_download else args.branch
patch_apply = PathApply(rpm_name=args.rpm_name, branch_rpm=branch, patch_path=args.patch_path,
- source_path=args.source_path)
- patch_apply.packing_source()
+ source_path=args.source_path,cve_num=args.cve_num)
+ status = patch_apply.packing_source()
+ patch_apply.clear_trash()
+ logger.info("End of adaptation Process,resule is %s"%status)
\ No newline at end of file
diff --git a/cve-agency-manager/cve_tracking/conf/setting.py b/cve-agency-manager/cve_tracking/conf/setting.py
index d9a5152c628cbccd4d8edc68081d1be7d070c5dc..9e3fd8e157c1f6557b786c262aa7b61820f0cbcd 100644
--- a/cve-agency-manager/cve_tracking/conf/setting.py
+++ b/cve-agency-manager/cve_tracking/conf/setting.py
@@ -25,7 +25,7 @@ class DefaultConfig:
# Log related
# **********
# level of LOG
- LOGGER_LEVEL = "INFO"
+ LOGGER_LEVEL = "ERROR"
# Log file path
LOGGER_PATH = "/var/log/cve_tracking"
# Log file name
@@ -48,14 +48,14 @@ class DefaultConfig:
# Feedback issue related configuration
# **********
- FEEDBACK_PLATFORM = "github"
+ FEEDBACK_PLATFORM = "gitee"
# Feedback to the owner of the issue
- FEEDBACK_ISSUE_OWNER = "liheavy"
+ FEEDBACK_ISSUE_OWNER = "song-jintang"
# Feedback to the repo of the issue
- FEEDBACK_ISSUE_REPO = "cve_tracking"
+ FEEDBACK_ISSUE_REPO = "song-jintang"
# The owner of the comment function submits the comment
- DEFAULT_OWNER = "src-openeuler"
+ DEFAULT_OWNER = "song-jintang"
class Config:
@@ -163,8 +163,11 @@ class YamlConfiguration:
else:
regulars = filter(lambda x: x["label"] == label, self.regex)
return [regx for reg in regulars for regx in reg["regular"]]
-
+
def token(self, name):
+ """
+ get and match gitee github gitlab token
+ """
token = filter(lambda x: x["name"] == name, self.authentication)
if token:
return list(token)[-1]["token"]
diff --git a/cve-agency-manager/cve_tracking/config.ini b/cve-agency-manager/cve_tracking/config.ini
index ccd7f5809e7a834c6673b7d29e00ee934f3ae169..e4638ea91ef21d20eb9ec9ad0326ae3d8d2628d7 100644
--- a/cve-agency-manager/cve_tracking/config.ini
+++ b/cve-agency-manager/cve_tracking/config.ini
@@ -13,6 +13,7 @@
;Log related configuration
[LOG]
;level of LOG
+; LOGGER_LEVEL = CRITICAL
LOGGER_LEVEL = INFO
; Log file path
LOGGER_PATH = /var/log/cve_tracking
@@ -33,7 +34,7 @@ SOURCE_CODE_PATH = /opt/cve_tracking/source_code
;Comment function related configuration
[COMMENT]
;The owner of the comment function submits the comment
-DEFAULT_OWNER = src-openeuler
+DEFAULT_OWNER = song-jintang
;Feedback issue related configuration
[FEEDBACK]
@@ -41,6 +42,6 @@ DEFAULT_OWNER = src-openeuler
;Please configure the token of the corresponding platform synchronously in cve-tracking.yaml
FEEDBACK_PLATFORM = gitee
;Feedback to the owner of the issue
-FEEDBACK_ISSUE_OWNER = liheavy
+FEEDBACK_ISSUE_OWNER = song-jintang
;Feedback to the repo of the issue
-FEEDBACK_ISSUE_REPO = cve_tracking
\ No newline at end of file
+FEEDBACK_ISSUE_REPO = kernel
\ No newline at end of file
diff --git a/cve-agency-manager/cve_tracking/core/comment/issue_comment.py b/cve-agency-manager/cve_tracking/core/comment/issue_comment.py
index f5fdfbc467ee0b4b6b6e5302fa4496d305f35430..1137aac91ad163f516c3a59d7566d3a15dc07f80 100644
--- a/cve-agency-manager/cve_tracking/core/comment/issue_comment.py
+++ b/cve-agency-manager/cve_tracking/core/comment/issue_comment.py
@@ -36,7 +36,7 @@ class Table:
def _rowspan(num, platform):
return """{} | """.format(str(num), platform)
- def span_content(self, pr_url, pr_status, commits):
+ def span_content(self, packing, pr_url, pr_status, commits, application):
"""
Splicing comment content
:param pr_url: pull request url
@@ -45,15 +45,21 @@ class Table:
:return: content spliced
"""
hyperlink_commits = [f"{commit}" for commit in commits]
- contents = self._td(pr_url) + self._td(pr_status) + self._td("
".join(hyperlink_commits))
+ if packing:
+ contents = self._td(pr_url) + self._td(pr_status) + self._td("
".join(hyperlink_commits))+self._td(application)
+ else:
+ contents = self._td(pr_url) + self._td(pr_status) + self._td("
".join(hyperlink_commits))
return contents
- def _table_name(self):
+ def _table_name(self,packing):
"""
Splicing header
:return: header
"""
- table_names = ["参考网址", "关联pr", "状态", "补丁链接"]
+ if packing:
+ table_names = ["参考网址", "关联pr", "状态", "补丁链接","验证结果 "]
+ else:
+ table_names = ["参考网址", "关联pr", "状态", "补丁链接"]
table_names = " ".join(map(self._th, table_names))
return self._tr(table_names)
@@ -64,7 +70,8 @@ class Table:
number += len(issues["issue"].get("prs", list()))
return number
- def table(self, lst):
+ def table(self, lst,packing):
+
"""
Splicing table
:param lst: cve info
@@ -82,13 +89,14 @@ class Table:
if not details:
none_tr = self._tr(self._td(platform) + self._td("") * 3)
contents += none_tr
-
for issue_info in details:
for pr in issue_info["issue"].get("prs", list()):
content = self.span_content(
+ packing,
pr["url"],
pr["status"],
pr.get("commits", []),
+ pr.get("application")
)
if platform_index == 0:
rowspan_content = rowspan_content + content
@@ -97,11 +105,10 @@ class Table:
else:
span_content = self._tr(content)
contents += span_content
-
- return self._table(self._table_name() + contents)
+ return self._table(self._table_name(packing) + contents)
-async def issue_comment(patch_details, number, gitee: Gitee):
+async def issue_comment(patch_details, number, gitee: Gitee,packing):
"""
Call gitee's api to add issue comments
:param patch_details: patch detail
@@ -109,7 +116,7 @@ async def issue_comment(patch_details, number, gitee: Gitee):
:param gitee: gitee instance
:return: None
"""
- table = Table().table(patch_details)
+ table = Table().table(patch_details,packing)
instruction = "\n" * 2 + "> 说明：抱歉，当前工具暂未找到推荐补丁，请人工查找或者之后评论'/find-patch'尝试再次查找。\n" \
"若人工查找到补丁，烦请在此issue下评论 '/report-patch 参考网址 补丁链接1,补丁链接2' 便于我们不断优化工具，不胜感激。\n" \
"如 /report-patch https://security-tracker.debian.org/tracker/CVE-2021-3997 https://github.com/systemd/systemd/commit/5b1cf7a9be37e20133c0208005274ce4a5b5c6a1"
diff --git a/cve-agency-manager/cve_tracking/core/crawler/patch.py b/cve-agency-manager/cve_tracking/core/crawler/patch.py
index 97c44ffc27da8c19f73360112a68e3b685eb96c3..ef1c4384d0f4a04183268b77065bc547cf0050d7 100644
--- a/cve-agency-manager/cve_tracking/core/crawler/patch.py
+++ b/cve-agency-manager/cve_tracking/core/crawler/patch.py
@@ -13,7 +13,6 @@
import re
import asyncio
from collections import namedtuple
-
from constant import Constant
from core.platform import CvePlatform
from logger import logger
@@ -36,8 +35,12 @@ class Patch:
self.pr_status_dict = dict()
self.github_api = Github()
self.gitlab_api = Gitlab()
- self._SummaryInfo = namedtuple("SummaryInfo", ["pr_list", "issues_list"])
+ self._SummaryInfo = namedtuple(
+ "SummaryInfo", ["pr_list", "issues_list"])
self._PatchDetail = namedtuple("PatchDetail", ["platform", "details"])
+ self.zero_num = 0
+ self.platform_num = 2
+ self.platform_re_num = 2
@property
def issue_relation_info(self):
@@ -47,44 +50,75 @@ class Patch:
def pr_relation_info(self):
return {"url": None, "status": None, "commits": []}
+ def get_detail(self, detail, detail_num):
+ """
+ String splicing
+ """
+ if detail["details"] != [] and detail["details"][self.zero_num]["issue"]["prs"] != [] and detail["details"][self.zero_num]["issue"]["prs"][self.zero_num]["commits"] != []:
+ commit_num = 0
+ for commit in detail["details"][self.zero_num]["issue"]["prs"][self.zero_num]["commits"]:
+ self.patch_detail_list[detail_num]["details"][self.zero_num]["issue"]["prs"][self.zero_num]["commits"][commit_num] = settings.get_platform()[
+ self.platform_num]["redirect"][self.platform_re_num]["prefix"] + commit
+ commit_num += 1
+ return detail, detail_num
+
async def find_patches_detail(self):
"""
Further analyze the obtained patch information to get the final patch link
:return: self.patch_detail_list
"""
- await self._find_patches_info()
- # commit,pr,issue summary structure,convenient for deduplication and concurrent search
- summary_info = self._SummaryInfo(pr_list=list(), issues_list=list())
- for patch_info in self.patch_info_list:
- summary_info.pr_list.extend(patch_info.get("pr", []))
- summary_info.issues_list.extend(patch_info.get("issue", []))
-
- # Find issue linked pr and add it to the pr list obtained earlier
- if summary_info.issues_list:
- issue_task_list = await self._create_async_task(
- self._get_issue_link_pr, summary_info.issues_list
- )
- issue_done_task, _ = await asyncio.wait(issue_task_list)
- for task in issue_done_task:
- summary_info.pr_list.extend(task.result())
-
- if summary_info.pr_list:
- # Query pr status
- pr_status_task_list = await self._create_async_task(
- self._get_pr_status, summary_info.pr_list
- )
- await asyncio.wait(pr_status_task_list)
- # Find pr contain commits
- pr_task_list = await self._create_async_task(
- self._get_pr_contain_commits, summary_info.pr_list
+ specialrpms = settings.specialrpm
+ sp_name_list = [sp_detail.get("name") for sp_detail in specialrpms]
+ if self.rpm_name not in sp_name_list:
+ await self._find_patches_info()
+ # commit,pr,issue summary structure,convenient for deduplication and concurrent search
+ summary_info = self._SummaryInfo(
+ pr_list=list(), issues_list=list())
+ for patch_info in self.patch_info_list:
+ summary_info.pr_list.extend(patch_info.get("pr", []))
+ summary_info.issues_list.extend(patch_info.get("issue", []))
+
+ # Find issue linked pr and add it to the pr list obtained earlier
+ if summary_info.issues_list:
+ issue_task_list = await self._create_async_task(
+ self._get_issue_link_pr, summary_info.issues_list
+ )
+ issue_done_task, _ = await asyncio.wait(issue_task_list)
+ for task in issue_done_task:
+ summary_info.pr_list.extend(task.result())
+
+ if summary_info.pr_list:
+ # Query pr status
+ pr_status_task_list = await self._create_async_task(
+ self._get_pr_status, summary_info.pr_list
+ )
+ await asyncio.wait(pr_status_task_list)
+ # Find pr contain commits
+ pr_task_list = await self._create_async_task(
+ self._get_pr_contain_commits, summary_info.pr_list
+ )
+ await asyncio.wait(pr_task_list)
+ # Generate the final information that needs to be output
+ self._convert_path_detail()
+ logger.info(
+ f'Find the patch information of cve "{self.cve_num}" as: \n{self.patch_detail_list}'
)
- await asyncio.wait(pr_task_list)
-
- # Generate the final information that needs to be output
- self._convert_path_detail()
- logger.info(
- f'Find the patch information of cve "{self.cve_num}" as: \n{self.patch_detail_list}'
- )
+ # Adding a Special Platform
+ else:
+ for sp_name in specialrpms:
+ if self.rpm_name == sp_name.get("name") and sp_name.get("request") is False:
+ # patch_detail_list.append()
+ sp_name["patch_url"] = sp_name.get(
+ "patch_url").format(self.cve_num)
+ self.patch_detail_list.append({'platform': '{}'.format(sp_name["url"]), 'details': [{'issue': {
+ 'url': None, 'prs': [{'url': None, 'status': None, 'commits': ['{}'.format(sp_name["patch_url"])]}]}}]})
+
+ detail_num = 0
+ # 修改ghostscript，path链接
+ if self.rpm_name == "ghostscript":
+ for detail in self.patch_detail_list:
+ detail, detail_num = self.get_detail(detail, detail_num)
+ detail_num += 1
return self.patch_detail_list
async def _find_patches_info(self):
@@ -136,19 +170,22 @@ class Patch:
elif Constant.GITLAB in issue:
try:
issue_info = issue.split("/")
- self.gitlab_api.set_attr(owner=issue_info[-5], repo=issue_info[-4])
+ self.gitlab_api.set_attr(
+ owner=issue_info[-5], repo=issue_info[-4])
self._set_gitlab_host(issue)
issue_num = issue_info[-1]
response = await self.gitlab_api.issue_relevance_pull(
issue_id=issue_num
)
- pr_list = self._convert_api_response(response, url_key="web_url")
+ pr_list = self._convert_api_response(
+ response, url_key="web_url")
except IndexError:
logger.warning(
f"The issue link: {issue} does not conform to the standard format"
)
else:
- logger.warning(f"Issue {issue} failed to match the relevant code platform")
+ logger.warning(
+ f"Issue {issue} failed to match the relevant code platform")
# Record the relationship between issue and pr to facilitate subsequent patch information processing
logger.info(f"According to issue {issue} get pr: {str(pr_list)}")
@@ -196,7 +233,8 @@ class Patch:
)
commits_list = self._convert_api_response(response, "web_url")
- logger.info(f"According to pull {pr} get commits: {str(commits_list)}")
+ logger.info(
+ f"According to pull {pr} get commits: {str(commits_list)}")
self.issue_pr_dict[pr] = commits_list
@@ -214,7 +252,8 @@ class Patch:
elif Constant.GITLAB in pr:
self.gitlab_api.set_attr(owner=pr_info[-5], repo=pr_info[-4])
else:
- logger.warning(f"Pr {pr} failed to match the relevant code platform")
+ logger.warning(
+ f"Pr {pr} failed to match the relevant code platform")
return None
except IndexError:
logger.warning(
@@ -245,7 +284,8 @@ class Patch:
:param issue_pr: issue or pull
:return: None
"""
- _host_match = re.search(pattern=Constant.GITLAB_HOST_REGEX, string=issue_pr)
+ _host_match = re.search(
+ pattern=Constant.GITLAB_HOST_REGEX, string=issue_pr)
_host = _host_match.group()
self.gitlab_api.host = _host
@@ -294,4 +334,4 @@ class Patch:
patch_detail.details.append(issue_relation_info)
- self.patch_detail_list.append(dict(patch_detail._asdict()))
+ self.patch_detail_list.append(dict(patch_detail._asdict()))
\ No newline at end of file
diff --git a/cve-agency-manager/cve_tracking/core/download/save.py b/cve-agency-manager/cve_tracking/core/download/save.py
index 329b4cec08fd874f34895101fe9ab342ec0b7e10..5ef1f813e334807bb24232b52ec37cb20e7ac43a 100644
--- a/cve-agency-manager/cve_tracking/core/download/save.py
+++ b/cve-agency-manager/cve_tracking/core/download/save.py
@@ -15,7 +15,7 @@ import re
from conf import CONFIG, settings
from logger import logger
from request import http
-
+from core.verification.apply import PathApply
def _file(cve, file):
"""
@@ -28,7 +28,7 @@ def _file(cve, file):
os.makedirs(file, exist_ok=True)
file_name = (
"fix-cve-"
- + str(sum([os.path.isfile(list_index) for list_index in os.listdir(file)]) + 1)
+ + str(sum([os.path.isfile(file+"/"+list_index) for list_index in os.listdir(file)]) + 1)
+ ".patch"
)
return os.path.join(file, file_name)
@@ -50,7 +50,7 @@ def _patch_url(url):
return url + target_regex[-1]
-def save_patch(patch_details, cve, path=CONFIG.PATCH_SAVE_PATH):
+def save_patch(patch_details, cve, path=CONFIG.PATCH_SAVE_PATH,args=None):
"""
Download and save the patch file
:param patch_details: patch info
@@ -63,15 +63,32 @@ def save_patch(patch_details, cve, path=CONFIG.PATCH_SAVE_PATH):
for issue_info in patch.get("details", list()):
for pr in issue_info["issue"].get("prs", list()):
commits.update(pr.get("commits", []))
-
- # download patch
- for url in commits:
- out_file = _file(cve=cve, file=path)
- patch_url = _patch_url(url=url) or url
- file = http.download(url=patch_url, out_fname=out_file)
- if not file:
- logger.error("Failed to download the file: %s" % patch_url)
-
+ if not args.packing:
+ # download patch
+ for url in commits:
+ out_file = _file(cve=cve, file=path)
+ patch_url = _patch_url(url=url) or url
+ file = http.download(url=patch_url, out_fname=out_file)
+ if not file:
+ logger.error("Failed to download the file: %s" % patch_url)
+ return None
+ else:
+ patch_url_state={}
+ for url in commits:
+ patch_url_state[url] = ""
+ for url in commits:
+ out_file = _file(cve=cve, file=path)
+ patch_url = _patch_url(url=url) or url
+ file = http.download(url=patch_url, out_fname=out_file)
+ if not file:
+ logger.error("Failed to download the file: %s" % patch_url)
+ patch_url_state[url] = "不适配"
+ patch_apply = PathApply(rpm_name=args.rpm_name, patch_path=args.patch_path,
+ source_path=args.source_path, branch_rpm=args.branch,cve_num=args.cve_num)
+ stat = patch_apply.packing_source()
+ patch_apply.clear_trash()
+ patch_url_state[url] = stat
+ return patch_url_state
__all__ = ("save_patch",)
diff --git a/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py b/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py
index ec2250d065d5082eb85ba97e0f8e9299f450a669..5932369acd7f60bdf56c2c9f5437057f0f04e520 100644
--- a/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py
+++ b/cve-agency-manager/cve_tracking/core/feedback/issue_mode.py
@@ -17,6 +17,8 @@ from util.github_api import Github
from util.gitlab_api import Gitlab
+
+
class IssueMode:
"""
Feedback by creating an issue
diff --git a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py
index a057edf0c56d225d74f38b5097b62c02b0064571..fe516576f68961e2433055855d759dcc9ccb7816 100644
--- a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py
+++ b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py
@@ -17,8 +17,6 @@ from collections import namedtuple
from functools import wraps
from bs4 import BeautifulSoup
-
-from constant import Constant
from exception import RequestError
from logger import logger
from request import http
@@ -68,7 +66,7 @@ class CvePlatform:
self.cve_num = cve_num
self.base_url = base_url
self._format = format_text
- self._Patch = namedtuple("Patch", ["platform", "commits", "pr", "issue"])
+ self._patch = namedtuple("Patch", ["platform", "commits", "pr", "issue"])
@property
def crawler_url(self):
@@ -84,16 +82,16 @@ class CvePlatform:
Structure of cve patch information
:return: instance of _Patch
"""
- return self._Patch(platform=self.crawler_url, commits=[], pr=[], issue=[])
+ return self._patch(platform=self.crawler_url, commits=[], pr=[], issue=[])
- async def _rule_redirect(self, response):
+ async def _rule_redirect(self, _response_text):
"""
Page multi layer jump data parsing
:param response: http response data
:return: response data
"""
for redirect_rule in self._platform.get("redirect", []):
- format_text = self.format_text(response.text)
+ format_text = self.format_text(_response_text)
target_val = list(
set(
re.findall(
@@ -108,7 +106,6 @@ class CvePlatform:
response = await self._method(redirect_rule)(
url, data=redirect_rule.get("body")
)
-
return response
@staticmethod
@@ -125,26 +122,29 @@ class CvePlatform:
_response = await self._method(self._platform)(
self.crawler_url, data=self._platform.get("body")
)
+ _response_text = _response.text
if "redirect" in self._platform:
- _response = await self._rule_redirect(response=_response)
+ redirect_response = await self._rule_redirect(_response_text=_response_text)
+ if not redirect_response.text:
+ redirect_response.text = ""
+ _response_text += redirect_response.text
except RequestError:
return None
-
- if _response.error or not _response.text:
+ if _response.error or not _response_text:
logger.error(
f"Failed to access URL {self.crawler_url}, detail: {_response.error}"
)
return None
-
+
formatted_text = (
- self.format_text(_response.text)
+ self.format_text(_response_text)
if self._format == "text"
- else self.json(_response.text)
+ else self.json(_response_text)
)
if formatted_text is None:
- logger.error(f"Failed to format string {_response.text}")
+ logger.error(f"Failed to format string {_response_text}")
return None
-
+
patch_info_dict = self.match_patch(
formatted_text, dict(self.patch_info._asdict())
)
@@ -162,12 +162,14 @@ class CvePlatform:
:return: None
"""
# Matching url de-duplication
+
for key, value in patch_info.items():
if isinstance(value, list):
patch_info[key] = list(set(value))
-
logger.info(f"Find patch: {patch_info}")
+
return patch_info
+ # def specialrpms_match_patch():
@staticmethod
def format_text(text):
@@ -190,9 +192,9 @@ class CvePlatform:
try:
text_dict = json.loads(text)
return json.dumps(text_dict, indent=4)
- except JSONDecodeError as e:
+ except JSONDecodeError as errors:
logger.error(
f"The format of the content obtained by bugzilla website is incorrect, "
- f"content is {text}, message is {e.msg}"
+ f"content is {text}, message is {errors.msg}"
)
return None
diff --git a/cve-agency-manager/cve_tracking/core/verification/apply.py b/cve-agency-manager/cve_tracking/core/verification/apply.py
index 6cf5b533c147d9a2893c33f686542492f8a09376..58e5e4d31da88f98b1cf86fa60aa676e5fb7ed9e 100644
--- a/cve-agency-manager/cve_tracking/core/verification/apply.py
+++ b/cve-agency-manager/cve_tracking/core/verification/apply.py
@@ -11,7 +11,6 @@
# See the Mulan PSL v2 for more details.
# ******************************************************************************/
import os.path
-import stat
import subprocess
from exception import ConfigNotFoundError, PathNotExistError, PathEmptyError
@@ -24,13 +23,12 @@ class PathApply:
"""
Patch verification class
"""
-
- def __init__(self, rpm_name, patch_path, source_path, branch_rpm):
+ def __init__(self, rpm_name, patch_path, source_path, branch_rpm,cve_num):
self.rpm_name = rpm_name
self.patch_path = patch_path
self.source_path = source_path
self.branch_rpm = branch_rpm
-
+ self.cve_num = cve_num
def packing_source(self):
"""
Package verification implementation class
@@ -43,15 +41,23 @@ class PathApply:
run_shell = os.path.join(CURRENT_PATH, 'run.sh')
try:
output = subprocess.check_output(
- [run_shell, self.rpm_name, self.branch_rpm, self.patch_path, self.source_path],
+ [run_shell, self.rpm_name, self.branch_rpm, self.patch_path, self.source_path,self.cve_num],
stderr=subprocess.STDOUT,
shell=False)
apply_result = output.decode('utf-8')
logger.info(f'Patch verification process:\n{apply_result}')
- logger.info(f'Apply patch successfully, log is {self.source_path}/{self.rpm_name}/result.log')
- except subprocess.CalledProcessError as e:
- logger.info(f'Patch verification process:\n{e.output.decode("utf-8")}')
- logger.error(f'Apply patch failed, log is {self.source_path}/{self.rpm_name}/result.log')
+ return "完全适配"
+ except subprocess.CalledProcessError as grepexc:
+ apply_result = grepexc.output.decode('utf-8')
+ if "200" in apply_result:
+ logger.info(f'Patch verification process:\n{apply_result}')
+ return "完全适配"
+ elif "202" in apply_result:
+ logger.info(f'Patch verification process:\n{apply_result}')
+ return "不完全适配"
+ else:
+ logger.info(f'Patch verification process:\n{apply_result}')
+ return "不适配"
def _validate_path(self):
"""
@@ -84,3 +90,14 @@ class PathApply:
for file in os.listdir(CURRENT_PATH):
if str(file).endswith('.sh'):
os.chmod(os.path.join(CURRENT_PATH, file), 0o0755)
+
+ def clear_trash(self):
+ """
+ clear patch_path,rpmbuild/SPEC and rpmbuild/SOURCE
+ """
+ run_shell = os.path.join(CURRENT_PATH, 'trash.sh')
+ try:
+ subprocess.check_output([run_shell,self.patch_path],stderr=subprocess.STDOUT,shell=False)
+ logger.info(f'Clear successfully !')
+ except subprocess.CalledProcessError as errors:
+ logger.error('Clear failed %s!!'%errors)
\ No newline at end of file
diff --git a/cve-agency-manager/cve_tracking/core/verification/common.sh b/cve-agency-manager/cve_tracking/core/verification/common.sh
old mode 100644
new mode 100755
diff --git a/cve-agency-manager/cve_tracking/core/verification/packing.sh b/cve-agency-manager/cve_tracking/core/verification/packing.sh
old mode 100644
new mode 100755
index 8ab7db464c3137f4c8dafd12c1bc3ddbdf2ed956..75312a3e80d890e424f9aa9cce2280ec08cc2007
--- a/cve-agency-manager/cve_tracking/core/verification/packing.sh
+++ b/cve-agency-manager/cve_tracking/core/verification/packing.sh
@@ -78,6 +78,7 @@ function update_spec() {
echo "[INFO] Start to update spec file"
cd ${source_path}
spec_file=$(ls | grep *.spec | grep -v *.bak | grep -v *.update)
+ spec_f = $(pwd)
if [[ -z ${spec_file} ]]; then
echo "[ERROR] spec file is not found"
exit 1
diff --git a/cve-agency-manager/cve_tracking/core/verification/run.sh b/cve-agency-manager/cve_tracking/core/verification/run.sh
old mode 100644
new mode 100755
index 9b9f073edda7b76eed86b0f1e610949001b6ed0f..c7e9e594b2abf179e846dba6b4f809c48ee317ea
--- a/cve-agency-manager/cve_tracking/core/verification/run.sh
+++ b/cve-agency-manager/cve_tracking/core/verification/run.sh
@@ -1,63 +1,236 @@
#!/bin/bash
# shellcheck disable=SC2068
# shellcheck disable=SC2164
-# shellcheck disable=SC2002
-# shellcheck disable=SC2045
-# shellcheck disable=SC2115
+# shellcheck disable=SC2010
+# shellcheck disable=SC2063
+# shellcheck disable=SC2062
+# shellcheck disable=SC2035
rpm_name=$1
-remote_branch=$2
-patch_path=$3
+rpm_branch=$2
+patches_path=$3
source_path=$4
+cve_num=$5
+path_file_name=""
+install_retry_count=0
+# The rpm package that needs to be installed when compiling
+requires_rpms=""
+spec_file=""
+home_path=$(
+ cd ~
+ pwd -P
+)
+root_build_path=${home_path}/"rpmbuild"
current_path=$(
cd "$(dirname "$0")"
pwd
)
. ${current_path}/common.sh
-function check() {
- # Check input legitimacy
- if [[ -z ${rpm_name} ]] || [[ -z ${remote_branch} ]]; then
- echo "[ERROR] rpm name or gitee branch not specified"
+function check_patch() {
+ if [[ -z ${patches_path} ]]; then
+ echo "400[ERROR] Input patch is null"
+ exit 1
+ fi
+ for patch in ${patches_path[@]}/$cve_num; do
+ if [[ ! -d ${patch} ]]; then
+ echo "[ERROR] ${patch} is not found"
+ exit 1
+ fi
+ done
+ if [ "`ls -A ${patches_path}/$cve_num`" = "" ];then
+ echo "400补丁未下载"
+ exit 1
+ fi
+ for patch in `ls ${patches_path[@]}/$cve_num`; do
+ if [[ ${patch} == fix*.tmp ]]; then
+ echo "400补丁下载错误"
+ rm -f $patches_path/$cve_num/$patch
+ exit 1
+ fi
+ done
+
+}
+
+function check_local() {
+ if [[ ! -d ${source_path} ]]; then
+ echo "400[ERROR] The source package path ${source_path} does not exist"
exit 1
fi
- # Check if the path exists
- if [[ ! -d ${patch_path} ]] || [[ ! -d ${source_path} ]]; then
- echo "[ERROR] The patch file directory: ${patch_path} or the source code download directory: ${source_path} does not exist is not exist"
+}
+
+function check_remote() {
+ if [[ -z ${rpm_name} ]] || [[ -z ${rpm_branch} ]]; then
+ echo "400[ERROR] Incorrect input,Please use [ /bin/bash add_patch.sh 'name of package' 'branch of package' 'path of patch file']"
exit 1
fi
}
-function run() {
- patch_files=$(ls ${patch_path})
- patch_ids=""
- new_patches=""
- for patch_file in ${patch_files[@]}; do
- patch_file=${patch_path}/${patch_file}
- git_patch_ids=$(cat "${patch_file}" | git patch-id --stable)
- if [[ ! ${patch_ids} =~ ${git_patch_ids} ]]; then
- new_patches="${new_patches} ${patch_file}"
- patch_ids="${patch_ids} ${git_patch_ids}"
+function pre_env_build() {
+ echo "[INFO] Create root path of rpmbuild"
+ if [[ ! -d ${root_build_path} ]]; then
+ mkdir -p ${root_build_path}/{BUILD,BUILDROOT,RPMS,SPECS,SOURCES,SRPMS}
+ fi
+}
+
+function git_clone() {
+ echo "[INFO] Start to git clone ${rpm_name}"
+ cd ${source_path}
+ git clone -b ${rpm_branch} https://gitee.com/src-openeuler/${rpm_name}.git >/dev/null 2>&1
+ rpm_path=$(ls | grep ${rpm_name})
+ if [[ -z ${rpm_path} ]]; then
+ echo "400[ERROR] Access https://gitee.com/src-openeuler/${rpm_name}.git clone ${rpm_name} failed, please check path ${source_path}"
+ exit 1
+ fi
+
+ source_path=${source_path}/${rpm_name}
+}
+
+function update_spec() {
+ echo "[INFO] Start to update spec file"
+ cd ${root_build_path}/SOURCES
+ spec_file=$(ls | grep *.spec | grep -v *.bak | grep -v *.update)
+ if [[ -z ${spec_file} ]]; then
+ echo "400[ERROR] spec file is not found"
+ exit 1
+ fi
+ # backup spec
+ /bin/cp -rf ${spec_file} ${spec_file}.bak
+ # update Release
+ release_version=$(grep "Release:" ${spec_file} | awk -F " " '{print $NF}' | tr -cd "[0-9]")
+ new_release=$(expr ${release_version} + 1)
+ sed -i "s/Release:.*${release_version}/Release: ${new_release}/" ${spec_file}
+ # add Patch***
+ last_patch=$(grep "Patch.*:" ${spec_file} | sed -n '$p')
+ if [[ -z ${last_patch} ]]; then
+ source_row=$(grep -n "Source.*:" ${spec_file} | sed -n '$p' | awk -F ':' '{print $1}')
+ sed -i -e "${source_row}G;${source_row}a Patch0000: ${path_file_name}" ${spec_file}
+ else
+ last_patch_row=$(grep -n "${last_patch}" ${spec_file} | awk -F ':' '{print $1}')
+ last_patch_num=$(echo ${last_patch} | awk -F ':' '{print $1}' | awk -F 'Patch' '{print $2}')
+ patch_name_len=${#last_patch_num}
+ new_patch_num=$(expr ${last_patch_num} + 1)
+ new_patch_num=$(printf "%0${patch_name_len}d" ${new_patch_num})
+ sed -i "${last_patch_row}a Patch${new_patch_num}: ${path_file_name}" ${spec_file}
+ fi
+ # add %patch
+ last_patch_apply=$(grep "%patch.* " ${spec_file} | sed -n '$p')
+ if [[ -n ${last_patch_apply} ]]; then
+ last_patch_apply_row=$(grep -n "${last_patch_apply}" ${spec_file} | awk -F ':' '{print $1}')
+ last_patch_apply_num=$(echo ${last_patch_apply} | awk -F ' ' '{print $1}' | awk -F 'patch' '{print $2}')
+ ignore_level_num=$(echo ${last_patch_apply} | awk -F ' ' '{print $2}')
+ new_patch_apply_num=$(expr ${last_patch_apply_num} + 1)
+ sed -i "${last_patch_apply_row}a %patch${new_patch_apply_num} ${ignore_level_num} " ${spec_file}
+ fi
+ # add changelog
+ change_log_row=$(grep -n '%changelog' ${spec_file} | sed -n '$p' | awk -F ':' '{print $1}')
+ date_now=$(env LANG=en_US.UTF-8 date "+%a %b %d %Y")
+ version=$(grep 'Version:' ${spec_file} | awk -F ' ' '{print $NF}')
+ log_description="- add ${path_file_name}"
+ log_title="* ${date_now} robot - ${version}-${new_release}"
+ sed -i "${change_log_row}G" ${spec_file}
+ sed -i "${change_log_row}a ${log_description}" ${spec_file}
+ sed -i "${change_log_row}a ${log_title}" ${spec_file}
+ /bin/cp -rf ${root_build_path}/SOURCES/*.spec ${root_build_path}/SPECS
+ echo "[INFO] Update spec file success"
+}
+
+function mv_source_file() {
+ echo "[INFO] Copy source file to ${root_build_path}"
+ cd ${source_path}
+ spec_file=$(ls | grep *.spec | grep -v *.bak | grep -v *.update)
+ /bin/cp -rf ${patches_path}/${cve_num}/* ${root_build_path}/SOURCES
+ /bin/cp -rf * ${root_build_path}/SOURCES
+}
+
+function rpm_build() {
+ echo "[INFO] Start to rpmbuild"
+ install_rpm rpm-build rpm
+ apt-get -y build-dep ${rpm_name} >/dev/null 2>&1
+ dnf builddep -y ${root_build_path}/SPECS/${spec_file} >/dev/null 2>&1
+ if [[ $? -eq 0 ]]; then
+ echo "[INFO] build dependencies success !!!"
+ fi
+ rpmbuild -bp ${root_build_path}/SPECS/${spec_file} >./result.log 2>&1
+ if [[ $? -eq 0 ]]; then
+ log_path="${root_build_path}/SOURCES/result.log"
+ fix_load=$(awk '(/fix-cve/){print $0}' $log_path)
+ if [[ -n ${fix_load} ]];then
+ echo "[INFO] build success !!!"
+ exit 0
else
- echo "[INFO] ${patch_file} duplicate content"
+ echo "400补丁未能正确加载,请检查spec文件"
+ exit 1
fi
- done
- # Perform code download, modify spec file, package verification operation
- /bin/bash ${current_path}/packing.sh "${rpm_name}" "${remote_branch}" "${new_patches}" "${source_path}"
- add_patch_result=$?
- if [[ ${add_patch_result} -eq 0 ]]; then
- echo "[INFO] rpm: \"${rpm_name}\" of branch: \"${remote_branch}\" apply patch \"${new_patches}\" successfully"
- exit 0
+ elif [[ -n $(grep "Failed build dependencies" ./result.log) ]] && [[ ${install_retry_count} -le 2 ]]; then
+ requires_rpms=$(grep -r "is needed by" ./result.log | awk -F " " '{print $1}')
+ echo "${requires_rpms}" >./requires_rpms.log
+ for rpm in ${requires_rpms[@]}; do
+ install_rpm ${rpm} ${rpm}
+ done
+ install_retry_count=$(expr ${install_retry_count} + 1)
+ rpm_build
else
- echo "[ERROR] rpm: \"${rpm_name}\" of branch: \"${remote_branch}\" failed to apply patch \"${new_patches}\""
- exit 1
+ # mv -f ${source_path}/${spec_file} ${source_path}/${spec_file}.update
+ echo "[ERROR] build failed,log is ${root_build_path}/SOURCES/result.log"
+ # exit 1
fi
}
-function main() {
- # Executive total entrance
- check
- install_rpm git git
- run
+function get_error_msq(){
+ echo "[INFO] Start to get_error_msq"
+ log_path="${root_build_path}/SOURCES/result.log"
+ error_msg=$(awk '(/^[0-9]{1,}/){print $0}' $log_path)
+ insert_num=$(awk '(/^[0-9]{1,}/){print $1}' $log_path)
+ local_num=$(awk '(/^[0-9]{1,}/){print $4}' $log_path)
+ fix_load=$(awk '(/fix-cve/){print $0}' $log_path)
+ download_right=$(awk '(/Only garbage was found in the patch input/){print $0}' $log_path)
+ if [[ -n ${fix_load} ]];then
+ :
+ if [[ -z ${download_right} ]];then
+ :
+ if [[ -n ${error_msg} ]]; then
+ :
+ if [[ "$insert_num" == "$local_num" ]];then
+ echo "400版本不受影响或补丁错误"
+ else
+ if [[ ${#insert_num} == 1 ]];then
+ echo "202补丁部分可用"
+ else
+ echo "400版本不受影响或补丁错误"
+ fi
+ fi
+ fi
+ else
+ echo "400补丁未能正确加载，请检查补丁内容是否正确下载"
+ fi
+ else
+ echo "400补丁未能正确加载,请检查spec文件中"
+ fi
+ exit 1
}
-main
+function main() {
+ check_patch
+ if [[ ${rpm_branch} == "no" ]]; then
+ check_local
+ pre_env_build
+ else
+ check_remote
+ pre_env_build
+ git_clone
+ fi
+ mv_source_file
+ if [[ -n `ls $patches_path/$cve_num` ]]; then
+ for patch in `ls ${patches_path[@]}/$cve_num`; do
+ if [[ ${patch} == fix-cve-*.patch ]]; then
+ path_file_name=$(echo "${patch}" | awk -F "/" '{print $NF}' | awk -F "\\" '{print $NF}')
+ update_spec
+ fi
+ done
+ rpm_build
+ get_error_msq
+ else
+ echo "400补丁未能下载，请尝试手动下载"
+ fi
+}
+main
\ No newline at end of file
diff --git a/cve-agency-manager/cve_tracking/core/verification/trash.sh b/cve-agency-manager/cve_tracking/core/verification/trash.sh
new file mode 100755
index 0000000000000000000000000000000000000000..306ed0c2aeb6345bfa87e9aa111ffe965359cfec
--- /dev/null
+++ b/cve-agency-manager/cve_tracking/core/verification/trash.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# shellcheck disable=SC2068
+# shellcheck disable=SC2164
+# shellcheck disable=SC2010
+# shellcheck disable=SC2063
+# shellcheck disable=SC2062
+# shellcheck disable=SC2035
+patch_path=$1
+home_path=$(
+ cd ~
+ pwd -P
+)
+root_build_path=${home_path}/"rpmbuild"
+rm $root_build_path/SOURCES/* -rf
+rm $root_build_path/SPECS/* -rf
+rm $patch_path/* -r
\ No newline at end of file
diff --git a/cve-agency-manager/cve_tracking/cve-tracking.yaml b/cve-agency-manager/cve_tracking/cve-tracking.yaml
index 6e2873990201c4c190e3de1ec29ab7ba0f0b5660..45798377f67c4a586a5a0a40233af3fa961e6319 100644
--- a/cve-agency-manager/cve_tracking/cve-tracking.yaml
+++ b/cve-agency-manager/cve_tracking/cve-tracking.yaml
@@ -1,4 +1,7 @@
# Regular expression to match cve patch info
+# rpm列表
+# platform:
+# - ghostscript
# 匹配cve补丁信息的正则表达式
regex:
# A tag that matches the type of content, currently only "commit" "pr" "issue" is supported
@@ -7,15 +10,26 @@ regex:
# Regular expressions that match specific types of content, you can set multiple matching rules
# 匹配特定类型内容的正则表达式，可以设置多个匹配规则
regular:
- - http[s]?://(?:[-\w.\/;?])+(?:/rev|/ci|/commit[s]?)/(?:\?id=)?[0-9a-z]{8,40}
- - http[s]?://(?:[-\w.\/;=&?])+a=commit(?:[\w;&?])+h=[0-9a-z]{8,40}
+ - http[s]?://(?:[-\w.\/;?\~?])+(?:/rev|/ci|/commit[s]?|/linus|/qtbase/\+|/c|/revision)/(?:\?id=)?[0-9a-z-]{1,40}
+ - /\?p=ghostpdl.git;a=commitdiff;h=[a-z0-9]{40};?(?:hp=[0-9a-z;=]{40})?
+ - http[s]?://[\w.\/\+]+patch_to_fix_[0-9]{0,8}
- label: pr
regular:
- http[s]?://(?:[-\w.\/;?])+(?:pull[s]?|merge_requests)/[1-9][0-9]*
- label: issue
regular:
- - http[s]?://(?:[-\w.\/;?])+issues/[0-9A-Z]+
+ - http[s]?://(?:[-\w.\/;?])+issue[s]?/[0-9A-Z]+
+specialrpm:
+ - name: strongswan
+ url: https://download.strongswan.org/security
+ patch_url: https://download.strongswan.org/security/{}/
+ request: false
+ - name: log4j12
+ url: https://sources.debian.org/src/apache-log4j1.2
+ patch_url: https://sources.debian.org/src/apache-log4j1.2/1.2.17-11/debian/patches/{}.patch/
+ request: false
+
# Platforms to find bug fixes for cve, such as Debian, Bugzilla, Nvd, etc.
# 查找cve漏洞修复的平台，例如 Debian、Bugzilla、Nvd等
platform:
@@ -24,7 +38,7 @@ platform:
- name: Cnnvd
# The request address for finding cve details, "{cve num}" is a placeholder for string replacement in python (required)
# 查找cve详情信息的请求地址，“{cve_num}”是python中字符串替换的占位符(必配项)
- url: http://cnnvd.org.cn/web/vulnerability/queryLds.tag?qcvCnnvdid={cve_num}
+ url: http://www.cnnvd.org.cn/web/vulnerability/queryLds.tag?qcvCnnvdid={cve_num}
# The method of sending the request, the default is get request, and it can also be specified as post request.
# When the request method is get, it can not be configured here.
# 发送请求的方式，默认为get请求，还可以指定为post请求，当请求方式为get时，此处可不配置
@@ -43,7 +57,7 @@ platform:
# The prefix of the jump address, generally the domain name of the jump page, if the jump address matched by
# the regular expression has a complete domain name + path, this configuration item can be left blank
# 跳转地址的前缀，一般情况下为跳转页面的域名，如果正则表达式匹配到的跳转地址中有完整的域名+路径，此配置项可以不填写
- - prefix: http://cnnvd.org.cn
+ - prefix: http://www.cnnvd.org.cn
# The regular expression that matches the page jump address, if it matches multiple values, get the last match
# 匹配页面跳转地址的正则表达式，如果匹配多个值，则获取最后一个匹配项
regex: /web/xxk/ldxqById\.tag\?CNNVD=CNNVD[0-9-]+
@@ -54,11 +68,30 @@ platform:
# If the request method is post and there is a request body, the body is required and must be in json format
# 若请求方式为post且存在请求体时，body为必填项，且必须为json格式
body:
-
+ - name: opencve
+ url: https://www.opencve.io/cve/{cve_num}
+ format: text
+
- name: Debian
url: https://security-tracker.debian.org/tracker/{cve_num}
+ method: get
+ # body:
format: text
-
+ redirect:
+ - prefix: https://bugs.launchpad.net
+ regex: /mailman/\+bug/[0-9]{0,8}
+ method: get
+ body:
+ - prefix: https://community.otrs.com
+ regex: /security-advisory-[0-9]{4}-[0-9]{2}
+ method: get
+ body:
+ - prefix: https://git.ghostscript.com
+ regex: /\?p=ghostpdl.git;a=(?:commit|commitdiff);h=[a-z0-9]{40};?(?:hp=[0-9a-z;=]{40})?
+ method: get
+ body:
+
+
- name: Ubuntn
url: https://ubuntu.com/security/{cve_num}
format: text
@@ -75,6 +108,10 @@ platform:
url: https://bugzilla.suse.com/show_bug.cgi?id={cve_num}
format: text
+ - name: Redhat
+ url: https://bugzilla.redhat.com/show_bug.cgi?id={cve_num}
+ format: text
+
# Private token for api access, currently only supports "gitee" "github" "gitlab"
# api访问时的私人令牌，当前只支持"gitee"/"github"/"gitlab"
authentication:
@@ -83,9 +120,9 @@ authentication:
- name: gitee
# A private token for accessing the api, which can be manually obtained and changed later
# 访问api的私人令牌，可以手动获取后更改
- token:
+ token: 2be60f57f6eaef9efbac94f342685b59
- name: github
- token:
+ token: 9c2290c567bd16064219cbbc0f61c8dc
- name: gitlab
token: glpat-24DGxkuTA8nTR1YQsXvZ
diff --git "a/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" "b/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png"
index a1da01a4592a933bece141ed77d370be3b86eaa4..fa3ce2cdea002ca03ce3ad418cf98bacbb756511 100644
Binary files "a/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" and "b/cve-agency-manager/cve_tracking/doc/images/\345\274\200\345\217\221\350\247\206\345\233\276.png" differ
diff --git "a/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" "b/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png"
index 57922ae14b4161e6be7d2cb2b8b5a833f4f73b4d..a3227baaa958cab69241265db28a2e3f821ec441 100644
Binary files "a/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" and "b/cve-agency-manager/cve_tracking/doc/images/\346\211\251\345\261\225\350\256\276\350\256\241.png" differ
diff --git a/cve-agency-manager/cve_tracking/main.py b/cve-agency-manager/cve_tracking/main.py
index 88cf3239db7b6e967da680881a13c0a6218835cc..8eb4003d77d9011a532309f7cf120cd69807e987 100644
--- a/cve-agency-manager/cve_tracking/main.py
+++ b/cve-agency-manager/cve_tracking/main.py
@@ -1,33 +1,32 @@
-#!/usr/bin/python3
-# ******************************************************************************
-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved.
-# licensed under the Mulan PSL v2.
-# You can use this software according to the terms and conditions of the Mulan PSL v2.
-# You may obtain a copy of Mulan PSL v2 at:
-# http://license.coscl.org.cn/MulanPSL2
-# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
-# PURPOSE.
-# See the Mulan PSL v2 for more details.
-# ******************************************************************************/
-"""
-Tool execution total entrance
-"""
-import asyncio
-
-from cli.base import CveTrackingCommand
-from conf import CONFIG
-from exception import CveException
-from logger import logger
-
-try:
- # Register all subcommands
- for sub_class in CveTrackingCommand.__subclasses__():
- CveTrackingCommand.register_command(sub_class())
-
- # Parallel execution of corresponding functions
- loop = asyncio.get_event_loop()
- loop.run_until_complete(CveTrackingCommand.run())
-except CveException as e:
- logger.error(
- f'Execution failed, message is : {e.message}, detail in {CONFIG.LOGGER_PATH}/{CONFIG.LOGGER_FILE_NAME}')
+#!/usr/bin/python3
+# ******************************************************************************
+# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved.
+# licensed under the Mulan PSL v2.
+# You can use this software according to the terms and conditions of the Mulan PSL v2.
+# You may obtain a copy of Mulan PSL v2 at:
+# http://license.coscl.org.cn/MulanPSL2
+# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR
+# PURPOSE.
+# See the Mulan PSL v2 for more details.
+# ******************************************************************************/
+"""
+Tool execution total entrance
+"""
+import asyncio
+from cli.base import CveTrackingCommand
+from conf import CONFIG
+from exception import CveException
+from logger import logger
+
+try:
+ # Register all subcommands
+ for sub_class in CveTrackingCommand.__subclasses__():
+ CveTrackingCommand.register_command(sub_class())
+
+ # Parallel execution of corresponding functions
+ loop = asyncio.get_event_loop()
+ loop.run_until_complete(CveTrackingCommand.run())
+except CveException as errors:
+ logger.error(
+ f'Execution failed, message is : {errors.message}, detail in {CONFIG.LOGGER_PATH}/{CONFIG.LOGGER_FILE_NAME}')
\ No newline at end of file
diff --git a/cve-agency-manager/cve_tracking/request.py b/cve-agency-manager/cve_tracking/request.py
index b2859f473cc90ed60456d3495f95ef8da3b6bbe1..4e16b5578f77546ba131638a3e9baa6823abf3e9 100644
--- a/cve-agency-manager/cve_tracking/request.py
+++ b/cve-agency-manager/cve_tracking/request.py
@@ -11,20 +11,21 @@
# See the Mulan PSL v2 for more details.
# ******************************************************************************/
import json
-from asyncio import TimeoutError
from http.client import RemoteDisconnected
from urllib.error import HTTPError, URLError
-
+from asyncio import TimeoutError
import aiohttp
import wget
from fake_useragent import UserAgent
from retrying import retry
-
from constant import Constant
from exception import RequestError
class Response:
+ """
+ Process the Response
+ """
def __init__(self, error=None, response: aiohttp.ClientResponse = None, text=None) -> None:
self.error = error
self.response = response
@@ -32,14 +33,23 @@ class Response:
@property
def success(self):
+ """
+ return successful response and status
+ """
return self.response and (self.response.status == 200 or self.response.status == 201)
@property
def status_code(self):
+ """
+ return reponse and status
+ """
return self.response and self.response.status
@property
def json(self):
+ """
+ return json
+ """
if not self.response:
return None
try:
@@ -70,6 +80,9 @@ class Http:
@property
def headers(self):
+ """
+ constitute headers
+ """
headers = {
"User-Agent": self.user_agent.random
}
@@ -84,6 +97,9 @@ class Http:
"Content-Type", "application/json")
async def _get(self, url, params=None, **kwargs):
+ """
+ get Methods using
+ """
self._set_headers(request_header=kwargs)
@retry(stop_max_attempt_number=self._max_attempt_number, stop_max_delay=self._stop_max_delay)
@@ -94,6 +110,9 @@ class Http:
@classmethod
async def get(cls, url, **kwargs) -> "Response":
+ """
+ call _get
+ """
async with cls() as self:
await self._get(url, **kwargs)
return self._response
@@ -104,12 +123,14 @@ class Http:
self._response.text = await self._response.response.text()
except aiohttp.ClientError as error:
self._response.error = error
- except TimeoutError as e:
- raise RequestError(f'Call url timeout: {e}')
+ except TimeoutError as error:
+ raise RequestError(f'Call url timeout: {error}')
async def _post(self, url, data=None, **kwargs):
+ """
+ post Methods using
+ """
# self._set_headers(request_header=kwargs)
-
@retry(stop_max_attempt_number=self._max_attempt_number, stop_max_delay=self._stop_max_delay)
async def _http_request():
return await self.session.post(url, data=data, timeout=Constant.CALL_MAX_DELAY, **kwargs)
@@ -118,6 +139,9 @@ class Http:
@classmethod
async def post(cls, url, data=None, **kwargs) -> "Response":
+ """
+ call _post
+ """
async with cls() as self:
await self._post(url, data, **kwargs)
return self._response
@@ -127,19 +151,25 @@ class Http:
def getfile():
file_name = wget.download(url, out_fname)
if not file_name:
- raise
+ raise
return file_name
return getfile()
@staticmethod
def download(url, out_fname, **kwargs):
+ """
+ patch url downlad
+ """
max_attempt_number = kwargs.get("max_attempt_number", Constant.MAX_RETRY)
stop_max_delay = kwargs.get("stop_max_delay", Constant.MAX_DOWNLOAD_DELAY)
@retry(stop_max_attempt_number=max_attempt_number, stop_max_delay=stop_max_delay)
def _dw():
- file_name = wget.download(url, out_fname)
+ try:
+ file_name = wget.download(url, out_fname)
+ except Exception as error:
+ return None
if not file_name:
raise RequestError(f'Call url:"{url}" failed')
return file_name
diff --git a/cve-agency-manager/cve_tracking/util/__init__.py b/cve-agency-manager/cve_tracking/util/__init__.py
index 5a7ebcdc275716a52600da9e60838866db1f34a9..064ef9bbef4a868c6d1f2ccca6216fa092ff8d75 100644
--- a/cve-agency-manager/cve_tracking/util/__init__.py
+++ b/cve-agency-manager/cve_tracking/util/__init__.py
@@ -33,8 +33,8 @@ class Api:
"""
try:
response = await http.post(url=url, data=values, **kwargs)
- except RequestError as e:
- logger.error(f'Call url {url} failed, message: {e.message}')
+ except RequestError as errors:
+ logger.error(f'Call url {url} failed, message: {errors.message}')
return None
if not response.success:
@@ -55,8 +55,8 @@ class Api:
"""
try:
response = await http.get(url=url, params=params, **kwargs)
- except RequestError as e:
- logger.error(f'Call url {url} failed, message: {e.message}')
+ except RequestError as errors:
+ logger.error(f'Call url {url} failed, message: {errors.message}')
return None
if not response.success: