From 9ad22dc0eb77675da4170d70489fb1006e6584d8 Mon Sep 17 00:00:00 2001 From: liheavy Date: Tue, 28 Dec 2021 11:05:32 +0800 Subject: [PATCH 1/2] =?UTF-8?q?readme=E6=9B=B4=E6=96=B0=EF=BC=8C=E7=BC=96?= =?UTF-8?q?=E8=AF=91=E4=BE=9D=E8=B5=96=E6=8F=90=E5=89=8D=E5=AE=89=E8=A3=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cve-agency-manager/cve_tracking/README.md | 84 +++++++++++++++++-- .../cve_tracking/core/verification/packing.sh | 4 +- 2 files changed, 79 insertions(+), 9 deletions(-) diff --git a/cve-agency-manager/cve_tracking/README.md b/cve-agency-manager/cve_tracking/README.md index c9903c9..b8b953f 100644 --- a/cve-agency-manager/cve_tracking/README.md +++ b/cve-agency-manager/cve_tracking/README.md @@ -1,23 +1,91 @@ # cve_tracking #### 介绍 -cve补丁自动获取工具 +cve补丁自动获取工具，该工具将会根据cve和rpm包自动在上游社区查找补丁并反馈结果，同时也可以下载查找到的补丁以及验证补丁的可用性。 #### 软件架构 -软件架构说明 +python可执行代码 #### 安装教程 -1. xxxx -2. xxxx -3. xxxx +1. 下载代码 + + ```shell + git clone https://gitee.com/openeuler/cve-manager.git + ``` + +2. 进入工具执行目录 + + ```shell + cd xxx(上述代码下载目录)/cve-manager/cve-agency-manager/cve_tracking + ``` + +3. 在config.ini中设置GITEE_TOKEN（gitee的私人令牌）和GITLAB_TOKEN（gitlab的私人令牌，默认设置了一个临时令牌，可临时使用），GITHUB_TOKEN（github的私人令牌）可不设置。 + +4. 安装依赖包 + + ``` + pip3 install -r requirements.txt + ``` + +3. 根据使用说明执行工具 #### 使用说明 -1. xxxx -2. xxxx -3. xxxx +1. 补丁查找及评论issue + + ```shell + python3 main.py comment -c cve_num -r rpm_name -i issue_num + ``` + + > 参数说明： + > + > -c cve的编号 + > + > -r rpm包名称 + > + > -i 需要评论的issue编号 + > + > 注意：默认仓库为src-openeuler，如果要更改，请修改main.py同目录下constant.py中的DEFAULT_OWNER的值。 + +2. 补丁查找及下载(验证) + + ``` + python3 main.py -c cve_num -r rpm_name [-f patch_save_path] [-s source_path] [-p] [-b branch] + ``` + + > 参数说明： + > + > -c cve的编号 + > + > -r rpm包名称 + > + > -f 补丁文件的下载目录，不设置默认为/opt/cve_tracking/patches + > + > -s 源码包下载路径，不设置默认为/opt/cve_tracking/source_code + > + > -b 源码包所在的gitee的src-openeuler仓库的分支，默认为master + > + > -p 是否进行补丁应用，默认为不应用，若需要应用，添加该参数。 + +3. 补丁验证 + + ``` + python3 main.py packing -r rpm_name -f patch_save_path -s source_path -b branch [-nd] + ``` + + > 参数说明： + > + > -r rpm包名称 + > + > -f 补丁文件路径 + > + > -s 源码包路径，如果无需下载指定为本地源码包的路径；如果需要下载指定为需要下载源码包的路径即可 + > + > -b 源码包所在gitee中src-openeuler仓库的分支，不设置默认为master + > + > -nd 是否需要下载源代码，默认为需要下载，若无需下载添加该参数 #### 参与贡献 diff --git a/cve-agency-manager/cve_tracking/core/verification/packing.sh b/cve-agency-manager/cve_tracking/core/verification/packing.sh index 9b8f8ee..8ab7db4 100644 --- a/cve-agency-manager/cve_tracking/core/verification/packing.sh +++ b/cve-agency-manager/cve_tracking/core/verification/packing.sh @@ -103,7 +103,7 @@ function update_spec() { fi # add %patch last_patch_apply=$(grep "%patch.* " ${spec_file} | sed -n '$p') - if [[ -n ${last_patch_apply} ]];then + if [[ -n ${last_patch_apply} ]]; then last_patch_apply_row=$(grep -n "${last_patch_apply}" ${spec_file} | awk -F ':' '{print $1}') last_patch_apply_num=$(echo ${last_patch_apply} | awk -F ' ' '{print $1}' | awk -F 'patch' '{print $2}') ignore_level_num=$(echo ${last_patch_apply} | awk -F ' ' '{print $2}') @@ -135,6 +135,8 @@ function mv_source_file() { function rpm_build() { echo "[INFO] Start to rpmbuild" install_rpm rpm-build rpm + apt-get build-dep ${rpm_name} >/dev/null 2>&1 + dnf builddep ${root_build_path}/SPECS/${spec_file} >/dev/null 2>&1 rpmbuild -bp ${root_build_path}/SPECS/${spec_file} >./result.log 2>&1 if [[ $? -eq 0 ]]; then echo "[INFO] build success !!!" -- Gitee From 16333de7ab6787a6a4454a2573672adc309a5820 Mon Sep 17 00:00:00 2001 From: liheavy Date: Mon, 10 Jan 2022 21:03:23 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E9=94=99=E8=AF=AF?= =?UTF-8?q?=E6=B3=A8=E9=87=8A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../cve_tracking/conf/setting.py | 18 +++++++++++++ cve-agency-manager/cve_tracking/config.ini | 11 ++++---- .../core/platform/cve_platform.py | 18 ++++++------- .../cve_tracking/cve-tracking.yaml | 27 ++++++++++++++++--- ...76\350\256\241\346\226\207\346\241\243.md" | 3 +-- .../cve_tracking/requirements.txt | 1 - 6 files changed, 57 insertions(+), 21 deletions(-) diff --git a/cve-agency-manager/cve_tracking/conf/setting.py b/cve-agency-manager/cve_tracking/conf/setting.py index dcb5660..d9a5152 100644 --- a/cve-agency-manager/cve_tracking/conf/setting.py +++ b/cve-agency-manager/cve_tracking/conf/setting.py @@ -54,6 +54,7 @@ class DefaultConfig: # Feedback to the repo of the issue FEEDBACK_ISSUE_REPO = "cve_tracking" + # The owner of the comment function submits the comment DEFAULT_OWNER = "src-openeuler" @@ -109,6 +110,9 @@ class Config: class YamlConfiguration: + """ + yaml file parsing class + """ yaml = os.path.join( os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "cve-tracking.yaml" ) @@ -137,6 +141,11 @@ class YamlConfiguration: return self.__dict__[name] def get_platform(self, name=None): + """ + Get the configured cve platform + :param name: specify platform name + :return: platform name + """ if name is None: return self.platform platform = filter(lambda x: x["name"] == name, self.platform) @@ -144,6 +153,11 @@ class YamlConfiguration: return list(platform)[-1] def get_regex(self, label=None): + """ + Regular expression to get patch + :param label: issue or pr or commit + :return: regex + """ if label is None: regulars = [reg for reg in self.regex] else: @@ -157,4 +171,8 @@ class YamlConfiguration: @property def configuration(self): + """ + All configuration items + :return: dict + """ return self.__dict__ diff --git a/cve-agency-manager/cve_tracking/config.ini b/cve-agency-manager/cve_tracking/config.ini index f4b3f75..3983d70 100644 --- a/cve-agency-manager/cve_tracking/config.ini +++ b/cve-agency-manager/cve_tracking/config.ini @@ -30,6 +30,11 @@ PATCH_SAVE_PATH = /opt/cve_tracking/patches ;Source code package and patch file storage path, used to download gitee source code package and package verification. SOURCE_CODE_PATH = /opt/cve_tracking/source_code +;Comment function related configuration +[COMMENT] +;The owner of the comment function submits the comment +DEFAULT_OWNER = src-openeuler + ;Feedback issue related configuration [FEEDBACK] ;Feedback issue platform,Currently supports "gitee" "github" "gitlab",default is "gitee" @@ -38,8 +43,4 @@ FEEDBACK_PLATFORM = github ;Feedback to the owner of the issue FEEDBACK_ISSUE_OWNER = liheavy ;Feedback to the repo of the issue -FEEDBACK_ISSUE_REPO = cve_tracking - -[WAREHOUSE] - -DEFAULT_OWNER = src-openeuler \ No newline at end of file +FEEDBACK_ISSUE_REPO = cve_tracking \ No newline at end of file diff --git a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py index 37d23c1..a057edf 100644 --- a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py +++ b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py @@ -86,27 +86,27 @@ class CvePlatform: """ return self._Patch(platform=self.crawler_url, commits=[], pr=[], issue=[]) - async def _rule_redirct(self, response): + async def _rule_redirect(self, response): """ Page multi layer jump data parsing :param response: http response data :return: response data """ - for redirct_rule in self._platform.get("redirct", []): + for redirect_rule in self._platform.get("redirect", []): format_text = self.format_text(response.text) target_val = list( set( re.findall( - pattern=redirct_rule.get("regex", ""), string=format_text or "" + pattern=redirect_rule.get("regex", ""), string=format_text or "" ) ) ) try: - url = redirct_rule["prefix"] + target_val[-1] + url = redirect_rule["prefix"] + target_val[-1] except IndexError: - url = redirct_rule["prefix"] - response = await self._method(redirct_rule)( - url, data=redirct_rule.get("body") + url = redirect_rule["prefix"] + response = await self._method(redirect_rule)( + url, data=redirect_rule.get("body") ) return response @@ -125,8 +125,8 @@ class CvePlatform: _response = await self._method(self._platform)( self.crawler_url, data=self._platform.get("body") ) - if "redirct" in self._platform: - _response = await self._rule_redirct(response=_response) + if "redirect" in self._platform: + _response = await self._rule_redirect(response=_response) except RequestError: return None diff --git a/cve-agency-manager/cve_tracking/cve-tracking.yaml b/cve-agency-manager/cve_tracking/cve-tracking.yaml index a456929..9783d9a 100644 --- a/cve-agency-manager/cve_tracking/cve-tracking.yaml +++ b/cve-agency-manager/cve_tracking/cve-tracking.yaml @@ -1,7 +1,10 @@ +# Regular expression to match cve patch info # 匹配cve补丁信息的正则表达式 regex: + # A tag that matches the type of content, currently only "commit" "pr" "issue" is supported # 匹配内容的类型的标签，当前只支持 "commit"/"pr"/"issue" - label: commit + # Regular expressions that match specific types of content, you can set multiple matching rules # 匹配特定类型内容的正则表达式，可以设置多个匹配规则 regular: - http[s]?://(?:[-\w.\/;?])+(?:/rev|/ci|/commit[s]?)/(?:\?id=)?[0-9a-z]{8,40} @@ -13,28 +16,42 @@ regex: regular: - http[s]?://(?:[-\w.\/;?])+issues/[0-9A-Z]+ +# Platforms to find bug fixes for cve, such as Debian, Bugzilla, Nvd, etc. # 查找cve漏洞修复的平台，例如 Debian、Bugzilla、Nvd等 platform: - # The name of the platform, similar to a label + # Platform name (required) # 平台的名称(必配项) - name: Cnnvd - # Request address to find CVE information,{cve_num} is a placeholder for string substitution in Python + # The request address for finding cve details, "{cve num}" is a placeholder for string replacement in python (required) # 查找cve详情信息的请求地址，“{cve_num}”是python中字符串替换的占位符(必配项) url: http://cnnvd.org.cn/web/vulnerability/queryLds.tag?qcvCnnvdid={cve_num} + # The method of sending the request, the default is get request, and it can also be specified as post request. + # When the request method is get, it can not be configured here. # 发送请求的方式，默认为get请求，还可以指定为post请求，当请求方式为get时，此处可不配置 method: get + # If the request method is post and there is a request body, the body is required and must be in json format. + # When the request method is get, it can be omitted here. # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式，当请求方式为get时，此处可不配置 body: + # The way to parse the response body, currently only "text" or "json" is supported (required) # 解析响应体的方式，当前只支持“text”或“json”(必配项) format: text + # When the request cannot directly obtain the cve information (multiple jumps according to the specific link in + # the page), you can specify the method of page jump to obtain further information # 当请求无法直接获取cve信息时（根据页面中特定链接多次跳转），可以指定页面跳转的方式来获取进一步的信息 - redirct: + redirect: + # The prefix of the jump address, generally the domain name of the jump page, if the jump address matched by + # the regular expression has a complete domain name + path, this configuration item can be left blank # 跳转地址的前缀，一般情况下为跳转页面的域名，如果正则表达式匹配到的跳转地址中有完整的域名+路径，此配置项可以不填写 - prefix: http://cnnvd.org.cn + # The regular expression that matches the page jump address, if it matches multiple values, get the last match # 匹配页面跳转地址的正则表达式，如果匹配多个值，则获取最后一个匹配项 regex: /web/xxk/ldxqById\.tag\?CNNVD=CNNVD[0-9-]+ + # The method of sending the request, the default is get request, and it can also be specified as post request. + # When the request is get, it can not be configured here. # 送请求的方法，默认为get请求，还可以指定为post请求，当请求为get时，此处可不配置 method: get + # If the request method is post and there is a request body, the body is required and must be in json format # 若请求方式为post且存在请求体时，body为必填项，且必须为json格式 body: @@ -58,11 +75,13 @@ platform: url: https://bugzilla.suse.com/show_bug.cgi?id={cve_num} format: text -# Private token for API access +# Private token for api access, currently only supports "gitee" "github" "gitlab" # api访问时的私人令牌，当前只支持"gitee"/"github"/"gitlab" authentication: + # api name, this value cannot be changed # api名称，此值不可更改 - name: gitee + # A private token for accessing the api, which can be manually obtained and changed later # 访问api的私人令牌，可以手动获取后更改 token: - name: github diff --git "a/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" index 914f2f5..46a91a7 100644 --- "a/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -19,7 +19,6 @@ | 组件 | 组件描述 | 可获得性 | | ------------------------------ | ------------------------------------------- | --------------- | | python3 | Python程序运行环境，需要python3.6以上版本。 | openEuler已集成 | -| python3-requests | python发送http/https请求程序库 | openEuler已集成 | | python3-retrying | requests请求重试依赖程序库 | openEuler已集成 | | python3-concurrent-log-handler | python日志程序库，包括日志记录，日志转储 | openEuler已集成 | | python3-pyyaml | python处理yaml文件程序库 | openEuler已集成 | @@ -169,7 +168,7 @@ platform: # 解析响应体的方式，当前只支持“text”或“json”(必配项) format: text # 当请求无法直接获取cve信息时（根据页面中特定链接多次跳转），可以指定页面跳转的方式来获取进一步的信息 - redirct: + redirect: # 跳转地址的前缀，一般情况下为跳转页面的域名，如果正则表达式匹配到的跳转地址中有完整的域名+路径，此配置项可以不填写 - prefix: http://cnnvd.org.cn # 匹配页面跳转地址的正则表达式，如果匹配多个值，则获取最后一个匹配项 diff --git a/cve-agency-manager/cve_tracking/requirements.txt b/cve-agency-manager/cve_tracking/requirements.txt index c1697f7..99850e4 100644 --- a/cve-agency-manager/cve_tracking/requirements.txt +++ b/cve-agency-manager/cve_tracking/requirements.txt @@ -1,5 +1,4 @@ asyncio -requests bs4 aiohttp wget -- Gitee