diff --git a/cve-agency-manager/cve_tracking/conf/setting.py b/cve-agency-manager/cve_tracking/conf/setting.py index dcb5660551dfacda8a65567357945a5525906484..d9a5152c628cbccd4d8edc68081d1be7d070c5dc 100644 --- a/cve-agency-manager/cve_tracking/conf/setting.py +++ b/cve-agency-manager/cve_tracking/conf/setting.py @@ -54,6 +54,7 @@ class DefaultConfig: # Feedback to the repo of the issue FEEDBACK_ISSUE_REPO = "cve_tracking" + # The owner of the comment function submits the comment DEFAULT_OWNER = "src-openeuler" @@ -109,6 +110,9 @@ class Config: class YamlConfiguration: + """ + yaml file parsing class + """ yaml = os.path.join( os.path.dirname(os.path.abspath(os.path.dirname(__file__))), "cve-tracking.yaml" ) @@ -137,6 +141,11 @@ class YamlConfiguration: return self.__dict__[name] def get_platform(self, name=None): + """ + Get the configured cve platform + :param name: specify platform name + :return: platform name + """ if name is None: return self.platform platform = filter(lambda x: x["name"] == name, self.platform) @@ -144,6 +153,11 @@ class YamlConfiguration: return list(platform)[-1] def get_regex(self, label=None): + """ + Regular expression to get patch + :param label: issue or pr or commit + :return: regex + """ if label is None: regulars = [reg for reg in self.regex] else: @@ -157,4 +171,8 @@ class YamlConfiguration: @property def configuration(self): + """ + All configuration items + :return: dict + """ return self.__dict__ diff --git a/cve-agency-manager/cve_tracking/config.ini b/cve-agency-manager/cve_tracking/config.ini index f4b3f755aa24715e620fce02b75ba1533f1d7b76..3983d70ab4f2a0e086f615a7047a9cee5bca9e7e 100644 --- a/cve-agency-manager/cve_tracking/config.ini +++ b/cve-agency-manager/cve_tracking/config.ini @@ -30,6 +30,11 @@ PATCH_SAVE_PATH = /opt/cve_tracking/patches ;Source code package and patch file storage path, used to download gitee source code package and package verification. SOURCE_CODE_PATH = /opt/cve_tracking/source_code +;Comment function related configuration +[COMMENT] +;The owner of the comment function submits the comment +DEFAULT_OWNER = src-openeuler + ;Feedback issue related configuration [FEEDBACK] ;Feedback issue platform,Currently supports "gitee" "github" "gitlab",default is "gitee" @@ -38,8 +43,4 @@ FEEDBACK_PLATFORM = github ;Feedback to the owner of the issue FEEDBACK_ISSUE_OWNER = liheavy ;Feedback to the repo of the issue -FEEDBACK_ISSUE_REPO = cve_tracking - -[WAREHOUSE] - -DEFAULT_OWNER = src-openeuler \ No newline at end of file +FEEDBACK_ISSUE_REPO = cve_tracking \ No newline at end of file diff --git a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py index 37d23c10fce2e4e59b5ec10f074a36c02e8f37fb..a057edf0c56d225d74f38b5097b62c02b0064571 100644 --- a/cve-agency-manager/cve_tracking/core/platform/cve_platform.py +++ b/cve-agency-manager/cve_tracking/core/platform/cve_platform.py @@ -86,27 +86,27 @@ class CvePlatform: """ return self._Patch(platform=self.crawler_url, commits=[], pr=[], issue=[]) - async def _rule_redirct(self, response): + async def _rule_redirect(self, response): """ Page multi layer jump data parsing :param response: http response data :return: response data """ - for redirct_rule in self._platform.get("redirct", []): + for redirect_rule in self._platform.get("redirect", []): format_text = self.format_text(response.text) target_val = list( set( re.findall( - pattern=redirct_rule.get("regex", ""), string=format_text or "" + pattern=redirect_rule.get("regex", ""), string=format_text or "" ) ) ) try: - url = redirct_rule["prefix"] + target_val[-1] + url = redirect_rule["prefix"] + target_val[-1] except IndexError: - url = redirct_rule["prefix"] - response = await self._method(redirct_rule)( - url, data=redirct_rule.get("body") + url = redirect_rule["prefix"] + response = await self._method(redirect_rule)( + url, data=redirect_rule.get("body") ) return response @@ -125,8 +125,8 @@ class CvePlatform: _response = await self._method(self._platform)( self.crawler_url, data=self._platform.get("body") ) - if "redirct" in self._platform: - _response = await self._rule_redirct(response=_response) + if "redirect" in self._platform: + _response = await self._rule_redirect(response=_response) except RequestError: return None diff --git a/cve-agency-manager/cve_tracking/cve-tracking.yaml b/cve-agency-manager/cve_tracking/cve-tracking.yaml index a456929ea95a18bcc6d3c8d211a1adb1337fea83..9783d9a550f375401ddb52a208ef83bbd574c0a1 100644 --- a/cve-agency-manager/cve_tracking/cve-tracking.yaml +++ b/cve-agency-manager/cve_tracking/cve-tracking.yaml @@ -1,7 +1,10 @@ +# Regular expression to match cve patch info # 匹配cve补丁信息的正则表达式 regex: + # A tag that matches the type of content, currently only "commit" "pr" "issue" is supported # 匹配内容的类型的标签,当前只支持 "commit"/"pr"/"issue" - label: commit + # Regular expressions that match specific types of content, you can set multiple matching rules # 匹配特定类型内容的正则表达式,可以设置多个匹配规则 regular: - http[s]?://(?:[-\w.\/;?])+(?:/rev|/ci|/commit[s]?)/(?:\?id=)?[0-9a-z]{8,40} @@ -13,28 +16,42 @@ regex: regular: - http[s]?://(?:[-\w.\/;?])+issues/[0-9A-Z]+ +# Platforms to find bug fixes for cve, such as Debian, Bugzilla, Nvd, etc. # 查找cve漏洞修复的平台,例如 Debian、Bugzilla、Nvd等 platform: - # The name of the platform, similar to a label + # Platform name (required) # 平台的名称(必配项) - name: Cnnvd - # Request address to find CVE information,{cve_num} is a placeholder for string substitution in Python + # The request address for finding cve details, "{cve num}" is a placeholder for string replacement in python (required) # 查找cve详情信息的请求地址,“{cve_num}”是python中字符串替换的占位符(必配项) url: http://cnnvd.org.cn/web/vulnerability/queryLds.tag?qcvCnnvdid={cve_num} + # The method of sending the request, the default is get request, and it can also be specified as post request. + # When the request method is get, it can not be configured here. # 发送请求的方式,默认为get请求,还可以指定为post请求,当请求方式为get时,此处可不配置 method: get + # If the request method is post and there is a request body, the body is required and must be in json format. + # When the request method is get, it can be omitted here. # 若请求方式为post且存在请求体时,body为必填项,且必须为json格式,当请求方式为get时,此处可不配置 body: + # The way to parse the response body, currently only "text" or "json" is supported (required) # 解析响应体的方式,当前只支持“text”或“json”(必配项) format: text + # When the request cannot directly obtain the cve information (multiple jumps according to the specific link in + # the page), you can specify the method of page jump to obtain further information # 当请求无法直接获取cve信息时(根据页面中特定链接多次跳转),可以指定页面跳转的方式来获取进一步的信息 - redirct: + redirect: + # The prefix of the jump address, generally the domain name of the jump page, if the jump address matched by + # the regular expression has a complete domain name + path, this configuration item can be left blank # 跳转地址的前缀,一般情况下为跳转页面的域名,如果正则表达式匹配到的跳转地址中有完整的域名+路径,此配置项可以不填写 - prefix: http://cnnvd.org.cn + # The regular expression that matches the page jump address, if it matches multiple values, get the last match # 匹配页面跳转地址的正则表达式,如果匹配多个值,则获取最后一个匹配项 regex: /web/xxk/ldxqById\.tag\?CNNVD=CNNVD[0-9-]+ + # The method of sending the request, the default is get request, and it can also be specified as post request. + # When the request is get, it can not be configured here. # 送请求的方法,默认为get请求,还可以指定为post请求,当请求为get时,此处可不配置 method: get + # If the request method is post and there is a request body, the body is required and must be in json format # 若请求方式为post且存在请求体时,body为必填项,且必须为json格式 body: @@ -58,11 +75,13 @@ platform: url: https://bugzilla.suse.com/show_bug.cgi?id={cve_num} format: text -# Private token for API access +# Private token for api access, currently only supports "gitee" "github" "gitlab" # api访问时的私人令牌,当前只支持"gitee"/"github"/"gitlab" authentication: + # api name, this value cannot be changed # api名称,此值不可更改 - name: gitee + # A private token for accessing the api, which can be manually obtained and changed later # 访问api的私人令牌,可以手动获取后更改 token: - name: github diff --git "a/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" "b/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" index 914f2f5c7f0b26791e348c799635ef08cc6a405d..46a91a718af1a9cfa713127f5bbdf31d4a51ca5f 100644 --- "a/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" +++ "b/cve-agency-manager/cve_tracking/doc/CVE\350\241\245\344\270\201\350\207\252\345\212\250\350\216\267\345\217\226\345\267\245\345\205\267\350\256\276\350\256\241\346\226\207\346\241\243.md" @@ -19,7 +19,6 @@ | 组件 | 组件描述 | 可获得性 | | ------------------------------ | ------------------------------------------- | --------------- | | python3 | Python程序运行环境,需要python3.6以上版本。 | openEuler已集成 | -| python3-requests | python发送http/https请求程序库 | openEuler已集成 | | python3-retrying | requests请求重试依赖程序库 | openEuler已集成 | | python3-concurrent-log-handler | python日志程序库,包括日志记录,日志转储 | openEuler已集成 | | python3-pyyaml | python处理yaml文件程序库 | openEuler已集成 | @@ -169,7 +168,7 @@ platform: # 解析响应体的方式,当前只支持“text”或“json”(必配项) format: text # 当请求无法直接获取cve信息时(根据页面中特定链接多次跳转),可以指定页面跳转的方式来获取进一步的信息 - redirct: + redirect: # 跳转地址的前缀,一般情况下为跳转页面的域名,如果正则表达式匹配到的跳转地址中有完整的域名+路径,此配置项可以不填写 - prefix: http://cnnvd.org.cn # 匹配页面跳转地址的正则表达式,如果匹配多个值,则获取最后一个匹配项 diff --git a/cve-agency-manager/cve_tracking/requirements.txt b/cve-agency-manager/cve_tracking/requirements.txt index c1697f7e8de4b843ef1e29900b5f455dd6e8918b..99850e426a2b7813c13c6c4c2aca28681b20da15 100644 --- a/cve-agency-manager/cve_tracking/requirements.txt +++ b/cve-agency-manager/cve_tracking/requirements.txt @@ -1,5 +1,4 @@ asyncio -requests bs4 aiohttp wget