From 19f879b19c2debacc04a0d276c97b472cd5c1d22 Mon Sep 17 00:00:00 2001 From: lhwerico Date: Thu, 1 Jul 2021 16:40:04 +0800 Subject: [PATCH 1/2] add cve-tracking tool --- tools/cve-tracking/.gitignore | 8 + tools/cve-tracking/README.md | 111 ++ tools/cve-tracking/requirements.txt | 7 + tools/cve-tracking/setup.py | 27 + tools/cve-tracking/src/__init__.py | 12 + tools/cve-tracking/src/cve.py | 17 + tools/cve-tracking/src/cve/__init__.py | 16 + tools/cve-tracking/src/cve/apply.py | 146 +++ tools/cve-tracking/src/cve/bugzilla.py | 66 ++ tools/cve-tracking/src/cve/cli.py | 103 ++ tools/cve-tracking/src/cve/engine.py | 381 ++++++ tools/cve-tracking/src/cve/gitee.py | 469 ++++++++ tools/cve-tracking/src/cve/httprequest.py | 212 ++++ tools/cve-tracking/src/cve/logger.py | 113 ++ tools/cve-tracking/src/cve/pipe.py | 371 ++++++ tools/cve-tracking/src/cve/plantform.py | 596 ++++++++++ tools/cve-tracking/src/cve/settings.py | 38 + tools/cve-tracking/src/cve/user-agent.json | 1251 ++++++++++++++++++++ tools/cve-tracking/src/shell/__init__.py | 12 + tools/cve-tracking/src/shell/add_patch.sh | 212 ++++ tools/cve-tracking/src/shell/start.sh | 104 ++ tools/cve-tracking/test/__init__.py | 12 + 22 files changed, 4284 insertions(+) create mode 100644 tools/cve-tracking/.gitignore create mode 100644 tools/cve-tracking/README.md create mode 100644 tools/cve-tracking/requirements.txt create mode 100644 tools/cve-tracking/setup.py create mode 100644 tools/cve-tracking/src/__init__.py create mode 100644 tools/cve-tracking/src/cve.py create mode 100644 tools/cve-tracking/src/cve/__init__.py create mode 100644 tools/cve-tracking/src/cve/apply.py create mode 100644 tools/cve-tracking/src/cve/bugzilla.py create mode 100644 tools/cve-tracking/src/cve/cli.py create mode 100644 tools/cve-tracking/src/cve/engine.py create mode 100644 tools/cve-tracking/src/cve/gitee.py create mode 100644 tools/cve-tracking/src/cve/httprequest.py create mode 100644 tools/cve-tracking/src/cve/logger.py create mode 100644 tools/cve-tracking/src/cve/pipe.py create mode 100644 tools/cve-tracking/src/cve/plantform.py create mode 100644 tools/cve-tracking/src/cve/settings.py create mode 100644 tools/cve-tracking/src/cve/user-agent.json create mode 100644 tools/cve-tracking/src/shell/__init__.py create mode 100644 tools/cve-tracking/src/shell/add_patch.sh create mode 100644 tools/cve-tracking/src/shell/start.sh create mode 100644 tools/cve-tracking/test/__init__.py diff --git a/tools/cve-tracking/.gitignore b/tools/cve-tracking/.gitignore new file mode 100644 index 0000000..5570ffd --- /dev/null +++ b/tools/cve-tracking/.gitignore @@ -0,0 +1,8 @@ +.DS_Store +*/.DS_Store +*.pyc +*.vscode +.idea/cve_patch.iml +.idea/inspectionProfiles/Project_Default.xml +.idea/inspectionProfiles/profiles_settings.xml +.idea/modules.xml diff --git a/tools/cve-tracking/README.md b/tools/cve-tracking/README.md new file mode 100644 index 0000000..c3a5f26 --- /dev/null +++ b/tools/cve-tracking/README.md @@ -0,0 +1,111 @@ +# cve-tracking初版使用说明 + +## 一、代码下载： + +```shell +git clone https://gitee.com/gongzt/cve.git +``` + +## 二、权限修改： + +```shell +chmod 755 -R xxx/cve(下载代码所在路径) +``` + +> root@localhost:/opt/project/cve_tracking +> ▶ **git clone https://gitee.com/gongzt/cve.git** +> Cloning into 'cve'... +> remote: Enumerating objects: 840, done. +> remote: Counting objects: 100% (840/840), done. +> remote: Compressing objects: 100% (801/801), done. +> remote: Total 840 (delta 578), reused 64 (delta 34), pack-reused 0 +> Receiving objects: 100% (840/840), 176.65 KiB | 977.00 KiB/s, done. +> Resolving deltas: 100% (578/578), done. +> +> root@localhost:/opt/project/cve_tracking +> ▶ **chmod 755 -R cve** + +## 三、执行代码 + +首先需要安装依赖的python插件（仅第一次使用时需要）： + +```shell + pip3 install -r xxx/requirements.txt +``` + +### 1、只查找和下载补丁，不执行打补丁操作 + +特点：执行快，占用空间少 + +- 执行方法： + +```shell +python3 xxx/cve/src/cve/cve.py -cve CVE编号 -name 软件包名 -branch 软件包分支(支持多个) +``` + +- 执行后结果： + +下载文件和日志记录在：/tmp/cve_tracking/match_cve下，文件以 ”软件包名-cve号“ 命名 + +> root@localhost:/tmp/cve_tracking/match_cve +> ▶ ll +> total 0 +> drwx------. 3 root root 80 Jun 30 16:49 freerdp-CVE-2020-11523 + +进入路径后： repair-verdict.txt为查找日志，包含cve信息和补丁链接信息，下载成功的补丁文件在以”软件包版本号“命名的文件夹中。 + +> root@localhost:cve_tracking/match_cve/freerdp-CVE-2020-11523 +> ▶ ll +> total 64K +> drwx------. 2 root root 60 Jun 30 16:49 0.0.0 +> -rw-------. 1 root root 486 Jun 30 16:49 repair-verdict.txt +> +> root@localhost:cve_tracking/match_cve/freerdp-CVE-2020-11523 +> ▶ cd 0.0.0 +> +> root@localhost:match_cve/freerdp-CVE-2020-11523/0.0.0 +> ▶ ll +> total 64K +> -rw-------. 1 root root 1.7K Jun 30 16:49 CVE-2020-11523_1.patch + +### 2、查找和下载补丁并执行打补丁操作 + +特点：自动执行rpmbuild实验补丁是否可用，执行慢，占用空间大（主要为安装依赖包） + +- 执行方法： + +```shell +python3 xxx/cve/src/cve/cve.py -cve CVE编号 -name 软件包名 -branch 软件包分支（支持多个） -cmd +``` + +- 执行后结果： + +下载文件和查找日志和上面一致 + +源码编译结果也在 repair-verdict.txt 文件中 + +源码包下载路径和补丁拷贝路径为： + +/tmp/cve_tracking/download_source + +> root@localhost:/tmp/cve_tracking/download_source +> ▶ ll +> total 0 +> drwx------. 3 root root 260 Jun 30 17:07 freerdp + +具体的编译日志为进入对应软件包文件夹后rpmbuild.log： + +> root@localhost:cve_tracking/download_source/freerdp master ✗ 153d2h ⚑ ◒ +> ▶ ll +> total 7.2M +> -rw-------. 1 root root 1.7K Jun 30 17:07 CVE-2020-11523_1.patch +> -rw-------. 1 root root 1.1K Jun 30 17:07 Fix-freerdp-shadow-cli-exit-codes-for-help-and-version.patch +> -rw-------. 1 root root 4.9K Jun 30 17:07 Fix-xfreerdp-exit-codesfor-help-and-similar-option-1.patch +> -rw-------. 1 root root 4.3K Jun 30 17:07 Fix-xfreerdp-exit-codesfor-help-and-similar-option-2.patch +> -rw-------. 1 root root 692 Jun 30 17:07 Fix-xfreerdp-exit-codesfor-help-and-similar-option-3.patch +> -rw-------. 1 root root 6.6M Jun 30 17:07 FreeRDP-2.2.0.tar.gz +> -rw-------. 1 root root 5.6K Jun 30 17:07 freerdp.spec +> -rw-------. 1 root root 5.5K Jun 30 17:07 freerdp.spec.bak +> -rw-------. 1 root root 78 Jun 30 17:07 freerdp.yaml +> -rw-------. 1 root root 1.3K Jun 30 17:07 rpmbuild.log + diff --git a/tools/cve-tracking/requirements.txt b/tools/cve-tracking/requirements.txt new file mode 100644 index 0000000..641c91d --- /dev/null +++ b/tools/cve-tracking/requirements.txt @@ -0,0 +1,7 @@ +requests==2.24.0 +retrying==1.3.3 +fuzzywuzzy==0.18.0 +concurrent_log_handler==0.9.19 +beautifulsoup4==4.0.1 +fake-useragent==0.1.11 +pyyaml==5.4.1 \ No newline at end of file diff --git a/tools/cve-tracking/setup.py b/tools/cve-tracking/setup.py new file mode 100644 index 0000000..5ae0bfc --- /dev/null +++ b/tools/cve-tracking/setup.py @@ -0,0 +1,27 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +from setuptools import setup, find_packages + + +setup( + name='automatic-cve', + version='0.0.1', + packages=find_packages("src"), + package_dir={"": "src"}, + install_requires=['requests (==2.21.0)', + 'retrying(==1.3.3)'], + license='Automatic acquisition of CVE', + long_description=open('README.md', encoding='utf-8').read(), + author='gongzt', + zip_safe=False +) diff --git a/tools/cve-tracking/src/__init__.py b/tools/cve-tracking/src/__init__.py new file mode 100644 index 0000000..6851032 --- /dev/null +++ b/tools/cve-tracking/src/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ diff --git a/tools/cve-tracking/src/cve.py b/tools/cve-tracking/src/cve.py new file mode 100644 index 0000000..dd4654e --- /dev/null +++ b/tools/cve-tracking/src/cve.py @@ -0,0 +1,17 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import sys +from cve.cli import main + + +main(sys.argv[1:], exit_code=True) diff --git a/tools/cve-tracking/src/cve/__init__.py b/tools/cve-tracking/src/cve/__init__.py new file mode 100644 index 0000000..0bd4560 --- /dev/null +++ b/tools/cve-tracking/src/cve/__init__.py @@ -0,0 +1,16 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ + +from .cli import main + +__version__ = "0.0.1" diff --git a/tools/cve-tracking/src/cve/apply.py b/tools/cve-tracking/src/cve/apply.py new file mode 100644 index 0000000..fede608 --- /dev/null +++ b/tools/cve-tracking/src/cve/apply.py @@ -0,0 +1,146 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import os +import subprocess +from .gitee import Gitee +from .logger import logger + +CURRENT_PATCH = os.path.realpath(os.path.dirname(__file__)) +CLEAR = "clear" + + +def apply_patch(patches_path, branches): + """ + Call the script to test compile + :param patches_path: path of cve_patches + :param branches: branch of source code + :return: apply result + """ + shell_file = os.path.join(os.path.dirname(CURRENT_PATCH), "shell/start.sh") + try: + branch_str = "master" + if isinstance(branches, list) and branches: + branch_str = " ".join(str(branch) for branch in branches) + if isinstance(branches, str) and branches: + branch_str = branches + + output = subprocess.check_output( + [shell_file, patches_path, branch_str], + stderr=subprocess.STDOUT, + shell=False, + ) + result = output.decode("utf-8") + except subprocess.CalledProcessError: + result = "All branch apply failed" + + # _clear_env(shell_file) + return result + + +def _clear_env(shell_file): + """ + Clear tmp path and file + :param shell_file: shell + :return: None + """ + try: + subprocess.run([shell_file, CLEAR], stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + shell=False) + except subprocess.CalledProcessError: + return + + +def row(func): + """ + Combine the rows of the table + """ + + def wrapper(*args, **kwargs): + cols = func(*args, **kwargs) + return "" + "".join([str(col) for col in cols]) + "" + + return wrapper + + +class Col: + """ + Generates a cell for a row in Table + """ + + def __init__(self, val, span=0, header=False) -> None: + self._val = val + self._span = span + self.header = header + + def __str__(self) -> str: + row_span = "" + if self._span: + row_span = f' rowspan="{self._span}" ' + if self.header: + return "" + self._val + "" + + return "" + self._val + "" + + +class AutoComment: + """ + Automatically submit the content of comments + """ + + def __init__(self, header=None) -> None: + self._gitee = Gitee() + self._line_feed = "\n" + self._header = header or ["Patchs URL", "Branch", "Apply Result"] + + @row + def _row(self, cols, spans=None, header=False): + tds = [Col(val=col, header=header) for col in cols] + if spans: + tds.insert(0, Col(val=spans[0], span=spans[1])) + + return tds + + def _create_table(self, header, body: dict): + rows = [self._row(cols=header, header=True)] + for index, key in enumerate(body["apply_result"]): + row_val = [key, body["apply_result"][key]] + if index == 0: + rows.append( + self._row( + cols=row_val, + spans=(self._line_feed.join(body["urls"]), len(body)), + ) + ) + continue + rows.append(self._row(cols=row_val)) + + return "" + self._line_feed.join(rows) + "
" + + def comment(self, number, body, repo, owner="src-openeuler", table=True): + """Comments on PR""" + comments = self._gitee.get_issue_comments(repo=repo, number=number, owner=owner) + if comments: + for comment in comments: + if all([header in comment.get("body", "") for header in self._header]): + logger.info( + "Issue %s in repository %s already has a fixed comment content ." + % (number, repo) + ) + return + if table: + body = self._create_table(header=self._header, body=body) + body += self._line_feed * 2 + "> 说明：补丁链接和应用结果仅供初步排查参考，实际应用请人工再次确认。" + self._gitee.create_issue_comment( + repo=repo, number=number, body=body, owner=owner + ) diff --git a/tools/cve-tracking/src/cve/bugzilla.py b/tools/cve-tracking/src/cve/bugzilla.py new file mode 100644 index 0000000..c6fc327 --- /dev/null +++ b/tools/cve-tracking/src/cve/bugzilla.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ + + +class Api: + """ + bugzilla api + """ + + def __init__(self, base_url): + self.host = base_url + + def _parse_query_param(self, params): + """ + Parse the request parameters of the query + """ + try: + query_str = [] + for key, value in params.items(): + query_str.append("{key}={value}".format(key=key, value=value)) + return "&".join(query_str) + except AttributeError: + return query_str + + def get_bug(self, fields, **kwargs): + """ + To get information about a particular bug using its ID or alias + + You can also use Search Bugs to return more than one bug at a time + by specifying bug IDs as the search terms + + """ + + if isinstance(fields, (str, int)): + rest_api = '/rest/bug/{}'.format(fields) + if isinstance(fields, dict): + rest_api = "/rest/bug?{query_param}".format( + query_param=self._parse_query_param(params=fields)) + + return self.host + rest_api + + def get_comments(self, fields, is_comment_id=False, **kwargs): + """ + To get all comments for a particular bug using the bug ID or alias + + To get a specific comment based on the comment ID + + """ + if is_comment_id: + rest_api = '/rest/bug/comment/{comment_id}'.format( + comment_id=fields) + else: + rest_api = '/rest/bug/{id_or_alias}/comment'.format( + id_or_alias=fields) + + return self.host + rest_api diff --git a/tools/cve-tracking/src/cve/cli.py b/tools/cve-tracking/src/cve/cli.py new file mode 100644 index 0000000..7e74f41 --- /dev/null +++ b/tools/cve-tracking/src/cve/cli.py @@ -0,0 +1,103 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import sys +from argparse import ArgumentParser +from .engine import crawler +from .settings import DEFAULT_SAVE_PATH + + +def register_parser(parser): + """ + Register command line arguments + """ + parser.add_argument( + "-cve", help="The version number of CVE", default=None, action="store" + ) + parser.add_argument( + "-name", help="Name of the affected package", default=None, action="store" + ) + + parser.add_argument( + "-v", + nargs="*", + default=["0.0.0"], + help="The version number of the affected package", + ) + + parser.add_argument( + "-o", + help="The path to output the CVE result", + default=DEFAULT_SAVE_PATH, + action="store", + ) + + parser.add_argument( + "-f", help="CVE file path for batch operation", default=None, action="store" + ) + + parser.add_argument( + "-cmd", + help="Whether a shell script to validate the patch needs to be executed", + default=False, + action="store_true", + ) + + parser.add_argument( + "-branch", + nargs="*", + default=["master"], + help="A list of branches currently affected by CVE", + ) + parser.add_argument( + "-issue", + help="Issue id", + default=None, + action="store", + ) + + parser.set_defaults(func=crawler.run) + + return parser + + +def _run(args, option_parser=ArgumentParser): + parser = register_parser( + parser=option_parser(description="CVE vulnerability repair management") + ) + if not args: + print(parser.parse_args(["-h"])) + return 0 + if ("-cve" not in args or "-name" not in args) and "-f" not in args: + print("Both the cve number and package name are required, please enter them") + return 0 + try: + parser_args = parser.parse_args(args=args) + parser_args.func(parser_args) + except Exception as e: + return 1 + else: + return 0 + + +def main(args, exit_code=False): + """ + CVE gets command-line entry + """ + errcode = _run(args) + if exit_code: + sys.exit(errcode) + return errcode + + +if __name__ == "__main__": + main(sys.argv[1:], exit_code=True) diff --git a/tools/cve-tracking/src/cve/engine.py b/tools/cve-tracking/src/cve/engine.py new file mode 100644 index 0000000..c9fdbee --- /dev/null +++ b/tools/cve-tracking/src/cve/engine.py @@ -0,0 +1,381 @@ +#!/usr/bin/python3 +import os +import csv +import re +import shutil +import signal +import time +from concurrent.futures import ThreadPoolExecutor +from queue import Queue +from collections import deque +from threading import Thread +from typing import Iterable +import requests + +from .apply import apply_patch +from .logger import logger +from .httprequest import RemoteService +from .plantform import Ubuntu, Nvd, Debian, Bugzilla, Crawl +from .pipe import RequestRepeat, RequestsUrl, SavePipe, FileHandle +from .settings import ( + INTERNAL_SERVER, + ENABLE_PLANTFORM, + MAX_WORKERS, + MAX_QUEUE, + RECORD_FILE, +) +from .apply import AutoComment + + +class Cardiac: + """ + The engine center + """ + + download_queue = Queue(maxsize=MAX_QUEUE) + save_queue = Queue(maxsize=MAX_QUEUE) + engine_queue = Queue(maxsize=MAX_QUEUE) + pool = ThreadPoolExecutor(max_workers=MAX_WORKERS) + download_thread = deque(maxlen=MAX_QUEUE) + plantform = [Bugzilla, Debian, Ubuntu, Nvd] + + def __init__(self) -> None: + self._active = True + self.cve_infos = list() + self._request_fingerprint = RequestRepeat() + + def _consume(self, cve, obj): + if not issubclass(obj, Crawl): + raise RuntimeError("") + + crawl = obj(**cve) + return RequestsUrl( + url=crawl.start_url, crawl=crawl, callback=crawl.parse, start=True + ) + + def _record_base_info(self, cve): + text = "CVE Information : {cve}".format(cve=str(cve)) + return SavePipe(text=text, crawl=Crawl(**cve)) + + def _req_flow(self): + """ + Sign up for various platforms + Returns: + + """ + for cve in self.cve_infos: + for plantform in self.plantform: + if plantform.__name__ not in ENABLE_PLANTFORM: + continue + yield self._consume(cve, obj=plantform) + + yield self._record_base_info(cve) + + def _internal_server(self, reuqest, pkg, commitid): + first_char = pkg.lower()[0] + success = False + response = reuqest.request( + url=INTERNAL_SERVER, + method="post", + json={ + "git_repo": f"upstream/{first_char}/{pkg}/{pkg}.git", + "git_command": ["git-show", commitid, "--pretty=email"], + }, + max_retry=1, + ) + if response.text and response.status_code == requests.codes["ok"]: + success = True + return success, response + + def _downloader(self, req_flow): + """ + downloader + Returns: + + """ + request = RemoteService() + + def _github(request, response=None, success=False): + if not success: + response = request.request(url=req_flow.url, method="get", timeout=15) + + if response.status_code != requests.codes["ok"]: + self._download_failed(request=req_flow) + return + + callback = req_flow.callback(response) + if isinstance(callback, Iterable): + self._iterback(callback) + + if hasattr(req_flow, "is_patch") and getattr(req_flow, "is_patch"): + success, response = self._internal_server( + request, + req_flow.crawl.pkg, + req_flow.url.split("/")[-1].replace(".patch", ""), + ) + _github(request, response=response, success=success) + else: + _github(request) + + def _download_failed(self, request): + if hasattr(request, "start") and getattr(request, "start"): + logger.warning("Start page %s failed to open" % request.url) + return + text = "File download failed:%s" % request.url + logger.error(text) + self.engine_queue.put(SavePipe(text=text, crawl=request.crawl)) + + def _iterback(self, callback): + while self._active: + try: + self.engine_queue.put(callback.send(None)) + except StopIteration: + break + + def download(self): + """ + downloader + """ + while self._active: + request_flow = self.download_queue.get() + if not request_flow: + break + if self._request_fingerprint.request_seen(request_flow): + continue + future = self.pool.submit(self._downloader, request_flow) + self.download_thread.append(future) + + def save_pipe(self): + """ + Save data pipe + """ + + while self._active: + save_handle = self.save_queue.get() + if not save_handle: + break + + callback = save_handle.save_process(save_handle.crawl) + if isinstance(callback, Iterable): + self._iterback(callback) + + def clear(self, queues): + """Clear the stack of messages""" + if not isinstance(queues, (list, tuple, set)): + return + for _queue in queues: + if not isinstance(_queue, Queue): + continue + _queue.queue.clear() + + def stop(self): + """ + Stop the service + """ + + self._active = False + self.download_queue.put_nowait(None) + self.engine_queue.put_nowait(None) + self.save_queue.put_nowait(None) + return not self._active + + def _read_csvfile(self, file, out_path, return_content=False): + """ + Read the CSV file + Args: + file: CSV file + + Returns: + + """ + if not os.path.exists(file) or not os.path.isfile(file): + logger.info("file not found:{}".format(file)) + return + csv_contents = [] + try: + csv_data = csv.reader(open(file, encoding="utf-8")) + for cveinfo in csv_data: + if len(cveinfo) != 3: + continue + cve, pkg, versions = cveinfo + versions = re.sub("\[|\]", "", versions).split(",") + self._set_cve(cve=cve, pkg=pkg, versions=versions, out_path=out_path) + csv_contents.append([pkg, cve]) + except csv.Error as error: + logger.error(error) + + if return_content: + return csv_contents + + def _set_cve(self, cve, pkg, versions, out_path): + if isinstance(versions, str): + versions = [versions] + self.cve_infos.append( + {"cve": cve, "pkg": pkg, "versions": versions, "out_path": out_path} + ) + + def start(self, args): + """ + Start the service + """ + # Get information such as CVE + if args.f: + self._read_csvfile(file=args.f, out_path=args.o) + if args.cve: + self._set_cve(cve=args.cve, pkg=args.name, versions=args.v, out_path=args.o) + + if not self.cve_infos: + logger.info("There is no CVE information queried this time") + self.stop() + + req_flow = self._req_flow() + while self._active: + try: + flow = next(req_flow) + self.engine_queue.put(flow) + except StopIteration: + break + + def scheduler(self): + """ + Task scheduler + """ + while self._active: + task = self.engine_queue.get() + if not task: + break + if isinstance(task, RequestsUrl): + self.download_queue.put(task) + + if isinstance(task, SavePipe): + self.save_queue.put(task) + + +class CrawlerProcess: + """ + The process of data crawling + """ + + def __init__(self) -> None: + self.engine = Cardiac() + self._thread_container = [self._downloader, self._scheduler, self._save_pipe] + self.end_signal = False + self._interval = 6 + + @property + def finish(self): + """Task queue consumption completed""" + return all( + [ + self.engine.download_queue.empty(), + self.engine.save_queue.empty(), + self.engine.engine_queue.empty(), + ] + ) + + def _heartbeat(self): + """ + Heartbeat mechanism + Returns: + + """ + while True: + time.sleep(self._interval) + while len(self.engine.download_thread) > 0: + thread = self.engine.download_thread.popleft() + if not thread.done() and not self.end_signal: + self.engine.download_thread.append(thread) + if self.end_signal or self.finish: + self.end_signal = True + self.engine.clear( + queues=[ + self.engine.download_queue, + self.engine.save_queue, + self.engine.engine_queue, + ] + ) + self.engine.stop() + break + + def _stop(self, signum, frame): + print("\nIt's in the process of closing .....") + self.end_signal = True + self._interval = 0 + self.engine.download_thread.clear() + + @property + def _downloader(self): + return Thread(target=self.engine.download) + + @property + def _scheduler(self): + return Thread(target=self.engine.scheduler) + + @property + def _save_pipe(self): + return Thread(target=self.engine.save_pipe) + + def _extract_info(self, args): + file_handle = FileHandle( + folder=args.o, + branch=args.branch, + cve=args.cve, + pkg=args.name, + csv_file=args.f, + read_csv=self.engine._read_csvfile, + ) + apply_result = None + if args.cmd: + apply_result = apply_patch(args.o, args.branch) + + file_handle.format_text(apply_result=apply_result) + if args.issue: + path = os.path.join(args.o, args.name + "-" + args.cve, RECORD_FILE) + self._comment( + file_handle, + path, + args.issue, + args.name, + cmd=args.cmd, + branch=args.branch, + cve=args.cve, + ) + + def _comment(self, file_handle, path, issue, repo, **kwargs): + auto_comment = AutoComment() + + body = file_handle.extract_text(path) + if not kwargs["cmd"]: + body["apply_result"] = {branch: "Not apply" for branch in kwargs["branch"]} + if kwargs["cmd"] and not body["apply_result"]: + body["apply_result"] = { + branch: "Apply failed" for branch in kwargs["branch"] + } + + if not body.get("urls"): + logger.warning( + "%s has not found a valid patch file yet ." % kwargs.get("cve", "") + ) + return + auto_comment.comment(number=issue, body=body, repo=repo) + + def run(self, args): + """ + Entrance to program execution + """ + # Clears all files in the specified path + if os.path.exists(args.o): + shutil.rmtree(args.o) + signal.signal(signal.SIGINT, self._stop) + # Start Multiple Tasks + while self._thread_container: + task = self._thread_container.pop() + task.start() + + self.engine.start(args) + self._heartbeat() + if self.end_signal: + self._extract_info(args) + + +crawler = CrawlerProcess() diff --git a/tools/cve-tracking/src/cve/gitee.py b/tools/cve-tracking/src/cve/gitee.py new file mode 100644 index 0000000..68aa6b9 --- /dev/null +++ b/tools/cve-tracking/src/cve/gitee.py @@ -0,0 +1,469 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +This is a helper script for working with gitee.com +""" +import sys +import os +import json +import base64 +import urllib +import urllib.request +import urllib.parse +import urllib.error +from datetime import datetime +from fake_useragent import UserAgent +import yaml +from .logger import logger +from .settings import GITEE_AUTH + + +class Gitee: + """ + Gitee is a helper class to abstract gitee.com api + """ + + user_agent = UserAgent( + path=os.path.join(os.path.dirname(__file__), "user-agent.json") + ) + + def __init__(self): + self.token = {"access_token": os.getenv(GITEE_AUTH["token"]), "user": os.getenv(GITEE_AUTH["account"])} + + self.headers = {"User-Agent": self.user_agent.random} + self.src_openeuler_url = "https://gitee.com/src-openeuler/{repo}/raw/{br}/" + self.advisor_url = "https://gitee.com/openeuler/openEuler-Advisor/raw/master/" + self.time_format = "%Y-%m-%dT%H:%M:%S%z" + + def post_gitee(self, url, values, headers=None): + """ + POST into gitee API + """ + if headers is None: + headers = self.headers.copy() + data = urllib.parse.urlencode(values).encode("utf-8") + req = urllib.request.Request( + url=url, data=data, headers=headers, method="POST") + try: + result = urllib.request.urlopen(req) + return result.read().decode("utf-8") + except urllib.error.HTTPError as err: + logger.warning( + "reuqest url: %s status code: %s headers: %s " + % (url, str(err.code), str(err.headers)) + ) + return False + + def fork_repo(self, repo, owner="src-openeuler"): + """ + Fork repository in gitee + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/forks" + url = url_template.format(owner=owner, repo=repo) + values = {} + values["access_token"] = self.token["access_token"] + return self.post_gitee(url, values) + + def create_issue( + self, + repo, + version="", + branch="master", + owner="src-openeuler", + title=None, + body=None, + ): + """ + Create issue in gitee + """ + title = title or "Upgrade {pkg} to {ver} in {br}".format( + pkg=repo, ver=version, br=branch + ) + body = ( + body + or """This issue is automatically created by openEuler-Advisor. + Please check the correspond PR is accepted before close it. + Thanks. + Yours openEuler-Advisor.""" + ) + return self._post_issue(repo, title, body, owner=owner) + + def create_issue_comment(self, repo, owner, number, body): + """ + create issue comment + """ + url_template = ( + "https://gitee.com/api/v5/repos/{owner}/{repo}/issues/{number}/comments" + ) + url = url_template.format(owner=owner, repo=repo, number=number) + values = {} + values["access_token"] = self.token["access_token"] + values["body"] = body + return self.post_gitee(url, values) + + def get_reviewers(self, repo, owner="src-openeuler"): + """ + Get reviewers of pkg + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{pkg}/collaborators" + url = url_template.format(owner=owner, pkg=repo) + return self.get_gitee_dict(url) + + def create_pr( + self, + repo, + version="", + issue=None, + title=None, + branch="master", + owner="src-openeuler", + body=None, + ): + """ + Create PR in gitee + """ + # assignees = "" + # reviewer_info = self.get_reviewers(repo) + # if reviewer_info: + # reviewer_list = json.loads(reviewer_info) + # assignees = ",".join(reviewer["login"] + # for reviewer in reviewer_list) + url_template = "https://gitee.com/api/v5/repos/{owner}/{pkg}/pulls" + url = url_template.format(owner=owner, pkg=repo) + values = {} + values["access_token"] = self.token["access_token"] + values["title"] = title or "Upgrade {pkg} to {ver}".format( + pkg=repo, ver=version + ) + values["head"] = "{hd}:{br}".format(hd=self.token["user"], br=branch) + values["base"] = branch + # values["assignees"] = assignees + if issue: + values["issue"] = issue + values["body"] = ( + body + or """This is a automatically created PR by openEuler-Advisor. + Please be noted that it's not throughly tested. + Review carefully before accept this PR. + Thanks. + Yours openEuler-Advisor.""" + ) + return self.post_gitee(url, values) + + def create_pr_comment(self, repo, number, body, owner="src-openeuler"): + """ + Post comment to the given specific PR + """ + url_template = ( + "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls/{number}/comments" + ) + url = url_template.format(owner=owner, repo=repo, number=number) + values = {} + values["access_token"] = self.token["access_token"] + values["body"] = body + return self.post_gitee(url, values) + + def get_gitee(self, url, headers=None): + """ + GET from gitee api + """ + if headers is None: + req = urllib.request.Request(url=url, headers=self.headers) + else: + req = urllib.request.Request(url=url, headers=headers) + try: + result = urllib.request.urlopen(req) + return result.read().decode("utf-8") + except urllib.error.HTTPError as e: + logger.warning( + "reuqest url: %s status code: %s messages: %s " + % (url, str(e.code), str(e.reason)) + ) + return None + except urllib.error.URLError as e: + logger.warning("reuqest url: %s messages: %s " % + (url, str(e.reason))) + return None + + def get_pr(self, repo, num, owner="src-openeuler"): + """ + Get detailed information of the given specific PR + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls/{number}" + url = url_template.format(owner=owner, repo=repo, number=num) + return self.get_gitee_json(url) + + def get_gitee_json(self, url): + """ + Get and load gitee json response + """ + json_resp = [] + headers = self.headers.copy() + headers["Content-Type"] = "application/json;charset=UTF-8" + resp = self.get_gitee(url, headers) + if resp: + json_resp = json.loads(resp) + return json_resp + + def get_branch_info(self, branch): + """ + Get upgrade branch info + """ + upgrade_branches_url = ( + self.advisor_url + "advisors/helper/upgrade_branches.yaml" + ) + resp = self.get_gitee(upgrade_branches_url) + if not resp: + print("ERROR: upgrade_branches.yaml may not exist.") + sys.exit(1) + branches_info = yaml.load(resp, Loader=yaml.Loader) + for br_info in branches_info["branches"]: + if branch == br_info["name"]: + return br_info + print("WARNING: Don't support branch: {} in auto-upgrade.".format(branch)) + sys.exit(1) + + def get_spec_exception(self, repo): + """ + Get well known spec file exception + """ + specfile_exception_url = ( + self.advisor_url + "advisors/helper/specfile_exceptions.yaml" + ) + resp = self.get_gitee(specfile_exception_url) + if not resp: + print("ERROR: specfile_exceptions.yaml may not exist.") + sys.exit(1) + excpt_list = yaml.load(resp, Loader=yaml.Loader) + if repo in excpt_list: + return excpt_list[repo] + return None + + def get_version_exception(self): + """ + Get version recommend exceptions + """ + version_exception_url = ( + self.advisor_url + "advisors/helper/version_exceptions.yaml" + ) + resp = self.get_gitee(version_exception_url) + if not resp: + print("ERROR: version_exceptions.yaml may not exist.") + sys.exit(1) + excpt = yaml.load(resp, Loader=yaml.Loader) + return excpt + + def get_spec(self, pkg, branch="master"): + """ + Get openeuler spec file for specific package + """ + specurl = self.src_openeuler_url + "{repo}.spec" + specurl = specurl.format(repo=pkg, br=branch) + excpt = self.get_spec_exception(pkg) + if excpt: + specurl = urllib.parse.urljoin( + specurl, os.path.join(excpt["dir"], excpt["file"]) + ) + resp = self.get_gitee(specurl) + return resp + + def get_yaml(self, pkg): + """ + Get upstream yaml metadata for specific package + """ + yamlurl = self.advisor_url + "upstream-info/{}.yaml".format(pkg) + resp = self.get_gitee(yamlurl) + if not resp: + yamlurl = self.src_openeuler_url + "{repo}.yaml" + yamlurl = yamlurl.format(repo=pkg, br="master") + resp = self.get_gitee(yamlurl) + if not resp: + print( + "WARNING: {}.yaml can't be found in upstream-info and repo.".format( + pkg + ) + ) + return resp + + def get_community(self, repo): + """ + Get yaml data from community repo + """ + yamlurl = ( + "https://gitee.com/api/v5/repos/openeuler/community/contents/" + "repository/{repo}.yaml".format(repo=repo) + ) + resp = self.get_gitee_json(yamlurl) + resp_str = base64.b64decode(resp["content"]) + return resp_str + + def get_issues(self, pkg, prj="src-openeuler"): + """ + List all open issues of pkg + """ + issues_url = "https://gitee.com/api/v5/repos/{prj}/{pkg}/issues?".format( + prj=prj, pkg=pkg + ) + parameters = "state=open&sort=created&direction=desc&page=1&per_page=20" + return self.get_gitee_json(issues_url + parameters) + + def get_issue_comments(self, repo, number, owner="src-openeuler"): + """ + Get comments of specific issue + """ + issues_url = "https://gitee.com/api/v5/repos/{owner}/{repo}/issues/{number}/comments?page=1&per_page=50&order=asc".format( + repo=repo, owner=owner, number=number + ) + return self.get_gitee_json(issues_url) + + def get_issue_merged_branches(self, issue_id, repo, owner="src-openeuler"): + """ + Get merged branches of specific issue + """ + issues_url = "https://gitee.com/api/v5/repos/{owner}/issues/{number}/pull_requests?".format( + owner=owner, number=issue_id + ) + + param = "&repo={}".format(repo) + return self.get_gitee_dict(issues_url, param) + + def _post_issue(self, pkg, title, body, owner="src-openeuler"): + """ + Post new issue + """ + issues_url = "https://gitee.com/api/v5/repos/{owner}/issues".format( + owner=owner) + parameters = {} + parameters["access_token"] = self.token["access_token"] + parameters["repo"] = pkg + parameters["title"] = title + parameters["body"] = body + return self.post_gitee(issues_url, parameters) + + def post_issue_comment(self, pkg, number, comment, prj="src-openeuler"): + """ + Post comment of issue + """ + issues_url = ( + "https://gitee.com/api/v5/repos/{prj}/{pkg}/issues/{number}/" + "comments".format(prj=prj, pkg=pkg, number=number) + ) + parameters = {} + parameters["access_token"] = self.token["access_token"] + parameters["body"] = comment + self.post_gitee(issues_url, parameters) + + def get_gitee_datetime(self, time_string): + """ + Get datetime of gitee + """ + result = datetime.strptime(time_string, self.time_format) + return result.replace(tzinfo=None) + + def get_gitee_dict(self, url, param=""): + url += param + token_param = "access_token={}".format(self.token["access_token"]) + separator = "?" + if param: + separator = "&" + url += separator + token_param + return self.get_gitee_json(url) + + def get_contributors(self, repo, owner): + """ + Get contributors of owner/repo + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/contributors" + url = url_template.format(owner=owner, repo=repo) + return self.get_gitee_dict(url) + + def get_branches(self, repo, owner): + """ + Get branches of owner/repo + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/branches" + url = url_template.format(owner=owner, repo=repo) + return self.get_gitee_dict(url) + + def get_commits( + self, repo, owner, sha="", author="", since="", until="", page=1, per_page=20 + ): + """ + Get commits of owner/repo + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/commits" + url = url_template.format(owner=owner, repo=repo) + param = "?page={}&per_page={}".format(page, per_page) + if sha: + param += "&sha={}".format(sha) + if author: + param += "&author={}".format(author) + if since: + param += "&since={}".format(since) + if until: + param += "&until={}".format(until) + return self.get_gitee_dict(url, param) + + def get_one_commit(self, repo, sha, owner): + """ + Get one commit of owner/repo + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/commits/{sha}" + url = url_template.format(owner=owner, repo=repo, sha=sha) + return self.get_gitee_dict(url) + + def get_pr_list( + self, repo, owner, head="", base="", state="all", page=1, per_page=20 + ): + """ + Get PR list of owner/repo + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls" + url = url_template.format(owner=owner, repo=repo) + param = "?state={}&sort=created&direction=desc&page={}&per_page={}".format( + state, page, per_page + ) + if head: + param += "&head={}".format(head) + if base: + param += "&base={}".format(base) + return self.get_gitee_dict(url, param) + + def get_pr_comments(self, repo, number, owner, page=1, per_page=20): + """ + Get PR comments of owner/repo + """ + url_template = ( + "https://gitee.com/api/v5/repos/{owner}/{repo}/pulls/{number}/comments" + ) + url = url_template.format(owner=owner, repo=repo, number=number) + param = "?page={}&per_page={}".format(page, per_page) + return self.get_gitee_dict(url, param) + + def get_repos(self, org, repo_type, page=1, per_page=20): + """ + Get repos of org + """ + url_template = "https://gitee.com/api/v5/orgs/{org}/repos" + url = url_template.format(org=org) + param = "?type={}&page={}&per_page={}".format( + repo_type, page, per_page) + return self.get_gitee_dict(url, param) + + def get_repo(self, repo, owner): + """ + Get repo + """ + url_template = "https://gitee.com/api/v5/repos/{owner}/{repo}" + url = url_template.format(owner=owner, repo=repo) + return self.get_gitee(url) diff --git a/tools/cve-tracking/src/cve/httprequest.py b/tools/cve-tracking/src/cve/httprequest.py new file mode 100644 index 0000000..cc948f6 --- /dev/null +++ b/tools/cve-tracking/src/cve/httprequest.py @@ -0,0 +1,212 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +import os +import re +from functools import wraps +import requests +from fake_useragent import UserAgent +from requests.exceptions import RequestException, HTTPError +from retrying import retry + + +class RemoteService: + """ + HTTP request service + + Attributes: + _max_delay:Maximum interval time + _retry:Retry count + _body:Request body + _response:Response info + _request_error: request error + """ + + user_agent = UserAgent( + path=os.path.join(os.path.dirname(__file__), "user-agent.json") + ) + url_regex = re.compile( + r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" + ) + + def __init__(self, max_delay=1000): + self._retry = 3 + if not isinstance(max_delay, int): + raise TypeError("Max_delay is the number of milliseconds of the integer ") + self._max_delay = max_delay + self._body = None + self._response = None + self._request_error = None + self._query_param = ( + "params", + "data", + "cookies", + "files", + "auth", + "timeout", + "allow_redirects", + "proxies", + "hooks", + "stream", + "verify", + "cert", + "json", + ) + self._headers = {"User-Agent": self.user_agent.random} + + @property + def response(self): + return self._response + + @property + def status_code(self): + """ + Description: status code of the response + """ + if self._response is None: + return requests.codes["internal_server_error"] + return self._response.status_code + + @property + def content(self): + """ + Description: original content of the response + """ + return self._response.content if self._response else None + + @property + def text(self): + """ + Description: content of the decoded response + """ + return self._response.content.decode("utf-8") if self._response else None + + @property + def errors(self): + """ + Description: The error message content of the request + """ + return self._request_error + + def _redirected(self, response): + if not response: + raise HTTPError("No address after redirect") + url = re.findall(self.url_regex, response) + if not url: + raise HTTPError("Incorrect redirect address") + return url[0] + + def _dispatch(self, method, url, **kwargs): + """ + Description: Request remote services in different ways + + Args: + method: request method， GET、 POST 、PUT 、DELETE + url:Remote request address + kwargs:parameters associated with the request + """ + + @retry(stop_max_attempt_number=self._retry, stop_max_delay=self._max_delay) + def http(url): + response = method(url, **kwargs) + if response.status_code == requests.codes["found"]: + url = self._redirected(response.text) + response = http(url) + + if response.status_code != requests.codes["ok"]: + _msg = ( + "There is an exception with the remote service [%s]，" + "Please try again later.The HTTP error code is：%s" + % (url, str(response.status_code)) + ) + raise HTTPError(_msg) + return response + + method = getattr(self, method, None) + if method is None: + raise RequestException( + "Request mode error, temporarily only support POST, GET" + ) + try: + self._response = http(url) + except RequestException as error: + raise RequestException(str(error)) from error + + def request(self, url, method, body=None, max_retry=3, **kwargs): + """ + Description: Request a remote http service + + Args: + url: http service address + method: mode of request ,only GET、 POST、 DELETE、 PUT is supported + body: Request body content + max_retry: The number of times the request failed to retry + kwargs: Request the relevant parameters + """ + if not isinstance(max_retry, int): + raise TypeError("MAX_RETRY must be an integer greater than zero ") + self._retry = max_retry + self._body = body + try: + self._dispatch(method=method, url=url, **kwargs) + except RequestException as error: + self._request_error = str(error) + self._response = requests.Response() + + return self._response + + def get(self, url, **kwargs): + """ + Description: HTTP get request method + + Args: + kwargs: requests parameters + url: requested remote address + """ + query_param = {key: kwargs.get(key, None) for key in self._query_param} + + response = requests.get(url=url, headers=self._headers, **query_param) + return response + + def post(self, url, **kwargs): + """ + Description: HTTP post request method + + Args: + kwargs: requests parameters + url: requested remote address + """ + data = kwargs.get("data") or self._body + query_param = {key: kwargs.get(key, None) for key in self._query_param} + if data: + query_param["data"] = data + response = requests.post(url=url, headers=self._headers, **query_param) + return response + + +def get(func): + """ + A decorator for HTTP GET requests + """ + + @wraps(func) + def _wrapper(*args, **kwargs): + rest_api = func(*args, **kwargs) + _remote = RemoteService() + _remote.request(url=rest_api, method="get", **kwargs) + if _remote.status_code == requests.codes["ok"]: + response = (requests.codes["ok"], _remote.text) + else: + response = (_remote.status_code, _remote.errors) + return response + + return _wrapper diff --git a/tools/cve-tracking/src/cve/logger.py b/tools/cve-tracking/src/cve/logger.py new file mode 100644 index 0000000..22b2137 --- /dev/null +++ b/tools/cve-tracking/src/cve/logger.py @@ -0,0 +1,113 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ +""" +Logging related +""" +import logging +import os +import pathlib +import pprint +from concurrent_log_handler import ConcurrentRotatingFileHandler +from .settings import LOG_DIR + + +def get_spider_looger_str(website, pkg, versions, cve, patch_urls, other_urls): + """return logger str + + Args: + website (str): to spider web name + pkg (str): pkg name + versions (list): versions + cve (str): cve id + patch_urls (list): patch urls + other_urls (list): other urls + + Returns: + str: logger str + """ + if patch_urls or other_urls: + logger_info = ( + f"{website} result:\npkg_name:{pkg}\nversions:{pprint.pformat(versions)}\nCVE: {cve}\n" + f"get patch urls: {pprint.pformat(patch_urls)}\nget other urls {pprint.pformat(other_urls)}" + ) + else: + logger_info = f"{website} result: No information available" + return logger_info + + +class Log(object): + """ + operation log of the system + """ + + def __init__(self, name=__name__, path=None): + self.__current_rotating_file_handler = None + if not path: + path = os.path.dirname(__file__) + self.__path = os.path.join(path, "log_info.log") + + if not os.path.exists(self.__path): + try: + os.makedirs(os.path.split(self.__path)[0]) + except FileExistsError: + pathlib.Path(self.__path).touch(mode=0o644) + self.__max_bytes = 30000000 + self.__backup_count = 2 + self.__level = "INFO" + self.__logger = logging.getLogger(name) + self.__logger.setLevel(self.__level) + + def __init_handler(self): + self.__current_rotating_file_handler = ConcurrentRotatingFileHandler( + filename=self.__path, + mode="a", + maxBytes=self.__max_bytes, + backupCount=self.__backup_count, + encoding="utf-8", + use_gzip=True, + ) + self.__set_formatter() + self.__set_handler() + + def __set_formatter(self): + formatter = logging.Formatter( + "%(asctime)s-%(filename)s-[line:%(lineno)d]" + "-%(levelname)s-[ log details ]: %(message)s", + datefmt="%a, %d %b %Y %H:%M:%S", + ) + self.__current_rotating_file_handler.setFormatter(formatter) + + def __set_handler(self): + self.__current_rotating_file_handler.setLevel(self.__level) + self.__logger.addHandler(self.__current_rotating_file_handler) + + @property + def logger(self): + """ + Gets the logger property + """ + if not self.__current_rotating_file_handler: + self.__init_handler() + return self.__logger + + @property + def file_handler(self): + """ + The file handle to the log + """ + if not self.__current_rotating_file_handler: + self.__init_handler() + return self.__current_rotating_file_handler + + +logger = Log(__name__, path=LOG_DIR).logger diff --git a/tools/cve-tracking/src/cve/pipe.py b/tools/cve-tracking/src/cve/pipe.py new file mode 100644 index 0000000..872acfb --- /dev/null +++ b/tools/cve-tracking/src/cve/pipe.py @@ -0,0 +1,371 @@ +#!/usr/bin/python3 +import os +import re +import hashlib +import shutil +import ast +import pprint +from .logger import logger +from .settings import DEFAULT_SAVE_PATH, RECORD_FILE + +URL_REGEX = re.compile( + r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" +) + + +class FileHandle: + """ + File handle + """ + + def __init__( + self, folder, branch=None, cve=None, pkg=None, csv_file=None, read_csv=None + ) -> None: + if read_csv and not callable(read_csv): + raise ValueError( + "The read_csv parameter is a function that can be called to read CSV files" + ) + self.read_csv = read_csv or self._read_csv + self.folder = folder + self.cve = cve + self.pkg = pkg + self.cve_file = csv_file + self.branches = branch + self._apply = "All branch apply failed" + + def _read_csv(self, **kwargs): + raise NotImplementedError + + @staticmethod + def copy(source_path, target_path): + """ + Copy the files in the directory to each version folder + Args: + source_path: source path + target_path: target path + Returns: + folder: folder path + + """ + os.makedirs(target_path, exist_ok=True) + files_num = sum([os.path.isfile(listx) for listx in os.listdir(target_path)]) + files = os.listdir(source_path) + for index, file in enumerate(files, start=1): + try: + mv_file_name = file.split("_")[0] + "_{num}.patch".format( + num=files_num + index + ) + shutil.copyfile( + os.path.join(source_path, file), + os.path.join(target_path, mv_file_name), + ) + except shutil.Error: + logger.warning( + "The file '%s' of the same name exists in the folder : %s" + % (file, os.path.join(target_path, mv_file_name)) + ) + + def _parse_text(self, path): + """ + parse text content + Args: + root: root + cve_folder: cve_folder + + Returns: + repair_content: repair_content + """ + with open(path, "r", encoding="utf-8") as file: + suspected_spatch = [] + not_found_patch = [] + download_failed = [] + finds = [] + find_path = {} + for line in file.readlines(): + if "cve information" in line.lower(): + cve_info = line.replace("CVE Information : ", "") + cve_dict = ast.literal_eval(cve_info) + if "Uncertain" in line: + suspected_spatch.append(line.split("：")[1].split("\n")[0]) + if "no patch information" in line: + not_found_patch.append(line.split(" result:")[0]) + if "File download failed" in line: + download_failed.append(line.split("failed:")[1].split("\n")[0]) + if "find patch information" in line: + find = line.split("：")[1].split("\n")[0] + if find not in finds: + finds.append(find) + find_path[line.split("in ")[1].split("：")[0]] = finds + repair_content = { + "pkg_name": cve_dict.get("pkg"), + "cve": cve_dict.get("cve"), + "versions": cve_dict.get("versions"), + "Not found patch": ["Platform", list(set(not_found_patch))], + "Seen patch files": ["Platform Patchs", find_path], + "Download failed": ["Download Patchs URl", list(set(download_failed))], + "Suspected Patchs URL": [ + "Suspected Patchs URL", + list(set(suspected_spatch)), + ], + } + return repair_content + + def _write_text_body(self, heard, patch_content, file): + """ + write_text_body + Args: + heard: heard + patch_content: patch_content + file: file + """ + file.write("=" * 20 + heard + "=" * 20 + "\n") + if not patch_content[1]: + file.write("There is no information" + "\n\n") + return + if isinstance(patch_content[1], dict): + for platform, patchs in patch_content[1].items(): + seen_patch = patch_content[0].split(" ") + for patch in patchs: + file.write( + seen_patch[0] + + " " + + platform + + " " + + seen_patch[1] + + ": " + + patch + + "\n\n" + ) + else: + for val in patch_content[1]: + file.write(patch_content[0] + ": " + val + "\n\n") + + def _wirte_text_format(self, repair_content, path, apply_result, pkg): + """ + wirte text format + Args: + repair_content: repair_content + root: root + cve_folder: cve_folder + """ + + pkg_name = repair_content.get("pkg_name") + cve = repair_content.get("cve") + versions = repair_content.get("versions") + + with open(path, "w", encoding="utf-8") as file: + file.write( + f"package name: {pkg_name}\n\nCVE number: {cve}\n\nversions: {pprint.pformat(versions)}\n\n" + ) + for heard, patch_content in repair_content.items(): + if heard not in ["pkg_name", "cve", "versions"]: + self._write_text_body(heard, patch_content, file) + + # apply results of patch + if apply_result: + for apply_info in self._write_apply_result(pkg, apply_result): + file.write(apply_info) + + def _write_apply_result(self, pkg_name, apply_result): + + add_content = list("=" * 20 + "Apply Result" + "=" * 20 + "\n") + if apply_result == self._apply: + add_content.append(apply_result) + else: + for branch in self.branches: + for result in apply_result.split("\n"): + if re.match( + f"^\[INFO\] {pkg_name}.*{branch}.*successfully$", result + ): + add_content.append(f"Apply result: {branch} : apply success") + if re.match(f"^\[ERROR\] {pkg_name}.*{branch}.*failed", result): + add_content.append(f"Apply result: {branch} : apply failed") + + return add_content + + def _overwrite_text(self, folder, pkg_name, cve_num, apply_result): + """ + write content + Args: + args: parameter + pkg_name: pkg_name + cve_num: cve_num + apply_result: apply patch result + """ + file_path = os.path.join(folder, pkg_name + "-" + cve_num, RECORD_FILE) + if not os.path.exists(file_path): + logger.error("The specified file does not exist : %s" % file_path) + return + try: + + format_text = self._parse_text(file_path) + self._wirte_text_format(format_text, file_path, apply_result, pkg_name) + except IOError as error: + logger.error(error) + + def format_text(self, apply_result=None): + """ + Format the contents recorded in the file to present the process in a uniform format + Args: + args : parameter + """ + cve_infos = [] + if self.cve_file: + try: + cve_infos.extend( + self.read_csv( + os.path.join(self.cve_file), + out_path=self.folder, + return_content=True, + ) + ) + except TypeError: + raise ValueError( + "The return value of the read_csv function must be iterable" + ) + if all([self.pkg, self.cve]): + cve_infos.append([self.pkg, self.cve]) + + # Traversal process the searched patch information + for cve in cve_infos: + pkg_name, cve_num = cve + self._overwrite_text(self.folder, pkg_name, cve_num, apply_result) + + def extract_text(self, path): + """ + Extracting the comment content from the text + """ + patchs = list() + apply_result = [] + with open(path, "r", encoding="utf-8") as file: + lines = file.readlines() + for index, line in enumerate(lines, start=1): + if all([p in line for p in ("Platform", "Patchs")]): + patchs.extend(re.findall(URL_REGEX, line)) + if "Apply Result" in line: + apply_result = lines[index:] + break + if self._apply in apply_result: + apply_result = [branch + ":" + self._apply for branch in self.branches] + + return { + "urls": list(set(patchs)), + "apply_result": dict([tuple(apply.split(":")) for apply in apply_result]), + } + + +class RequestsUrl: + """ + File download save method + """ + + def __init__(self, url, crawl, callback=None, **kwargs) -> None: + self.crawl = crawl + self.url = url + self.callback = callback or self._wget + self.__dict__.update(**kwargs) + + def _wget(self, response, **kwargs): + """ + Send the request + Args: + response: response + **kwargs: + + Returns: + + """ + folder = self.crawl.folder or DEFAULT_SAVE_PATH + tmp_path = os.path.join(folder, "tmp-folder") + try: + os.makedirs(tmp_path, exist_ok=True) + with open( + os.path.join( + tmp_path, + self.crawl.cve + + "_{r}.patch".format( + r=hashlib.sha1(self.url.encode()).hexdigest() + ), + ), + "w", + encoding="utf-8", + ) as file: + file.write(response.text) + except IOError as error: + logger.error(error) + else: + folders = self.crawl.create_folders(self.crawl.versions) + if isinstance(folders, str): + folders = [folders] + + for target_folder in folders: + FileHandle.copy(tmp_path, target_folder) + finally: + if os.path.exists(tmp_path): + shutil.rmtree(tmp_path) + + +class RequestRepeat: + """ + Request address duplicate determination + """ + + def __init__(self) -> None: + self._fingerprints = set() + + def _to_bytes(self, text, encoding="utf-8"): + if isinstance(text, bytes): + return text + return text.encode(encoding) + + def _request_figerprint(self, request): + hash_fp = hashlib.sha1() + hash_fp.update(self._to_bytes(request.crawl.cve)) + hash_fp.update(self._to_bytes(request.url)) + return hash_fp.hexdigest() + + def request_seen(self, request): + """ + Check if the URL has already been processed + """ + hash_fp = self._request_figerprint(request=request) + if hash_fp in self._fingerprints: + return True + self._fingerprints.add(hash_fp) + + return False + + +class SavePipe: + """ + Data storage pipeline + """ + + def __init__(self, crawl, **kwargs) -> None: + """ + Initialize the class + Args: + crawl: + **kwargs: crawl + """ + self.crawl = crawl + self.__dict__.update(**kwargs) + + def save_process(self, crawl): + """ + Data storage pipeline + Args: + crawl: crawl + + Returns: + + """ + path = os.path.join(crawl.folder, RECORD_FILE) + try: + os.makedirs(crawl.folder, exist_ok=True) + if not hasattr(self, "text") or not self.text: + return + with open(path, "a", encoding="utf-8") as file: + file.write(getattr(self, "text", "") + "\n") + except IOError as error: + logger.error(error) diff --git a/tools/cve-tracking/src/cve/plantform.py b/tools/cve-tracking/src/cve/plantform.py new file mode 100644 index 0000000..9767226 --- /dev/null +++ b/tools/cve-tracking/src/cve/plantform.py @@ -0,0 +1,596 @@ +#!/usr/bin/python3 +import json +import re +import os +import pprint +from bs4 import BeautifulSoup +import requests +from .bugzilla import Api +from .settings import DEFAULT_SAVE_PATH +from .logger import logger, get_spider_looger_str +from .pipe import SavePipe, RequestsUrl + + +URL_REGEX = re.compile( + r"http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+" +) +VERSION_REGEX = r"(\d+[.-]?)+([.-]?[A-Za-z0-9]*)*" + +__all__ = ["Crawl", "Nvd", "Debian", "Ubuntu", "Bugzilla"] + + +class Crawl: + """ + CVE platform base class + """ + + def __init__(self, cve, pkg, versions, out_path, **kwargs) -> None: + self.cve = cve + self.pkg = pkg + self.versions = versions + self.fix_versions = None + self._folder = out_path + self.__dict__.update(**kwargs) + + def parse(self, response, **kwargs): + """ + All enabled entrances + Args: + response: response + **kwargs: kwargs + + Returns: + None + """ + raise NotImplementedError( + f"{self.__class__.__name__}.parse callback is not defined" + ) + + @staticmethod + def _re_escape(name: str): + """escape str + + Args: + name (str): pkgname + + Returns: + str: escape new pkg name + """ + if "+" in name: + name = name.replace("+", "\+") + if "^" in name: + name = name.replace("^", "\^") + if "$" in name: + name = name.replace("$", "\$") + if "." in name: + name = name.replace(".", "\.") + if "|" in name: + name = name.replace("|", "\|") + if "-" in name: + name = name.replace("-", "\-") + return name + + @property + def folder(self): + """ + The CVE folder in the specified directory + """ + folder = getattr(self, "_folder", DEFAULT_SAVE_PATH) + if hasattr(self, "pkg") and hasattr(self, "cve"): + folder = os.path.join(folder, self.pkg + "-" + self.cve) + return folder + + def _create_folder(self, version, folder=None): + """ + Create a folder Path + Args: + version: version + folder: The folder defaults to None + + Returns: + path: folder path + """ + if not folder: + folder = self.folder + + path = os.path.join(folder, str(version)) + os.makedirs(os.path.dirname(path), exist_ok=True) + return path + + def create_folders(self, versions): + """ + Creating Folder Collection + Args: + versions: Multiple versions + + Returns: + folder: folder path + """ + if not versions: + return self.folder + + for version in versions: + return self._create_folder(version) + + def git(self, func): + """ + git + """ + git_regex = "^http[s]?://git.({pkg_name}|.*).(org|net)(.*{pkg_name}([0-9]+)?(.git)?/commit.*|(/gitweb)?/\?p={pkg_name}([0-9]+)?.git;a=commit.*)".format( + pkg_name=Crawl._re_escape(self.pkg) + ) + + def wrap(*args, **kwargs): + url = kwargs.get("url", "") + if re.match(git_regex, url, flags=re.IGNORECASE): + url = re.sub("commit|commitdiff|commitdiff_plain", "patch", url) + return url + if re.match( + "^https://git.*raw/([0-9a-z]*$|.*\.patch$)", url, flags=re.IGNORECASE + ): + return url + + return func(*args, **kwargs) + + return wrap + + def github(self, func): + """ + github + """ + github_regex = "^http[s]?://github.com.*{pkg_name}(.*)?/commit.*".format( + pkg_name=Crawl._re_escape(self.pkg) + ) + + def wrap(*args, **kwargs): + url = kwargs.get("url", "") + if re.match(github_regex, url, flags=re.IGNORECASE): + url = re.sub("\s+(.*)|\?.*", "", url) + return url + ".patch" + return func(*args, **kwargs) + + return wrap + + def gitlab(self, func): + """ + gitlab + """ + gitlab_regex = ( + "^http[s]?://gitlab(.*.org|.com).*{pkg_name}([0-9]+)?(/-)?/commit.*".format( + pkg_name=Crawl._re_escape(self.pkg) + ) + ) + + def wrap(*args, **kwargs): + url = kwargs.get("url", "") + if re.match(gitlab_regex, url, flags=re.IGNORECASE): + url = re.sub("\s+(.*)|\?.*", "", url) + return url + ".patch" + return func(*args, **kwargs) + + return wrap + + def match(self, url): + """ + match url + """ + + @self.github + @self.git + @self.gitlab + def _match_url(url): + return None + + return _match_url(url=url) + + +class Versions: + """ + Version number processing + """ + + separator = (".", "-") + _connector = "&" + + def _order(self, version, separator=None): + """ + Version of the cutting + Args: + version: version + separator: separator + + Returns: + + """ + if not separator: + separator = self._connector + return tuple([int(v) for v in version.split(separator) if v.isdigit()]) + + def _similar(self, text, compare_text): + """ + Similarity comparison + Args: + text: text + compare_text: compare_text + + Returns: + + """ + _text = text if len(text) < len(compare_text) else compare_text + weight = 0 + for index, _ in enumerate(_text): + if text[index] != compare_text[index]: + weight = index + break + return weight / len(text) * 1.0 * 100 + + def similarity(self, text, compare_queue): + """ + Compare the similarity of two strings + """ + if isinstance(compare_queue, str): + return self._similar(text, compare_queue) + ratio = [self._similar(text, compare_text) for compare_text in compare_queue] + return ratio + + def match_version(self, pkg_name, fix_text): + """ + Match the version number of the software from the repaired text + """ + versions = list() + for pkg_info in fix_text.split(","): + if pkg_name.lower() in pkg_info.lower(): + _v = re.search(VERSION_REGEX, pkg_info) + versions.append(_v.group()) + return versions + + def lgt(self, version, compare_version): + """ + Returns true if the size of the compared version is greater + than that of the compared version, or false otherwise + + """ + for separator in self.separator: + version = self._connector.join([v for v in version.split(separator)]) + compare_version = self._connector.join( + [v for v in compare_version.split(separator)] + ) + version = self._order(version) + compare_version = self._order(compare_version) + return version >= compare_version + + +class Nvd(Crawl): + """ + nvd plantform + """ + + nvd_host = "https://nvd.nist.gov" + + def __init__(self, cve, pkg, versions, out_path=DEFAULT_SAVE_PATH): + super(Nvd, self).__init__( + cve=cve, pkg=pkg, versions=versions, out_path=out_path + ) + self.start_url = self.nvd_host + "/vuln/detail/" + cve + + def get_urls(self, resp_str, pkg): + """ + Gets the URL of patch + Args: + resp_str: resp_str + pkg: pkg + + Returns: + + """ + patch_urls = set() + other_urls = set() + pkg = Crawl._re_escape(pkg) + + soup = BeautifulSoup(resp_str, "html.parser") + urls = [ + res.text + for res in soup.find_all( + name="td", attrs={"data-testid": re.compile("vuln-hyperlinks-link-\d+")} + ) + ] + for url in urls: + seen_url = self.match(url=url) + if seen_url: + patch_urls.add(seen_url) + else: + other_urls.add(url) + + logger_str = get_spider_looger_str( + "Nvd", self.pkg, self.versions, self.cve, patch_urls, other_urls + ) + logger.info(logger_str) + + return patch_urls, other_urls + + def parse(self, response, **kwargs): + """ + Parse response data + Args: + response: response + **kwargs: + + Returns: + + """ + patch_urls, other_urls = self.get_urls(response.text, self.pkg) + if not patch_urls: + if not other_urls: + text = "NVD result:There is no patch information and it has not been repaired" + yield SavePipe(crawl=self, text=text) + else: + for other_url in other_urls: + text = "Uncertain patch information in Nvd：%s" % other_url + yield SavePipe(crawl=self, text=text) + else: + for url in patch_urls: + text = "find patch information in Nvd：%s" % url + yield SavePipe(crawl=self, text=text) + yield RequestsUrl(url=url, crawl=self, is_patch=True) + + +class Debian(Crawl): + """ + Get the PR address in the Debian website + """ + + def __init__(self, cve, versions, pkg, out_path=DEFAULT_SAVE_PATH): + """ + Initialize attribute + Args: + cve_num: cve number + """ + super(Debian, self).__init__( + cve=cve, pkg=pkg, versions=versions, out_path=out_path + ) + self.start_url = "https://security-tracker.debian.org/tracker/" + self.cve + + def parse(self, response, **kwargs): + """ + Parse response data + Args: + response: response + **kwargs: + + Returns: + + """ + + patch_urls = [] + other_urls = [] + soup = BeautifulSoup(response.text, "html.parser") + pre = soup.pre + + if pre: + links = pre.find_all("a") + for link in links: + _url = link.get("href") + seen_url = self.match(url=_url) + if seen_url: + patch_urls.append(seen_url) + else: + other_urls.append(_url) + logger_str = get_spider_looger_str( + "Debian", self.pkg, self.versions, self.cve, patch_urls, other_urls + ) + logger.info(logger_str) + for url in other_urls: + text = "Uncertain patch information in Debian：%s" % url + yield SavePipe(crawl=self, text=text) + for patch_url in patch_urls: + yield RequestsUrl(url=patch_url, crawl=self, is_patch=True) + text = "find patch information in Debian：%s" % patch_url + yield SavePipe(crawl=self, text=text) + if not any([other_urls, patch_urls]): + text = "Debian result:There is no patch information and it has not been repaired" + yield SavePipe(crawl=self, text=text) + + +class Ubuntu(Crawl): + """ + Get the PR address in the Ubuntu website + """ + + def __init__(self, cve, versions, pkg, out_path=DEFAULT_SAVE_PATH): + """ + Initialize attribute + Args: + cve: cve number + """ + super(Ubuntu, self).__init__( + cve=cve, pkg=pkg, versions=versions, out_path=out_path + ) + self.start_url = "https://ubuntu.com/security/" + self.cve + + def parse(self, response, **kwargs): + """ + Parse the webpage and extract the url + Args: + response:webpage + Returns: + + """ + + git_list, other_list = [], [] + try: + soup = BeautifulSoup(response.text, "lxml") + content = soup.find(colspan="2") + for url in content.find_all("a"): + patch_url = url.get("href") + seen_url = self.match(url=patch_url) + if seen_url: + git_list.append(seen_url) + else: + other_list.append(patch_url) + + logger_str = get_spider_looger_str( + "Ubuntu", self.pkg, self.versions, self.cve, git_list, other_list + ) + logger.info(logger_str) + + for patch in git_list: + yield RequestsUrl(url=patch, crawl=self, is_patch=True) + text = "find patch information in Ubuntu：%s" % patch + yield SavePipe(crawl=self, text=text) + for url in other_list: + text = "Uncertain patch information in Ubuntu：%s" % url + yield SavePipe(crawl=self, text=text) + except: + logger.info( + "Ubuntu result:There is no patch information and it has not been repaired" + ) + yield SavePipe( + crawl=self, + text="Ubuntu result:There is no patch information and it has not been repaired", + ) + + +class Bugzilla(Crawl): + """ + Bug fix + """ + + bugzilla_api = Api(base_url="https://bugzilla.redhat.com") + + def __init__(self, cve, pkg, versions, out_path=DEFAULT_SAVE_PATH): + self._v = Versions() + super(Bugzilla, self).__init__( + cve=cve, pkg=pkg, versions=versions, out_path=out_path + ) + self.start_url = self.bugzilla_api.get_bug(fields=dict(alias=self.cve)) + + @staticmethod + def load_json(content): + """ + Loading JSON data + + Args: + content: The JSON content returned by the HTTP request + """ + if not content: + return dict() + try: + json_data = json.loads(content) + except json.JSONDecodeError: + json_data = dict() + return json_data + + def _extract_fixed_in_version(self, bugs_info): + """ + extract fixed in version + + Args: + bugs_info: Description of CVE information + pkg_name: The package name of the fix + """ + fixed_in_versions = [] + for bug in bugs_info.get("bugs", []): + fixed_in_versions.extend( + self._v.match_version(self.pkg, bug.get("cf_fixed_in", "")) + ) + + return fixed_in_versions + + def _get_comments(self): + """ + Get CVE comment information + + """ + status_code, response = self.bugzilla_api.get_comments(fields=self.cve) + comments = dict() + if status_code == requests.codes["ok"]: + comments = Bugzilla.load_json(response) + return comments + + def _filter_patch_remote(self, remote_url): + """ + Filter out the patch pack path in GitHub based on the existing policy + + Args: + remote_url:The remote address + """ + if isinstance(remote_url, str): + remote_url = [remote_url] + patch_urls = list() + + for url in remote_url: + seen_url = self.match(url=url) + if seen_url: + patch_urls.append(seen_url) + + return patch_urls + + def contrast_version(self, fix_versions, warehouse_versions): + """ + Fix or affect version alignment + """ + _fix_version = list() + for _version in warehouse_versions: + version_similarity = self._v.similarity(_version, fix_versions) + compared_version = fix_versions[ + version_similarity.index(max(version_similarity)) + ] + if self._v.lgt(_version, compared_version): + _fix_version.append(_version) + + return _fix_version + + def _extract_url(self, comments): + """ + Extracting comment information + Args: + comments: + + Returns: + + """ + pulls = [] + for _, comment in comments["bugs"].items(): + for comment_info in comment["comments"]: + pulls.extend(re.findall(URL_REGEX, comment_info["text"])) + return pulls + + def parse_comments(self, response, **kwargs): + """ + Parse the bug comment content + """ + comments = Bugzilla.load_json(response.text) + if not comments: + logger.info("Bugzilla result:{} has no comment".format(self.cve)) + yield SavePipe( + crawl=self, + text="Bugzilla result:There is no patch information and it has not been repaired", + ) + + patchs = self._filter_patch_remote(remote_url=self._extract_url(comments)) + if patchs: + logger.info(f"Bugzilla result: found patch urls :{pprint.pformat(patchs)}") + for patch in patchs: + text = "find patch information in Bugzilla：%s" % patch + yield SavePipe(crawl=self, text=text) + yield RequestsUrl(url=patch, crawl=self, is_patch=True) + else: + logger.info("Bugzilla result:{} is not fixed yet".format(self.cve)) + yield SavePipe( + crawl=self, + text="Bugzilla result:no patch information in Bugzilla and it has not been fixed yet", + ) + + def parse(self, response, **kwargs): + """ + Get the bug information for parsing + """ + self.fix_versions = self._extract_fixed_in_version( + bugs_info=Bugzilla.load_json(response.text) + ) + + yield RequestsUrl( + url=self.bugzilla_api.get_comments(fields=self.cve), + crawl=self, + callback=self.parse_comments, + ) diff --git a/tools/cve-tracking/src/cve/settings.py b/tools/cve-tracking/src/cve/settings.py new file mode 100644 index 0000000..a9468cd --- /dev/null +++ b/tools/cve-tracking/src/cve/settings.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ + + +# The default path to save CVE information +DEFAULT_SAVE_PATH = "/tmp/cve_tracking/match_cve" + +# The save path of the log file +LOG_DIR = "/tmp/cve_tracking/cve-log" + +# Internal source package information request address +INTERNAL_SERVER = "http://124.160.11.57:8100/git_command" + +# Enable patch lookup platform +ENABLE_PLANTFORM = ["Bugzilla", "Debian", "Ubuntu", "Nvd"] + +# The maximum number of threads allowed to live at the same time in the downloaded thread pool +MAX_WORKERS = 16 + +# Maximum number in the queue +MAX_QUEUE = 100 + +# The name of the file after platform information integration, which records the important information for finding CVE patch + +RECORD_FILE = "repair-verdict.txt" + +# Code cloud related API authorization operations, You need to set a global environment variable on the server, where you specify the name of the environment variable +GITEE_AUTH = {"token": "gitee_token", "account": "gitee_account"} diff --git a/tools/cve-tracking/src/cve/user-agent.json b/tools/cve-tracking/src/cve/user-agent.json new file mode 100644 index 0000000..92352d5 --- /dev/null +++ b/tools/cve-tracking/src/cve/user-agent.json @@ -0,0 +1,1251 @@ +{ + "browsers": { + "chrome": [ + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.1 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2226.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.4; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2225.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2062.124 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 4.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36", + "Mozilla/5.0 (X11; OpenBSD i386) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.125 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2309.372 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.47 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1866.237 Safari/537.36", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36 Mozilla/5.0 (iPad; U; CPU OS 3_2 like Mac OS X; en-us) AppleWebKit/531.21.10 (KHTML, like Gecko) Version/4.0.4 Mobile/7B334b Safari/531.21.10", + "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/33.0.1750.517 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1667.0 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.16 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1623.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/30.0.1599.17 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36", + "Mozilla/5.0 (X11; CrOS i686 4319.74.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.57 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.2 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1464.0 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.90 Safari/537.36", + "Mozilla/5.0 (X11; NetBSD) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", + "Mozilla/5.0 (X11; CrOS i686 3912.101.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.116 Safari/537.36", + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1312.60 Safari/537.17", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/537.17 (KHTML, like Gecko) Chrome/24.0.1309.0 Safari/537.17", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15", + "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.14 (KHTML, like Gecko) Chrome/24.0.1292.0 Safari/537.14" + ], + "opera": [ + "Opera/9.80 (X11; Linux i686; Ubuntu/14.10) Presto/2.12.388 Version/12.16", + "Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14", + "Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14", + "Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02", + "Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00", + "Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00", + "Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00", + "Opera/12.0(Windows NT 5.1;U;en)Presto/22.9.168 Version/12.00", + "Mozilla/5.0 (Windows NT 5.1) Gecko/20100101 Firefox/14.0 Opera/12.0", + "Opera/9.80 (Windows NT 6.1; WOW64; U; pt) Presto/2.10.229 Version/11.62", + "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.10.229 Version/11.62", + "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; fr) Presto/2.9.168 Version/11.52", + "Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; de) Presto/2.9.168 Version/11.52", + "Opera/9.80 (Windows NT 5.1; U; en) Presto/2.9.168 Version/11.51", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; de) Opera 11.51", + "Opera/9.80 (X11; Linux x86_64; U; fr) Presto/2.9.168 Version/11.50", + "Opera/9.80 (X11; Linux i686; U; hu) Presto/2.9.168 Version/11.50", + "Opera/9.80 (X11; Linux i686; U; ru) Presto/2.8.131 Version/11.11", + "Opera/9.80 (X11; Linux i686; U; es-ES) Presto/2.8.131 Version/11.11", + "Mozilla/5.0 (Windows NT 5.1; U; en; rv:1.8.1) Gecko/20061208 Firefox/5.0 Opera 11.11", + "Opera/9.80 (X11; Linux x86_64; U; bg) Presto/2.8.131 Version/11.10", + "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.8.99 Version/11.10", + "Opera/9.80 (Windows NT 5.1; U; zh-tw) Presto/2.8.131 Version/11.10", + "Opera/9.80 (Windows NT 6.1; Opera Tablet/15165; U; en) Presto/2.8.149 Version/11.1", + "Opera/9.80 (X11; Linux x86_64; U; Ubuntu/10.10 (maverick); pl) Presto/2.7.62 Version/11.01", + "Opera/9.80 (X11; Linux i686; U; ja) Presto/2.7.62 Version/11.01", + "Opera/9.80 (X11; Linux i686; U; fr) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; sv) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; en-US) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.1; U; cs) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 6.0; U; pl) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 5.1; U;) Presto/2.7.62 Version/11.01", + "Opera/9.80 (Windows NT 5.1; U; cs) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US; rv:1.9.2.13) Gecko/20101213 Opera/9.80 (Windows NT 6.1; U; zh-tw) Presto/2.7.62 Version/11.01", + "Mozilla/5.0 (Windows NT 6.1; U; nl; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", + "Mozilla/5.0 (Windows NT 6.1; U; de; rv:1.9.1.6) Gecko/20091201 Firefox/3.5.6 Opera 11.01", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; de) Opera 11.01", + "Opera/9.80 (X11; Linux x86_64; U; pl) Presto/2.7.62 Version/11.00", + "Opera/9.80 (X11; Linux i686; U; it) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; zh-cn) Presto/2.6.37 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; pl) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; ko) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; fi) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1; U; en-GB) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.1 x64; U; en) Presto/2.7.62 Version/11.00", + "Opera/9.80 (Windows NT 6.0; U; en) Presto/2.7.39 Version/11.00" + ], + "firefox": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:40.0) Gecko/20100101 Firefox/40.1", + "Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10; rv:33.0) Gecko/20100101 Firefox/33.0", + "Mozilla/5.0 (X11; Linux i586; rv:31.0) Gecko/20100101 Firefox/31.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:31.0) Gecko/20130401 Firefox/31.0", + "Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0", + "Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0", + "Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0", + "Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:27.0) Gecko/20121011 Firefox/27.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/25.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv:25.0) Gecko/20100101 Firefox/25.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:24.0) Gecko/20100101 Firefox/24.0", + "Mozilla/5.0 (Windows NT 6.0; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:24.0) Gecko/20100101 Firefox/24.0", + "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/23.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20130406 Firefox/23.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:23.0) Gecko/20131011 Firefox/23.0", + "Mozilla/5.0 (Windows NT 6.2; rv:22.0) Gecko/20130405 Firefox/22.0", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:22.0) Gecko/20130328 Firefox/22.0", + "Mozilla/5.0 (Windows NT 6.1; rv:22.0) Gecko/20130405 Firefox/22.0", + "Mozilla/5.0 (Microsoft Windows NT 6.2.9200.0); rv:22.0) Gecko/20130405 Firefox/22.0", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:16.0.1) Gecko/20121011 Firefox/21.0.1", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:21.0.0) Gecko/20121011 Firefox/21.0.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20130331 Firefox/21.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (X11; Linux i686; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.2; WOW64; rv:21.0) Gecko/20130514 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.2; rv:21.0) Gecko/20130326 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130401 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130331 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130330 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130328 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130401 Firefox/21.0", + "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20130331 Firefox/21.0", + "Mozilla/5.0 (Windows NT 5.1; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 5.0; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:21.0) Gecko/20100101 Firefox/21.0", + "Mozilla/5.0 (Windows NT 6.2; Win64; x64;) Gecko/20100101 Firefox/20.0", + "Mozilla/5.0 (Windows x86; rv:19.0) Gecko/20100101 Firefox/19.0", + "Mozilla/5.0 (Windows NT 6.1; rv:6.0) Gecko/20100101 Firefox/19.0", + "Mozilla/5.0 (Windows NT 6.1; rv:14.0) Gecko/20100101 Firefox/18.0.1", + "Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0) Gecko/20100101 Firefox/18.0", + "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0.6" + ], + "internetexplorer": [ + "Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; AS; rv:11.0) like Gecko", + "Mozilla/5.0 (compatible, MSIE 11, Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko", + "Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 7.0; InfoPath.3; .NET CLR 3.1.40767; Trident/6.0; en-IN)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", + "Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)", + "Mozilla/5.0 (compatible; MSIE 10.0; Macintosh; Intel Mac OS X 10_7_3; Trident/6.0)", + "Mozilla/4.0 (Compatible; MSIE 8.0; Windows NT 5.2; Trident/6.0)", + "Mozilla/4.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)", + "Mozilla/1.22 (compatible; MSIE 10.0; Windows 3.1)", + "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))", + "Mozilla/5.0 (Windows; U; MSIE 9.0; Windows NT 9.0; en-US)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; Media Center PC 6.0; InfoPath.3; MS-RTC LM 8; Zune 4.7", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; InfoPath.3; MS-RTC LM 8; .NET4.0C; .NET4.0E)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; chromeframe/12.0.742.112)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 2.0.50727; SLCC2; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 4.0; Tablet PC 2.0; InfoPath.3; .NET4.0C; .NET4.0E)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; yie8)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; InfoPath.2; .NET CLR 1.1.4322; .NET4.0C; Tablet PC 2.0)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; FunWebProducts)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/13.0.782.215)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0; chromeframe/11.0.696.57)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0) chromeframe/10.0.648.205", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.1; SV1; .NET CLR 2.8.52393; WOW64; en-US)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/5.0; chromeframe/11.0.696.57)", + "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0; Trident/4.0; GTB7.4; InfoPath.3; SV1; .NET CLR 3.1.76908; WOW64; en-US)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; GTB7.4; InfoPath.2; SV1; .NET CLR 3.3.69573; WOW64; en-US)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.8.36217; WOW64; en-US)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; .NET CLR 2.7.58687; SLCC2; Media Center PC 5.0; Zune 3.4; Tablet PC 3.6; InfoPath.3)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.2; Trident/4.0; Media Center PC 4.0; SLCC1; .NET CLR 3.0.04320)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 1.1.4322)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.1; SLCC1; .NET CLR 1.1.4322)", + "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 5.0; Trident/4.0; InfoPath.1; SV1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 3.0.04506.30)", + "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.0; Trident/4.0; FBSMTWB; .NET CLR 2.0.34861; .NET CLR 3.0.3746.3218; .NET CLR 3.5.33652; msn OptimizedIE8;ENUS)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.2; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; Media Center PC 6.0; InfoPath.2; MS-RTC LM 8", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; Media Center PC 6.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET4.0C)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.3; .NET4.0C; .NET4.0E; .NET CLR 3.5.30729; .NET CLR 3.0.30729; MS-RTC LM 8)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; InfoPath.2)", + "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; Zune 3.0)" + ], + "safari": [ + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_3) AppleWebKit/537.75.14 (KHTML, like Gecko) Version/7.0.3 Safari/7046A194A", + "Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10", + "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; de-at) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; da-dk) AppleWebKit/533.21.1 (KHTML, like Gecko) Version/5.0.5 Safari/533.21.1", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; tr-TR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; ko-KR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; fr-FR) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; cs-CZ) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; PPC Mac OS X 10_5_8; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; zh-cn) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; sv-se) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ko-kr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; ja-jp) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; it-it) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-fr) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; es-es) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-us) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; en-gb) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; de-de) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; sv-SE) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; hu-HU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; de-DE) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; ja-JP) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; it-IT) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_7; en-us) AppleWebKit/534.16+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; fr-ch) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; de-de) AppleWebKit/534.15+ (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_5; ar) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Android 2.2; Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.3 Safari/533.19.4", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; zh-HK) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 6.1; en-US) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; tr-TR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; nb-NO) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 6.0; fr-FR) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-TW) AppleWebKit/533.19.4 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Windows; U; Windows NT 5.1; ru-RU) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5", + "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; zh-cn) AppleWebKit/533.18.1 (KHTML, like Gecko) Version/5.0.2 Safari/533.18.5" + ] + }, + "randomize": { + "344": "chrome", + "819": "firefox", + "346": "chrome", + "347": "chrome", + "340": "chrome", + "341": "chrome", + "342": "chrome", + "343": "chrome", + "810": "internetexplorer", + "811": "internetexplorer", + "812": "internetexplorer", + "813": "firefox", + "348": "chrome", + "349": "chrome", + "816": "firefox", + "817": "firefox", + "737": "chrome", + "719": "chrome", + "718": "chrome", + "717": "chrome", + "716": "chrome", + "715": "chrome", + "714": "chrome", + "713": "chrome", + "712": "chrome", + "711": "chrome", + "710": "chrome", + "421": "chrome", + "129": "chrome", + "420": "chrome", + "423": "chrome", + "422": "chrome", + "425": "chrome", + "619": "chrome", + "424": "chrome", + "427": "chrome", + "298": "chrome", + "299": "chrome", + "296": "chrome", + "297": "chrome", + "294": "chrome", + "295": "chrome", + "292": "chrome", + "293": "chrome", + "290": "chrome", + "291": "chrome", + "591": "chrome", + "590": "chrome", + "593": "chrome", + "592": "chrome", + "595": "chrome", + "594": "chrome", + "597": "chrome", + "596": "chrome", + "195": "chrome", + "194": "chrome", + "197": "chrome", + "196": "chrome", + "191": "chrome", + "190": "chrome", + "193": "chrome", + "192": "chrome", + "270": "chrome", + "271": "chrome", + "272": "chrome", + "273": "chrome", + "274": "chrome", + "275": "chrome", + "276": "chrome", + "277": "chrome", + "278": "chrome", + "279": "chrome", + "569": "chrome", + "497": "chrome", + "524": "chrome", + "525": "chrome", + "526": "chrome", + "527": "chrome", + "520": "chrome", + "521": "chrome", + "522": "chrome", + "523": "chrome", + "528": "chrome", + "529": "chrome", + "449": "chrome", + "448": "chrome", + "345": "chrome", + "443": "chrome", + "442": "chrome", + "441": "chrome", + "440": "chrome", + "447": "chrome", + "446": "chrome", + "445": "chrome", + "444": "chrome", + "47": "chrome", + "108": "chrome", + "109": "chrome", + "102": "chrome", + "103": "chrome", + "100": "chrome", + "101": "chrome", + "106": "chrome", + "107": "chrome", + "104": "chrome", + "105": "chrome", + "902": "firefox", + "903": "firefox", + "39": "chrome", + "38": "chrome", + "906": "firefox", + "907": "firefox", + "904": "firefox", + "905": "firefox", + "33": "chrome", + "32": "chrome", + "31": "chrome", + "30": "chrome", + "37": "chrome", + "36": "chrome", + "35": "chrome", + "34": "chrome", + "641": "chrome", + "640": "chrome", + "643": "chrome", + "642": "chrome", + "645": "chrome", + "644": "chrome", + "438": "chrome", + "439": "chrome", + "436": "chrome", + "437": "chrome", + "434": "chrome", + "435": "chrome", + "432": "chrome", + "433": "chrome", + "430": "chrome", + "431": "chrome", + "826": "firefox", + "339": "chrome", + "338": "chrome", + "335": "chrome", + "334": "chrome", + "337": "chrome", + "336": "chrome", + "331": "chrome", + "330": "chrome", + "333": "chrome", + "332": "chrome", + "559": "chrome", + "745": "chrome", + "854": "firefox", + "818": "firefox", + "856": "firefox", + "857": "firefox", + "850": "firefox", + "851": "firefox", + "852": "firefox", + "0": "chrome", + "858": "firefox", + "859": "firefox", + "748": "chrome", + "6": "chrome", + "43": "chrome", + "99": "chrome", + "98": "chrome", + "91": "chrome", + "90": "chrome", + "93": "chrome", + "92": "chrome", + "95": "chrome", + "94": "chrome", + "97": "chrome", + "96": "chrome", + "814": "firefox", + "815": "firefox", + "153": "chrome", + "740": "chrome", + "741": "chrome", + "742": "chrome", + "743": "chrome", + "744": "chrome", + "558": "chrome", + "746": "chrome", + "747": "chrome", + "555": "chrome", + "554": "chrome", + "557": "chrome", + "556": "chrome", + "551": "chrome", + "550": "chrome", + "553": "chrome", + "552": "chrome", + "238": "chrome", + "239": "chrome", + "234": "chrome", + "235": "chrome", + "236": "chrome", + "237": "chrome", + "230": "chrome", + "231": "chrome", + "232": "chrome", + "233": "chrome", + "1": "chrome", + "155": "chrome", + "146": "chrome", + "147": "chrome", + "618": "chrome", + "145": "chrome", + "142": "chrome", + "143": "chrome", + "140": "chrome", + "141": "chrome", + "612": "chrome", + "613": "chrome", + "610": "chrome", + "611": "chrome", + "616": "chrome", + "617": "chrome", + "148": "chrome", + "149": "chrome", + "46": "chrome", + "154": "chrome", + "948": "safari", + "949": "safari", + "946": "safari", + "947": "safari", + "944": "safari", + "945": "safari", + "942": "safari", + "943": "safari", + "940": "safari", + "941": "safari", + "689": "chrome", + "688": "chrome", + "685": "chrome", + "684": "chrome", + "687": "chrome", + "686": "chrome", + "681": "chrome", + "680": "chrome", + "683": "chrome", + "682": "chrome", + "458": "chrome", + "459": "chrome", + "133": "chrome", + "132": "chrome", + "131": "chrome", + "130": "chrome", + "137": "chrome", + "136": "chrome", + "135": "chrome", + "134": "chrome", + "494": "chrome", + "495": "chrome", + "139": "chrome", + "138": "chrome", + "490": "chrome", + "491": "chrome", + "492": "chrome", + "493": "chrome", + "24": "chrome", + "25": "chrome", + "26": "chrome", + "27": "chrome", + "20": "chrome", + "21": "chrome", + "22": "chrome", + "23": "chrome", + "28": "chrome", + "29": "chrome", + "820": "firefox", + "407": "chrome", + "406": "chrome", + "405": "chrome", + "404": "chrome", + "403": "chrome", + "402": "chrome", + "401": "chrome", + "400": "chrome", + "933": "firefox", + "932": "firefox", + "931": "firefox", + "930": "firefox", + "937": "safari", + "452": "chrome", + "409": "chrome", + "408": "chrome", + "453": "chrome", + "414": "chrome", + "183": "chrome", + "415": "chrome", + "379": "chrome", + "378": "chrome", + "228": "chrome", + "829": "firefox", + "828": "firefox", + "371": "chrome", + "370": "chrome", + "373": "chrome", + "372": "chrome", + "375": "chrome", + "374": "chrome", + "377": "chrome", + "376": "chrome", + "708": "chrome", + "709": "chrome", + "704": "chrome", + "705": "chrome", + "706": "chrome", + "707": "chrome", + "700": "chrome", + "144": "chrome", + "702": "chrome", + "703": "chrome", + "393": "chrome", + "392": "chrome", + "88": "chrome", + "89": "chrome", + "397": "chrome", + "396": "chrome", + "395": "chrome", + "394": "chrome", + "82": "chrome", + "83": "chrome", + "80": "chrome", + "81": "chrome", + "86": "chrome", + "87": "chrome", + "84": "chrome", + "85": "chrome", + "797": "internetexplorer", + "796": "internetexplorer", + "795": "internetexplorer", + "794": "internetexplorer", + "793": "internetexplorer", + "792": "internetexplorer", + "791": "internetexplorer", + "790": "internetexplorer", + "749": "chrome", + "799": "internetexplorer", + "798": "internetexplorer", + "7": "chrome", + "170": "chrome", + "586": "chrome", + "587": "chrome", + "584": "chrome", + "585": "chrome", + "582": "chrome", + "583": "chrome", + "580": "chrome", + "581": "chrome", + "588": "chrome", + "589": "chrome", + "245": "chrome", + "244": "chrome", + "247": "chrome", + "246": "chrome", + "241": "chrome", + "614": "chrome", + "243": "chrome", + "242": "chrome", + "615": "chrome", + "249": "chrome", + "248": "chrome", + "418": "chrome", + "419": "chrome", + "519": "chrome", + "518": "chrome", + "511": "chrome", + "510": "chrome", + "513": "chrome", + "512": "chrome", + "515": "chrome", + "514": "chrome", + "517": "chrome", + "516": "chrome", + "623": "chrome", + "622": "chrome", + "621": "chrome", + "620": "chrome", + "627": "chrome", + "626": "chrome", + "625": "chrome", + "624": "chrome", + "450": "chrome", + "451": "chrome", + "629": "chrome", + "628": "chrome", + "454": "chrome", + "455": "chrome", + "456": "chrome", + "457": "chrome", + "179": "chrome", + "178": "chrome", + "177": "chrome", + "199": "chrome", + "175": "chrome", + "174": "chrome", + "173": "chrome", + "172": "chrome", + "171": "chrome", + "198": "chrome", + "977": "opera", + "182": "chrome", + "975": "opera", + "974": "opera", + "973": "opera", + "972": "opera", + "971": "opera", + "970": "opera", + "180": "chrome", + "979": "opera", + "978": "opera", + "656": "chrome", + "599": "chrome", + "654": "chrome", + "181": "chrome", + "186": "chrome", + "187": "chrome", + "184": "chrome", + "185": "chrome", + "652": "chrome", + "188": "chrome", + "189": "chrome", + "658": "chrome", + "653": "chrome", + "650": "chrome", + "651": "chrome", + "11": "chrome", + "10": "chrome", + "13": "chrome", + "12": "chrome", + "15": "chrome", + "14": "chrome", + "17": "chrome", + "16": "chrome", + "19": "chrome", + "18": "chrome", + "863": "firefox", + "862": "firefox", + "865": "firefox", + "864": "firefox", + "867": "firefox", + "866": "firefox", + "354": "chrome", + "659": "chrome", + "44": "chrome", + "883": "firefox", + "882": "firefox", + "881": "firefox", + "880": "firefox", + "887": "firefox", + "886": "firefox", + "885": "firefox", + "884": "firefox", + "889": "firefox", + "888": "firefox", + "116": "chrome", + "45": "chrome", + "657": "chrome", + "355": "chrome", + "322": "chrome", + "323": "chrome", + "320": "chrome", + "321": "chrome", + "326": "chrome", + "327": "chrome", + "324": "chrome", + "325": "chrome", + "328": "chrome", + "329": "chrome", + "562": "chrome", + "201": "chrome", + "200": "chrome", + "203": "chrome", + "202": "chrome", + "205": "chrome", + "204": "chrome", + "207": "chrome", + "206": "chrome", + "209": "chrome", + "208": "chrome", + "779": "internetexplorer", + "778": "internetexplorer", + "77": "chrome", + "76": "chrome", + "75": "chrome", + "74": "chrome", + "73": "chrome", + "72": "chrome", + "71": "chrome", + "70": "chrome", + "655": "chrome", + "567": "chrome", + "79": "chrome", + "78": "chrome", + "359": "chrome", + "358": "chrome", + "669": "chrome", + "668": "chrome", + "667": "chrome", + "666": "chrome", + "665": "chrome", + "664": "chrome", + "663": "chrome", + "662": "chrome", + "661": "chrome", + "660": "chrome", + "215": "chrome", + "692": "chrome", + "693": "chrome", + "690": "chrome", + "691": "chrome", + "696": "chrome", + "697": "chrome", + "694": "chrome", + "695": "chrome", + "698": "chrome", + "699": "chrome", + "542": "chrome", + "543": "chrome", + "540": "chrome", + "541": "chrome", + "546": "chrome", + "547": "chrome", + "544": "chrome", + "545": "chrome", + "8": "chrome", + "548": "chrome", + "549": "chrome", + "598": "chrome", + "869": "firefox", + "868": "firefox", + "120": "chrome", + "121": "chrome", + "122": "chrome", + "123": "chrome", + "124": "chrome", + "125": "chrome", + "126": "chrome", + "127": "chrome", + "128": "chrome", + "2": "chrome", + "219": "chrome", + "176": "chrome", + "214": "chrome", + "563": "chrome", + "928": "firefox", + "929": "firefox", + "416": "chrome", + "417": "chrome", + "410": "chrome", + "411": "chrome", + "412": "chrome", + "413": "chrome", + "920": "firefox", + "498": "chrome", + "922": "firefox", + "923": "firefox", + "924": "firefox", + "925": "firefox", + "926": "firefox", + "927": "firefox", + "319": "chrome", + "318": "chrome", + "313": "chrome", + "312": "chrome", + "311": "chrome", + "310": "chrome", + "317": "chrome", + "316": "chrome", + "315": "chrome", + "314": "chrome", + "921": "firefox", + "496": "chrome", + "832": "firefox", + "833": "firefox", + "830": "firefox", + "831": "firefox", + "836": "firefox", + "837": "firefox", + "834": "firefox", + "835": "firefox", + "838": "firefox", + "839": "firefox", + "3": "chrome", + "368": "chrome", + "369": "chrome", + "366": "chrome", + "367": "chrome", + "364": "chrome", + "365": "chrome", + "362": "chrome", + "363": "chrome", + "360": "chrome", + "361": "chrome", + "218": "chrome", + "380": "chrome", + "861": "firefox", + "382": "chrome", + "383": "chrome", + "384": "chrome", + "385": "chrome", + "386": "chrome", + "387": "chrome", + "388": "chrome", + "389": "chrome", + "784": "internetexplorer", + "785": "internetexplorer", + "786": "internetexplorer", + "787": "internetexplorer", + "780": "internetexplorer", + "781": "internetexplorer", + "782": "internetexplorer", + "381": "chrome", + "788": "internetexplorer", + "789": "internetexplorer", + "860": "firefox", + "151": "chrome", + "579": "chrome", + "578": "chrome", + "150": "chrome", + "573": "chrome", + "572": "chrome", + "571": "chrome", + "570": "chrome", + "577": "chrome", + "576": "chrome", + "575": "chrome", + "574": "chrome", + "60": "chrome", + "61": "chrome", + "62": "chrome", + "259": "chrome", + "64": "chrome", + "65": "chrome", + "66": "chrome", + "67": "chrome", + "68": "chrome", + "253": "chrome", + "250": "chrome", + "251": "chrome", + "256": "chrome", + "257": "chrome", + "254": "chrome", + "255": "chrome", + "499": "chrome", + "157": "chrome", + "156": "chrome", + "939": "safari", + "731": "chrome", + "730": "chrome", + "733": "chrome", + "938": "safari", + "735": "chrome", + "734": "chrome", + "508": "chrome", + "736": "chrome", + "506": "chrome", + "738": "chrome", + "504": "chrome", + "505": "chrome", + "502": "chrome", + "503": "chrome", + "500": "chrome", + "501": "chrome", + "630": "chrome", + "631": "chrome", + "632": "chrome", + "633": "chrome", + "469": "chrome", + "468": "chrome", + "636": "chrome", + "637": "chrome", + "465": "chrome", + "464": "chrome", + "467": "chrome", + "466": "chrome", + "461": "chrome", + "900": "firefox", + "463": "chrome", + "462": "chrome", + "901": "firefox", + "168": "chrome", + "169": "chrome", + "164": "chrome", + "165": "chrome", + "166": "chrome", + "167": "chrome", + "160": "chrome", + "161": "chrome", + "162": "chrome", + "163": "chrome", + "964": "safari", + "965": "safari", + "966": "safari", + "967": "safari", + "960": "safari", + "961": "safari", + "962": "safari", + "963": "safari", + "783": "internetexplorer", + "968": "safari", + "969": "opera", + "936": "firefox", + "935": "firefox", + "934": "firefox", + "908": "firefox", + "909": "firefox", + "722": "chrome", + "426": "chrome", + "878": "firefox", + "879": "firefox", + "876": "firefox", + "877": "firefox", + "874": "firefox", + "875": "firefox", + "872": "firefox", + "873": "firefox", + "870": "firefox", + "871": "firefox", + "9": "chrome", + "890": "firefox", + "891": "firefox", + "892": "firefox", + "893": "firefox", + "894": "firefox", + "647": "chrome", + "896": "firefox", + "897": "firefox", + "898": "firefox", + "899": "firefox", + "646": "chrome", + "649": "chrome", + "648": "chrome", + "357": "chrome", + "356": "chrome", + "809": "internetexplorer", + "808": "internetexplorer", + "353": "chrome", + "352": "chrome", + "351": "chrome", + "350": "chrome", + "803": "internetexplorer", + "802": "internetexplorer", + "801": "internetexplorer", + "800": "internetexplorer", + "807": "internetexplorer", + "806": "internetexplorer", + "805": "internetexplorer", + "804": "internetexplorer", + "216": "chrome", + "217": "chrome", + "768": "chrome", + "769": "chrome", + "212": "chrome", + "213": "chrome", + "210": "chrome", + "211": "chrome", + "762": "chrome", + "763": "chrome", + "760": "chrome", + "761": "chrome", + "766": "chrome", + "767": "chrome", + "764": "chrome", + "765": "chrome", + "40": "chrome", + "41": "chrome", + "289": "chrome", + "288": "chrome", + "4": "chrome", + "281": "chrome", + "280": "chrome", + "283": "chrome", + "282": "chrome", + "285": "chrome", + "284": "chrome", + "287": "chrome", + "286": "chrome", + "678": "chrome", + "679": "chrome", + "674": "chrome", + "675": "chrome", + "676": "chrome", + "677": "chrome", + "670": "chrome", + "671": "chrome", + "672": "chrome", + "673": "chrome", + "263": "chrome", + "262": "chrome", + "261": "chrome", + "260": "chrome", + "267": "chrome", + "266": "chrome", + "265": "chrome", + "264": "chrome", + "269": "chrome", + "268": "chrome", + "59": "chrome", + "58": "chrome", + "55": "chrome", + "54": "chrome", + "57": "chrome", + "56": "chrome", + "51": "chrome", + "258": "chrome", + "53": "chrome", + "52": "chrome", + "537": "chrome", + "536": "chrome", + "535": "chrome", + "63": "chrome", + "533": "chrome", + "532": "chrome", + "531": "chrome", + "530": "chrome", + "152": "chrome", + "539": "chrome", + "538": "chrome", + "775": "internetexplorer", + "774": "internetexplorer", + "982": "opera", + "983": "opera", + "980": "opera", + "981": "opera", + "777": "internetexplorer", + "984": "opera", + "50": "chrome", + "115": "chrome", + "252": "chrome", + "117": "chrome", + "776": "internetexplorer", + "111": "chrome", + "110": "chrome", + "113": "chrome", + "69": "chrome", + "771": "chrome", + "119": "chrome", + "118": "chrome", + "770": "chrome", + "773": "internetexplorer", + "772": "internetexplorer", + "429": "chrome", + "428": "chrome", + "534": "chrome", + "919": "firefox", + "918": "firefox", + "915": "firefox", + "914": "firefox", + "917": "firefox", + "916": "firefox", + "911": "firefox", + "910": "firefox", + "913": "firefox", + "912": "firefox", + "308": "chrome", + "309": "chrome", + "855": "firefox", + "300": "chrome", + "301": "chrome", + "302": "chrome", + "303": "chrome", + "304": "chrome", + "305": "chrome", + "306": "chrome", + "307": "chrome", + "895": "firefox", + "825": "firefox", + "824": "firefox", + "827": "firefox", + "847": "firefox", + "846": "firefox", + "845": "firefox", + "844": "firefox", + "843": "firefox", + "842": "firefox", + "841": "firefox", + "840": "firefox", + "821": "firefox", + "853": "firefox", + "849": "firefox", + "848": "firefox", + "823": "firefox", + "822": "firefox", + "240": "chrome", + "390": "chrome", + "732": "chrome", + "753": "chrome", + "752": "chrome", + "751": "chrome", + "750": "chrome", + "757": "chrome", + "756": "chrome", + "755": "chrome", + "754": "chrome", + "560": "chrome", + "561": "chrome", + "759": "chrome", + "758": "chrome", + "564": "chrome", + "565": "chrome", + "566": "chrome", + "701": "chrome", + "739": "chrome", + "229": "chrome", + "507": "chrome", + "227": "chrome", + "226": "chrome", + "225": "chrome", + "224": "chrome", + "223": "chrome", + "222": "chrome", + "221": "chrome", + "220": "chrome", + "114": "chrome", + "391": "chrome", + "726": "chrome", + "727": "chrome", + "724": "chrome", + "725": "chrome", + "568": "chrome", + "723": "chrome", + "720": "chrome", + "721": "chrome", + "728": "chrome", + "729": "chrome", + "605": "chrome", + "604": "chrome", + "607": "chrome", + "606": "chrome", + "601": "chrome", + "600": "chrome", + "603": "chrome", + "602": "chrome", + "159": "chrome", + "158": "chrome", + "112": "chrome", + "609": "chrome", + "608": "chrome", + "976": "opera", + "634": "chrome", + "399": "chrome", + "635": "chrome", + "959": "safari", + "958": "safari", + "398": "chrome", + "48": "chrome", + "49": "chrome", + "951": "safari", + "950": "safari", + "953": "safari", + "952": "safari", + "42": "chrome", + "954": "safari", + "957": "safari", + "956": "safari", + "638": "chrome", + "5": "chrome", + "639": "chrome", + "460": "chrome", + "489": "chrome", + "488": "chrome", + "487": "chrome", + "486": "chrome", + "485": "chrome", + "484": "chrome", + "483": "chrome", + "482": "chrome", + "481": "chrome", + "480": "chrome", + "509": "chrome", + "955": "safari", + "472": "chrome", + "473": "chrome", + "470": "chrome", + "471": "chrome", + "476": "chrome", + "477": "chrome", + "474": "chrome", + "475": "chrome", + "478": "chrome", + "479": "chrome" + } +} \ No newline at end of file diff --git a/tools/cve-tracking/src/shell/__init__.py b/tools/cve-tracking/src/shell/__init__.py new file mode 100644 index 0000000..3e26da5 --- /dev/null +++ b/tools/cve-tracking/src/shell/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ diff --git a/tools/cve-tracking/src/shell/add_patch.sh b/tools/cve-tracking/src/shell/add_patch.sh new file mode 100644 index 0000000..a0658e7 --- /dev/null +++ b/tools/cve-tracking/src/shell/add_patch.sh @@ -0,0 +1,212 @@ +#!/bin/bash +# shellcheck disable=SC2068 +# shellcheck disable=SC2164 +# shellcheck disable=SC2010 +# shellcheck disable=SC2063 +# shellcheck disable=SC2062 +# shellcheck disable=SC2035 +source_path="" +rpm_name="" +rpm_branch="" +patches_path="" +path_file_name="" +tmp_path=/tmp/cve_tracking/download_source +# The rpm package that needs to be installed when compiling +requires_rpms="" +spec_file="" +home_path=$( + cd ~ + pwd -P +) +root_build_path=${home_path}/"rpmbuild" + +function check_patch() { + if [[ -z ${patches_path} ]]; then + echo "[ERROR] Input patch is null" + exit 1 + fi + + for patch in ${patches_path[@]}; do + if [[ ! -f ${patch} ]]; then + echo "[ERROR] ${patch} is not found" + exit 1 + fi + done +} + +function check_local() { + if [[ ! -d ${source_path} ]]; then + echo "[ERROR] The source package path ${source_path} does not exist" + exit 1 + fi + + check_patch +} + +function check_remote() { + if [[ -z ${rpm_name} ]] || [[ -z ${rpm_branch} ]]; then + echo "[ERROR] Incorrect input,Please use [ /bin/bash add_patch.sh 'name of package' 'branch of package' 'path of patch file']" + exit 1 + fi + + check_patch +} + +function pre_env_download() { + if [ ! -d ${tmp_path} ]; then + mkdir -p ${tmp_path} + echo "[INFO] Create tmp path ${tmp_path}" + fi + cd ${tmp_path} + rm -rf "${rpm_name}" +} + +function pre_env_build() { + echo "[INFO] Create root path of rpmbuild" + if [[ ! -d ${root_build_path} ]]; then + mkdir -p ${root_build_path}/{BUILD,BUILDROOT,RPMS,SPECS,SOURCES,SRPMS} + fi +} + +function git_clone() { + echo "[INFO] Start to git clone ${rpm_name}" + git_status=$(rpm -qa git) + if [[ -z ${git_status} ]]; then + yum install git -y + git_status=$(rpm -qa git) + if [[ -z ${git_status} ]]; then + echo "[ERROR] Git install failed" + exit 1 + fi + fi + + cd ${tmp_path} + git clone -b ${rpm_branch} https://gitee.com/src-openeuler/${rpm_name}.git >/dev/null 2>&1 + rpm_path=$(ls | grep ${rpm_name}) + if [[ -z ${rpm_path} ]]; then + echo "[ERROR] git clone ${rpm_name} failed, please check path ${tmp_path}" + exit 1 + fi + + source_path=${tmp_path}/${rpm_name} +} + +function update_spec() { + echo "[INFO] Start to update spec file" + cd ${source_path} + spec_file=$(ls | grep *.spec | grep -v *.bak) + if [[ -z ${spec_file} ]]; then + echo "[ERROR] spec file is not found" + exit 1 + fi + # backup spec + /bin/cp ${spec_file} ${spec_file}.bak + # update Release + release_version=$(grep "Release:" ${spec_file} | awk -F " " '{print $NF}') + new_release=$(expr ${release_version} + 1) + sed -i "s/Release:.*${release_version}/Release: ${new_release}/" ${spec_file} + # add Patch*** + last_patch=$(grep "Patch.*:" ${spec_file} | sed -n '$p') + if [[ -z ${last_patch} ]]; then + source_row=$(grep -n "Source.*:" ${spec_file} | sed -n '$p' | awk -F ':' '{print $1}') + sed -ie "${source_row}G;${source_row}a Patch0000: ${path_file_name}" ${spec_file} + else + last_patch_row=$(grep -n "${last_patch}" ${spec_file} | awk -F ':' '{print $1}') + last_patch_num=$(echo ${last_patch} | awk -F ':' '{print $1}' | awk -F 'Patch' '{print $2}') + patch_name_len=${#last_patch_num} + new_patch_num=$(expr ${last_patch_num} + 1) + new_patch_num=$(printf "%0${patch_name_len}d" ${new_patch_num}) + sed -i "${last_patch_row}a Patch${new_patch_num}: ${path_file_name}" ${spec_file} + fi + # add changelog + change_log_row=$(grep -n '%changelog' ${spec_file} | sed -n '$p' | awk -F ':' '{print $1}') + date_now=$(date '+%a %b %d %Y') + version=$(grep 'Version:' ${spec_file} | awk -F ' ' '{print $NF}') + log_description="- add ${path_file_name}" + log_title="* ${date_now} robot - ${version}-${new_release}" + sed -i "${change_log_row}G" ${spec_file} + sed -i "${change_log_row}a ${log_description}" ${spec_file} + sed -i "${change_log_row}a ${log_title}" ${spec_file} + + echo "[INFO] Update spec file success" +} + +function mv_source_file() { + echo "[INFO] Copy source file to ${root_build_path}" + cd ${source_path} + spec_file=$(ls | grep *.spec | grep -v *.bak) + /bin/cp ${patches_path} ${source_path} + /bin/cp * ${root_build_path}/SOURCES + /bin/cp *.spec ${root_build_path}/SPECS +} + +function rpm_build() { + echo "[INFO] Start to rpmbuild" + rpmbuild_status=$(rpm -qa rpm-build) + if [[ -z ${rpmbuild_status} ]]; then + yum install rpm-build -y >/dev/null 2>&1 + rpmbuild_status=$(rpm -qa rpm-build) + if [[ -z ${rpmbuild_status} ]]; then + echo "[ERROR] Install rpm-build failed" + exit 1 + fi + fi + rpmbuild -bb ${root_build_path}/SPECS/${spec_file} >./rpmbuild.log 2>&1 + if [[ $? -eq 0 ]]; then + echo "[INFO] build success" + exit 0 + elif [[ -n $(grep "Failed build dependencies" ./rpmbuild.log) ]]; then + requires_rpms=$(grep -r "is needed by" ./rpmbuild.log | awk -F " " '{print $1}') + echo "${requires_rpms}" >./requires_rpms.log + for rpm in ${requires_rpms[@]}; do + yum install ${rpm} -y >/dev/null 2>&1 + if [[ $? -eq 0 ]]; then + echo "[INFO] Successfully install dependent package ${rpm}" + else + echo "[ERROR] Failed to install dependent package ${rpm}" + exit 1 + fi + done + rpm_build + else + echo "[ERROR] build failed,log is ${source_path}/rpmbuild.log" + exit 1 + fi +} + +function main() { + input_params_length=$# + if [[ ${input_params_length} -eq 2 ]]; then + source_path=$1 + patches_path=$2 + check_local + pre_env_build + for patch in ${patches_path[@]}; do + path_file_name=$(echo "${patch}" | awk -F "/" '{print $NF}' | awk -F "\\" '{print $NF}') + update_spec + done + elif [[ ${input_params_length} -eq 3 ]]; then + rpm_name=$1 + rpm_branch=$2 + patches_path=$3 + check_remote + pre_env_build + pre_env_download + git_clone + for patch in ${patches_path[@]}; do + path_file_name=$(echo "${patch}" | awk -F "/" '{print $NF}' | awk -F "\\" '{print $NF}') + update_spec + done + else + echo "[ERROR] Incorrect input,please use: + 1、 /bin/bash add_patch.sh 'Source package path' 'Patch file path' + 2、 /bin/bash add_patch.sh 'name of package' 'branch of package' 'path of patch file'" + exit 1 + fi + + mv_source_file + rpm_build +} + +main "$@" +exit $? diff --git a/tools/cve-tracking/src/shell/start.sh b/tools/cve-tracking/src/shell/start.sh new file mode 100644 index 0000000..2209374 --- /dev/null +++ b/tools/cve-tracking/src/shell/start.sh @@ -0,0 +1,104 @@ +#!/bin/bash +# shellcheck disable=SC2068 +# shellcheck disable=SC2164 +# shellcheck disable=SC2002 +# shellcheck disable=SC2045 +# shellcheck disable=SC2115 +rpms_path=$1 +clear_cli=$1 +remote_branch=$2 +current_path=$( + cd "$(dirname "$0")" + pwd +) + +function check() { + if [[ ! -d ${rpms_path} ]]; then + echo "[ERROR] ${rpms_path} is not exist" + exit 1 + fi + + if [[ -z ${remote_branch} ]]; then + echo "[ERROR] The remote branch is not specified" + exit 1 + fi +} + +function install_git() { + git_status=$(rpm -qa git) + if [[ -z ${git_status} ]]; then + yum install git -y + git_status=$(rpm -qa git) + if [[ -z ${git_status} ]]; then + echo "[ERROR] Git install failed" + exit 1 + fi + fi +} + +function run() { + check + + result_file=${rpms_path}/result.log + if [[ -f ${result_file} ]]; then + echo "" >"${result_file}" + fi + + rpm_cve_path=$(ls "${rpms_path}") + IFS=$'\n' + for rpm_cve in ${rpm_cve_path[@]}; do + if [[ -d ${rpms_path}/${rpm_cve} ]]; then + rpm_name=$(echo "${rpm_cve}" | awk -F '-CVE' '{print $1}') + patches=$(find "${rpms_path}/${rpm_cve}" -name "*.patch" | sort) + if [[ -z ${patches} ]]; then + echo "[WARRING] ${rpm_name} not found patch" | tee -a "${result_file}" + echo "=============================================================" + else + install_git + patch_ids="" + new_patches="" + for patch in ${patches[@]}; do + git_patch_ids=$(cat "${patch}" | git patch-id --stable) + if [[ ! ${patch_ids} =~ ${git_patch_ids} ]]; then + new_patches="${new_patches} ${patch}" + patch_ids="${patch_ids} ${git_patch_ids}" + fi + done + IFS=$' \t\n' + for branch in ${remote_branch[@]}; do + /bin/bash "${current_path}"/add_patch.sh "${rpm_name}" "${branch}" "${new_patches}" + add_patch_result=$? + if [[ ${add_patch_result} -eq 0 ]]; then + echo "[INFO] ${rpm_name} of ${branch} apply patch ${new_patches} successfully" | tee -a "${result_file}" + else + echo "[ERROR] ${rpm_name} of ${branch} failed to apply patch ${new_patches}" | tee -a "${result_file}" + fi + done + fi + fi + echo "=============================================================" + IFS=$'\n' + done +} + +function clear_env() { + echo "[INFO] clear env" + rm -rf /tmp/cve_tracking/download_source + build_root=$( + cd ~ + pwd -P + ) + rpmbuild_path=${build_root}/"rpmbuild" + + if [[ -d ${rpmbuild_path} ]]; then + for dir in $(ls ${rpmbuild_path}); do + rm -rf ${rpmbuild_path}/${dir}/* + done + fi +} + +if [[ ${clear_cli} == "clear" ]]; then + clear_env +else + run +fi diff --git a/tools/cve-tracking/test/__init__.py b/tools/cve-tracking/test/__init__.py new file mode 100644 index 0000000..6851032 --- /dev/null +++ b/tools/cve-tracking/test/__init__.py @@ -0,0 +1,12 @@ +#!/usr/bin/python3 +# ****************************************************************************** +# Copyright (c) Huawei Technologies Co., Ltd. 2020-2020. All rights reserved. +# licensed under the Mulan PSL v2. +# You can use this software according to the terms and conditions of the Mulan PSL v2. +# You may obtain a copy of Mulan PSL v2 at: +# http://license.coscl.org.cn/MulanPSL2 +# THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +# PURPOSE. +# See the Mulan PSL v2 for more details. +# ******************************************************************************/ -- Gitee From 3faa177b430d533678864a86e99cccdef1dac342 Mon Sep 17 00:00:00 2001 From: lhwerico Date: Thu, 1 Jul 2021 16:43:06 +0800 Subject: [PATCH 2/2] update doc --- tools/cve-tracking/README.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/tools/cve-tracking/README.md b/tools/cve-tracking/README.md index c3a5f26..38f8bfe 100644 --- a/tools/cve-tracking/README.md +++ b/tools/cve-tracking/README.md @@ -1,12 +1,15 @@ -# cve-tracking初版使用说明 +# cve-tracking -## 一、代码下载： +## 一、流程图 +![流程](https://images.gitee.com/uploads/images/2021/0701/114945_50b243d5_8079354.png "屏幕截图.png") + +## 二、代码下载： ```shell git clone https://gitee.com/gongzt/cve.git ``` -## 二、权限修改： +## 三、权限修改： ```shell chmod 755 -R xxx/cve(下载代码所在路径) @@ -25,7 +28,7 @@ chmod 755 -R xxx/cve(下载代码所在路径) > root@localhost:/opt/project/cve_tracking > ▶ **chmod 755 -R cve** -## 三、执行代码 +## 四、执行代码 首先需要安装依赖的python插件（仅第一次使用时需要）： -- Gitee