diff --git a/0001-add-file-sync-func.patch b/0001-add-file-sync-func.patch deleted file mode 100644 index a5d1b6abf548110e716082e93327983945eee224..0000000000000000000000000000000000000000 --- a/0001-add-file-sync-func.patch +++ /dev/null @@ -1,141 +0,0 @@ -From 0e2ad3cc141390e26d7a20bccf5ccc0660f7c172 Mon Sep 17 00:00:00 2001 -From: smjiao -Date: Thu, 7 Sep 2023 16:24:42 +0800 -Subject: [PATCH] add file sync func -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - zeus/conf/constant.py | 2 ++ - zeus/config_manager/view.py | 45 ++++++++++++++++++++++++++++++++-- - zeus/function/verify/config.py | 9 +++++++ - zeus/url.py | 6 ++++- - 4 files changed, 59 insertions(+), 3 deletions(-) - -diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py -index 2fbf1be..3175c65 100644 ---- a/zeus/conf/constant.py -+++ b/zeus/conf/constant.py -@@ -33,6 +33,7 @@ CERES_CVE_REPO_SET = "aops-ceres apollo --set-repo '%s'" - CERES_CVE_SCAN = "aops-ceres apollo --scan '%s'" - CERES_CVE_FIX = "aops-ceres apollo --fix '%s'" - CERES_CVE_ROLLBACK = "aops-ceres apollo --rollback '%s'" -+CERES_SYNC_CONF = "aops-ceres sync --conf '%s'" - - # zeus route - ADD_HOST = "/manage/host/add" -@@ -54,6 +55,7 @@ DELETE_GROUP = "/manage/host/group/delete" - GET_GROUP = "/manage/host/group/get" - - COLLECT_CONFIG = '/manage/config/collect' -+SYNC_CONFIG = '/manage/config/sync' - - USER_LOGIN = "/manage/account/login" - LOGOUT = "/manage/account/logout" -diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py -index cd83e35..baeef7e 100644 ---- a/zeus/config_manager/view.py -+++ b/zeus/config_manager/view.py -@@ -21,10 +21,11 @@ from typing import List, Dict - from vulcanus.multi_thread_handler import MultiThreadHandler - from vulcanus.restful.resp import state - from vulcanus.restful.response import BaseResponse --from zeus.conf.constant import CERES_COLLECT_FILE -+from zeus.conf import configuration -+from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF - from zeus.database.proxy.host import HostProxy - from zeus.function.model import ClientConnectArgs --from zeus.function.verify.config import CollectConfigSchema -+from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema - from zeus.host_manager.ssh import execute_command_and_parse_its_result - - -@@ -218,3 +219,43 @@ class CollectConfig(BaseResponse): - return self.response( - state.SUCCEED, None, self.generate_target_data_format(multi_thread.get_result(), host_id_with_config_file) - ) -+ -+ -+class SyncConfig(BaseResponse): -+ -+ @staticmethod -+ def sync_config_content(host_info: Dict, sync_config_info: Dict): -+ command = CERES_SYNC_CONF % json.dumps(sync_config_info) -+ status, content = execute_command_and_parse_its_result( -+ ClientConnectArgs(host_info.get("host_ip"), host_info.get("ssh_port"), -+ host_info.get("ssh_user"), host_info.get("pkey")), command) -+ return status -+ -+ @BaseResponse.handle(schema=SyncConfigSchema, token=False) -+ def put(self, **params): -+ -+ sync_config_info = dict() -+ sync_config_info['file_path'] = params.get('file_path') -+ sync_config_info['content'] = params.get('content') -+ -+ sync_result = { -+ "file_path": sync_config_info['file_path'], -+ "sync_result": False -+ } -+ -+ # Query host address from database -+ proxy = HostProxy(configuration) -+ if not proxy.connect(): -+ return self.response(code=state.DATABASE_CONNECT_ERROR, data={"resp": sync_result}) -+ -+ status, host_list = proxy.get_host_info( -+ {"username": "admin", "host_list": [params.get('host_id')]}, True) -+ if status != state.SUCCEED or len(host_list) == 1: -+ return self.response(code=status, data={"resp": sync_result}) -+ -+ host_info = host_list[0] -+ status = self.sync_config_content(host_info, sync_config_info) -+ if status == state.SUCCEED: -+ sync_result['sync_result'] = True -+ return self.response(code=state.SUCCEED, data={"resp": sync_result}) -+ return self.response(code=state.UNKNOWN_ERROR, data={"resp": sync_result}) -diff --git a/zeus/function/verify/config.py b/zeus/function/verify/config.py -index 230b65d..6e5bf64 100644 ---- a/zeus/function/verify/config.py -+++ b/zeus/function/verify/config.py -@@ -36,3 +36,12 @@ class CollectConfigSchema(Schema): - """ - - infos = fields.List(fields.Nested(ConfigSchema(), required=True), required=True, validate=lambda s: len(s) > 0) -+ -+ -+class SyncConfigSchema(Schema): -+ """ -+ validators for SyncConfigSchema -+ """ -+ host_id = fields.Integer(required=True, validate=lambda s: s > 0) -+ file_path = fields.String(required=True, validate=lambda s: len(s) > 0) -+ content = fields.String(required=True, validate=lambda s: len(s) > 0) -diff --git a/zeus/url.py b/zeus/url.py -index e0cf1de..597dcc7 100644 ---- a/zeus/url.py -+++ b/zeus/url.py -@@ -49,6 +49,7 @@ from zeus.conf.constant import ( - REFRESH_TOKEN, - UPDATE_HOST, - USER_LOGIN, -+ SYNC_CONFIG, - ) - from zeus.config_manager import view as config_view - from zeus.host_manager import view as host_view -@@ -83,7 +84,10 @@ SPECIFIC_URLS = { - (host_view.DeleteHostGroup, DELETE_GROUP), - (host_view.GetHostGroup, GET_GROUP), - ], -- "CONFIG_URLS": [(config_view.CollectConfig, COLLECT_CONFIG)], -+ "CONFIG_URLS": [ -+ (config_view.CollectConfig, COLLECT_CONFIG), -+ (config_view.SyncConfig, SYNC_CONFIG) -+ ], - 'AGENT_URLS': [ - (agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO), - (agent_view.SetAgentPluginStatus, AGENT_PLUGIN_SET), --- -Gitee - diff --git a/0001-fix-metric-proxy-init-failed-error.patch b/0001-fix-metric-proxy-init-failed-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..ca84d256b08f50312c9910ecd4ba85a97a7a5661 --- /dev/null +++ b/0001-fix-metric-proxy-init-failed-error.patch @@ -0,0 +1,46 @@ +From 833a9d721bed5b3e64ea49710a477a52b74c5255 Mon Sep 17 00:00:00 2001 +From: zhangdaolong +Date: Thu, 12 Oct 2023 14:28:06 +0800 +Subject: [PATCH 1/1] fix metric proxy init failed error +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + zeus/database/proxy/metric.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/zeus/database/proxy/metric.py b/zeus/database/proxy/metric.py +index e899e26..5fa75e9 100644 +--- a/zeus/database/proxy/metric.py ++++ b/zeus/database/proxy/metric.py +@@ -21,6 +21,7 @@ from prometheus_api_client import PrometheusApiClientException + from vulcanus.database.proxy import PromDbProxy + from vulcanus.log.log import LOGGER + from vulcanus.restful.resp.state import SUCCEED, DATABASE_QUERY_ERROR, NO_DATA, PARAM_ERROR, PARTIAL_SUCCEED ++from zeus.conf import configuration + + + class MetricProxy(PromDbProxy): +@@ -28,16 +29,15 @@ class MetricProxy(PromDbProxy): + Proxy of prometheus time series database + """ + +- def __init__(self, configuration, host=None, port=None): ++ def __init__(self, host=None, port=None): + """ + Init MetricProxy + + Args: +- configuration (Config) + host (str) + port (int) + """ +- PromDbProxy.__init__(self, configuration, host, port) ++ PromDbProxy.__init__(self, host, port) + self.default_instance_port = configuration.agent.get('DEFAULT_INSTANCE_PORT') or 9100 + self.query_range_step = configuration.prometheus.get('QUERY_RANGE_STEP') or "15s" + +-- +2.33.0 + diff --git a/0002-add-key-authentication-for-add-host-api.patch b/0002-add-key-authentication-for-add-host-api.patch new file mode 100644 index 0000000000000000000000000000000000000000..8a3c9e166e9936d4616345c3720d024990c7c2f2 --- /dev/null +++ b/0002-add-key-authentication-for-add-host-api.patch @@ -0,0 +1,298 @@ +From 7a8164696bb913a75cf79cf6b57c9973530efefa Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Sun, 15 Oct 2023 16:37:55 +0800 +Subject: [PATCH 1/1] add a way about key authentication for add host api +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + database/zeus.sql | 2 +- + zeus/conf/constant.py | 6 +-- + zeus/database/table.py | 2 +- + zeus/function/verify/host.py | 4 +- + zeus/host_manager/view.py | 99 ++++++++++++++++++++++++++++-------- + 5 files changed, 85 insertions(+), 28 deletions(-) + +diff --git a/database/zeus.sql b/database/zeus.sql +index 3dc9f3c..7db734e 100644 +--- a/database/zeus.sql ++++ b/database/zeus.sql +@@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS `host` ( + `os_version` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, + `ssh_user` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, + `ssh_port` int(11) NULL DEFAULT NULL, +- `pkey` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, ++ `pkey` varchar(4096) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, + `status` int(11) NULL DEFAULT NULL, + `user` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, + `host_group_id` int(11) NULL DEFAULT NULL, +diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py +index 3175c65..bf8792a 100644 +--- a/zeus/conf/constant.py ++++ b/zeus/conf/constant.py +@@ -90,9 +90,9 @@ CHECK_IDENTIFY_SCENE = "/check/scene/identify" + CHECK_WORKFLOW_HOST_EXIST = '/check/workflow/host/exist' + + # host template file content +-HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,host_name,host_group_name,management +-test_ip_1,22,root,password,test_host,test_host_group,False +-test_ip_2,22,root,password,test_host,test_host_group,False ++HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,ssh_pkey,host_name,host_group_name,management ++test_ip_1,22,root,password,ssh_pkey,test_host,test_host_group,False ++test_ip_2,22,root,password,ssh_pkey,test_host,test_host_group,False + """ + + +diff --git a/zeus/database/table.py b/zeus/database/table.py +index 9596492..265eb45 100644 +--- a/zeus/database/table.py ++++ b/zeus/database/table.py +@@ -59,7 +59,7 @@ class Host(Base, MyBase): # pylint: disable=R0903 + os_version = Column(String(40)) + ssh_user = Column(String(40), default="root") + ssh_port = Column(Integer(), default=22) +- pkey = Column(String(2048)) ++ pkey = Column(String(4096)) + status = Column(Integer(), default=2) + + user = Column(String(40), ForeignKey('user.username')) +diff --git a/zeus/function/verify/host.py b/zeus/function/verify/host.py +index b054d62..d09eedd 100644 +--- a/zeus/function/verify/host.py ++++ b/zeus/function/verify/host.py +@@ -103,11 +103,12 @@ class AddHostSchema(Schema): + """ + + ssh_user = fields.String(required=True, validate=lambda s: len(s) > 0) +- password = fields.String(required=True, validate=lambda s: len(s) > 0) ++ password = fields.String(required=True, allow_none=True, validate=lambda s: len(s) >= 0) + host_name = fields.String( + required=True, validate=[validate.Length(min=1, max=50), ValidateRules.space_character_check] + ) + host_ip = fields.IP(required=True) ++ ssh_pkey = fields.String(required=True, allow_none=True, validate=lambda s: 4096 >= len(s) >= 0) + ssh_port = fields.Integer(required=True, validate=lambda s: 65535 >= s > 0) + host_group_name = fields.String(required=True, validate=lambda s: len(s) > 0) + management = fields.Boolean(required=True) +@@ -133,3 +134,4 @@ class UpdateHostSchema(Schema): + host_name = fields.String(required=False, validate=lambda s: len(s) > 0) + host_group_name = fields.String(required=False, validate=lambda s: len(s) > 0) + management = fields.Boolean(required=False) ++ ssh_pkey = fields.String(required=False, validate=lambda s: 4096 >= len(s) >= 0) +diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py +index 768d2cd..95e1434 100644 +--- a/zeus/host_manager/view.py ++++ b/zeus/host_manager/view.py +@@ -16,12 +16,13 @@ Author: + Description: Restful APIs for host + """ + import json +-from io import BytesIO ++from io import BytesIO, StringIO + from typing import Iterable, List, Tuple, Union + import socket + + import gevent + import paramiko ++from paramiko.ssh_exception import SSHException + from flask import request, send_file + from marshmallow import Schema + from marshmallow.fields import Boolean +@@ -333,7 +334,8 @@ class AddHost(BaseResponse): + "host_ip":"127.0.0.1", + "ssh_port":"22", + "management":false, +- "username": "admin" ++ "username": "admin", ++ "ssh_pkey": "RSA key" + } + + Returns: +@@ -363,6 +365,7 @@ class AddHost(BaseResponse): + "ssh_port": host_info.get("ssh_port"), + "user": host_info.get("username"), + "management": host_info.get("management"), ++ "pkey": host_info.get("ssh_pkey"), + } + ) + if host in hosts: +@@ -384,7 +387,8 @@ class AddHost(BaseResponse): + "host_ip":"127.0.0.1", + "ssh_port":"22", + "management":false, +- "username": "admin" ++ "username": "admin", ++ "ssh_pkey": "RSA key" + } + + Returns: +@@ -396,15 +400,55 @@ class AddHost(BaseResponse): + if status != state.SUCCEED: + return self.response(code=status) + +- status, private_key = save_ssh_public_key_to_client( +- params.get('host_ip'), params.get('ssh_port'), params.get('ssh_user'), params.get('password') +- ) +- if status == state.SUCCEED: +- host.pkey = private_key +- host.status = HostStatus.ONLINE ++ if params.get("ssh_pkey"): ++ status = verify_ssh_login_info( ++ ClientConnectArgs( ++ params.get("host_ip"), params.get("ssh_port"), params.get("ssh_user"), params.get("ssh_pkey") ++ ) ++ ) ++ host.status = HostStatus.ONLINE if status == state.SUCCEED else HostStatus.UNESTABLISHED ++ else: ++ status, private_key = save_ssh_public_key_to_client( ++ params.get('host_ip'), params.get('ssh_port'), params.get('ssh_user'), params.get('password') ++ ) ++ if status == state.SUCCEED: ++ host.pkey = private_key ++ host.status = HostStatus.ONLINE + return self.response(code=self.proxy.add_host(host)) + + ++def verify_ssh_login_info(ssh_login_info: ClientConnectArgs) -> str: ++ """ ++ Verify that the ssh login information is correct ++ ++ Args: ++ ssh_login_info(ClientConnectArgs): e.g ++ ClientConnectArgs(host_ip='127.0.0.1', ssh_port=22, ssh_user='root', pkey=RSAKey string) ++ ++ Returns: ++ status code ++ """ ++ try: ++ client = SSH( ++ ip=ssh_login_info.host_ip, ++ username=ssh_login_info.ssh_user, ++ port=ssh_login_info.ssh_port, ++ pkey=paramiko.RSAKey.from_private_key(StringIO(ssh_login_info.pkey)), ++ ) ++ client.close() ++ except socket.error as error: ++ LOGGER.error(error) ++ return state.SSH_CONNECTION_ERROR ++ except SSHException as error: ++ LOGGER.error(error) ++ return state.SSH_AUTHENTICATION_ERROR ++ except Exception as error: ++ LOGGER.error(error) ++ return state.SSH_CONNECTION_ERROR ++ ++ return state.SUCCEED ++ ++ + def save_ssh_public_key_to_client(ip: str, port: int, username: str, password: str) -> tuple: + """ + generate RSA key pair,save public key to the target host machine +@@ -465,7 +509,7 @@ class GetHostTemplateFile(BaseResponse): + file = BytesIO() + file.write(HOST_TEMPLATE_FILE_CONTENT.encode('utf-8')) + file.seek(0) +- response = send_file(file,mimetype="application/octet-stream") ++ response = send_file(file, mimetype="application/octet-stream") + response.headers['Content-Disposition'] = 'attachment; filename=template.csv' + return response + +@@ -574,6 +618,7 @@ class AddHostBatch(BaseResponse): + continue + + password = host_info.pop("password") ++ pkey = host_info.pop("ssh_pkey", None) + host_info.update( + {"host_group_id": group_id_info.get(host_info['host_group_name']), "user": data["username"]} + ) +@@ -585,7 +630,7 @@ class AddHostBatch(BaseResponse): + ) + continue + +- valid_host.append((host, password)) ++ valid_host.append((host, password, pkey)) + return valid_host + + def save_key_to_client(self, host_connect_infos: List[tuple]) -> list: +@@ -598,8 +643,8 @@ class AddHostBatch(BaseResponse): + Returns: + host object list + """ +- # 30 connections are created at a time. +- tasks = [host_connect_infos[index : index + 30] for index in range(0, len(host_connect_infos), 30)] ++ # 100 connections are created at a time. ++ tasks = [host_connect_infos[index : index + 100] for index in range(0, len(host_connect_infos), 100)] + result = [] + + for task in tasks: +@@ -612,18 +657,23 @@ class AddHostBatch(BaseResponse): + return result + + @staticmethod +- def update_rsa_key_to_host(host: Host, password: str) -> Host: ++ def update_rsa_key_to_host(host: Host, password: str = None, pkey: str = None) -> Host: + """ + save ssh public key to client and update its private key in host + + Args: + host(Host): host object + password(str): password for ssh login ++ pkey(str): rsa key for ssh login + + Returns: + host object + """ +- status, pkey = save_ssh_public_key_to_client(host.host_ip, host.ssh_port, host.ssh_user, password) ++ if pkey: ++ status = verify_ssh_login_info(ClientConnectArgs(host.host_ip, host.ssh_port, host.ssh_user, pkey)) ++ else: ++ status, pkey = save_ssh_public_key_to_client(host.host_ip, host.ssh_port, host.ssh_user, password) ++ + if status == state.SUCCEED: + host.status = HostStatus.ONLINE + host.pkey = pkey +@@ -654,7 +704,7 @@ class AddHostBatch(BaseResponse): + new_host.update(update_info) + self.add_result.append(new_host) + else: +- for host, _ in hosts: ++ for host, _, _ in hosts: + new_host = { + "host_ip": host.host_ip, + "ssh_port": host.ssh_port, +@@ -789,9 +839,14 @@ class UpdateHost(BaseResponse): + """ + ssh_user = params.get("ssh_user") or self.host.ssh_user + ssh_port = params.get("ssh_port") or self.host.ssh_port +- status, private_key = save_ssh_public_key_to_client( +- self.host.host_ip, ssh_port, ssh_user, params.pop("password", None) +- ) ++ private_key = params.pop("ssh_pkey", None) ++ if private_key: ++ status = verify_ssh_login_info(ClientConnectArgs(self.host.host_ip, ssh_port, ssh_user, private_key)) ++ else: ++ status, private_key = save_ssh_public_key_to_client( ++ self.host.host_ip, ssh_port, ssh_user, params.pop("password", None) ++ ) ++ + params.update( + { + "ssh_user": ssh_user, +@@ -876,10 +931,10 @@ class UpdateHost(BaseResponse): + return self.response(code=state.PARAM_ERROR, message="there is a duplicate host ssh address in database!") + + if params.get("ssh_user") or params.get("ssh_port"): +- if not params.get("password"): +- return self.response(code=state.PARAM_ERROR, message="please update password") ++ if not params.get("password") or not params.get("ssh_pkey"): ++ return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.") + self._save_ssh_key(params) +- elif params.get("password"): ++ elif params.get("password") or params.get("ssh_pkey"): + self._save_ssh_key(params) + + return self.response(callback.update_host_info(params.pop("host_id"), params)) +-- +2.33.0 + diff --git a/0002-update-callback-request-headers.patch b/0002-update-callback-request-headers.patch deleted file mode 100644 index b73eacad00bb12804535cede8129f1ca5f4cb84b..0000000000000000000000000000000000000000 --- a/0002-update-callback-request-headers.patch +++ /dev/null @@ -1,68 +0,0 @@ -From 51be015c84619925873ff377a72827b7c9770632 Mon Sep 17 00:00:00 2001 -From: rabbitali -Date: Mon, 18 Sep 2023 12:18:52 +0800 -Subject: [PATCH 1/1] update callback request headers -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - zeus/vulnerability_manage/view.py | 14 +++++--------- - 1 file changed, 5 insertions(+), 9 deletions(-) - -diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py -index 1bfed63..b5360e2 100644 ---- a/zeus/vulnerability_manage/view.py -+++ b/zeus/vulnerability_manage/view.py -@@ -119,7 +119,7 @@ def generate_tasks(tasks: list, host_infos: dict, **kwargs: dict) -> list: - - class BaseExcuteTask: - def __init__(self) -> None: -- self._header = {'content-type': 'application/json', 'access_token': request.headers.get('access_token')} -+ self._header = {"exempt_authentication": configuration.individuation.get("EXEMPT_AUTHENTICATION")} - - - class ExecuteRepoSetTask(BaseResponse, BaseExcuteTask): -@@ -234,7 +234,7 @@ class ExecuteRepoSetTask(BaseResponse, BaseExcuteTask): - return self.response(code=status_code) - - self._callback_url = params.get('callback') -- self._header = {'content-type': 'application/json', 'access_token': request.headers.get('access_token')} -+ self._header["local_account"] = params.get("username") - self._task_id = params.get("task_id") - self._task_name = params.get("task_name") - self._task_type = params.get("task_type") -@@ -337,13 +337,7 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask): - return self.response(code=status_code) - # parse args - self._callback_url = params.get('callback') -- if params.get("timed"): -- self._header.update( -- { -- "exempt_authentication": configuration.individuation.get("EXEMPT_AUTHENTICATION"), -- "local_account": params.get("username"), -- } -- ) -+ self._header["local_account"] = params.get("username") - self._task_id = params.get("task_id") - self._check_items = params.get('check_items') - # Execute task -@@ -465,6 +459,7 @@ class ExecuteCveFixTask(BaseResponse, BaseExcuteTask): - self._task_id = params.get("task_id") - self._task_name = params.get("task_name") - self._task_type = params.get("task_type") -+ self._header["local_account"] = params.get("username") - self._accepted = params.get('accepted') - self._takeover = params.get('takeover') - self._check_items = params.get('check_items') -@@ -582,6 +577,7 @@ class ExecuteCveRollbackTask(BaseResponse, BaseExcuteTask): - # Generate tasks - self._callback_url = params.get('callback') - self._task_id = params.get("task_id") -+ self._header["local_account"] = params.get("username") - self._task_name = params.get("task_name") - self._task_type = params.get("task_type") - --- -2.33.1.windows.1 - diff --git a/0003-fix-bash-file-sync-error.patch b/0003-fix-bash-file-sync-error.patch deleted file mode 100644 index 7a6837404fcbccba14f955d6a06b24d3a5740caf..0000000000000000000000000000000000000000 --- a/0003-fix-bash-file-sync-error.patch +++ /dev/null @@ -1,220 +0,0 @@ -From 80272352b3067ebb2cb3011cfbeeef5e9d464fa6 Mon Sep 17 00:00:00 2001 -From: smjiao -Date: Mon, 18 Sep 2023 20:37:50 +0800 -Subject: [PATCH 1/1] fix bash file sync error -MIME-Version: 1.0 -Content-Type: text/plain; charset=UTF-8 -Content-Transfer-Encoding: 8bit - ---- - zeus/config_manager/view.py | 48 ++++++++++++++++++++----- - zeus/host_manager/ssh.py | 72 +++++++++++++++++++++++++++++++------ - 2 files changed, 102 insertions(+), 18 deletions(-) - -diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py -index baeef7e..6779153 100644 ---- a/zeus/config_manager/view.py -+++ b/zeus/config_manager/view.py -@@ -16,6 +16,7 @@ Author: - Description: Restful APIs for host - """ - import json -+import os - from typing import List, Dict - - from vulcanus.multi_thread_handler import MultiThreadHandler -@@ -26,7 +27,7 @@ from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF - from zeus.database.proxy.host import HostProxy - from zeus.function.model import ClientConnectArgs - from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema --from zeus.host_manager.ssh import execute_command_and_parse_its_result -+from zeus.host_manager.ssh import execute_command_and_parse_its_result, execute_command_sftp_result - - - class CollectConfig(BaseResponse): -@@ -222,15 +223,46 @@ class CollectConfig(BaseResponse): - - - class SyncConfig(BaseResponse): -- - @staticmethod -- def sync_config_content(host_info: Dict, sync_config_info: Dict): -- command = CERES_SYNC_CONF % json.dumps(sync_config_info) -- status, content = execute_command_and_parse_its_result( -+ def sync_config_by_execute_command_sftp(host_info: Dict, sync_config_info: Dict, local_path: str, -+ remote_path: str): -+ content = sync_config_info.get("content") -+ with open(local_path, "w", encoding="UTF-8") as f: -+ f.write(content) -+ status = execute_command_sftp_result( - ClientConnectArgs(host_info.get("host_ip"), host_info.get("ssh_port"), -- host_info.get("ssh_user"), host_info.get("pkey")), command) -+ host_info.get("ssh_user"), host_info.get("pkey")), local_path, remote_path) - return status - -+ @staticmethod -+ def sync_config_content(host_info: Dict, sync_config_info: Dict): -+ join_path = "/tmp" -+ if sync_config_info.get("file_path") == "/etc/profile": -+ local_path = os.path.join(join_path, "profile") -+ remote_path = "/etc/profile" -+ status = SyncConfig.sync_config_by_execute_command_sftp(host_info, sync_config_info, local_path, -+ remote_path) -+ return status -+ elif sync_config_info.get("file_path") == "/etc/rc.local": -+ local_path = os.path.join(join_path, "rc.local") -+ remote_path = "/etc/rc.local" -+ status = SyncConfig.sync_config_by_execute_command_sftp(host_info, sync_config_info, local_path, -+ remote_path) -+ return status -+ elif sync_config_info.get("file_path") == "/etc/bashrc": -+ local_path = os.path.join(join_path, "bashrc") -+ remote_path = "/etc/bashrc" -+ status = SyncConfig.sync_config_by_execute_command_sftp(host_info, sync_config_info, local_path, -+ remote_path) -+ return status -+ else: -+ command = CERES_SYNC_CONF % json.dumps(sync_config_info) -+ -+ status, content = execute_command_and_parse_its_result( -+ ClientConnectArgs(host_info.get("host_ip"), host_info.get("ssh_port"), -+ host_info.get("ssh_user"), host_info.get("pkey")), command) -+ return status -+ - @BaseResponse.handle(schema=SyncConfigSchema, token=False) - def put(self, **params): - -@@ -244,13 +276,13 @@ class SyncConfig(BaseResponse): - } - - # Query host address from database -- proxy = HostProxy(configuration) -+ proxy = HostProxy() - if not proxy.connect(): - return self.response(code=state.DATABASE_CONNECT_ERROR, data={"resp": sync_result}) - - status, host_list = proxy.get_host_info( - {"username": "admin", "host_list": [params.get('host_id')]}, True) -- if status != state.SUCCEED or len(host_list) == 1: -+ if status != state.SUCCEED: - return self.response(code=status, data={"resp": sync_result}) - - host_info = host_list[0] -diff --git a/zeus/host_manager/ssh.py b/zeus/host_manager/ssh.py -index 4c3a259..11f6383 100644 ---- a/zeus/host_manager/ssh.py -+++ b/zeus/host_manager/ssh.py -@@ -15,6 +15,7 @@ from io import StringIO - from typing import Tuple - - import paramiko -+from paramiko import sftp - - from vulcanus.log.log import LOGGER - from vulcanus.restful.resp import state -@@ -57,7 +58,13 @@ class SSH: - """ - - def __init__(self, ip, username, port, password=None, pkey=None): -- self._client_args = {'hostname': ip, 'username': username, 'port': port, "password": password, "pkey": pkey} -+ self._client_args = { -+ 'hostname': ip, -+ 'username': username, -+ 'port': port, -+ "password": password, -+ "pkey": pkey -+ } - self._client = self.client() - - def client(self): -@@ -71,15 +78,15 @@ class SSH: - - def execute_command(self, command: str, timeout: float = None) -> tuple: - """ -- create a ssh client, execute command and parse result -+ create a ssh client, execute command and parse result - -- Args: -- command(str): shell command -- timeout(float): the maximum time to wait for the result of command execution -+ Args: -+ command(str): shell command -+ timeout(float): the maximum time to wait for the result of command execution - -- Returns: -- tuple: -- status, result, error message -+ Returns: -+ tuple: -+ status, result, error message - """ - open_channel = self._client.get_transport().open_session(timeout=timeout) - open_channel.set_combine_stderr(False) -@@ -110,13 +117,14 @@ def execute_command_and_parse_its_result(connect_args: ClientConnectArgs, comman - status, result - """ - if not connect_args.pkey: -- return state.SSH_AUTHENTICATION_ERROR, f"ssh authentication failed when connect host " f"{connect_args.host_ip}" -+ return state.SSH_AUTHENTICATION_ERROR, f"ssh authentication failed when connect host " \ -+ f"{connect_args.host_ip}" - try: - client = SSH( - ip=connect_args.host_ip, - username=connect_args.ssh_user, - port=connect_args.ssh_port, -- pkey=paramiko.RSAKey.from_private_key(StringIO(connect_args.pkey)), -+ pkey=paramiko.RSAKey.from_private_key(StringIO(connect_args.pkey)) - ) - exit_status, stdout, stderr = client.execute_command(command, connect_args.timeout) - except socket.error as error: -@@ -131,3 +139,47 @@ def execute_command_and_parse_its_result(connect_args: ClientConnectArgs, comman - return state.SUCCEED, stdout - LOGGER.error(stderr) - return state.EXECUTE_COMMAND_ERROR, stderr -+ -+ -+def execute_command_sftp_result(connect_args: ClientConnectArgs, local_path=None, remote_path=None): -+ """ -+ create a ssh client, execute command and parse result -+ -+ Args: -+ connect_args(ClientConnectArgs): e.g -+ ClientArgs(host_ip='127.0.0.1', ssh_port=22, ssh_user='root', pkey=RSAKey string) -+ command(str): shell command -+ -+ Returns: -+ tuple: -+ status, result -+ """ -+ global sftp_client, client -+ if not connect_args.pkey: -+ return state.SSH_AUTHENTICATION_ERROR, f"ssh authentication failed when connect host " \ -+ f"{connect_args.host_ip}" -+ try: -+ client = SSH( -+ ip=connect_args.host_ip, -+ username=connect_args.ssh_user, -+ port=connect_args.ssh_port, -+ pkey=paramiko.RSAKey.from_private_key(StringIO(connect_args.pkey)) -+ ) -+ sftp_client = client.client().open_sftp() -+ -+ # Specifies the path to the local file and the remote file -+ # Upload files to a remote server -+ sftp_client.put(local_path, remote_path) -+ return state.SUCCEED -+ except socket.error as error: -+ LOGGER.error(error) -+ return state.SSH_CONNECTION_ERROR, "SSH.Connection.Error" -+ except paramiko.ssh_exception.SSHException as error: -+ LOGGER.error(error) -+ return state.SSH_AUTHENTICATION_ERROR, "SSH.Authentication.Error" -+ except Exception as error: -+ LOGGER.error(error) -+ return state.SSH_AUTHENTICATION_ERROR, "SSH.Authentication.Error" -+ finally: -+ sftp_client.close() -+ client.close() --- -2.33.1.windows.1 - diff --git a/0003-fix-python-prometheus-api-client-import-error.patch b/0003-fix-python-prometheus-api-client-import-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..c447ca44a979e715590f93fe342e4c8ca1f50713 --- /dev/null +++ b/0003-fix-python-prometheus-api-client-import-error.patch @@ -0,0 +1,643 @@ +From cca4f59f2c68b20804fae06692179b8d3e181327 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Mon, 6 Nov 2023 09:36:24 +0800 +Subject: [PATCH] fix python-prometheus-api-client import error + +--- + zeus/conf/constant.py | 4 - + zeus/database/proxy/metric.py | 458 -------------------------------- + zeus/function/verify/metric.py | 28 -- + zeus/metric_manager/__init__.py | 12 - + zeus/metric_manager/view.py | 51 ---- + zeus/url.py | 9 - + 6 files changed, 562 deletions(-) + delete mode 100644 zeus/database/proxy/metric.py + delete mode 100644 zeus/function/verify/metric.py + delete mode 100644 zeus/metric_manager/__init__.py + delete mode 100644 zeus/metric_manager/view.py + +diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py +index bf8792a..304ed7e 100644 +--- a/zeus/conf/constant.py ++++ b/zeus/conf/constant.py +@@ -72,10 +72,6 @@ EXECUTE_CVE_FIX = '/manage/vulnerability/cve/fix' + EXECUTE_CVE_SCAN = '/manage/vulnerability/cve/scan' + EXECUTE_CVE_ROLLBACK = "/manage/vulnerability/cve/rollback" + +-# metric config +-QUERY_METRIC_NAMES = '/manage/host/metric/names' +-QUERY_METRIC_DATA = '/manage/host/metric/data' +-QUERY_METRIC_LIST = '/manage/host/metric/list' + + # auth login + GITEE_OAUTH = "https://gitee.com/oauth/authorize" +diff --git a/zeus/database/proxy/metric.py b/zeus/database/proxy/metric.py +deleted file mode 100644 +index 5fa75e9..0000000 +--- a/zeus/database/proxy/metric.py ++++ /dev/null +@@ -1,458 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +-""" +-Time: 2022-07-27 +-Author: YangYunYi +-Description: Query raw data from Prometheus +-""" +-from typing import Dict, Tuple, List, Optional +-import datetime +-from prometheus_api_client import PrometheusApiClientException +-from vulcanus.database.proxy import PromDbProxy +-from vulcanus.log.log import LOGGER +-from vulcanus.restful.resp.state import SUCCEED, DATABASE_QUERY_ERROR, NO_DATA, PARAM_ERROR, PARTIAL_SUCCEED +-from zeus.conf import configuration +- +- +-class MetricProxy(PromDbProxy): +- """ +- Proxy of prometheus time series database +- """ +- +- def __init__(self, host=None, port=None): +- """ +- Init MetricProxy +- +- Args: +- host (str) +- port (int) +- """ +- PromDbProxy.__init__(self, host, port) +- self.default_instance_port = configuration.agent.get('DEFAULT_INSTANCE_PORT') or 9100 +- self.query_range_step = configuration.prometheus.get('QUERY_RANGE_STEP') or "15s" +- +- @staticmethod +- def __metric_dict2str(metric: Dict) -> str: +- """ +- Trans metric dict to string +- Args: +- metric (Dict): +- { +- "__name__": "metric name1", +- 'instance': '172.168.128.164:9100', +- 'job': 'prometheus', +- 'label1': 'label_value1' +- ... +- } +- +- Returns: +- metric_str(str): 'metric_name1{label1="value1", label2="values"}' +- """ +- +- label_str = "" +- if "__name__" not in metric.keys(): +- return label_str +- sorted_label_list = sorted(metric.items(), reverse=False) +- for label in sorted_label_list: +- # The job label is usually "prometheus" in this framework and +- # has no effect on subsequent data requests, so remove it to save space. +- # __name__ label move to the front. +- metric_key = label[0] +- metric_value = label[1] +- if metric_key in ["__name__", "job"]: +- continue +- label_str += "%s=\"%s\"," % (metric_key, metric_value) +- +- if label_str == "": +- # It's a metric with only a name and no personalized label +- ret = metric["__name__"] +- else: +- # Remove the comma of the last element +- ret = "%s{%s}" % (metric["__name__"], label_str[:-1]) +- return ret +- +- def query_metric_names(self, query_ip: str) -> Tuple[int, dict]: +- """ +- Query data +- Args: +- query_ip(str): query ip address +- +- Returns: +- int: status code +- dict: e.g +- { +- "results": ["metric1", "metric2"] +- } +- """ +- +- query_ip = query_ip["query_ip"] +- query_host = {"host_id": "query_host_id", "host_ip": query_ip} +- +- query_metric_names = [] +- res = {'results': query_metric_names} +- +- host_ip = query_host["host_ip"] +- host_port = query_host.get("instance_port", self.default_instance_port) +- +- ret, metric_list = self.query_metric_list_of_host(host_ip, host_port) +- +- if ret != SUCCEED: +- return ret, res +- +- for metric in metric_list: +- metric = metric.split('{')[0] +- if metric not in query_metric_names: +- query_metric_names.append(metric) +- +- return ret, res +- +- def query_metric_list(self, data: Dict[str, str]) -> Tuple[int, dict]: +- """ +- Query metric list +- Args: +- data(dict): param e.g +- { +- "query_ip": "str", +- "metric_names": ["metric1", "metric2"] +- } +- +- Returns: +- int: status code +- dict: e.g +- { +- 'results': { +- "metric1": [ +- "metric1{label1="label1_value", label2="label2_value", ..., }", +- "metric1{label1="label1_value", label2="label2_value", ..., }", +- ], +- "metric2": [ +- "metric2{label1="label1_value", label2="label2_value", ..., }", +- "metric2{label1="label1_value", label2="label2_value", ..., }", +- ] +- } +- } +- """ +- +- query_ip = data.get('query_ip') +- query_metric = data.get('metric_names') +- +- query_host = {"host_id": "query_host_id", "host_ip": query_ip} +- +- query_metric_list = {} +- res = {'results': query_metric_list} +- +- host_ip = query_host["host_ip"] +- host_port = query_host.get("instance_port", self.default_instance_port) +- +- ret, metric_list = self.query_metric_list_of_host( +- host_ip, +- host_port, +- ) +- +- if ret != SUCCEED: +- LOGGER.warning("Host metric list query error") +- return ret, res +- +- for query_metric_name in query_metric: +- query_metric_list[query_metric_name] = [] +- for metric in metric_list: +- metric_name = metric.split('{')[0] +- if metric_name != query_metric_name: +- continue +- query_metric_list[query_metric_name].append(metric) +- +- return ret, res +- +- def query_metric_data(self, data: Dict[str, str]) -> Tuple[int, dict]: +- """ +- Query metric data +- Args: +- data(dict): param e.g +- { +- "time_range": "list", +- "query_ip": "str", +- "query_info": { +- "metric1": [ +- "metric1{label1="label1_value", label2="label2_value", ..., }", +- "metric1{label1="label1_value", label2="label2_value", ..., }", +- ], +- "metric2": [ +- "metric2{label1="label1_value", label2="label2_value", ..., }", +- "metric2{label1="label1_value", label2="label2_value", ..., }", +- ] +- } +- } +- +- Returns: +- int: status code +- dict: e.g +- { +- 'results': { +- "metric1": { +- "metric1{label1="label1_value", label2="label2_value", ..., }": [ +- [1658926441, '0'], +- [1658926441, '0'] +- ], +- "metric1{label1="label1_value", label2="label2_value", ..., }": [ +- [1658926441, '0'], +- [1658926441, '0'] +- ], +- }, +- "metric1": { +- "metric2{label1="label1_value", label2="label2_value", ..., }": [ +- [1658926441, '0'], +- [1658926441, '0'] +- ], +- "metric2{label1="label1_value", label2="label2_value", ..., }": [ +- [1658926441, '0'], +- [1658926441, '0'] +- ], +- }, +- } +- } +- """ +- time_range = data.get('time_range') +- query_ip = data.get('query_ip') +- query_info = data.get('query_info') +- +- query_host = {"host_id": "query_host_id", "host_ip": query_ip} +- +- host_ip = query_host["host_ip"] +- host_port = query_host.get("instance_port", self.default_instance_port) +- +- query_host_list = [query_host] +- query_data = {} +- res = {'results': query_data} +- +- # adjust query range step based on query time range +- # the default query range step is 15 seconds +- QUERY_RANGE_STEP = 15 +- query_range_step = QUERY_RANGE_STEP +- query_range_seconds = time_range[1] - time_range[0] +- total_query_times = query_range_seconds / query_range_step +- +- while total_query_times > 11000: +- query_range_step += 15 +- total_query_times = query_range_seconds / query_range_step +- +- def add_two_dim_dict(thedict, key_a, key_b, val): +- if key_a in thedict: +- thedict[key_a].update({key_b: val}) +- else: +- thedict.update({key_a: {key_b: val}}) +- +- if not query_info: +- return SUCCEED, res +- +- for metric_name, metric_list in query_info.items(): +- if not metric_list: +- _, metric_list = self.query_metric_list_of_host(host_ip, host_port, metric_name) +- if not metric_list: +- query_data[metric_name] = [] +- for metric_info in metric_list: +- data_status, monitor_data = self.query_data( +- time_range=time_range, +- host_list=query_host_list, +- metric=metric_info, +- adjusted_range_step=query_range_step, +- ) +- values = [] +- if data_status == SUCCEED: +- values = monitor_data[query_host["host_id"]][metric_info] +- add_two_dim_dict(query_data, metric_name, metric_info, values) +- +- return SUCCEED, res +- +- def query_data( +- self, +- time_range: List[int], +- host_list: list, +- metric: Optional[str] = None, +- adjusted_range_step: Optional[int] = None, +- ) -> Tuple[int, Dict]: +- """ +- Query data +- Args: +- time_range(list): time range +- host_list(list): host list, If the port is not specified, the default value is used +- [{"host_id": "id1", "host_ip": "172.168.128.164", "instance_port": 9100}, +- {"host_id": "id1", "host_ip": "172.168.128.164", "instance_port": 8080}, +- {"host_id": "id2", "host_ip": "172.168.128.165"}] +- +- Returns: +- ret(int): query ret +- host_data_list(dict): host data list ret +- { +- 'id1': { +- 'metric1'{instance="172.168.128.164:9100",label1="value2"}': +- [[time1, 'value1'], +- [time2, 'value2'], +- 'metric12{instance="172.168.128.164:8080"}': [], => get data list is empty +- 'metric13{instance="172.168.128.164:8080"}': None => get no data +- }, +- 'id2': None => get no metric list of this host +- } +- """ +- +- host_data_list = {} +- if not host_list: +- return PARAM_ERROR, host_data_list +- +- status = SUCCEED +- for host in host_list: +- host_id = host["host_id"] +- host_ip = host["host_ip"] +- host_port = host.get("instance_port", self.default_instance_port) +- if host_id not in host_data_list.keys(): +- host_data_list[host_id] = None +- +- ret, metric_list = self.query_metric_list_of_host(host_ip, host_port, metric) +- if ret != SUCCEED: +- status = PARTIAL_SUCCEED +- host_data_list[host_id] = None +- continue +- ret, data_list = self.__query_data_by_host(metric_list, time_range, adjusted_range_step) +- if ret != SUCCEED: +- status = PARTIAL_SUCCEED +- if not host_data_list[host_id]: +- host_data_list[host_id] = data_list +- else: +- host_data_list[host_id].update(data_list) +- +- return status, host_data_list +- +- def __parse_metric_data(self, metric_data: List) -> List[str]: +- """ +- Parse metric data from prometheus to name<-> label_config dict +- Args: +- metric_data(List): metric list data from prometheus +- [{'metric':{ +- "__name__": "metric_name1", +- 'instance': '172.168.128.164:9100', +- 'job': 'prometheus', +- 'label1': 'label_value1' +- ... +- }, +- 'value': [1658926441.526, '0'] +- }] +- +- Returns: +- metric_list(List[str]): +- [ +- 'metric_name1{label1="value1", label2="value2"}' +- ] +- """ +- +- metric_list = [] +- for metric_item in metric_data: +- metric_dict = metric_item["metric"] +- metric_str = self.__metric_dict2str(metric_dict) +- if metric_str == "": +- continue +- metric_list.append(metric_str) +- +- return metric_list +- +- def query_metric_list_of_host( +- self, host_ip: str, host_port: Optional[int] = None, metric: Optional[str] = None +- ) -> Tuple[int, List[str]]: +- """ +- Query metric list of a host +- Args: +- host_ip(str): host ip +- host_port(int): host port +- +- Returns: +- ret(int): query ret +- metric_list(list): metric list ret +- [ +- 'metric_name1{label1="value1", label2="value2"}' +- ] +- """ +- if not host_port: +- host_port = self.default_instance_port +- query_str = "{instance=\"%s:%s\"}" % (host_ip, str(host_port)) +- if metric is not None: +- if metric.find('{') != -1: +- query_str = metric +- else: +- query_str = metric + query_str +- try: +- data = self._prom.custom_query(query=query_str) +- if not data: +- if query_str.startswith('{'): +- LOGGER.error( +- "Query metric list result is empty. " +- "Can not get metric list of host %s:%s " % (host_ip, host_port) +- ) +- return NO_DATA, [] +- metric_list = self.__parse_metric_data(data) +- return SUCCEED, metric_list +- +- except (ValueError, TypeError, PrometheusApiClientException) as error: +- LOGGER.error("host %s:%d Prometheus query metric list failed. %s" % (host_ip, host_port, error)) +- return DATABASE_QUERY_ERROR, [] +- +- def __query_data_by_host( +- self, metrics_list: List[str], time_range: List[int], adjusted_range_step: Optional[int] = None +- ) -> Tuple[int, Dict]: +- """ +- Query data of a host +- Args: +- metrics_list(list): metric list of this host +- time_range(list): time range to query +- +- Returns: +- ret(int): query ret +- data_list(list): data list ret +- { +- 'metric1'{instance="172.168.128.164:9100",label1="value2"}': +- [[time1, 'value1'], +- [time2, 'value2'], +- 'metric12{instance="172.168.128.164:8080"}': [], => get data list is empty +- 'metric13{instance="172.168.128.164:8080"}': None => get no data +- } +- +- """ +- start_time = datetime.datetime.fromtimestamp(time_range[0]) +- end_time = datetime.datetime.fromtimestamp(time_range[1]) +- +- query_range_step = self.query_range_step +- if adjusted_range_step is not None: +- query_range_step = adjusted_range_step +- +- data_list = {} +- ret = SUCCEED +- for metric in metrics_list: +- try: +- data = self._prom.custom_query_range( +- query=metric, start_time=start_time, end_time=end_time, step=query_range_step +- ) +- if not data or "values" not in data[0]: +- LOGGER.debug( +- "Query data result is empty. " +- "metric %s in %d-%d doesn't record in the prometheus " % (metric, time_range[0], time_range[1]) +- ) +- data_list[metric] = None +- ret = PARTIAL_SUCCEED +- continue +- data_list[metric] = data[0]["values"] +- +- except (ValueError, TypeError, PrometheusApiClientException) as error: +- LOGGER.error( +- "Prometheus metric %s in %d-%d query data failed. %s" +- % (metric, time_range[0], time_range[1], error) +- ) +- data_list[metric] = None +- ret = PARTIAL_SUCCEED +- return ret, data_list +diff --git a/zeus/function/verify/metric.py b/zeus/function/verify/metric.py +deleted file mode 100644 +index 5df28a2..0000000 +--- a/zeus/function/verify/metric.py ++++ /dev/null +@@ -1,28 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +-from marshmallow import Schema, fields +- +- +-class QueryHostMetricNamesSchema(Schema): +- query_ip = fields.String(required=True) +- +- +-class QueryHostMetricDataSchema(Schema): +- time_range = fields.List(fields.Integer, required=True) +- query_ip = fields.String(required=True) +- query_info = fields.Dict() +- +- +-class QueryHostMetricListSchema(Schema): +- query_ip = fields.String(required=True) +- metric_names = fields.List(fields.String) +diff --git a/zeus/metric_manager/__init__.py b/zeus/metric_manager/__init__.py +deleted file mode 100644 +index e90ecf9..0000000 +--- a/zeus/metric_manager/__init__.py ++++ /dev/null +@@ -1,12 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +diff --git a/zeus/metric_manager/view.py b/zeus/metric_manager/view.py +deleted file mode 100644 +index 4d98cf2..0000000 +--- a/zeus/metric_manager/view.py ++++ /dev/null +@@ -1,51 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +-from vulcanus.restful.response import BaseResponse +-from zeus.database.proxy.metric import MetricProxy +-from zeus.function.verify.metric import QueryHostMetricDataSchema, QueryHostMetricListSchema, QueryHostMetricNamesSchema +- +- +-class QueryHostMetricNames(BaseResponse): +- """ +- Interface for query host metric names from web. +- Restful API: GET +- """ +- +- @BaseResponse.handle(schema=QueryHostMetricNamesSchema, proxy=MetricProxy) +- def get(self, callback: MetricProxy, **params): +- status_code, result = callback.query_metric_names(params) +- return self.response(code=status_code, data=result) +- +- +-class QueryHostMetricData(BaseResponse): +- """ +- Interface for query host metric data from web. +- Restful API: POST +- """ +- +- @BaseResponse.handle(schema=QueryHostMetricDataSchema, proxy=MetricProxy) +- def post(self, callback: MetricProxy, **params): +- status_code, result = callback.query_metric_data(params) +- return self.response(code=status_code, data=result) +- +- +-class QueryHostMetricList(BaseResponse): +- """ +- Interface for query host metric list from web. +- Restful API: POST +- """ +- +- @BaseResponse.handle(schema=QueryHostMetricListSchema, proxy=MetricProxy) +- def post(self, callback: MetricProxy, **params): +- status_code, result = callback.query_metric_list(params) +- return self.response(code=status_code, data=result) +diff --git a/zeus/url.py b/zeus/url.py +index 597dcc7..3ec8d21 100644 +--- a/zeus/url.py ++++ b/zeus/url.py +@@ -43,9 +43,6 @@ from zeus.conf.constant import ( + LOGOUT, + QUERY_HOST, + QUERY_HOST_DETAIL, +- QUERY_METRIC_DATA, +- QUERY_METRIC_LIST, +- QUERY_METRIC_NAMES, + REFRESH_TOKEN, + UPDATE_HOST, + USER_LOGIN, +@@ -53,7 +50,6 @@ from zeus.conf.constant import ( + ) + from zeus.config_manager import view as config_view + from zeus.host_manager import view as host_view +-from zeus.metric_manager import view as metric_view + from zeus.vulnerability_manage import view as vulnerability_view + + URLS = [] +@@ -100,11 +96,6 @@ SPECIFIC_URLS = { + (vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX), + (vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK), + ], +- 'METRIC': [ +- (metric_view.QueryHostMetricNames, QUERY_METRIC_NAMES), +- (metric_view.QueryHostMetricData, QUERY_METRIC_DATA), +- (metric_view.QueryHostMetricList, QUERY_METRIC_LIST), +- ], + } + + for _, value in SPECIFIC_URLS.items(): +-- +2.33.0 + diff --git a/0004-bugfix-cve-fix-result-parsing-error.patch b/0004-bugfix-cve-fix-result-parsing-error.patch deleted file mode 100644 index 40248bef7e5b75672ee4d8ce69ab609dfa6c7702..0000000000000000000000000000000000000000 --- a/0004-bugfix-cve-fix-result-parsing-error.patch +++ /dev/null @@ -1,29 +0,0 @@ -From 21a8a9db54175547675bda5214d98ffa5a654d03 Mon Sep 17 00:00:00 2001 -From: rabbitali -Date: Tue, 19 Sep 2023 16:58:56 +0800 -Subject: [PATCH 1/1] bugfix: cve fix result parsing error - ---- - zeus/vulnerability_manage/view.py | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py -index b5360e2..345978f 100644 ---- a/zeus/vulnerability_manage/view.py -+++ b/zeus/vulnerability_manage/view.py -@@ -416,10 +416,11 @@ class ExecuteCveFixTask(BaseResponse, BaseExcuteTask): - cve_fix_result_json = json.loads(cve_fix_result) - if cve_fix_result_json.pop("code") != state.SUCCEED: - request_body["status"] = CveTaskStatus.FAIL -+ else: -+ request_body["status"] = CveTaskStatus.SUCCEED - - request_body.update( - { -- "status": CveTaskStatus.SUCCEED, - "check_items": cve_fix_result_json.get("check_items"), - "cves": cve_fix_result_json.get("cves"), - } --- -2.33.0 - diff --git a/0004-update-the-template-file-contents-for-adding-hosts.patch b/0004-update-the-template-file-contents-for-adding-hosts.patch new file mode 100644 index 0000000000000000000000000000000000000000..ca6c95daa91791c2276f736f2ed14d7c4e63efeb --- /dev/null +++ b/0004-update-the-template-file-contents-for-adding-hosts.patch @@ -0,0 +1,33 @@ +From 36d5b7a26f9f470e0b9a593edb7f198cd9022c47 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Thu, 26 Oct 2023 18:43:37 +0800 +Subject: [PATCH] update the template file contents for adding hosts + +--- + zeus/conf/constant.py | 10 ++++++++-- + 1 file changed, 8 insertions(+), 2 deletions(-) + +diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py +index bf8792a..df2948d 100644 +--- a/zeus/conf/constant.py ++++ b/zeus/conf/constant.py +@@ -91,8 +91,14 @@ CHECK_WORKFLOW_HOST_EXIST = '/check/workflow/host/exist' + + # host template file content + HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,ssh_pkey,host_name,host_group_name,management +-test_ip_1,22,root,password,ssh_pkey,test_host,test_host_group,False +-test_ip_2,22,root,password,ssh_pkey,test_host,test_host_group,False ++127.0.0.1,22,root,password,private key,test_host,test_host_group,FALSE ++127.0.0.1,23,root,password,private key,test_host,test_host_group,FALSE ++,,,,,,, ++"提示:",,,,,,, ++"1. 除登录密码与SSH登录秘钥外,其余信息都应提供有效值",,,,,,, ++"2. 登录密码与SSH登录秘钥可选择一种填入,当两者都提供时,以SSH登录秘钥为准",,,,,,, ++"3. 添加的主机信息不应存在重复信息(主机名称重复或者主机IP+端口重复)",,,,,,, ++"4. 上传本文件前,请删除此部分提示内容",,,,,,, + """ + + +-- +2.33.0 + diff --git a/0005-bugfix-update-host-api-request-error.patch b/0005-bugfix-update-host-api-request-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..d87c727716c67f34eb3e348182516b89d59aa261 --- /dev/null +++ b/0005-bugfix-update-host-api-request-error.patch @@ -0,0 +1,33 @@ +From 36f98b43bd571ac9f2f4f9a9fe658684d591d52e Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Fri, 27 Oct 2023 15:21:54 +0800 +Subject: [PATCH] bugfix: update host api request error when changing username + +--- + zeus/host_manager/view.py | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py +index 95e1434..10418d1 100644 +--- a/zeus/host_manager/view.py ++++ b/zeus/host_manager/view.py +@@ -930,11 +930,11 @@ class UpdateHost(BaseResponse): + LOGGER.warning(f"there is a duplicate host address in database " f"when update host {self.host.host_id}!") + return self.response(code=state.PARAM_ERROR, message="there is a duplicate host ssh address in database!") + +- if params.get("ssh_user") or params.get("ssh_port"): +- if not params.get("password") or not params.get("ssh_pkey"): +- return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.") +- self._save_ssh_key(params) +- elif params.get("password") or params.get("ssh_pkey"): ++ if params.get("password") or params.get("ssh_pkey"): + self._save_ssh_key(params) ++ return self.response(callback.update_host_info(params.pop("host_id"), params)) ++ ++ if params.get("ssh_user") or params.get("ssh_port"): ++ return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.") + + return self.response(callback.update_host_info(params.pop("host_id"), params)) +-- +2.33.0 + diff --git a/0006-optimize-import.patch b/0006-optimize-import.patch new file mode 100644 index 0000000000000000000000000000000000000000..25676a3dc476ddf701d7a022c2cc1b57ebdf18bd --- /dev/null +++ b/0006-optimize-import.patch @@ -0,0 +1,139 @@ +From 6ba93db6c012b7547e80fc71e7dc3dd96d6f0aa4 Mon Sep 17 00:00:00 2001 +From: xuyongliang_01 +Date: Wed, 18 Oct 2023 02:49:04 +0000 +Subject: [PATCH 1/5] optimize Import +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +Signed-off-by: xuyongliang_01 +--- + zeus/config_manager/view.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py +index 6779153..0aa9682 100644 +--- a/zeus/config_manager/view.py ++++ b/zeus/config_manager/view.py +@@ -22,7 +22,6 @@ from typing import List, Dict + from vulcanus.multi_thread_handler import MultiThreadHandler + from vulcanus.restful.resp import state + from vulcanus.restful.response import BaseResponse +-from zeus.conf import configuration + from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF + from zeus.database.proxy.host import HostProxy + from zeus.function.model import ClientConnectArgs +-- +Gitee + + +From a43c526f7e8dcda9fe9b46ff6e1f80f370954259 Mon Sep 17 00:00:00 2001 +From: xuyongliang_01 +Date: Wed, 18 Oct 2023 02:56:33 +0000 +Subject: [PATCH 2/5] update zeus/tests/host_manager/test_add_host.py. + +Signed-off-by: xuyongliang_01 +--- + zeus/tests/host_manager/test_add_host.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/zeus/tests/host_manager/test_add_host.py b/zeus/tests/host_manager/test_add_host.py +index c51be95..6b8b1b0 100644 +--- a/zeus/tests/host_manager/test_add_host.py ++++ b/zeus/tests/host_manager/test_add_host.py +@@ -17,7 +17,6 @@ from io import BytesIO + from unittest import mock + + import paramiko +-import sqlalchemy + from paramiko import AuthenticationException + from sqlalchemy.orm.collections import InstrumentedList + +-- +Gitee + + +From d03456b27fdc16c868c0b871b3f43786cc383146 Mon Sep 17 00:00:00 2001 +From: xuyongliang_01 +Date: Wed, 18 Oct 2023 02:58:24 +0000 +Subject: [PATCH 3/5] update zeus/host_manager/ssh.py. + +Signed-off-by: xuyongliang_01 +--- + zeus/host_manager/ssh.py | 1 - + 1 file changed, 1 deletion(-) + +diff --git a/zeus/host_manager/ssh.py b/zeus/host_manager/ssh.py +index 11f6383..a4e7628 100644 +--- a/zeus/host_manager/ssh.py ++++ b/zeus/host_manager/ssh.py +@@ -15,7 +15,6 @@ from io import StringIO + from typing import Tuple + + import paramiko +-from paramiko import sftp + + from vulcanus.log.log import LOGGER + from vulcanus.restful.resp import state +-- +Gitee + + +From 887b9525192140c5142b62c14b91c1656b3a657a Mon Sep 17 00:00:00 2001 +From: xuyongliang_01 +Date: Wed, 18 Oct 2023 02:59:12 +0000 +Subject: [PATCH 4/5] update zeus/tests/host_manager/test_delete_host.py. + +Signed-off-by: xuyongliang_01 +--- + zeus/tests/host_manager/test_delete_host.py | 2 -- + 1 file changed, 2 deletions(-) + +diff --git a/zeus/tests/host_manager/test_delete_host.py b/zeus/tests/host_manager/test_delete_host.py +index 59a4098..5a0aea6 100644 +--- a/zeus/tests/host_manager/test_delete_host.py ++++ b/zeus/tests/host_manager/test_delete_host.py +@@ -12,8 +12,6 @@ + # ******************************************************************************/ + from unittest import mock + +-import sqlalchemy +- + from vulcanus.exceptions import DatabaseConnectionFailed + from vulcanus.restful.resp import state + from vulcanus.restful.response import BaseResponse +-- +Gitee + + +From 089c24550ae9f5720629376692344870316e97a6 Mon Sep 17 00:00:00 2001 +From: xuyongliang_01 +Date: Wed, 18 Oct 2023 03:00:38 +0000 +Subject: [PATCH 5/5] update zeus/vulnerability_manage/view.py. + +Signed-off-by: xuyongliang_01 +--- + zeus/vulnerability_manage/view.py | 3 +-- + 1 file changed, 1 insertion(+), 2 deletions(-) + +diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py +index 345978f..c56f672 100644 +--- a/zeus/vulnerability_manage/view.py ++++ b/zeus/vulnerability_manage/view.py +@@ -15,12 +15,11 @@ import time + import threading + from typing import Dict, Tuple + +-from flask import Response, request ++from flask import Response + import sqlalchemy + import gevent + + from vulcanus.log.log import LOGGER +-from vulcanus.multi_thread_handler import MultiThreadHandler + from vulcanus.restful.resp import state + from vulcanus.restful.response import BaseResponse + from zeus.conf import configuration +-- +Gitee + diff --git a/0007-add-an-api-for-querying-file-list.patch b/0007-add-an-api-for-querying-file-list.patch new file mode 100644 index 0000000000000000000000000000000000000000..def8749f8e6a0272755edcf07fb6b0af7a9bad22 --- /dev/null +++ b/0007-add-an-api-for-querying-file-list.patch @@ -0,0 +1,392 @@ +From 318ad494e275d0404b2f1af768506a25c27028e2 Mon Sep 17 00:00:00 2001 +From: liulei <450962@qq.com> +Date: Tue, 31 Oct 2023 10:17:37 +0800 +Subject: [PATCH 1/2] Add an interface for querying file list +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + zeus/conf/constant.py | 2 + + zeus/config_manager/view.py | 42 +++++++++++- + zeus/function/verify/config.py | 8 +++ + zeus/vulnerability_manage/url.py | 113 +++++++++++++++++++++++++++++++ + 4 files changed, 163 insertions(+), 2 deletions(-) + create mode 100644 zeus/vulnerability_manage/url.py + +diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py +index df2948d..44065be 100644 +--- a/zeus/conf/constant.py ++++ b/zeus/conf/constant.py +@@ -34,6 +34,7 @@ CERES_CVE_SCAN = "aops-ceres apollo --scan '%s'" + CERES_CVE_FIX = "aops-ceres apollo --fix '%s'" + CERES_CVE_ROLLBACK = "aops-ceres apollo --rollback '%s'" + CERES_SYNC_CONF = "aops-ceres sync --conf '%s'" ++CERES_OBJECT_FILE_CONF = "aops-ceres ragdoll --list '%s'" + + # zeus route + ADD_HOST = "/manage/host/add" +@@ -56,6 +57,7 @@ GET_GROUP = "/manage/host/group/get" + + COLLECT_CONFIG = '/manage/config/collect' + SYNC_CONFIG = '/manage/config/sync' ++OBJECT_FILE_CONFIG = '/manage/config/objectfile' + + USER_LOGIN = "/manage/account/login" + LOGOUT = "/manage/account/logout" +diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py +index 0aa9682..bde6dbf 100644 +--- a/zeus/config_manager/view.py ++++ b/zeus/config_manager/view.py +@@ -22,10 +22,10 @@ from typing import List, Dict + from vulcanus.multi_thread_handler import MultiThreadHandler + from vulcanus.restful.resp import state + from vulcanus.restful.response import BaseResponse +-from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF ++from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF, OBJECT_FILE_CONF, CERES_OBJECT_FILE_CONF + from zeus.database.proxy.host import HostProxy + from zeus.function.model import ClientConnectArgs +-from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema ++from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema, ObjectFileConfigSchema + from zeus.host_manager.ssh import execute_command_and_parse_its_result, execute_command_sftp_result + + +@@ -290,3 +290,41 @@ class SyncConfig(BaseResponse): + sync_result['sync_result'] = True + return self.response(code=state.SUCCEED, data={"resp": sync_result}) + return self.response(code=state.UNKNOWN_ERROR, data={"resp": sync_result}) ++ ++ ++class ObjectFileConfig(BaseResponse): ++ ++ @staticmethod ++ def object_file_config_content(host_info: Dict, file_directory: str): ++ command = CERES_OBJECT_FILE_CONF % file_directory ++ status, content = execute_command_and_parse_its_result( ++ ClientConnectArgs(host_info.get("host_ip"), host_info.get("ssh_port"), ++ host_info.get("ssh_user"), host_info.get("pkey")), command) ++ return status, content ++ ++ @BaseResponse.handle(schema=ObjectFileConfigSchema, token=False) ++ def post(self, **params): ++ object_file_result = { ++ "object_file_paths": list(), ++ "object_file_result": False ++ } ++ # Query host address from database ++ proxy = HostProxy() ++ if not proxy.connect(): ++ return self.response(code=state.DATABASE_CONNECT_ERROR, data={"resp": object_file_result}) ++ ++ status, host_list = proxy.get_host_info( ++ {"username": "admin", "host_list": [params.get('host_id')]}, True) ++ if status != state.SUCCEED: ++ return self.response(code=status, data={"resp": object_file_result}) ++ ++ host_info = host_list[0] ++ status, content = self.object_file_config_content(host_info, params.get('file_directory')) ++ if status == state.SUCCEED: ++ object_file_result['object_file_result'] = True ++ content_res = json.loads(content) ++ if content_res.get("resp"): ++ resp = content_res.get("resp") ++ object_file_result['object_file_paths'] = resp ++ return self.response(code=state.SUCCEED, data={"resp": object_file_result}) ++ return self.response(code=state.UNKNOWN_ERROR, data={"resp": object_file_result}) +diff --git a/zeus/function/verify/config.py b/zeus/function/verify/config.py +index 6e5bf64..1ef7b97 100644 +--- a/zeus/function/verify/config.py ++++ b/zeus/function/verify/config.py +@@ -45,3 +45,11 @@ class SyncConfigSchema(Schema): + host_id = fields.Integer(required=True, validate=lambda s: s > 0) + file_path = fields.String(required=True, validate=lambda s: len(s) > 0) + content = fields.String(required=True, validate=lambda s: len(s) > 0) ++ ++ ++class ObjectFileConfigSchema(Schema): ++ """ ++ validators for ObjectFileConfigSchema ++ """ ++ host_id = fields.Integer(required=True, validate=lambda s: s > 0) ++ file_directory = fields.String(required=True, validate=lambda s: len(s) > 0) +diff --git a/zeus/vulnerability_manage/url.py b/zeus/vulnerability_manage/url.py +new file mode 100644 +index 0000000..fe464ac +--- /dev/null ++++ b/zeus/vulnerability_manage/url.py +@@ -0,0 +1,113 @@ ++#!/usr/bin/python3 ++# ****************************************************************************** ++# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. ++# licensed under the Mulan PSL v2. ++# You can use this software according to the terms and conditions of the Mulan PSL v2. ++# You may obtain a copy of Mulan PSL v2 at: ++# http://license.coscl.org.cn/MulanPSL2 ++# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR ++# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR ++# PURPOSE. ++# See the Mulan PSL v2 for more details. ++# ******************************************************************************/ ++""" ++Time: ++Author: ++Description: url set ++""" ++from zeus.account_manager import view as account_view ++from zeus.agent_manager import view as agent_view ++from zeus.conf.constant import ( ++ ADD_GROUP, ++ ADD_HOST, ++ ADD_HOST_BATCH, ++ ADD_USER, ++ AGENT_METRIC_SET, ++ AGENT_PLUGIN_INFO, ++ AGENT_PLUGIN_SET, ++ AUTH_REDIRECT_URL, ++ BIND_AUTH_ACCOUNT, ++ CHANGE_PASSWORD, ++ COLLECT_CONFIG, ++ DELETE_GROUP, ++ DELETE_HOST, ++ EXECUTE_CVE_FIX, ++ EXECUTE_CVE_ROLLBACK, ++ EXECUTE_CVE_SCAN, ++ EXECUTE_REPO_SET, ++ GET_GROUP, ++ GET_HOST_COUNT, ++ GET_HOST_TEMPLATE_FILE, ++ GITEE_AUTH_LOGIN, ++ HOST_SCENE_GET, ++ LOGOUT, ++ QUERY_HOST, ++ QUERY_HOST_DETAIL, ++ QUERY_METRIC_DATA, ++ QUERY_METRIC_LIST, ++ QUERY_METRIC_NAMES, ++ REFRESH_TOKEN, ++ UPDATE_HOST, ++ USER_LOGIN, ++ SYNC_CONFIG, ++ OBJECT_FILE_CONFIG ++) ++from zeus.config_manager import view as config_view ++from zeus.host_manager import view as host_view ++from zeus.metric_manager import view as metric_view ++from zeus.vulnerability_manage import view as vulnerability_view ++ ++URLS = [] ++ ++SPECIFIC_URLS = { ++ "ACCOUNT_URLS": [ ++ (account_view.Login, USER_LOGIN), ++ (account_view.ChangePassword, CHANGE_PASSWORD), ++ (account_view.AddUser, ADD_USER), ++ (account_view.GiteeAuthLogin, GITEE_AUTH_LOGIN), ++ (account_view.AuthRedirectUrl, AUTH_REDIRECT_URL), ++ (account_view.BindAuthAccount, BIND_AUTH_ACCOUNT), ++ (account_view.RefreshToken, REFRESH_TOKEN), ++ (account_view.Logout, LOGOUT), ++ ], ++ "HOST_URLS": [ ++ (host_view.AddHost, ADD_HOST), ++ (host_view.AddHostBatch, ADD_HOST_BATCH), ++ (host_view.DeleteHost, DELETE_HOST), ++ (host_view.UpdateHost, UPDATE_HOST), ++ (host_view.GetHost, QUERY_HOST), ++ (host_view.GetHostInfo, QUERY_HOST_DETAIL), ++ (host_view.GetHostCount, GET_HOST_COUNT), ++ (host_view.GetHostTemplateFile, GET_HOST_TEMPLATE_FILE), ++ ], ++ "HOST_GROUP_URLS": [ ++ (host_view.AddHostGroup, ADD_GROUP), ++ (host_view.DeleteHostGroup, DELETE_GROUP), ++ (host_view.GetHostGroup, GET_GROUP), ++ ], ++ "CONFIG_URLS": [ ++ (config_view.CollectConfig, COLLECT_CONFIG), ++ (config_view.SyncConfig, SYNC_CONFIG), ++ (config_view.ObjectFileConfig, OBJECT_FILE_CONFIG) ++ ], ++ 'AGENT_URLS': [ ++ (agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO), ++ (agent_view.SetAgentPluginStatus, AGENT_PLUGIN_SET), ++ (agent_view.SetAgentMetricStatus, AGENT_METRIC_SET), ++ (agent_view.GetHostScene, HOST_SCENE_GET), ++ ], ++ 'CVE_URLS': [ ++ (vulnerability_view.ExecuteRepoSetTask, EXECUTE_REPO_SET), ++ (vulnerability_view.ExecuteCveScanTask, EXECUTE_CVE_SCAN), ++ (vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX), ++ (vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK), ++ ], ++ 'METRIC': [ ++ (metric_view.QueryHostMetricNames, QUERY_METRIC_NAMES), ++ (metric_view.QueryHostMetricData, QUERY_METRIC_DATA), ++ (metric_view.QueryHostMetricList, QUERY_METRIC_LIST), ++ ], ++} ++ ++for _, value in SPECIFIC_URLS.items(): ++ URLS.extend(value) +-- +Gitee + + +From bf654cd0d5086b29ab3e5b27cdc42cc0f8cc24ed Mon Sep 17 00:00:00 2001 +From: liulei <450962@qq.com> +Date: Tue, 31 Oct 2023 10:29:27 +0800 +Subject: [PATCH 2/2] Add an interface for querying file list +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + zeus/url.py | 4 +- + zeus/vulnerability_manage/url.py | 113 ------------------------------- + 2 files changed, 3 insertions(+), 114 deletions(-) + delete mode 100644 zeus/vulnerability_manage/url.py + +diff --git a/zeus/url.py b/zeus/url.py +index 597dcc7..fe464ac 100644 +--- a/zeus/url.py ++++ b/zeus/url.py +@@ -50,6 +50,7 @@ from zeus.conf.constant import ( + UPDATE_HOST, + USER_LOGIN, + SYNC_CONFIG, ++ OBJECT_FILE_CONFIG + ) + from zeus.config_manager import view as config_view + from zeus.host_manager import view as host_view +@@ -86,7 +87,8 @@ SPECIFIC_URLS = { + ], + "CONFIG_URLS": [ + (config_view.CollectConfig, COLLECT_CONFIG), +- (config_view.SyncConfig, SYNC_CONFIG) ++ (config_view.SyncConfig, SYNC_CONFIG), ++ (config_view.ObjectFileConfig, OBJECT_FILE_CONFIG) + ], + 'AGENT_URLS': [ + (agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO), +diff --git a/zeus/vulnerability_manage/url.py b/zeus/vulnerability_manage/url.py +deleted file mode 100644 +index fe464ac..0000000 +--- a/zeus/vulnerability_manage/url.py ++++ /dev/null +@@ -1,113 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +-""" +-Time: +-Author: +-Description: url set +-""" +-from zeus.account_manager import view as account_view +-from zeus.agent_manager import view as agent_view +-from zeus.conf.constant import ( +- ADD_GROUP, +- ADD_HOST, +- ADD_HOST_BATCH, +- ADD_USER, +- AGENT_METRIC_SET, +- AGENT_PLUGIN_INFO, +- AGENT_PLUGIN_SET, +- AUTH_REDIRECT_URL, +- BIND_AUTH_ACCOUNT, +- CHANGE_PASSWORD, +- COLLECT_CONFIG, +- DELETE_GROUP, +- DELETE_HOST, +- EXECUTE_CVE_FIX, +- EXECUTE_CVE_ROLLBACK, +- EXECUTE_CVE_SCAN, +- EXECUTE_REPO_SET, +- GET_GROUP, +- GET_HOST_COUNT, +- GET_HOST_TEMPLATE_FILE, +- GITEE_AUTH_LOGIN, +- HOST_SCENE_GET, +- LOGOUT, +- QUERY_HOST, +- QUERY_HOST_DETAIL, +- QUERY_METRIC_DATA, +- QUERY_METRIC_LIST, +- QUERY_METRIC_NAMES, +- REFRESH_TOKEN, +- UPDATE_HOST, +- USER_LOGIN, +- SYNC_CONFIG, +- OBJECT_FILE_CONFIG +-) +-from zeus.config_manager import view as config_view +-from zeus.host_manager import view as host_view +-from zeus.metric_manager import view as metric_view +-from zeus.vulnerability_manage import view as vulnerability_view +- +-URLS = [] +- +-SPECIFIC_URLS = { +- "ACCOUNT_URLS": [ +- (account_view.Login, USER_LOGIN), +- (account_view.ChangePassword, CHANGE_PASSWORD), +- (account_view.AddUser, ADD_USER), +- (account_view.GiteeAuthLogin, GITEE_AUTH_LOGIN), +- (account_view.AuthRedirectUrl, AUTH_REDIRECT_URL), +- (account_view.BindAuthAccount, BIND_AUTH_ACCOUNT), +- (account_view.RefreshToken, REFRESH_TOKEN), +- (account_view.Logout, LOGOUT), +- ], +- "HOST_URLS": [ +- (host_view.AddHost, ADD_HOST), +- (host_view.AddHostBatch, ADD_HOST_BATCH), +- (host_view.DeleteHost, DELETE_HOST), +- (host_view.UpdateHost, UPDATE_HOST), +- (host_view.GetHost, QUERY_HOST), +- (host_view.GetHostInfo, QUERY_HOST_DETAIL), +- (host_view.GetHostCount, GET_HOST_COUNT), +- (host_view.GetHostTemplateFile, GET_HOST_TEMPLATE_FILE), +- ], +- "HOST_GROUP_URLS": [ +- (host_view.AddHostGroup, ADD_GROUP), +- (host_view.DeleteHostGroup, DELETE_GROUP), +- (host_view.GetHostGroup, GET_GROUP), +- ], +- "CONFIG_URLS": [ +- (config_view.CollectConfig, COLLECT_CONFIG), +- (config_view.SyncConfig, SYNC_CONFIG), +- (config_view.ObjectFileConfig, OBJECT_FILE_CONFIG) +- ], +- 'AGENT_URLS': [ +- (agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO), +- (agent_view.SetAgentPluginStatus, AGENT_PLUGIN_SET), +- (agent_view.SetAgentMetricStatus, AGENT_METRIC_SET), +- (agent_view.GetHostScene, HOST_SCENE_GET), +- ], +- 'CVE_URLS': [ +- (vulnerability_view.ExecuteRepoSetTask, EXECUTE_REPO_SET), +- (vulnerability_view.ExecuteCveScanTask, EXECUTE_CVE_SCAN), +- (vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX), +- (vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK), +- ], +- 'METRIC': [ +- (metric_view.QueryHostMetricNames, QUERY_METRIC_NAMES), +- (metric_view.QueryHostMetricData, QUERY_METRIC_DATA), +- (metric_view.QueryHostMetricList, QUERY_METRIC_LIST), +- ], +-} +- +-for _, value in SPECIFIC_URLS.items(): +- URLS.extend(value) +-- +Gitee + diff --git a/0008-bugfix-email-config-does-not-take-effect.patch b/0008-bugfix-email-config-does-not-take-effect.patch new file mode 100644 index 0000000000000000000000000000000000000000..e11149d973c9430334ddd11404d42ff5bb211e21 --- /dev/null +++ b/0008-bugfix-email-config-does-not-take-effect.patch @@ -0,0 +1,75 @@ +From 2ccba1565c7d2ad5e4bb821f05f09a3b63edbd8b Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Tue, 31 Oct 2023 18:03:40 +0800 +Subject: [PATCH] bugfix: email config does not take effect + +--- + zeus/vulnerability_manage/view.py | 31 ++++++++++++++++++------------- + 1 file changed, 18 insertions(+), 13 deletions(-) + +diff --git a/zeus/vulnerability_manage/view.py b/zeus/vulnerability_manage/view.py +index c56f672..34a33c9 100644 +--- a/zeus/vulnerability_manage/view.py ++++ b/zeus/vulnerability_manage/view.py +@@ -15,7 +15,7 @@ import time + import threading + from typing import Dict, Tuple + +-from flask import Response ++from flask import Response, request + import sqlalchemy + import gevent + +@@ -301,14 +301,13 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask): + you can turn it off by modifying email config configuration in aops-private-config.ini, but it will still be + triggered during scheduled cve scan. + """ +- if configuration.email.get("ENABLED") or self._header.get("exempt_authentication"): +- self.get_response( +- "post", +- f'http://{configuration.apollo.get("IP")}:{ configuration.apollo.get("PORT")}{VUL_TASK_CVE_SCAN_NOTICE}', +- {}, +- self._header, +- timeout=10, +- ) ++ self.get_response( ++ "post", ++ f'http://{configuration.apollo.get("IP")}:{ configuration.apollo.get("PORT")}{VUL_TASK_CVE_SCAN_NOTICE}', ++ {}, ++ self._header, ++ timeout=10, ++ ) + + @BaseResponse.handle(schema=CveScanSchema) + def post(self, **params) -> Response: +@@ -330,6 +329,13 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask): + Returns: + response body + """ ++ ++ def execute_task(host_infos, need_to_send_email): ++ gevent.joinall([gevent.spawn(self._execute_task, host) for host in host_infos]) ++ if need_to_send_email: ++ LOGGER.info("Plan to request the interface for sending emails") ++ self._execute_send_email() ++ + # Query host basic info from database + status_code, host_infos = query_host_basic_info(params.get('total_hosts'), params.get('username')) + if status_code != state.SUCCEED: +@@ -340,10 +346,9 @@ class ExecuteCveScanTask(BaseResponse, BaseExcuteTask): + self._task_id = params.get("task_id") + self._check_items = params.get('check_items') + # Execute task +- threading.Thread( +- target=lambda: gevent.joinall([gevent.spawn(self._execute_task, host) for host in host_infos.values()]) +- ).start() +- threading.Thread(target=self._execute_send_email).start() ++ need_to_send_email = request.headers.get("exempt_authentication") or configuration.email.get("ENABLED") ++ threading.Thread(target=execute_task, args=(host_infos.values(), need_to_send_email)).start() ++ + return self.response(code=state.SUCCEED) + + +-- +Gitee + diff --git a/0009-fix-import-error.patch b/0009-fix-import-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..389fcfb47ae08099121aa870004ee7ebc16881ac --- /dev/null +++ b/0009-fix-import-error.patch @@ -0,0 +1,25 @@ +From 1e49007ac150ef9d51f8e1feff106ebb22dd9e96 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Sat, 4 Nov 2023 00:28:30 +0800 +Subject: [PATCH] fix import error + +--- + zeus/config_manager/view.py | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/zeus/config_manager/view.py b/zeus/config_manager/view.py +index bde6dbf..b012c62 100644 +--- a/zeus/config_manager/view.py ++++ b/zeus/config_manager/view.py +@@ -22,7 +22,7 @@ from typing import List, Dict + from vulcanus.multi_thread_handler import MultiThreadHandler + from vulcanus.restful.resp import state + from vulcanus.restful.response import BaseResponse +-from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF, OBJECT_FILE_CONF, CERES_OBJECT_FILE_CONF ++from zeus.conf.constant import CERES_COLLECT_FILE, CERES_SYNC_CONF, CERES_OBJECT_FILE_CONF + from zeus.database.proxy.host import HostProxy + from zeus.function.model import ClientConnectArgs + from zeus.function.verify.config import CollectConfigSchema, SyncConfigSchema, ObjectFileConfigSchema +-- +2.33.0 + diff --git a/aops-zeus-v1.3.0.tar.gz b/aops-zeus-v1.3.1.tar.gz similarity index 52% rename from aops-zeus-v1.3.0.tar.gz rename to aops-zeus-v1.3.1.tar.gz index 6882ff18ce1c90f986aea04bb1c1bd31f9061dd4..da95fe85b025d16ed980909ec8644da992876a23 100644 Binary files a/aops-zeus-v1.3.0.tar.gz and b/aops-zeus-v1.3.1.tar.gz differ diff --git a/aops-zeus.spec b/aops-zeus.spec index 00f2c2eb984d988d40a9d45409a5d24e0b523fd3..ddda8980f1074c40d1c4437cc99f42681017587d 100644 --- a/aops-zeus.spec +++ b/aops-zeus.spec @@ -1,21 +1,26 @@ Name: aops-zeus -Version: v1.3.0 -Release: 4 +Version: v1.3.1 +Release: 5 Summary: A host and user manager service which is the foundation of aops. License: MulanPSL2 URL: https://gitee.com/openeuler/%{name} Source0: %{name}-%{version}.tar.gz -Patch0001: 0001-add-file-sync-func.patch -Patch0002: 0002-update-callback-request-headers.patch -Patch0003: 0003-fix-bash-file-sync-error.patch -Patch0004: 0004-bugfix-cve-fix-result-parsing-error.patch +Patch0001: 0001-fix-metric-proxy-init-failed-error.patch +Patch0002: 0002-add-key-authentication-for-add-host-api.patch +Patch0003: 0003-fix-python-prometheus-api-client-import-error.patch +Patch0004: 0004-update-the-template-file-contents-for-adding-hosts.patch +Patch0005: 0005-bugfix-update-host-api-request-error.patch +Patch0006: 0006-optimize-import.patch +Patch0007: 0007-add-an-api-for-querying-file-list.patch +Patch0008: 0008-bugfix-email-config-does-not-take-effect.patch +Patch0009: 0009-fix-import-error.patch BuildRequires: python3-setuptools -Requires: aops-vulcanus >= v1.2.0 +Requires: aops-vulcanus >= v1.3.0 Requires: python3-marshmallow >= 3.13.0 python3-flask python3-flask-restful python3-gevent Requires: python3-requests python3-uWSGI python3-sqlalchemy python3-werkzeug python3-PyMySQL -Requires: python3-paramiko >= 2.11.0 python3-redis python3-prometheus-api-client +Requires: python3-paramiko >= 2.11.0 python3-redis Provides: aops-zeus Conflicts: aops-manager @@ -49,6 +54,25 @@ cp -r database %{buildroot}/opt/aops/ %changelog +* Mon Nov 06 2023 wenxin - v1.3.1-5 +- Bugfix: email config does not take effect +- Add an interface for querying file list + +* Fri Oct 27 2023 liulei<450962@qq.com> - v1.3.1-4 +- Bugfix: update host api request error when changing username + +* Thu Oct 26 2023 wenxin - v1.3.1-3 +- update the template file contents for adding hosts + +* Wed Oct 18 2023 wenxin - v1.3.1-2 +- fix bug: metric proxy init failed +- add a way about key authentication for add host api +- remove python3-prometheus-api-client + +* Thu Sep 21 2023 wenxin - v1.3.1-1 +- update spec requires +- update version info in setup.py file + * Tue Sep 19 2023 wenxin - v1.3.0-4 - bugfix: cve fix result parsing error