diff --git a/0001-fix-metric-proxy-init-failed-error.patch b/0001-fix-metric-proxy-init-failed-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..ca84d256b08f50312c9910ecd4ba85a97a7a5661 --- /dev/null +++ b/0001-fix-metric-proxy-init-failed-error.patch @@ -0,0 +1,46 @@ +From 833a9d721bed5b3e64ea49710a477a52b74c5255 Mon Sep 17 00:00:00 2001 +From: zhangdaolong +Date: Thu, 12 Oct 2023 14:28:06 +0800 +Subject: [PATCH 1/1] fix metric proxy init failed error +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + zeus/database/proxy/metric.py | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/zeus/database/proxy/metric.py b/zeus/database/proxy/metric.py +index e899e26..5fa75e9 100644 +--- a/zeus/database/proxy/metric.py ++++ b/zeus/database/proxy/metric.py +@@ -21,6 +21,7 @@ from prometheus_api_client import PrometheusApiClientException + from vulcanus.database.proxy import PromDbProxy + from vulcanus.log.log import LOGGER + from vulcanus.restful.resp.state import SUCCEED, DATABASE_QUERY_ERROR, NO_DATA, PARAM_ERROR, PARTIAL_SUCCEED ++from zeus.conf import configuration + + + class MetricProxy(PromDbProxy): +@@ -28,16 +29,15 @@ class MetricProxy(PromDbProxy): + Proxy of prometheus time series database + """ + +- def __init__(self, configuration, host=None, port=None): ++ def __init__(self, host=None, port=None): + """ + Init MetricProxy + + Args: +- configuration (Config) + host (str) + port (int) + """ +- PromDbProxy.__init__(self, configuration, host, port) ++ PromDbProxy.__init__(self, host, port) + self.default_instance_port = configuration.agent.get('DEFAULT_INSTANCE_PORT') or 9100 + self.query_range_step = configuration.prometheus.get('QUERY_RANGE_STEP') or "15s" + +-- +2.33.0 + diff --git a/0002-add-key-authentication-for-add-host-api.patch b/0002-add-key-authentication-for-add-host-api.patch new file mode 100644 index 0000000000000000000000000000000000000000..8a3c9e166e9936d4616345c3720d024990c7c2f2 --- /dev/null +++ b/0002-add-key-authentication-for-add-host-api.patch @@ -0,0 +1,298 @@ +From 7a8164696bb913a75cf79cf6b57c9973530efefa Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Sun, 15 Oct 2023 16:37:55 +0800 +Subject: [PATCH 1/1] add a way about key authentication for add host api +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + database/zeus.sql | 2 +- + zeus/conf/constant.py | 6 +-- + zeus/database/table.py | 2 +- + zeus/function/verify/host.py | 4 +- + zeus/host_manager/view.py | 99 ++++++++++++++++++++++++++++-------- + 5 files changed, 85 insertions(+), 28 deletions(-) + +diff --git a/database/zeus.sql b/database/zeus.sql +index 3dc9f3c..7db734e 100644 +--- a/database/zeus.sql ++++ b/database/zeus.sql +@@ -42,7 +42,7 @@ CREATE TABLE IF NOT EXISTS `host` ( + `os_version` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, + `ssh_user` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, + `ssh_port` int(11) NULL DEFAULT NULL, +- `pkey` varchar(2048) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, ++ `pkey` varchar(4096) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, + `status` int(11) NULL DEFAULT NULL, + `user` varchar(40) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NULL DEFAULT NULL, + `host_group_id` int(11) NULL DEFAULT NULL, +diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py +index 3175c65..bf8792a 100644 +--- a/zeus/conf/constant.py ++++ b/zeus/conf/constant.py +@@ -90,9 +90,9 @@ CHECK_IDENTIFY_SCENE = "/check/scene/identify" + CHECK_WORKFLOW_HOST_EXIST = '/check/workflow/host/exist' + + # host template file content +-HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,host_name,host_group_name,management +-test_ip_1,22,root,password,test_host,test_host_group,False +-test_ip_2,22,root,password,test_host,test_host_group,False ++HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,ssh_pkey,host_name,host_group_name,management ++test_ip_1,22,root,password,ssh_pkey,test_host,test_host_group,False ++test_ip_2,22,root,password,ssh_pkey,test_host,test_host_group,False + """ + + +diff --git a/zeus/database/table.py b/zeus/database/table.py +index 9596492..265eb45 100644 +--- a/zeus/database/table.py ++++ b/zeus/database/table.py +@@ -59,7 +59,7 @@ class Host(Base, MyBase): # pylint: disable=R0903 + os_version = Column(String(40)) + ssh_user = Column(String(40), default="root") + ssh_port = Column(Integer(), default=22) +- pkey = Column(String(2048)) ++ pkey = Column(String(4096)) + status = Column(Integer(), default=2) + + user = Column(String(40), ForeignKey('user.username')) +diff --git a/zeus/function/verify/host.py b/zeus/function/verify/host.py +index b054d62..d09eedd 100644 +--- a/zeus/function/verify/host.py ++++ b/zeus/function/verify/host.py +@@ -103,11 +103,12 @@ class AddHostSchema(Schema): + """ + + ssh_user = fields.String(required=True, validate=lambda s: len(s) > 0) +- password = fields.String(required=True, validate=lambda s: len(s) > 0) ++ password = fields.String(required=True, allow_none=True, validate=lambda s: len(s) >= 0) + host_name = fields.String( + required=True, validate=[validate.Length(min=1, max=50), ValidateRules.space_character_check] + ) + host_ip = fields.IP(required=True) ++ ssh_pkey = fields.String(required=True, allow_none=True, validate=lambda s: 4096 >= len(s) >= 0) + ssh_port = fields.Integer(required=True, validate=lambda s: 65535 >= s > 0) + host_group_name = fields.String(required=True, validate=lambda s: len(s) > 0) + management = fields.Boolean(required=True) +@@ -133,3 +134,4 @@ class UpdateHostSchema(Schema): + host_name = fields.String(required=False, validate=lambda s: len(s) > 0) + host_group_name = fields.String(required=False, validate=lambda s: len(s) > 0) + management = fields.Boolean(required=False) ++ ssh_pkey = fields.String(required=False, validate=lambda s: 4096 >= len(s) >= 0) +diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py +index 768d2cd..95e1434 100644 +--- a/zeus/host_manager/view.py ++++ b/zeus/host_manager/view.py +@@ -16,12 +16,13 @@ Author: + Description: Restful APIs for host + """ + import json +-from io import BytesIO ++from io import BytesIO, StringIO + from typing import Iterable, List, Tuple, Union + import socket + + import gevent + import paramiko ++from paramiko.ssh_exception import SSHException + from flask import request, send_file + from marshmallow import Schema + from marshmallow.fields import Boolean +@@ -333,7 +334,8 @@ class AddHost(BaseResponse): + "host_ip":"127.0.0.1", + "ssh_port":"22", + "management":false, +- "username": "admin" ++ "username": "admin", ++ "ssh_pkey": "RSA key" + } + + Returns: +@@ -363,6 +365,7 @@ class AddHost(BaseResponse): + "ssh_port": host_info.get("ssh_port"), + "user": host_info.get("username"), + "management": host_info.get("management"), ++ "pkey": host_info.get("ssh_pkey"), + } + ) + if host in hosts: +@@ -384,7 +387,8 @@ class AddHost(BaseResponse): + "host_ip":"127.0.0.1", + "ssh_port":"22", + "management":false, +- "username": "admin" ++ "username": "admin", ++ "ssh_pkey": "RSA key" + } + + Returns: +@@ -396,15 +400,55 @@ class AddHost(BaseResponse): + if status != state.SUCCEED: + return self.response(code=status) + +- status, private_key = save_ssh_public_key_to_client( +- params.get('host_ip'), params.get('ssh_port'), params.get('ssh_user'), params.get('password') +- ) +- if status == state.SUCCEED: +- host.pkey = private_key +- host.status = HostStatus.ONLINE ++ if params.get("ssh_pkey"): ++ status = verify_ssh_login_info( ++ ClientConnectArgs( ++ params.get("host_ip"), params.get("ssh_port"), params.get("ssh_user"), params.get("ssh_pkey") ++ ) ++ ) ++ host.status = HostStatus.ONLINE if status == state.SUCCEED else HostStatus.UNESTABLISHED ++ else: ++ status, private_key = save_ssh_public_key_to_client( ++ params.get('host_ip'), params.get('ssh_port'), params.get('ssh_user'), params.get('password') ++ ) ++ if status == state.SUCCEED: ++ host.pkey = private_key ++ host.status = HostStatus.ONLINE + return self.response(code=self.proxy.add_host(host)) + + ++def verify_ssh_login_info(ssh_login_info: ClientConnectArgs) -> str: ++ """ ++ Verify that the ssh login information is correct ++ ++ Args: ++ ssh_login_info(ClientConnectArgs): e.g ++ ClientConnectArgs(host_ip='127.0.0.1', ssh_port=22, ssh_user='root', pkey=RSAKey string) ++ ++ Returns: ++ status code ++ """ ++ try: ++ client = SSH( ++ ip=ssh_login_info.host_ip, ++ username=ssh_login_info.ssh_user, ++ port=ssh_login_info.ssh_port, ++ pkey=paramiko.RSAKey.from_private_key(StringIO(ssh_login_info.pkey)), ++ ) ++ client.close() ++ except socket.error as error: ++ LOGGER.error(error) ++ return state.SSH_CONNECTION_ERROR ++ except SSHException as error: ++ LOGGER.error(error) ++ return state.SSH_AUTHENTICATION_ERROR ++ except Exception as error: ++ LOGGER.error(error) ++ return state.SSH_CONNECTION_ERROR ++ ++ return state.SUCCEED ++ ++ + def save_ssh_public_key_to_client(ip: str, port: int, username: str, password: str) -> tuple: + """ + generate RSA key pair,save public key to the target host machine +@@ -465,7 +509,7 @@ class GetHostTemplateFile(BaseResponse): + file = BytesIO() + file.write(HOST_TEMPLATE_FILE_CONTENT.encode('utf-8')) + file.seek(0) +- response = send_file(file,mimetype="application/octet-stream") ++ response = send_file(file, mimetype="application/octet-stream") + response.headers['Content-Disposition'] = 'attachment; filename=template.csv' + return response + +@@ -574,6 +618,7 @@ class AddHostBatch(BaseResponse): + continue + + password = host_info.pop("password") ++ pkey = host_info.pop("ssh_pkey", None) + host_info.update( + {"host_group_id": group_id_info.get(host_info['host_group_name']), "user": data["username"]} + ) +@@ -585,7 +630,7 @@ class AddHostBatch(BaseResponse): + ) + continue + +- valid_host.append((host, password)) ++ valid_host.append((host, password, pkey)) + return valid_host + + def save_key_to_client(self, host_connect_infos: List[tuple]) -> list: +@@ -598,8 +643,8 @@ class AddHostBatch(BaseResponse): + Returns: + host object list + """ +- # 30 connections are created at a time. +- tasks = [host_connect_infos[index : index + 30] for index in range(0, len(host_connect_infos), 30)] ++ # 100 connections are created at a time. ++ tasks = [host_connect_infos[index : index + 100] for index in range(0, len(host_connect_infos), 100)] + result = [] + + for task in tasks: +@@ -612,18 +657,23 @@ class AddHostBatch(BaseResponse): + return result + + @staticmethod +- def update_rsa_key_to_host(host: Host, password: str) -> Host: ++ def update_rsa_key_to_host(host: Host, password: str = None, pkey: str = None) -> Host: + """ + save ssh public key to client and update its private key in host + + Args: + host(Host): host object + password(str): password for ssh login ++ pkey(str): rsa key for ssh login + + Returns: + host object + """ +- status, pkey = save_ssh_public_key_to_client(host.host_ip, host.ssh_port, host.ssh_user, password) ++ if pkey: ++ status = verify_ssh_login_info(ClientConnectArgs(host.host_ip, host.ssh_port, host.ssh_user, pkey)) ++ else: ++ status, pkey = save_ssh_public_key_to_client(host.host_ip, host.ssh_port, host.ssh_user, password) ++ + if status == state.SUCCEED: + host.status = HostStatus.ONLINE + host.pkey = pkey +@@ -654,7 +704,7 @@ class AddHostBatch(BaseResponse): + new_host.update(update_info) + self.add_result.append(new_host) + else: +- for host, _ in hosts: ++ for host, _, _ in hosts: + new_host = { + "host_ip": host.host_ip, + "ssh_port": host.ssh_port, +@@ -789,9 +839,14 @@ class UpdateHost(BaseResponse): + """ + ssh_user = params.get("ssh_user") or self.host.ssh_user + ssh_port = params.get("ssh_port") or self.host.ssh_port +- status, private_key = save_ssh_public_key_to_client( +- self.host.host_ip, ssh_port, ssh_user, params.pop("password", None) +- ) ++ private_key = params.pop("ssh_pkey", None) ++ if private_key: ++ status = verify_ssh_login_info(ClientConnectArgs(self.host.host_ip, ssh_port, ssh_user, private_key)) ++ else: ++ status, private_key = save_ssh_public_key_to_client( ++ self.host.host_ip, ssh_port, ssh_user, params.pop("password", None) ++ ) ++ + params.update( + { + "ssh_user": ssh_user, +@@ -876,10 +931,10 @@ class UpdateHost(BaseResponse): + return self.response(code=state.PARAM_ERROR, message="there is a duplicate host ssh address in database!") + + if params.get("ssh_user") or params.get("ssh_port"): +- if not params.get("password"): +- return self.response(code=state.PARAM_ERROR, message="please update password") ++ if not params.get("password") or not params.get("ssh_pkey"): ++ return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.") + self._save_ssh_key(params) +- elif params.get("password"): ++ elif params.get("password") or params.get("ssh_pkey"): + self._save_ssh_key(params) + + return self.response(callback.update_host_info(params.pop("host_id"), params)) +-- +2.33.0 + diff --git a/0003-fix-python-prometheus-api-client-import-error.patch b/0003-fix-python-prometheus-api-client-import-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..63cc76ebb8c3026d4eb640ceed319d2f5ab1a28c --- /dev/null +++ b/0003-fix-python-prometheus-api-client-import-error.patch @@ -0,0 +1,658 @@ +From 3ea95e9dcb73b0add7c7913dd64e67131ea5d9b0 Mon Sep 17 00:00:00 2001 +From: gongzt +Date: Wed, 18 Oct 2023 17:19:11 +0800 +Subject: fix python-prometheus-api-client import error +MIME-Version: 1.0 +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: 8bit + +--- + zeus/conf/constant.py | 4 - + zeus/database/proxy/metric.py | 458 -------------------------------- + zeus/function/verify/metric.py | 28 -- + zeus/metric_manager/__init__.py | 12 - + zeus/metric_manager/view.py | 51 ---- + zeus/url.py | 14 +- + 6 files changed, 1 insertion(+), 566 deletions(-) + delete mode 100644 zeus/database/proxy/metric.py + delete mode 100644 zeus/function/verify/metric.py + delete mode 100644 zeus/metric_manager/__init__.py + delete mode 100644 zeus/metric_manager/view.py + +diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py +index bf8792a..304ed7e 100644 +--- a/zeus/conf/constant.py ++++ b/zeus/conf/constant.py +@@ -72,10 +72,6 @@ EXECUTE_CVE_FIX = '/manage/vulnerability/cve/fix' + EXECUTE_CVE_SCAN = '/manage/vulnerability/cve/scan' + EXECUTE_CVE_ROLLBACK = "/manage/vulnerability/cve/rollback" + +-# metric config +-QUERY_METRIC_NAMES = '/manage/host/metric/names' +-QUERY_METRIC_DATA = '/manage/host/metric/data' +-QUERY_METRIC_LIST = '/manage/host/metric/list' + + # auth login + GITEE_OAUTH = "https://gitee.com/oauth/authorize" +diff --git a/zeus/database/proxy/metric.py b/zeus/database/proxy/metric.py +deleted file mode 100644 +index 5fa75e9..0000000 +--- a/zeus/database/proxy/metric.py ++++ /dev/null +@@ -1,458 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +-""" +-Time: 2022-07-27 +-Author: YangYunYi +-Description: Query raw data from Prometheus +-""" +-from typing import Dict, Tuple, List, Optional +-import datetime +-from prometheus_api_client import PrometheusApiClientException +-from vulcanus.database.proxy import PromDbProxy +-from vulcanus.log.log import LOGGER +-from vulcanus.restful.resp.state import SUCCEED, DATABASE_QUERY_ERROR, NO_DATA, PARAM_ERROR, PARTIAL_SUCCEED +-from zeus.conf import configuration +- +- +-class MetricProxy(PromDbProxy): +- """ +- Proxy of prometheus time series database +- """ +- +- def __init__(self, host=None, port=None): +- """ +- Init MetricProxy +- +- Args: +- host (str) +- port (int) +- """ +- PromDbProxy.__init__(self, host, port) +- self.default_instance_port = configuration.agent.get('DEFAULT_INSTANCE_PORT') or 9100 +- self.query_range_step = configuration.prometheus.get('QUERY_RANGE_STEP') or "15s" +- +- @staticmethod +- def __metric_dict2str(metric: Dict) -> str: +- """ +- Trans metric dict to string +- Args: +- metric (Dict): +- { +- "__name__": "metric name1", +- 'instance': '172.168.128.164:9100', +- 'job': 'prometheus', +- 'label1': 'label_value1' +- ... +- } +- +- Returns: +- metric_str(str): 'metric_name1{label1="value1", label2="values"}' +- """ +- +- label_str = "" +- if "__name__" not in metric.keys(): +- return label_str +- sorted_label_list = sorted(metric.items(), reverse=False) +- for label in sorted_label_list: +- # The job label is usually "prometheus" in this framework and +- # has no effect on subsequent data requests, so remove it to save space. +- # __name__ label move to the front. +- metric_key = label[0] +- metric_value = label[1] +- if metric_key in ["__name__", "job"]: +- continue +- label_str += "%s=\"%s\"," % (metric_key, metric_value) +- +- if label_str == "": +- # It's a metric with only a name and no personalized label +- ret = metric["__name__"] +- else: +- # Remove the comma of the last element +- ret = "%s{%s}" % (metric["__name__"], label_str[:-1]) +- return ret +- +- def query_metric_names(self, query_ip: str) -> Tuple[int, dict]: +- """ +- Query data +- Args: +- query_ip(str): query ip address +- +- Returns: +- int: status code +- dict: e.g +- { +- "results": ["metric1", "metric2"] +- } +- """ +- +- query_ip = query_ip["query_ip"] +- query_host = {"host_id": "query_host_id", "host_ip": query_ip} +- +- query_metric_names = [] +- res = {'results': query_metric_names} +- +- host_ip = query_host["host_ip"] +- host_port = query_host.get("instance_port", self.default_instance_port) +- +- ret, metric_list = self.query_metric_list_of_host(host_ip, host_port) +- +- if ret != SUCCEED: +- return ret, res +- +- for metric in metric_list: +- metric = metric.split('{')[0] +- if metric not in query_metric_names: +- query_metric_names.append(metric) +- +- return ret, res +- +- def query_metric_list(self, data: Dict[str, str]) -> Tuple[int, dict]: +- """ +- Query metric list +- Args: +- data(dict): param e.g +- { +- "query_ip": "str", +- "metric_names": ["metric1", "metric2"] +- } +- +- Returns: +- int: status code +- dict: e.g +- { +- 'results': { +- "metric1": [ +- "metric1{label1="label1_value", label2="label2_value", ..., }", +- "metric1{label1="label1_value", label2="label2_value", ..., }", +- ], +- "metric2": [ +- "metric2{label1="label1_value", label2="label2_value", ..., }", +- "metric2{label1="label1_value", label2="label2_value", ..., }", +- ] +- } +- } +- """ +- +- query_ip = data.get('query_ip') +- query_metric = data.get('metric_names') +- +- query_host = {"host_id": "query_host_id", "host_ip": query_ip} +- +- query_metric_list = {} +- res = {'results': query_metric_list} +- +- host_ip = query_host["host_ip"] +- host_port = query_host.get("instance_port", self.default_instance_port) +- +- ret, metric_list = self.query_metric_list_of_host( +- host_ip, +- host_port, +- ) +- +- if ret != SUCCEED: +- LOGGER.warning("Host metric list query error") +- return ret, res +- +- for query_metric_name in query_metric: +- query_metric_list[query_metric_name] = [] +- for metric in metric_list: +- metric_name = metric.split('{')[0] +- if metric_name != query_metric_name: +- continue +- query_metric_list[query_metric_name].append(metric) +- +- return ret, res +- +- def query_metric_data(self, data: Dict[str, str]) -> Tuple[int, dict]: +- """ +- Query metric data +- Args: +- data(dict): param e.g +- { +- "time_range": "list", +- "query_ip": "str", +- "query_info": { +- "metric1": [ +- "metric1{label1="label1_value", label2="label2_value", ..., }", +- "metric1{label1="label1_value", label2="label2_value", ..., }", +- ], +- "metric2": [ +- "metric2{label1="label1_value", label2="label2_value", ..., }", +- "metric2{label1="label1_value", label2="label2_value", ..., }", +- ] +- } +- } +- +- Returns: +- int: status code +- dict: e.g +- { +- 'results': { +- "metric1": { +- "metric1{label1="label1_value", label2="label2_value", ..., }": [ +- [1658926441, '0'], +- [1658926441, '0'] +- ], +- "metric1{label1="label1_value", label2="label2_value", ..., }": [ +- [1658926441, '0'], +- [1658926441, '0'] +- ], +- }, +- "metric1": { +- "metric2{label1="label1_value", label2="label2_value", ..., }": [ +- [1658926441, '0'], +- [1658926441, '0'] +- ], +- "metric2{label1="label1_value", label2="label2_value", ..., }": [ +- [1658926441, '0'], +- [1658926441, '0'] +- ], +- }, +- } +- } +- """ +- time_range = data.get('time_range') +- query_ip = data.get('query_ip') +- query_info = data.get('query_info') +- +- query_host = {"host_id": "query_host_id", "host_ip": query_ip} +- +- host_ip = query_host["host_ip"] +- host_port = query_host.get("instance_port", self.default_instance_port) +- +- query_host_list = [query_host] +- query_data = {} +- res = {'results': query_data} +- +- # adjust query range step based on query time range +- # the default query range step is 15 seconds +- QUERY_RANGE_STEP = 15 +- query_range_step = QUERY_RANGE_STEP +- query_range_seconds = time_range[1] - time_range[0] +- total_query_times = query_range_seconds / query_range_step +- +- while total_query_times > 11000: +- query_range_step += 15 +- total_query_times = query_range_seconds / query_range_step +- +- def add_two_dim_dict(thedict, key_a, key_b, val): +- if key_a in thedict: +- thedict[key_a].update({key_b: val}) +- else: +- thedict.update({key_a: {key_b: val}}) +- +- if not query_info: +- return SUCCEED, res +- +- for metric_name, metric_list in query_info.items(): +- if not metric_list: +- _, metric_list = self.query_metric_list_of_host(host_ip, host_port, metric_name) +- if not metric_list: +- query_data[metric_name] = [] +- for metric_info in metric_list: +- data_status, monitor_data = self.query_data( +- time_range=time_range, +- host_list=query_host_list, +- metric=metric_info, +- adjusted_range_step=query_range_step, +- ) +- values = [] +- if data_status == SUCCEED: +- values = monitor_data[query_host["host_id"]][metric_info] +- add_two_dim_dict(query_data, metric_name, metric_info, values) +- +- return SUCCEED, res +- +- def query_data( +- self, +- time_range: List[int], +- host_list: list, +- metric: Optional[str] = None, +- adjusted_range_step: Optional[int] = None, +- ) -> Tuple[int, Dict]: +- """ +- Query data +- Args: +- time_range(list): time range +- host_list(list): host list, If the port is not specified, the default value is used +- [{"host_id": "id1", "host_ip": "172.168.128.164", "instance_port": 9100}, +- {"host_id": "id1", "host_ip": "172.168.128.164", "instance_port": 8080}, +- {"host_id": "id2", "host_ip": "172.168.128.165"}] +- +- Returns: +- ret(int): query ret +- host_data_list(dict): host data list ret +- { +- 'id1': { +- 'metric1'{instance="172.168.128.164:9100",label1="value2"}': +- [[time1, 'value1'], +- [time2, 'value2'], +- 'metric12{instance="172.168.128.164:8080"}': [], => get data list is empty +- 'metric13{instance="172.168.128.164:8080"}': None => get no data +- }, +- 'id2': None => get no metric list of this host +- } +- """ +- +- host_data_list = {} +- if not host_list: +- return PARAM_ERROR, host_data_list +- +- status = SUCCEED +- for host in host_list: +- host_id = host["host_id"] +- host_ip = host["host_ip"] +- host_port = host.get("instance_port", self.default_instance_port) +- if host_id not in host_data_list.keys(): +- host_data_list[host_id] = None +- +- ret, metric_list = self.query_metric_list_of_host(host_ip, host_port, metric) +- if ret != SUCCEED: +- status = PARTIAL_SUCCEED +- host_data_list[host_id] = None +- continue +- ret, data_list = self.__query_data_by_host(metric_list, time_range, adjusted_range_step) +- if ret != SUCCEED: +- status = PARTIAL_SUCCEED +- if not host_data_list[host_id]: +- host_data_list[host_id] = data_list +- else: +- host_data_list[host_id].update(data_list) +- +- return status, host_data_list +- +- def __parse_metric_data(self, metric_data: List) -> List[str]: +- """ +- Parse metric data from prometheus to name<-> label_config dict +- Args: +- metric_data(List): metric list data from prometheus +- [{'metric':{ +- "__name__": "metric_name1", +- 'instance': '172.168.128.164:9100', +- 'job': 'prometheus', +- 'label1': 'label_value1' +- ... +- }, +- 'value': [1658926441.526, '0'] +- }] +- +- Returns: +- metric_list(List[str]): +- [ +- 'metric_name1{label1="value1", label2="value2"}' +- ] +- """ +- +- metric_list = [] +- for metric_item in metric_data: +- metric_dict = metric_item["metric"] +- metric_str = self.__metric_dict2str(metric_dict) +- if metric_str == "": +- continue +- metric_list.append(metric_str) +- +- return metric_list +- +- def query_metric_list_of_host( +- self, host_ip: str, host_port: Optional[int] = None, metric: Optional[str] = None +- ) -> Tuple[int, List[str]]: +- """ +- Query metric list of a host +- Args: +- host_ip(str): host ip +- host_port(int): host port +- +- Returns: +- ret(int): query ret +- metric_list(list): metric list ret +- [ +- 'metric_name1{label1="value1", label2="value2"}' +- ] +- """ +- if not host_port: +- host_port = self.default_instance_port +- query_str = "{instance=\"%s:%s\"}" % (host_ip, str(host_port)) +- if metric is not None: +- if metric.find('{') != -1: +- query_str = metric +- else: +- query_str = metric + query_str +- try: +- data = self._prom.custom_query(query=query_str) +- if not data: +- if query_str.startswith('{'): +- LOGGER.error( +- "Query metric list result is empty. " +- "Can not get metric list of host %s:%s " % (host_ip, host_port) +- ) +- return NO_DATA, [] +- metric_list = self.__parse_metric_data(data) +- return SUCCEED, metric_list +- +- except (ValueError, TypeError, PrometheusApiClientException) as error: +- LOGGER.error("host %s:%d Prometheus query metric list failed. %s" % (host_ip, host_port, error)) +- return DATABASE_QUERY_ERROR, [] +- +- def __query_data_by_host( +- self, metrics_list: List[str], time_range: List[int], adjusted_range_step: Optional[int] = None +- ) -> Tuple[int, Dict]: +- """ +- Query data of a host +- Args: +- metrics_list(list): metric list of this host +- time_range(list): time range to query +- +- Returns: +- ret(int): query ret +- data_list(list): data list ret +- { +- 'metric1'{instance="172.168.128.164:9100",label1="value2"}': +- [[time1, 'value1'], +- [time2, 'value2'], +- 'metric12{instance="172.168.128.164:8080"}': [], => get data list is empty +- 'metric13{instance="172.168.128.164:8080"}': None => get no data +- } +- +- """ +- start_time = datetime.datetime.fromtimestamp(time_range[0]) +- end_time = datetime.datetime.fromtimestamp(time_range[1]) +- +- query_range_step = self.query_range_step +- if adjusted_range_step is not None: +- query_range_step = adjusted_range_step +- +- data_list = {} +- ret = SUCCEED +- for metric in metrics_list: +- try: +- data = self._prom.custom_query_range( +- query=metric, start_time=start_time, end_time=end_time, step=query_range_step +- ) +- if not data or "values" not in data[0]: +- LOGGER.debug( +- "Query data result is empty. " +- "metric %s in %d-%d doesn't record in the prometheus " % (metric, time_range[0], time_range[1]) +- ) +- data_list[metric] = None +- ret = PARTIAL_SUCCEED +- continue +- data_list[metric] = data[0]["values"] +- +- except (ValueError, TypeError, PrometheusApiClientException) as error: +- LOGGER.error( +- "Prometheus metric %s in %d-%d query data failed. %s" +- % (metric, time_range[0], time_range[1], error) +- ) +- data_list[metric] = None +- ret = PARTIAL_SUCCEED +- return ret, data_list +diff --git a/zeus/function/verify/metric.py b/zeus/function/verify/metric.py +deleted file mode 100644 +index 5df28a2..0000000 +--- a/zeus/function/verify/metric.py ++++ /dev/null +@@ -1,28 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2022. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +-from marshmallow import Schema, fields +- +- +-class QueryHostMetricNamesSchema(Schema): +- query_ip = fields.String(required=True) +- +- +-class QueryHostMetricDataSchema(Schema): +- time_range = fields.List(fields.Integer, required=True) +- query_ip = fields.String(required=True) +- query_info = fields.Dict() +- +- +-class QueryHostMetricListSchema(Schema): +- query_ip = fields.String(required=True) +- metric_names = fields.List(fields.String) +diff --git a/zeus/metric_manager/__init__.py b/zeus/metric_manager/__init__.py +deleted file mode 100644 +index e90ecf9..0000000 +--- a/zeus/metric_manager/__init__.py ++++ /dev/null +@@ -1,12 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2021-2021. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +diff --git a/zeus/metric_manager/view.py b/zeus/metric_manager/view.py +deleted file mode 100644 +index 4d98cf2..0000000 +--- a/zeus/metric_manager/view.py ++++ /dev/null +@@ -1,51 +0,0 @@ +-#!/usr/bin/python3 +-# ****************************************************************************** +-# Copyright (c) Huawei Technologies Co., Ltd. 2022-2022. All rights reserved. +-# licensed under the Mulan PSL v2. +-# You can use this software according to the terms and conditions of the Mulan PSL v2. +-# You may obtain a copy of Mulan PSL v2 at: +-# http://license.coscl.org.cn/MulanPSL2 +-# THIS SOFTWARE IS PROVIDED ON AN 'AS IS' BASIS, WITHOUT WARRANTIES OF ANY KIND, EITHER EXPRESS OR +-# IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT, MERCHANTABILITY OR FIT FOR A PARTICULAR +-# PURPOSE. +-# See the Mulan PSL v2 for more details. +-# ******************************************************************************/ +-from vulcanus.restful.response import BaseResponse +-from zeus.database.proxy.metric import MetricProxy +-from zeus.function.verify.metric import QueryHostMetricDataSchema, QueryHostMetricListSchema, QueryHostMetricNamesSchema +- +- +-class QueryHostMetricNames(BaseResponse): +- """ +- Interface for query host metric names from web. +- Restful API: GET +- """ +- +- @BaseResponse.handle(schema=QueryHostMetricNamesSchema, proxy=MetricProxy) +- def get(self, callback: MetricProxy, **params): +- status_code, result = callback.query_metric_names(params) +- return self.response(code=status_code, data=result) +- +- +-class QueryHostMetricData(BaseResponse): +- """ +- Interface for query host metric data from web. +- Restful API: POST +- """ +- +- @BaseResponse.handle(schema=QueryHostMetricDataSchema, proxy=MetricProxy) +- def post(self, callback: MetricProxy, **params): +- status_code, result = callback.query_metric_data(params) +- return self.response(code=status_code, data=result) +- +- +-class QueryHostMetricList(BaseResponse): +- """ +- Interface for query host metric list from web. +- Restful API: POST +- """ +- +- @BaseResponse.handle(schema=QueryHostMetricListSchema, proxy=MetricProxy) +- def post(self, callback: MetricProxy, **params): +- status_code, result = callback.query_metric_list(params) +- return self.response(code=status_code, data=result) +diff --git a/zeus/url.py b/zeus/url.py +index 597dcc7..285e942 100644 +--- a/zeus/url.py ++++ b/zeus/url.py +@@ -43,9 +43,6 @@ from zeus.conf.constant import ( + LOGOUT, + QUERY_HOST, + QUERY_HOST_DETAIL, +- QUERY_METRIC_DATA, +- QUERY_METRIC_LIST, +- QUERY_METRIC_NAMES, + REFRESH_TOKEN, + UPDATE_HOST, + USER_LOGIN, +@@ -53,7 +50,6 @@ from zeus.conf.constant import ( + ) + from zeus.config_manager import view as config_view + from zeus.host_manager import view as host_view +-from zeus.metric_manager import view as metric_view + from zeus.vulnerability_manage import view as vulnerability_view + + URLS = [] +@@ -84,10 +80,7 @@ SPECIFIC_URLS = { + (host_view.DeleteHostGroup, DELETE_GROUP), + (host_view.GetHostGroup, GET_GROUP), + ], +- "CONFIG_URLS": [ +- (config_view.CollectConfig, COLLECT_CONFIG), +- (config_view.SyncConfig, SYNC_CONFIG) +- ], ++ "CONFIG_URLS": [(config_view.CollectConfig, COLLECT_CONFIG), (config_view.SyncConfig, SYNC_CONFIG)], + 'AGENT_URLS': [ + (agent_view.AgentPluginInfo, AGENT_PLUGIN_INFO), + (agent_view.SetAgentPluginStatus, AGENT_PLUGIN_SET), +@@ -100,11 +93,6 @@ SPECIFIC_URLS = { + (vulnerability_view.ExecuteCveFixTask, EXECUTE_CVE_FIX), + (vulnerability_view.ExecuteCveRollbackTask, EXECUTE_CVE_ROLLBACK), + ], +- 'METRIC': [ +- (metric_view.QueryHostMetricNames, QUERY_METRIC_NAMES), +- (metric_view.QueryHostMetricData, QUERY_METRIC_DATA), +- (metric_view.QueryHostMetricList, QUERY_METRIC_LIST), +- ], + } + + for _, value in SPECIFIC_URLS.items(): +-- +2.27.0 + diff --git a/0004-update-the-template-file-contents-for-adding-hosts.patch b/0004-update-the-template-file-contents-for-adding-hosts.patch new file mode 100644 index 0000000000000000000000000000000000000000..ca6c95daa91791c2276f736f2ed14d7c4e63efeb --- /dev/null +++ b/0004-update-the-template-file-contents-for-adding-hosts.patch @@ -0,0 +1,33 @@ +From 36d5b7a26f9f470e0b9a593edb7f198cd9022c47 Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Thu, 26 Oct 2023 18:43:37 +0800 +Subject: [PATCH] update the template file contents for adding hosts + +--- + zeus/conf/constant.py | 10 ++++++++-- + 1 file changed, 8 insertions(+), 2 deletions(-) + +diff --git a/zeus/conf/constant.py b/zeus/conf/constant.py +index bf8792a..df2948d 100644 +--- a/zeus/conf/constant.py ++++ b/zeus/conf/constant.py +@@ -91,8 +91,14 @@ CHECK_WORKFLOW_HOST_EXIST = '/check/workflow/host/exist' + + # host template file content + HOST_TEMPLATE_FILE_CONTENT = """host_ip,ssh_port,ssh_user,password,ssh_pkey,host_name,host_group_name,management +-test_ip_1,22,root,password,ssh_pkey,test_host,test_host_group,False +-test_ip_2,22,root,password,ssh_pkey,test_host,test_host_group,False ++127.0.0.1,22,root,password,private key,test_host,test_host_group,FALSE ++127.0.0.1,23,root,password,private key,test_host,test_host_group,FALSE ++,,,,,,, ++"提示:",,,,,,, ++"1. 除登录密码与SSH登录秘钥外,其余信息都应提供有效值",,,,,,, ++"2. 登录密码与SSH登录秘钥可选择一种填入,当两者都提供时,以SSH登录秘钥为准",,,,,,, ++"3. 添加的主机信息不应存在重复信息(主机名称重复或者主机IP+端口重复)",,,,,,, ++"4. 上传本文件前,请删除此部分提示内容",,,,,,, + """ + + +-- +2.33.0 + diff --git a/0005-bugfix-update-host-api-request-error.patch b/0005-bugfix-update-host-api-request-error.patch new file mode 100644 index 0000000000000000000000000000000000000000..d87c727716c67f34eb3e348182516b89d59aa261 --- /dev/null +++ b/0005-bugfix-update-host-api-request-error.patch @@ -0,0 +1,33 @@ +From 36f98b43bd571ac9f2f4f9a9fe658684d591d52e Mon Sep 17 00:00:00 2001 +From: rabbitali +Date: Fri, 27 Oct 2023 15:21:54 +0800 +Subject: [PATCH] bugfix: update host api request error when changing username + +--- + zeus/host_manager/view.py | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/zeus/host_manager/view.py b/zeus/host_manager/view.py +index 95e1434..10418d1 100644 +--- a/zeus/host_manager/view.py ++++ b/zeus/host_manager/view.py +@@ -930,11 +930,11 @@ class UpdateHost(BaseResponse): + LOGGER.warning(f"there is a duplicate host address in database " f"when update host {self.host.host_id}!") + return self.response(code=state.PARAM_ERROR, message="there is a duplicate host ssh address in database!") + +- if params.get("ssh_user") or params.get("ssh_port"): +- if not params.get("password") or not params.get("ssh_pkey"): +- return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.") +- self._save_ssh_key(params) +- elif params.get("password") or params.get("ssh_pkey"): ++ if params.get("password") or params.get("ssh_pkey"): + self._save_ssh_key(params) ++ return self.response(callback.update_host_info(params.pop("host_id"), params)) ++ ++ if params.get("ssh_user") or params.get("ssh_port"): ++ return self.response(code=state.PARAM_ERROR, message="please update password or authentication key.") + + return self.response(callback.update_host_info(params.pop("host_id"), params)) +-- +2.33.0 + diff --git a/aops-zeus.spec b/aops-zeus.spec index 6da50b1e1b73f84f948328a644adc251beaad898..fef8fa566fe6f97d373669e10e39370299ca4ecc 100644 --- a/aops-zeus.spec +++ b/aops-zeus.spec @@ -1,17 +1,22 @@ Name: aops-zeus Version: v1.3.1 -Release: 1 +Release: 4 Summary: A host and user manager service which is the foundation of aops. License: MulanPSL2 URL: https://gitee.com/openeuler/%{name} Source0: %{name}-%{version}.tar.gz +Patch0001: 0001-fix-metric-proxy-init-failed-error.patch +Patch0002: 0002-add-key-authentication-for-add-host-api.patch +Patch0003: 0003-fix-python-prometheus-api-client-import-error.patch +Patch0004: 0004-update-the-template-file-contents-for-adding-hosts.patch +Patch0005: 0005-bugfix-update-host-api-request-error.patch BuildRequires: python3-setuptools Requires: aops-vulcanus >= v1.3.0 Requires: python3-marshmallow >= 3.13.0 python3-flask python3-flask-restful python3-gevent Requires: python3-requests python3-uWSGI python3-sqlalchemy python3-werkzeug python3-PyMySQL -Requires: python3-paramiko >= 2.11.0 python3-redis python3-prometheus-api-client +Requires: python3-paramiko >= 2.11.0 python3-redis Provides: aops-zeus Conflicts: aops-manager @@ -45,6 +50,17 @@ cp -r database %{buildroot}/opt/aops/ %changelog +* Fri Oct 27 2023 wenxin - v1.3.1-4 +- Bugfix: update host api request error when changing username + +* Thu Oct 26 2023 wenxin - v1.3.1-3 +- update the template file contents for adding hosts + +* Wed Oct 18 2023 wenxin - v1.3.1-2 +- fix bug: metric proxy init failed +- add a way about key authentication for add host api +- remove python3-prometheus-api-client + * Thu Sep 21 2023 wenxin - v1.3.1-1 - update spec requires - update version info in setup.py file