From 52e1c19cc7eb82134106b20f4ccf840125f210df Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 19:06:44 +0800 Subject: [PATCH 001/196] feat(sdk): Add Framework Plugin Interface --- conf/config.yml | 3 + sysom_server/sdk/sysom_utils/__init__.py | 4 +- sysom_server/sdk/sysom_utils/config_parser.py | 21 ++++++ .../sdk/sysom_utils/framework_plug_mag.py | 72 +++++++++++++++++++ .../sdk/sysom_utils/framework_plugins.py | 69 ++++++++++++++++++ sysom_server/sdk/sysom_utils/yaml_concat.py | 2 +- 6 files changed, 169 insertions(+), 2 deletions(-) create mode 100644 sysom_server/sdk/sysom_utils/framework_plug_mag.py create mode 100644 sysom_server/sdk/sysom_utils/framework_plugins.py diff --git a/conf/config.yml b/conf/config.yml index 7c24acb7..6834b042 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -51,6 +51,9 @@ sysom_server: producer: cec_auto_mk_topic: true consumer: + cmg: + protocol: redis + special_param: logger: format: "%(asctime)s | %(levelname)s | %(message)s" level: "INFO" diff --git a/sysom_server/sdk/sysom_utils/__init__.py b/sysom_server/sdk/sysom_utils/__init__.py index b753d115..107fedfd 100644 --- a/sysom_server/sdk/sysom_utils/__init__.py +++ b/sysom_server/sdk/sysom_utils/__init__.py @@ -12,4 +12,6 @@ from .adddict import * from .node_manager import * from .yaml_concat import * from .config_parser import * -from .event_executor import * \ No newline at end of file +from .event_executor import * +from .framework_plug_mag import * +from .framework_plugins import * \ No newline at end of file diff --git a/sysom_server/sdk/sysom_utils/config_parser.py b/sysom_server/sdk/sysom_utils/config_parser.py index dde4227a..eb09d0d7 100644 --- a/sysom_server/sdk/sysom_utils/config_parser.py +++ b/sysom_server/sdk/sysom_utils/config_parser.py @@ -86,6 +86,9 @@ class ConfigParser: def get_service_config(self) -> Dict: return Dict(self._config.get(SYSOM_CONFIG_SECTION_SERVICE, {})) + def get_plugin_config(self, plugin_name) -> Dict: + return Dict(self.get_service_config().plugins.get(plugin_name, {})) + ############################################################################## # Helper functions ############################################################################## @@ -118,6 +121,24 @@ class ConfigParser: cec_url += "&".join(params) return cec_url + def get_cmg_url(self) -> str: + server_config = self.get_server_config() + cmg_config = server_config.cmg + special_param = {} + params = [] + dict_merge(special_param, cmg_config.special_param) + cmg_url = "" + if cmg_config.protocol == "redis": + redis_config = server_config.db.redis + cmg_url = ( + f"{cmg_config.protocol}://{redis_config.host}:{redis_config.port}?" + ) + if redis_config.username: + params.append(f"username={redis_config.username}") + if redis_config.password: + params.append(f"password={redis_config.password}") + return cmg_url + def get_local_channel_job_url(self) -> str: server_config = self.get_server_config() channel_job_config = server_config.channel_job diff --git a/sysom_server/sdk/sysom_utils/framework_plug_mag.py b/sysom_server/sdk/sysom_utils/framework_plug_mag.py new file mode 100644 index 00000000..14368c5a --- /dev/null +++ b/sysom_server/sdk/sysom_utils/framework_plug_mag.py @@ -0,0 +1,72 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/04/06 13:13 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File framework_plug_mag.py +Description: +""" +import threading +from typing import List, Optional, Type +from abc import abstractmethod, ABC +from .config_parser import ConfigParser + + +class StoppableThread(threading.Thread): + """Thread class with a stop() method + The thread itself has to check regularly for the stopped() condition. + """ + + def __init__(self, group=None, target=None, **kwargs): + super().__init__(group, target, **kwargs) + self._stop_event = threading.Event() + + def stop(self): + """Notify the thread to exit（async）""" + self._stop_event.set() + + def stopped(self): + """Determine whether the thread exits.""" + return self._stop_event.is_set() + + +class FrameworkPluginBase(ABC): + + @abstractmethod + def start(self): + pass + + @abstractmethod + def stop(self): + pass + + @abstractmethod + def join(self, timeout: Optional[float] = None): + pass + + +class FrameworkPlugMag(StoppableThread): + """A class for managing plugins in the framework layer + """ + + def __init__(self, config: ConfigParser) -> None: + self._config = config + self.setDaemon(True) + self._plugins: List[FrameworkPluginBase] = [] + + def load_plugin(self, plugin: FrameworkPluginBase): + self._plugins.append(plugin) + return self + + def load_plugin_cls(self, plugin_cls: Type[FrameworkPluginBase]): + self._plugins.append(plugin_cls(self._config)) + + def start(self) -> None: + for plugin in self._plugins: + plugin.start() + return super().start() + + def join(self, timeout: Optional[float] = None) -> None: + for plugin in self._plugins: + plugin.join(timeout) + return super().join(timeout) diff --git a/sysom_server/sdk/sysom_utils/framework_plugins.py b/sysom_server/sdk/sysom_utils/framework_plugins.py new file mode 100644 index 00000000..e71289dc --- /dev/null +++ b/sysom_server/sdk/sysom_utils/framework_plugins.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/04/06 13:13 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File framework_plugins.py +Description: +""" +from typing import Optional +from channel_job import ChannelJobExecutor +from cmg_base import dispatch_service_registry, ServiceRegistry, \ + ServiceInstance +from .config_parser import ConfigParser +from .event_executor import PluginEventExecutor +from .framework_plug_mag import FrameworkPluginBase + + +class NodeDispatcherPlugin(FrameworkPluginBase): + """Node Dispatcher Plugin + + NDP automatically implements the function of node file down and + initialization by reading the configuration + + Args: + FrameworkPluginBase (_type_): _description_ + """ + + def __init__(self, config: ConfigParser) -> None: + self._config = config + self._channel_job_executor = ChannelJobExecutor() + self._channel_job_executor.init_config( + self._config.get_local_channel_job_url() + ) + self._plugin_event_executor = PluginEventExecutor( + config, self._channel_job_executor + ) + + def start(self): + self._channel_job_executor.start() + self._plugin_event_executor.start() + + def stop(self): + self._channel_job_executor.stop() + self._plugin_event_executor.stop() + + def join(self, timeout: Optional[float] = None): + self._plugin_event_executor.join(timeout) + + +class CmgPlugin(FrameworkPluginBase): + def __init__(self, config: ConfigParser) -> None: + self._config = config + self._registry: ServiceRegistry = \ + dispatch_service_registry(self._config.get_cmg_url()) + self._service_instance: Optional[ServiceInstance] = None + cmg_plugin_config = self._config.get_service_config().plugins.cmg + if cmg_plugin_config: + self._service_instance = ServiceInstance(**dict(cmg_plugin_config)) + + def start(self): + if self._service_instance: + self._registry.register(self._service_instance) + + def stop(self): + if self._service_instance: + self._registry.unregister(self._service_instance.service_id) + + def join(self, timeout: Optional[float] = None): + return super().join(timeout) diff --git a/sysom_server/sdk/sysom_utils/yaml_concat.py b/sysom_server/sdk/sysom_utils/yaml_concat.py index acd29d55..1aa8fe46 100644 --- a/sysom_server/sdk/sysom_utils/yaml_concat.py +++ b/sysom_server/sdk/sysom_utils/yaml_concat.py @@ -16,7 +16,7 @@ class YamlConcatConstructor: def __call__(self, loader, node): if isinstance(node, yaml.nodes.SequenceNode): # type: ignore args = loader.construct_sequence(node) - return "".join(args) + return "".join([str(item) for item in args]) else: raise TypeError('Un-supported YAML node {!r}'.format(node)) -- Gitee From ea63d00bede0e879a301929a83bd8aedb8cd8400 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 19:07:41 +0800 Subject: [PATCH 002/196] feat(diagnosis): Diagnostic module access framework plug-in system --- .../sysom_diagnosis/apps/task/apps.py | 10 +++--- .../sysom_diagnosis/apps/task/urls.py | 3 +- .../sysom_diagnosis/apps/task/views.py | 5 ++- sysom_server/sysom_diagnosis/config.yml | 31 +++++++++++++++++-- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/sysom_server/sysom_diagnosis/apps/task/apps.py b/sysom_server/sysom_diagnosis/apps/task/apps.py index 27247741..515f3792 100644 --- a/sysom_server/sysom_diagnosis/apps/task/apps.py +++ b/sysom_server/sysom_diagnosis/apps/task/apps.py @@ -1,7 +1,7 @@ import sys from clogger import logger from django.apps import AppConfig -from sysom_utils import PluginEventExecutor +from sysom_utils import FrameworkPlugMag, NodeDispatcherPlugin, CmgPlugin class TaskConfig(AppConfig): @@ -18,10 +18,10 @@ class TaskConfig(AppConfig): default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) default_channel_job_executor.start() - # 初始化插件处理线程（自动处理节点端的初始化和清理操作） - PluginEventExecutor( - settings.YAML_CONFIG, default_channel_job_executor - ).start() + FrameworkPlugMag(settings.YAML_CONFIG) \ + .load_plugin_cls(NodeDispatcherPlugin) \ + .load_plugin_cls(CmgPlugin) \ + .start() # 这边微服务正式启动的时候执行一些处理代码 # 启动任务结果处理线程 diff --git a/sysom_server/sysom_diagnosis/apps/task/urls.py b/sysom_server/sysom_diagnosis/apps/task/urls.py index 6f8056af..566ccc13 100644 --- a/sysom_server/sysom_diagnosis/apps/task/urls.py +++ b/sysom_server/sysom_diagnosis/apps/task/urls.py @@ -15,7 +15,8 @@ router.register('tasks', views.TaskAPIView) urlpatterns = [ path('api/v1/tasks/offline_import/', views.TaskAPIView.as_view({'post': 'offline_import'})), - path('api/v1/', include(router.urls)), path('api/v2/tasks/', views.TaskAPIView.as_view({'post': 'create_task_v2'})), + path('api/v1/tasks/health_check/', views.TaskAPIView.as_view({'get': 'health_check'})), + path('api/v1/', include(router.urls)), re_path('^api/v1/tasks/(?P[a-zA-Z0-9]+)/(?P[a-zA-Z]+)/$', views.TaskAPIView.as_view({'get': 'get_task_svg'})), ] diff --git a/sysom_server/sysom_diagnosis/apps/task/views.py b/sysom_server/sysom_diagnosis/apps/task/views.py index c8c5f167..a0e4daa0 100644 --- a/sysom_server/sysom_diagnosis/apps/task/views.py +++ b/sysom_server/sysom_diagnosis/apps/task/views.py @@ -38,7 +38,7 @@ class TaskAPIView(CommonModelViewSet, def get_authenticators(self): # 判断请求是否是单查task task_id = self.kwargs.get(self.lookup_field, None) - if self.request.path.endswith("svg/") or (task_id is not None and self.request.method == 'GET'): + if self.request.path.endswith("health_check/") or (task_id is not None and self.request.method == 'GET'): return [] else: return [auth() for auth in self.authentication_classes] @@ -135,3 +135,6 @@ class TaskAPIView(CommonModelViewSet, except Exception as e: logger.exception(e) return ErrorResponse(msg=str(e)) + + def health_check(self, request, *args, **kwargs): + return success(result={}) diff --git a/sysom_server/sysom_diagnosis/config.yml b/sysom_server/sysom_diagnosis/config.yml index aea1fda6..c2781ffd 100644 --- a/sysom_server/sysom_diagnosis/config.yml +++ b/sysom_server/sysom_diagnosis/config.yml @@ -2,8 +2,7 @@ vars: SYSAK_DOWNLOAD_URL: &SYSAK_DOWNLOAD_URL https://mirrors.openanolis.cn/sysak/packages/ SYSAK_VERSION: &SYSAK_VERSION 1.3.0-2 SERVICE_NAME: &SERVICE_NAME sysom_diagnosis - SERVICE_CONSUMER_GROUP: - &SERVICE_CONSUMER_GROUP !concat [*SERVICE_NAME, "_consumer_group"] + SERVICE_CONSUMER_GROUP: !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] sysom_server: cec: @@ -16,6 +15,34 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7002 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + tags: + - Diagnosis + - Django + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/tasks/health_check/", + ] + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false # 节点测配置 sysom_node: -- Gitee From 9b8402cff61d2ec29ba2a7856cddd61212853d8f Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 26 Apr 2023 10:47:50 +0000 Subject: [PATCH 003/196] =?UTF-8?q?!1057=20=E7=89=B9=E6=80=A7=EF=BC=88diag?= =?UTF-8?q?nosis=EF=BC=89=EF=BC=9A=E8=AF=8A=E6=96=AD=E6=A8=A1=E5=9D=97?= =?UTF-8?q?=E6=8E=A5=E5=85=A5=E6=A1=86=E6=9E=B6=E6=8F=92=E4=BB=B6=E7=B3=BB?= =?UTF-8?q?=E7=BB=9F=20*=20feat(diagnosis):=20Diagnostic=20module=20access?= =?UTF-8?q?=20framework=20plug-in=20system?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../sysom_diagnosis/apps/task/apps.py | 10 +++--- .../sysom_diagnosis/apps/task/urls.py | 3 +- .../sysom_diagnosis/apps/task/views.py | 5 ++- sysom_server/sysom_diagnosis/config.yml | 31 +++++++++++++++++-- 4 files changed, 40 insertions(+), 9 deletions(-) diff --git a/sysom_server/sysom_diagnosis/apps/task/apps.py b/sysom_server/sysom_diagnosis/apps/task/apps.py index 27247741..515f3792 100644 --- a/sysom_server/sysom_diagnosis/apps/task/apps.py +++ b/sysom_server/sysom_diagnosis/apps/task/apps.py @@ -1,7 +1,7 @@ import sys from clogger import logger from django.apps import AppConfig -from sysom_utils import PluginEventExecutor +from sysom_utils import FrameworkPlugMag, NodeDispatcherPlugin, CmgPlugin class TaskConfig(AppConfig): @@ -18,10 +18,10 @@ class TaskConfig(AppConfig): default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) default_channel_job_executor.start() - # 初始化插件处理线程（自动处理节点端的初始化和清理操作） - PluginEventExecutor( - settings.YAML_CONFIG, default_channel_job_executor - ).start() + FrameworkPlugMag(settings.YAML_CONFIG) \ + .load_plugin_cls(NodeDispatcherPlugin) \ + .load_plugin_cls(CmgPlugin) \ + .start() # 这边微服务正式启动的时候执行一些处理代码 # 启动任务结果处理线程 diff --git a/sysom_server/sysom_diagnosis/apps/task/urls.py b/sysom_server/sysom_diagnosis/apps/task/urls.py index 6f8056af..566ccc13 100644 --- a/sysom_server/sysom_diagnosis/apps/task/urls.py +++ b/sysom_server/sysom_diagnosis/apps/task/urls.py @@ -15,7 +15,8 @@ router.register('tasks', views.TaskAPIView) urlpatterns = [ path('api/v1/tasks/offline_import/', views.TaskAPIView.as_view({'post': 'offline_import'})), - path('api/v1/', include(router.urls)), path('api/v2/tasks/', views.TaskAPIView.as_view({'post': 'create_task_v2'})), + path('api/v1/tasks/health_check/', views.TaskAPIView.as_view({'get': 'health_check'})), + path('api/v1/', include(router.urls)), re_path('^api/v1/tasks/(?P[a-zA-Z0-9]+)/(?P[a-zA-Z]+)/$', views.TaskAPIView.as_view({'get': 'get_task_svg'})), ] diff --git a/sysom_server/sysom_diagnosis/apps/task/views.py b/sysom_server/sysom_diagnosis/apps/task/views.py index c8c5f167..a0e4daa0 100644 --- a/sysom_server/sysom_diagnosis/apps/task/views.py +++ b/sysom_server/sysom_diagnosis/apps/task/views.py @@ -38,7 +38,7 @@ class TaskAPIView(CommonModelViewSet, def get_authenticators(self): # 判断请求是否是单查task task_id = self.kwargs.get(self.lookup_field, None) - if self.request.path.endswith("svg/") or (task_id is not None and self.request.method == 'GET'): + if self.request.path.endswith("health_check/") or (task_id is not None and self.request.method == 'GET'): return [] else: return [auth() for auth in self.authentication_classes] @@ -135,3 +135,6 @@ class TaskAPIView(CommonModelViewSet, except Exception as e: logger.exception(e) return ErrorResponse(msg=str(e)) + + def health_check(self, request, *args, **kwargs): + return success(result={}) diff --git a/sysom_server/sysom_diagnosis/config.yml b/sysom_server/sysom_diagnosis/config.yml index aea1fda6..c2781ffd 100644 --- a/sysom_server/sysom_diagnosis/config.yml +++ b/sysom_server/sysom_diagnosis/config.yml @@ -2,8 +2,7 @@ vars: SYSAK_DOWNLOAD_URL: &SYSAK_DOWNLOAD_URL https://mirrors.openanolis.cn/sysak/packages/ SYSAK_VERSION: &SYSAK_VERSION 1.3.0-2 SERVICE_NAME: &SERVICE_NAME sysom_diagnosis - SERVICE_CONSUMER_GROUP: - &SERVICE_CONSUMER_GROUP !concat [*SERVICE_NAME, "_consumer_group"] + SERVICE_CONSUMER_GROUP: !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] sysom_server: cec: @@ -16,6 +15,34 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7002 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + tags: + - Diagnosis + - Django + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/tasks/health_check/", + ] + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false # 节点测配置 sysom_node: -- Gitee From 32266fd9ef8b6b4bfe8eac6c21c8880a574ca129 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 19:25:22 +0800 Subject: [PATCH 004/196] feat(sysom-api): Api module access framework plug-in system --- sysom_server/sysom_api/apps/host/apps.py | 6 +++++ sysom_server/sysom_api/apps/host/urls.py | 2 ++ sysom_server/sysom_api/apps/host/views.py | 5 +++- sysom_server/sysom_api/config.yml | 28 +++++++++++++++++++++++ 4 files changed, 40 insertions(+), 1 deletion(-) diff --git a/sysom_server/sysom_api/apps/host/apps.py b/sysom_server/sysom_api/apps/host/apps.py index 8c279529..8fccc152 100644 --- a/sysom_server/sysom_api/apps/host/apps.py +++ b/sysom_server/sysom_api/apps/host/apps.py @@ -12,6 +12,7 @@ class HostConfig(AppConfig): def ready(self): if 'runserver' in sys.argv or 'manage.py' not in sys.argv: from django.conf import settings + from sysom_utils import FrameworkPlugMag, CmgPlugin # 这边微服务正式启动的时候执行一些处理代码 # 启动任务结果处理线程 default_channel_job_executor.init_config( @@ -25,6 +26,11 @@ class HostConfig(AppConfig): except Exception as e: logger.warning(f'主机心跳未启动: {e}') + # 启动框架插件系统 + FrameworkPlugMag(settings.YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() + from .cec_api import HostCecApi HostCecApi().start() else: diff --git a/sysom_server/sysom_api/apps/host/urls.py b/sysom_server/sysom_api/apps/host/urls.py index 30a518e4..13287432 100644 --- a/sysom_server/sysom_api/apps/host/urls.py +++ b/sysom_server/sysom_api/apps/host/urls.py @@ -19,6 +19,8 @@ urlpatterns = [ views.HostModelViewSet.as_view({'post': 'batch_del_host'})), path('api/v1/host/metrics', views.MetricsViewSet.as_view({'get': 'host_metrics'})), + path('api/v1/host/health_check/', + views.HostModelViewSet.as_view({'get': 'health_check'})), re_path(r'^api/v1/host/ip/(?P.*)/$', views.HostModelViewSet.as_view({'get': 'get_host'})), re_path(r'^api/v1/host/del/(?P.*)/$', diff --git a/sysom_server/sysom_api/apps/host/views.py b/sysom_server/sysom_api/apps/host/views.py index 1c7715d2..782ea4e2 100644 --- a/sysom_server/sysom_api/apps/host/views.py +++ b/sysom_server/sysom_api/apps/host/views.py @@ -145,7 +145,7 @@ class HostModelViewSet(CommonModelViewSet, self.perform_destroy(instance) self.client_deploy_cmd_delete(instance) return ser - + def perform_destroy(self, instance): """ 重新主机删除功能, 当删除主机出现IntegrityError, @@ -376,6 +376,9 @@ class HostModelViewSet(CommonModelViewSet, pattern = '^' + '\.'.join([p]*4) + '$' return bool(re.match(pattern, ip)) + def health_check(self, request, *args, **kwargs): + return success(result={}) + class MetricsViewSet(CommonModelViewSet): authentication_classes = [] diff --git a/sysom_server/sysom_api/config.yml b/sysom_server/sysom_api/config.yml index 50342514..f7768306 100644 --- a/sysom_server/sysom_api/config.yml +++ b/sysom_server/sysom_api/config.yml @@ -14,8 +14,36 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7001 heartbeat: HEARTBEAT_INTERVAL: 20 app_host: # Host init timeout (seconds) HOST_INIT_TIMEOUT: 600 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + tags: + - Diagnosis + - Django + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/host/health_check/", + ] + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false -- Gitee From 3cc5077a946d60fc4ccffdf3a145cc30bb11c07e Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 19:35:44 +0800 Subject: [PATCH 005/196] fix(sdk): FrameworkPlugMag not call Thread.__init__ error --- sysom_server/sdk/sysom_utils/framework_plug_mag.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sysom_server/sdk/sysom_utils/framework_plug_mag.py b/sysom_server/sdk/sysom_utils/framework_plug_mag.py index 14368c5a..91323e8f 100644 --- a/sysom_server/sdk/sysom_utils/framework_plug_mag.py +++ b/sysom_server/sdk/sysom_utils/framework_plug_mag.py @@ -50,20 +50,24 @@ class FrameworkPlugMag(StoppableThread): """ def __init__(self, config: ConfigParser) -> None: - self._config = config + super().__init__(target=self.run) self.setDaemon(True) + self._config = config self._plugins: List[FrameworkPluginBase] = [] def load_plugin(self, plugin: FrameworkPluginBase): self._plugins.append(plugin) return self - + def load_plugin_cls(self, plugin_cls: Type[FrameworkPluginBase]): self._plugins.append(plugin_cls(self._config)) + return self - def start(self) -> None: + def run(self): for plugin in self._plugins: plugin.start() + + def start(self) -> None: return super().start() def join(self, timeout: Optional[float] = None) -> None: -- Gitee From 0567c5294d4b9935a4ff1e5a7ec37c9c6a0747e0 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 19:46:12 +0800 Subject: [PATCH 006/196] refactor(migration): Use local channel job url --- sysom_server/sysom_migration/apps/migration/apps.py | 4 ++-- sysom_server/sysom_migration/conf/common.py | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/sysom_server/sysom_migration/apps/migration/apps.py b/sysom_server/sysom_migration/apps/migration/apps.py index c9f028e1..6e31bec8 100644 --- a/sysom_server/sysom_migration/apps/migration/apps.py +++ b/sysom_server/sysom_migration/apps/migration/apps.py @@ -11,10 +11,10 @@ class MigrationConfig(AppConfig): from django.conf import settings if ('runserver' in sys.argv or 'manage.py' not in sys.argv): from channel_job.job import default_channel_job_executor + from django.conf import settings # 初始化 channel_job sdk - config_url = f'{settings.SYSOM_CHANNEL_URL}/api/v1/channel/config/get?name=migration_setting' - default_channel_job_executor.initial_from_remote_server(config_url) + default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) default_channel_job_executor.start() logger.info(">>> Migration module loading success") diff --git a/sysom_server/sysom_migration/conf/common.py b/sysom_server/sysom_migration/conf/common.py index 39ffcfd3..c805ae13 100644 --- a/sysom_server/sysom_migration/conf/common.py +++ b/sysom_server/sysom_migration/conf/common.py @@ -142,6 +142,8 @@ MIG_IMP_REBOOT = 120 # Cec settings ################################################################## SYSOM_CEC_URL = YAML_CONFIG.get_cec_url(CecTarget.PRODUCER) +# channl_job SDK 需要的url +CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() ######################################################################################### # rest_framework settings -- Gitee From f912726d7ad8654bfd38b3be211c621bd69116cf Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 19:55:39 +0800 Subject: [PATCH 007/196] fix(sdk): ConfigParser get_cmg_url error, special_param is None --- sysom_server/sdk/sysom_utils/config_parser.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sysom_server/sdk/sysom_utils/config_parser.py b/sysom_server/sdk/sysom_utils/config_parser.py index eb09d0d7..adb9eb05 100644 --- a/sysom_server/sdk/sysom_utils/config_parser.py +++ b/sysom_server/sdk/sysom_utils/config_parser.py @@ -126,7 +126,8 @@ class ConfigParser: cmg_config = server_config.cmg special_param = {} params = [] - dict_merge(special_param, cmg_config.special_param) + if cmg_config.special_param: + dict_merge(special_param, cmg_config.special_param) cmg_url = "" if cmg_config.protocol == "redis": redis_config = server_config.db.redis -- Gitee From 043fe5adbb7029de2ad3c2318db6153f5ad405a0 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 19:58:16 +0800 Subject: [PATCH 008/196] remove(channel): Remove public channel url config --- sysom_server/sysom_channel/conf/common.py | 17 ----------------- sysom_server/sysom_channel/config.yml | 6 ------ sysom_server/sysom_channel/main.py | 13 ------------- 3 files changed, 36 deletions(-) diff --git a/sysom_server/sysom_channel/conf/common.py b/sysom_server/sysom_channel/conf/common.py index a2f9e23d..3011d3c7 100644 --- a/sysom_server/sysom_channel/conf/common.py +++ b/sysom_server/sysom_channel/conf/common.py @@ -65,23 +65,6 @@ SYSOM_CEC_CHANNEL_CONSUMER_GROUP = cec_config.consumer_group # 通道模块用于投递执行结果的主题 SYSOM_CEC_CHANNEL_RESULT_TOPIC = cec_config.topics.SYSOM_CEC_CHANNEL_RESULT_TOPIC -################################################################## -# Channel 对外配置 -################################################################## - -CHANNEL_PUBLIC_BASE_URL = service_config.channel.public.base_url - -# 迁移模块CEC配置 -SYSOM_MIGRATION_LISTEN_TOPIC = "SYSOM_MIGRATION_LISTEN_TOPIC" -SYSOM_MIGRATION_CONSUME_GROUP = "SYSOM_MIGRATION_CONSUME_GROUP" -SYSOM_MIGRATION_CEC_URL = f"{SYSOM_CEC_URL}&channel_job_target_topic={SYSOM_CEC_CHANNEL_TOPIC}&channel_job_listen_topic={SYSOM_MIGRATION_LISTEN_TOPIC}&channel_job_consumer_group={SYSOM_MIGRATION_CONSUME_GROUP}" - -# 主机模块CEC配置 -SYSOM_HOST_LISTEN_TOPIC = "SYSOM_HOST_LISTEN_TOPIC" -SYSOM_HOST_CONSUME_GROUP = "SYSOM_HOST_CONSUME_GROUP" -SYSOM_HOST_CEC_URL = f"{SYSOM_CEC_URL}&channel_job_target_topic={SYSOM_CEC_CHANNEL_TOPIC}&channel_job_listen_topic={SYSOM_HOST_LISTEN_TOPIC}&channel_job_consumer_group={SYSOM_HOST_CONSUME_GROUP}" - - ################################################################## # Logging config ################################################################## diff --git a/sysom_server/sysom_channel/config.yml b/sysom_server/sysom_channel/config.yml index be2afaf9..6e39e8b1 100644 --- a/sysom_server/sysom_channel/config.yml +++ b/sysom_server/sysom_channel/config.yml @@ -10,9 +10,3 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME - channel: - public: - proto: &CHANNEL_PUBLIC_PROTO http - host: &CHANNEL_PUBLIC_HOST 127.0.0.1 - port: &CHANNEL_PUBLIC_PORT "7003" - base_url: !concat [*CHANNEL_PUBLIC_PROTO, "://", *CHANNEL_PUBLIC_HOST, ":", *CHANNEL_PUBLIC_PORT] \ No newline at end of file diff --git a/sysom_server/sysom_channel/main.py b/sysom_server/sysom_channel/main.py index b4b00d0b..984279e9 100644 --- a/sysom_server/sysom_channel/main.py +++ b/sysom_server/sysom_channel/main.py @@ -80,19 +80,6 @@ def init_channel(): with SessionLocal() as db: ssh_setting = get_or_set_channel_setting( db, "ssh_key", ssh_key, "SysOM Channel auto generated key") - _ = get_or_set_channel_setting( - db, "cec_url", SYSOM_CEC_URL, "SysOM CEC URL") - _ = update_or_create( - db, "migration_setting", json.dumps({ - "cec_url": SYSOM_MIGRATION_CEC_URL, - "channel_base_url": CHANNEL_PUBLIC_BASE_URL - }), "SysOM migration channel setting") - _ = update_or_create( - db, "host_setting", json.dumps({ - "cec_url": SYSOM_HOST_CEC_URL, - "channel_file_base_url": CHANNEL_PUBLIC_BASE_URL - }), "SysOM host channel setting" - ) # 4. If ssh_key fetching or generation fails, the application does not start properly if ssh_setting is None: -- Gitee From 247cb5073b04f4b79027bea774913cdc93c607de Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 20:12:14 +0800 Subject: [PATCH 009/196] feat(channel): Channel module access framework plug-in system --- .../sysom_channel/app/routers/health.py | 21 +++++++++++++ sysom_server/sysom_channel/config.yml | 30 +++++++++++++++++++ sysom_server/sysom_channel/main.py | 10 ++++++- 3 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 sysom_server/sysom_channel/app/routers/health.py diff --git a/sysom_server/sysom_channel/app/routers/health.py b/sysom_server/sysom_channel/app/routers/health.py new file mode 100644 index 00000000..9b6f43fc --- /dev/null +++ b/sysom_server/sysom_channel/app/routers/health.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/04/17 19:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get("/check") +async def get_channel_config(): + return { + "code": 0, + "err_msg": "", + "data": "" + } diff --git a/sysom_server/sysom_channel/config.yml b/sysom_server/sysom_channel/config.yml index 6e39e8b1..46dd3da8 100644 --- a/sysom_server/sysom_channel/config.yml +++ b/sysom_server/sysom_channel/config.yml @@ -10,3 +10,33 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7003 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + host: *SERVICE_HOST + port: *SERVICE_PORT + tags: + - Diagnosis + - Django + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/channel/health/check", + ] + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false diff --git a/sysom_server/sysom_channel/main.py b/sysom_server/sysom_channel/main.py index 984279e9..9231bd46 100644 --- a/sysom_server/sysom_channel/main.py +++ b/sysom_server/sysom_channel/main.py @@ -16,7 +16,8 @@ from app.crud import create_setting, get_setting_by_name, update_or_create_chann from app.schemas import ChannelSetting from lib.ssh import AsyncSSH from conf.settings import * -from app.routers import file, config, cec_status +from app.routers import file, config, cec_status, health +from sysom_utils import FrameworkPlugMag, CmgPlugin app = FastAPI() @@ -25,6 +26,7 @@ app.mount("/public", StaticFiles(directory=STATIC_RESOURCE_PATH), name="public") app.include_router(file.router, prefix="/api/v1/channel/file") app.include_router(config.router, prefix="/api/v1/channel/config") app.include_router(cec_status.router, prefix="/api/v1/channel/cec_status") +app.include_router(health.router, prefix="/api/v1/channel/health") def init_channel(): @@ -96,11 +98,17 @@ def init_channel(): ChannelListener().start() except Exception as e: logger.exception(e) + +def init_frame_plugin_mag(): + FrameworkPlugMag(YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() @app.on_event("startup") async def on_start(): init_channel() + init_frame_plugin_mag() @app.on_event("shutdown") -- Gitee From e3ad13e182907412f9ef807a32ba68b73f177ecb Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 20:30:44 +0800 Subject: [PATCH 010/196] feat(hotfix): Hotfix module access framework plug-in system --- sysom_server/sysom_api/config.yml | 4 ++- sysom_server/sysom_channel/config.yml | 4 +-- sysom_server/sysom_diagnosis/config.yml | 2 ++ sysom_server/sysom_hotfix/apps/hotfix/apps.py | 6 +++- sysom_server/sysom_hotfix/apps/hotfix/urls.py | 1 + .../sysom_hotfix/apps/hotfix/views.py | 4 +++ sysom_server/sysom_hotfix/conf/common.py | 2 ++ sysom_server/sysom_hotfix/config.yml | 34 +++++++++++++++++-- 8 files changed, 51 insertions(+), 6 deletions(-) diff --git a/sysom_server/sysom_api/config.yml b/sysom_server/sysom_api/config.yml index f7768306..2a225cb0 100644 --- a/sysom_server/sysom_api/config.yml +++ b/sysom_server/sysom_api/config.yml @@ -26,8 +26,10 @@ sysom_service: node_dispatch: cmg: service_name: *SERVICE_NAME + host: *SERVICE_HOST + port: *SERVICE_PORT tags: - - Diagnosis + - Api - Django # Metadata of service metadata: diff --git a/sysom_server/sysom_channel/config.yml b/sysom_server/sysom_channel/config.yml index 46dd3da8..71f8bec2 100644 --- a/sysom_server/sysom_channel/config.yml +++ b/sysom_server/sysom_channel/config.yml @@ -20,8 +20,8 @@ sysom_service: host: *SERVICE_HOST port: *SERVICE_PORT tags: - - Diagnosis - - Django + - Channel + - FastApi # Metadata of service metadata: check: diff --git a/sysom_server/sysom_diagnosis/config.yml b/sysom_server/sysom_diagnosis/config.yml index c2781ffd..a77c2639 100644 --- a/sysom_server/sysom_diagnosis/config.yml +++ b/sysom_server/sysom_diagnosis/config.yml @@ -22,6 +22,8 @@ sysom_service: node_dispatch: cmg: service_name: *SERVICE_NAME + host: *SERVICE_HOST + port: *SERVICE_PORT tags: - Diagnosis - Django diff --git a/sysom_server/sysom_hotfix/apps/hotfix/apps.py b/sysom_server/sysom_hotfix/apps/hotfix/apps.py index 300db8cd..27e78740 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/apps.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/apps.py @@ -2,7 +2,6 @@ import sys from clogger import logger from django.apps import AppConfig from django.db.models.signals import post_migrate -from django.conf import settings from cec_base.admin import dispatch_admin from channel_job.job import default_channel_job_executor @@ -13,6 +12,7 @@ class HotfixConfig(AppConfig): def ready(self): from django.conf import settings if ('runserver' in sys.argv or 'manage.py' not in sys.argv): + from sysom_utils import FrameworkPlugMag, CmgPlugin try: # 初始化 channel_job sdk default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) @@ -25,6 +25,10 @@ class HotfixConfig(AppConfig): except Exception as e: logger.info(str(e)) logger.info(">> INIT_HOTFIX_VIEW : create hotfix_job cec topic failed") + # Framework Plug Mag init + FrameworkPlugMag(settings.YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() else: # nothing to do when execute database migrations pass diff --git a/sysom_server/sysom_hotfix/apps/hotfix/urls.py b/sysom_server/sysom_hotfix/apps/hotfix/urls.py index 96b3cd08..de5e4673 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/urls.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/urls.py @@ -30,5 +30,6 @@ urlpatterns = [ path('api/v1/hotfix/update_kernel_relation/', views.HotfixAPIView.as_view({'post': 'update_kernel_version'})), path('api/v1/hotfix/update_ostype/', views.HotfixAPIView.as_view({'post': 'update_ostype'})), path('api/v1/hotfix/rebuild_hotfix/',views.HotfixAPIView.as_view({'post': 'rebuild_hotfix'})), + path('api/v1/hotfix/health_check/', views.HealthViewset.as_view({'get': 'health_check'})), path('api/v1/', include(router.urls)), ] diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 059364c8..1e8c6302 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -485,3 +485,7 @@ class HotfixAPIView(GenericViewSet, except Exception as e: return other_response(msg=str(e), code=400) return success(result={"msg":"successfully update os_type object"},message="invoke update_ostype") + +class HealthViewset(CommonModelViewSet): + def health_check(self, request, *args, **kwargs): + return success(result={}) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/conf/common.py b/sysom_server/sysom_hotfix/conf/common.py index a97b9e71..8bb3de2c 100644 --- a/sysom_server/sysom_hotfix/conf/common.py +++ b/sysom_server/sysom_hotfix/conf/common.py @@ -117,6 +117,8 @@ HOST_URL = 'http://127.0.0.1:7001/api/v1/host/' ################################################################## SYSOM_CEC_URL = YAML_CONFIG.get_cec_url(CecTarget.PRODUCER) SYSOM_CEC_HOTFIX_TOPIC = "hotfix_job" +# channl_job SDK 需要的url +CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() ################################################################## # Config settings diff --git a/sysom_server/sysom_hotfix/config.yml b/sysom_server/sysom_hotfix/config.yml index c77bd8fd..f44827f4 100644 --- a/sysom_server/sysom_hotfix/config.yml +++ b/sysom_server/sysom_hotfix/config.yml @@ -1,7 +1,7 @@ vars: SERVICE_NAME: &SERVICE_NAME sysom_hotfix SERVICE_CONSUMER_GROUP: - &SERVICE_CONSUMER_GROUP !concat [*SERVICE_NAME, "_consumer_group"] + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] sysom_server: cec: @@ -13,4 +13,34 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME - service_dir: *SERVICE_NAME \ No newline at end of file + service_dir: *SERVICE_NAME + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7007 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + host: *SERVICE_HOST + port: *SERVICE_PORT + tags: + - Hotfix + - Django + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/hotfix/health_check/", + ] + interval: 2 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false -- Gitee From d9ad7f62cb45c733f48ca0b28874ca4e00561b7a Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 20:38:15 +0800 Subject: [PATCH 011/196] feat(migration): Migration module access framework plug-in system --- sysom_server/sysom_hotfix/config.yml | 2 +- .../sysom_migration/apps/migration/apps.py | 5 +++ .../sysom_migration/apps/migration/urls.py | 2 ++ .../sysom_migration/apps/migration/views.py | 4 +++ sysom_server/sysom_migration/config.yml | 32 ++++++++++++++++++- 5 files changed, 43 insertions(+), 2 deletions(-) diff --git a/sysom_server/sysom_hotfix/config.yml b/sysom_server/sysom_hotfix/config.yml index f44827f4..793b1685 100644 --- a/sysom_server/sysom_hotfix/config.yml +++ b/sysom_server/sysom_hotfix/config.yml @@ -39,7 +39,7 @@ sysom_service: *SERVICE_PORT, "/api/v1/hotfix/health_check/", ] - interval: 2 + interval: 10 timeout: 10 deregister: 25 header: diff --git a/sysom_server/sysom_migration/apps/migration/apps.py b/sysom_server/sysom_migration/apps/migration/apps.py index 6e31bec8..14aa18ce 100644 --- a/sysom_server/sysom_migration/apps/migration/apps.py +++ b/sysom_server/sysom_migration/apps/migration/apps.py @@ -12,9 +12,14 @@ class MigrationConfig(AppConfig): if ('runserver' in sys.argv or 'manage.py' not in sys.argv): from channel_job.job import default_channel_job_executor from django.conf import settings + from sysom_utils import FrameworkPlugMag, CmgPlugin # 初始化 channel_job sdk default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) default_channel_job_executor.start() + # Framework Plug Mag init + FrameworkPlugMag(settings.YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() logger.info(">>> Migration module loading success") diff --git a/sysom_server/sysom_migration/apps/migration/urls.py b/sysom_server/sysom_migration/apps/migration/urls.py index 152d934d..17f202dd 100644 --- a/sysom_server/sysom_migration/apps/migration/urls.py +++ b/sysom_server/sysom_migration/apps/migration/urls.py @@ -27,5 +27,7 @@ urlpatterns = [ path('api/v1/implementation/migrate/', views.MigImpView.as_view({'post': 'post_host_migrate'})), path('api/v1/implementation/migrate/all/', views.MigImpView.as_view({'post': 'post_all_migrate'})), + path('api/v1/migration/health_check/', views.HealthViewSet.as_view({'get': 'health_check'})), + path('api/v1/', include(router.urls)) ] diff --git a/sysom_server/sysom_migration/apps/migration/views.py b/sysom_server/sysom_migration/apps/migration/views.py index 7f85f155..0931404f 100644 --- a/sysom_server/sysom_migration/apps/migration/views.py +++ b/sysom_server/sysom_migration/apps/migration/views.py @@ -1057,3 +1057,7 @@ class MigImpView(CommonModelViewSet): mig_imp.status = 'fail' mig_imp.detail = '还原失败' mig_imp.save() + +class HealthViewSet(CommonModelViewSet): + def health_check(self, request, *args, **kwargs): + return success(result={}) \ No newline at end of file diff --git a/sysom_server/sysom_migration/config.yml b/sysom_server/sysom_migration/config.yml index ffa7b384..90924876 100644 --- a/sysom_server/sysom_migration/config.yml +++ b/sysom_server/sysom_migration/config.yml @@ -13,4 +13,34 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME - service_dir: *SERVICE_NAME \ No newline at end of file + service_dir: *SERVICE_NAME + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7006 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + host: *SERVICE_HOST + port: *SERVICE_PORT + tags: + - Migration + - Django + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/migration/health_check/", + ] + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false \ No newline at end of file -- Gitee From 25b50c8cd3fd9ee911e3b685378966da499f9cfc Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 20:45:33 +0800 Subject: [PATCH 012/196] feat(vmcore): Vmcore module access framework plug-in system --- sysom_server/sysom_vmcore/apps/vmcore/apps.py | 11 +++---- sysom_server/sysom_vmcore/apps/vmcore/urls.py | 2 ++ .../sysom_vmcore/apps/vmcore/views.py | 5 ++++ sysom_server/sysom_vmcore/config.yml | 30 +++++++++++++++++++ 4 files changed, 43 insertions(+), 5 deletions(-) diff --git a/sysom_server/sysom_vmcore/apps/vmcore/apps.py b/sysom_server/sysom_vmcore/apps/vmcore/apps.py index a021c740..67cb7819 100644 --- a/sysom_server/sysom_vmcore/apps/vmcore/apps.py +++ b/sysom_server/sysom_vmcore/apps/vmcore/apps.py @@ -15,18 +15,19 @@ class VmcoreConfig(AppConfig): def ready(self) -> None: from django.conf import settings - from sysom_utils import PluginEventExecutor if ('runserver' in sys.argv or 'manage.py' not in sys.argv): from channel_job.job import default_channel_job_executor + from sysom_utils import FrameworkPlugMag, CmgPlugin, NodeDispatcherPlugin # 初始化 channel_job sdk default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) default_channel_job_executor.start() - # 初始化插件处理线程（自动处理节点端的初始化和清理操作） - PluginEventExecutor( - settings.YAML_CONFIG, default_channel_job_executor - ).start() + # Framework Plug Mag init + FrameworkPlugMag(settings.YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .load_plugin_cls(NodeDispatcherPlugin) \ + .start() else: # 这边执行数据库迁移等操作的时候执行一些处理代码 pass diff --git a/sysom_server/sysom_vmcore/apps/vmcore/urls.py b/sysom_server/sysom_vmcore/apps/vmcore/urls.py index 1fa2744b..972925f5 100644 --- a/sysom_server/sysom_vmcore/apps/vmcore/urls.py +++ b/sysom_server/sysom_vmcore/apps/vmcore/urls.py @@ -15,5 +15,7 @@ urlpatterns = [ views.VmcoreDetail.as_view()), path(f'{settings.VMCORE_SERVICE_INTERFACE_PREFIX}vmcore_config_test/', views.VmcoreConfigTest.as_view()), + path(f'{settings.VMCORE_SERVICE_INTERFACE_PREFIX}health_check/', + views.HealthViewset.as_view({'get': 'health_check'})), path(settings.VMCORE_SERVICE_INTERFACE_PREFIX, include(router.urls)), ] diff --git a/sysom_server/sysom_vmcore/apps/vmcore/views.py b/sysom_server/sysom_vmcore/apps/vmcore/views.py index 93c0287e..b4ff9e9f 100644 --- a/sysom_server/sysom_vmcore/apps/vmcore/views.py +++ b/sysom_server/sysom_vmcore/apps/vmcore/views.py @@ -458,3 +458,8 @@ class VmcoreConfigTest(APIView): except Exception as e: logger.error(e) return other_response(message=str(e), code=400) + + +class HealthViewset(GenericViewSet): + def health_check(self, request, *args, **kwargs): + return success(result={}) \ No newline at end of file diff --git a/sysom_server/sysom_vmcore/config.yml b/sysom_server/sysom_vmcore/config.yml index 19737632..423c6da3 100644 --- a/sysom_server/sysom_vmcore/config.yml +++ b/sysom_server/sysom_vmcore/config.yml @@ -14,6 +14,36 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7004 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + host: *SERVICE_HOST + port: *SERVICE_PORT + tags: + - Vmcore + - Django + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/vmcore/health_check/", + ] + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false # 节点测配置 sysom_node: -- Gitee From cc92963e0f4dd67c74e4f240de9a45c0856e8212 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 17 Apr 2023 20:52:37 +0800 Subject: [PATCH 013/196] feat(vul): Vul module access framework plug-in system --- sysom_server/sysom_vul/apps/vul/apps.py | 10 +++++-- sysom_server/sysom_vul/apps/vul/urls.py | 1 + sysom_server/sysom_vul/apps/vul/views.py | 6 +++++ sysom_server/sysom_vul/config.yml | 34 ++++++++++++++++++++++-- 4 files changed, 47 insertions(+), 4 deletions(-) diff --git a/sysom_server/sysom_vul/apps/vul/apps.py b/sysom_server/sysom_vul/apps/vul/apps.py index 21356cb7..913ab961 100644 --- a/sysom_server/sysom_vul/apps/vul/apps.py +++ b/sysom_server/sysom_vul/apps/vul/apps.py @@ -22,10 +22,16 @@ class VulConfig(AppConfig): if ('runserver' in sys.argv or 'manage.py' not in sys.argv): from channel_job.job import default_channel_job_executor from .executor import VulListener + from sysom_utils import FrameworkPlugMag, CmgPlugin # 初始化 channel_job sdk default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) default_channel_job_executor.start() + # Framework Plug Mag init + FrameworkPlugMag(settings.YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() + try: VulListener().start() except Exception as e: @@ -39,7 +45,7 @@ class VulConfig(AppConfig): # setup 1 初始化vul addr 数据 self._initialize_vul_addr() - #　setup 2 复原sa相关host数据 + # 　setup 2 复原sa相关host数据 self._fix_migrate_hosts_field() def get_all_sa_bout_host(self, sender, **kwargs): @@ -69,7 +75,7 @@ class VulConfig(AppConfig): self._sa_host_cache[cve_id] = [ip] else: self._sa_host_cache[cve_id].append(ip) - + def _initialize_vul_addr(self): parser = { "cve_item_path": "data/data", diff --git a/sysom_server/sysom_vul/apps/vul/urls.py b/sysom_server/sysom_vul/apps/vul/urls.py index 68a80ff3..f79f3750 100644 --- a/sysom_server/sysom_vul/apps/vul/urls.py +++ b/sysom_server/sysom_vul/apps/vul/urls.py @@ -14,6 +14,7 @@ from apps.vul import views router = DefaultRouter() router.register('config', views.VulAddrViewSet) urlpatterns = [ + path('api/v1/vul/health_check/', views.HealthViewset.as_view({'get': 'health_check'})), path('api/v1/vul/', views.VulListView.as_view({'get': 'get_vul_list'})), path('api/v1/vul/', include(router.urls)), path('api/v1/vul/hist/', views.SaFixHistListView.as_view()), diff --git a/sysom_server/sysom_vul/apps/vul/views.py b/sysom_server/sysom_vul/apps/vul/views.py index 41e47d53..c118f0d6 100644 --- a/sysom_server/sysom_vul/apps/vul/views.py +++ b/sysom_server/sysom_vul/apps/vul/views.py @@ -419,3 +419,9 @@ class VulAddrViewSet(viewsets.ModelViewSet): except Exception as e: msg = f"Status Code: ERROR({e})" return msg + + +class HealthViewset(viewsets.GenericViewSet): + authentication_classes = [] + def health_check(self, request, *args, **kwargs): + return success(result={}) \ No newline at end of file diff --git a/sysom_server/sysom_vul/config.yml b/sysom_server/sysom_vul/config.yml index 52aba1da..b836255d 100644 --- a/sysom_server/sysom_vul/config.yml +++ b/sysom_server/sysom_vul/config.yml @@ -1,7 +1,7 @@ vars: SERVICE_NAME: &SERVICE_NAME sysom_vul SERVICE_CONSUMER_GROUP: - &SERVICE_CONSUMER_GROUP !concat [*SERVICE_NAME, "_consumer_group"] + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] sysom_server: cec: @@ -13,4 +13,34 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME - service_dir: *SERVICE_NAME \ No newline at end of file + service_dir: *SERVICE_NAME + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7005 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + host: *SERVICE_HOST + port: *SERVICE_PORT + tags: + - Vul + - Django + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/vul/health_check/", + ] + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false -- Gitee From 99198638e38570319a27d0cedf1553c64934669e Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 21 Apr 2023 14:21:36 +0800 Subject: [PATCH 014/196] refactor: Move environment-related functions to the infrastucture folder contains three functions: env, local_services and sdk 1. env => Python virutal env 2. local_services => MySQL, redis, nginx 3. sdk => Tool Library --- conf/config.yml | 3 - infrastructure/env/.pydistutils.cfg | 3 + infrastructure/env/clear.sh | 9 + infrastructure/env/init.sh | 102 ++++++++ infrastructure/env/pip.conf | 8 + infrastructure/env/requirements.txt | 40 +++ infrastructure/env/start.sh | 0 infrastructure/env/stop.sh | 0 infrastructure/local_services/clear.sh | 37 +++ infrastructure/local_services/init.sh | 115 +++++++++ infrastructure/local_services/nginx.conf | 68 ++++++ infrastructure/local_services/start.sh | 26 ++ infrastructure/local_services/stop.sh | 25 ++ infrastructure/local_services/sysom-redis.ini | 8 + infrastructure/local_services/sysom.conf | 141 +++++++++++ .../sdk/.gitignore | 0 .../sdk/cec_base/__init__.py | 0 .../sdk/cec_base/admin.py | 0 .../sdk/cec_base/base.py | 0 .../sdk/cec_base/cec_client.py | 0 .../sdk/cec_base/cli.py | 0 .../sdk/cec_base/consumer.py | 0 .../sdk/cec_base/event.py | 0 .../sdk/cec_base/exceptions.py | 0 .../sdk/cec_base/meta.py | 0 .../sdk/cec_base/producer.py | 0 .../sdk/cec_base/url.py | 0 .../sdk/cec_base/utils.py | 0 .../sdk/cec_redis/__init__.py | 0 .../sdk/cec_redis/admin_static.py | 0 .../sdk/cec_redis/common.py | 0 .../sdk/cec_redis/consume_status_storage.py | 0 .../sdk/cec_redis/heartbeat.py | 0 .../sdk/cec_redis/redis_admin.py | 0 .../sdk/cec_redis/redis_consumer.py | 0 .../sdk/cec_redis/redis_producer.py | 0 .../sdk/cec_redis/utils.py | 0 .../sdk/channel_job/__init__.py | 0 .../sdk/channel_job/exception.py | 0 .../sdk/channel_job/job.py | 0 .../sdk/channel_job/model.py | 0 infrastructure/sdk/clear.sh | 0 .../sdk/clogger/__init__.py | 0 .../sdk/clogger/clogger.py | 0 .../sdk/cmg_base/__init__.py | 0 .../sdk/cmg_base/exceptions.py | 0 .../sdk/cmg_base/load_balancing_strategy.py | 0 .../sdk/cmg_base/service_check.py | 0 .../sdk/cmg_base/service_discovery.py | 0 .../sdk/cmg_base/service_instance.py | 0 .../sdk/cmg_base/service_invoker.py | 0 .../sdk/cmg_base/service_registry.py | 0 .../sdk/cmg_base/url.py | 0 .../sdk/cmg_base/utils.py | 0 .../sdk/cmg_redis/__init__.py | 0 .../sdk/cmg_redis/common.py | 0 .../sdk/cmg_redis/redis_service_discovery.py | 0 .../sdk/cmg_redis/redis_service_registry.py | 0 .../sdk/cmg_redis/utils.py | 0 infrastructure/sdk/init.sh | 30 +++ .../sdk/setup_cec_base.py | 0 .../sdk/setup_cec_redis.py | 0 .../sdk/setup_channel_job.py | 0 .../sdk/setup_clogger.py | 0 .../sdk/setup_cmg_base.py | 0 .../sdk/setup_cmg_redis.py | 0 .../sdk/setup_sysom_utils.py | 0 infrastructure/sdk/start.sh | 0 infrastructure/sdk/stop.sh | 0 .../sdk/sysom_utils/__init__.py | 0 .../sdk/sysom_utils/adddict.py | 0 .../sdk/sysom_utils/config_parser.py | 0 .../sdk/sysom_utils/event_executor.py | 0 .../sdk/sysom_utils/framework_plug_mag.py | 0 .../sdk/sysom_utils/framework_plugins.py | 0 .../sdk/sysom_utils/node_manager.py | 0 .../sdk/sysom_utils/yaml_concat.py | 0 .../sdk/test_cec_redis/__init__.py | 0 .../sdk/test_cec_redis/test_heartbeat.py | 0 .../sdk/test_cec_redis/test_redis_admin.py | 0 .../test_cec_redis/test_redis_admin_async.py | 0 .../sdk/test_cec_redis/test_redis_consumer.py | 0 .../test_redis_consumer_async.py | 0 .../test_redis_multi_consumer.py | 0 .../sdk/test_cec_redis/test_redis_producer.py | 0 .../sdk/test_cec_redis/test_utils.py | 0 .../sdk/test_cmg_base/__init__.py | 0 .../sdk/test_cmg_base/test_health_check.py | 0 .../sdk/test_cmg_redis/__init__.py | 0 .../test_cmg_redis/test_service_discovery.py | 0 .../test_cmg_redis/test_service_registry.py | 0 script/deploy.sh | 227 ++++++++++++++++++ script/sysom.sh | 148 ++++++++++++ 93 files changed, 987 insertions(+), 3 deletions(-) create mode 100644 infrastructure/env/.pydistutils.cfg create mode 100755 infrastructure/env/clear.sh create mode 100755 infrastructure/env/init.sh create mode 100644 infrastructure/env/pip.conf create mode 100644 infrastructure/env/requirements.txt create mode 100755 infrastructure/env/start.sh create mode 100755 infrastructure/env/stop.sh create mode 100755 infrastructure/local_services/clear.sh create mode 100755 infrastructure/local_services/init.sh create mode 100644 infrastructure/local_services/nginx.conf create mode 100755 infrastructure/local_services/start.sh create mode 100755 infrastructure/local_services/stop.sh create mode 100644 infrastructure/local_services/sysom-redis.ini create mode 100644 infrastructure/local_services/sysom.conf rename {sysom_server => infrastructure}/sdk/.gitignore (100%) rename {sysom_server => infrastructure}/sdk/cec_base/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/admin.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/base.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/cec_client.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/cli.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/consumer.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/event.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/exceptions.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/meta.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/producer.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/url.py (100%) rename {sysom_server => infrastructure}/sdk/cec_base/utils.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/admin_static.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/common.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/consume_status_storage.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/heartbeat.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/redis_admin.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/redis_consumer.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/redis_producer.py (100%) rename {sysom_server => infrastructure}/sdk/cec_redis/utils.py (100%) rename {sysom_server => infrastructure}/sdk/channel_job/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/channel_job/exception.py (100%) rename {sysom_server => infrastructure}/sdk/channel_job/job.py (100%) rename {sysom_server => infrastructure}/sdk/channel_job/model.py (100%) create mode 100755 infrastructure/sdk/clear.sh rename {sysom_server => infrastructure}/sdk/clogger/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/clogger/clogger.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/exceptions.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/load_balancing_strategy.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/service_check.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/service_discovery.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/service_instance.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/service_invoker.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/service_registry.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/url.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_base/utils.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_redis/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_redis/common.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_redis/redis_service_discovery.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_redis/redis_service_registry.py (100%) rename {sysom_server => infrastructure}/sdk/cmg_redis/utils.py (100%) create mode 100755 infrastructure/sdk/init.sh rename {sysom_server => infrastructure}/sdk/setup_cec_base.py (100%) rename {sysom_server => infrastructure}/sdk/setup_cec_redis.py (100%) rename {sysom_server => infrastructure}/sdk/setup_channel_job.py (100%) rename {sysom_server => infrastructure}/sdk/setup_clogger.py (100%) rename {sysom_server => infrastructure}/sdk/setup_cmg_base.py (100%) rename {sysom_server => infrastructure}/sdk/setup_cmg_redis.py (100%) rename {sysom_server => infrastructure}/sdk/setup_sysom_utils.py (100%) create mode 100755 infrastructure/sdk/start.sh create mode 100755 infrastructure/sdk/stop.sh rename {sysom_server => infrastructure}/sdk/sysom_utils/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/sysom_utils/adddict.py (100%) rename {sysom_server => infrastructure}/sdk/sysom_utils/config_parser.py (100%) rename {sysom_server => infrastructure}/sdk/sysom_utils/event_executor.py (100%) rename {sysom_server => infrastructure}/sdk/sysom_utils/framework_plug_mag.py (100%) rename {sysom_server => infrastructure}/sdk/sysom_utils/framework_plugins.py (100%) rename {sysom_server => infrastructure}/sdk/sysom_utils/node_manager.py (100%) rename {sysom_server => infrastructure}/sdk/sysom_utils/yaml_concat.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/test_heartbeat.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/test_redis_admin.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/test_redis_admin_async.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/test_redis_consumer.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/test_redis_consumer_async.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/test_redis_multi_consumer.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/test_redis_producer.py (100%) rename {sysom_server => infrastructure}/sdk/test_cec_redis/test_utils.py (100%) rename {sysom_server => infrastructure}/sdk/test_cmg_base/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/test_cmg_base/test_health_check.py (100%) rename {sysom_server => infrastructure}/sdk/test_cmg_redis/__init__.py (100%) rename {sysom_server => infrastructure}/sdk/test_cmg_redis/test_service_discovery.py (100%) rename {sysom_server => infrastructure}/sdk/test_cmg_redis/test_service_registry.py (100%) create mode 100755 script/deploy.sh create mode 100755 script/sysom.sh diff --git a/conf/config.yml b/conf/config.yml index 6834b042..7c24acb7 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -51,9 +51,6 @@ sysom_server: producer: cec_auto_mk_topic: true consumer: - cmg: - protocol: redis - special_param: logger: format: "%(asctime)s | %(levelname)s | %(message)s" level: "INFO" diff --git a/infrastructure/env/.pydistutils.cfg b/infrastructure/env/.pydistutils.cfg new file mode 100644 index 00000000..2dd3a7cf --- /dev/null +++ b/infrastructure/env/.pydistutils.cfg @@ -0,0 +1,3 @@ +[easy_install] + +index_url = https://mirrors.aliyun.com/pypi/simple/ \ No newline at end of file diff --git a/infrastructure/env/clear.sh b/infrastructure/env/clear.sh new file mode 100755 index 00000000..1663cd0b --- /dev/null +++ b/infrastructure/env/clear.sh @@ -0,0 +1,9 @@ +#!/bin/bash +x + +VIRTUALENV_HOME="${INFRASTRUCTURE_HOME}/virtualenv" + +remove_virtualenv() { + rm -rf ${VIRTUALENV_HOME} +} + +remove_virtualenv \ No newline at end of file diff --git a/infrastructure/env/init.sh b/infrastructure/env/init.sh new file mode 100755 index 00000000..ed371cc3 --- /dev/null +++ b/infrastructure/env/init.sh @@ -0,0 +1,102 @@ +#!/bin/bash +#****************************************************************# +# ScriptName: init sysom env +# Author: huangtuquan +#***************************************************************# + +ALIYUN_MIRROR="https://mirrors.aliyun.com/pypi/simple/" +VIRTUALENV_HOME="${INFRASTRUCTURE_HOME}/virtualenv" + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +check_selinux_status() +{ + ###check selinux rpm### + rpm -qa | grep selinux-policy + if [ $? -eq 0 ] + then + cat /etc/selinux/config | grep "SELINUX=disabled" + if [ $? -eq 0 ] + then + echo "selinux disable..." + else + echo "selinux enable, please set selinux disable" + exit 1 + fi + else + echo "selinux rpm package not install" + fi +} + +touch_env_rpms() { + if [ -f /etc/alios-release ]; then + if [ ! -f /etc/yum.repos.d/epel.repo ]; then + wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo + fi + elif [ -f /etc/anolis-release ]; then + sed -i '/epel\/8\/Everything/{n;s/enabled=0/enabled=1/;}' /etc/yum.repos.d/AnolisOS-DDE.repo + fi + rpm -q --quiet python3 || yum install -y python3 +} + +check_requirements() { + echo "INFO: begin install requirements..." + if ! [ -d ${SERVER_HOME}/logs/ ]; then + mkdir -p ${SERVER_HOME}/logs/ || exit 1 + fi + + local requirements_log="${LOG_HOME}/requirements.log" + touch "$requirements_log" || exit + ### atomic-0.7.3 need cffi, we show install cffi first### + pip install --upgrade pip + pip install cffi + pip install -r requirements.txt -i "${ALIYUN_MIRROR}" |tee -a "${requirements_log}" || exit 1 + local pip_res=$? + if [ $pip_res -ne 0 ]; then + echo "ERROR: requirements not satisfied and auto install failed, please check ${requirements_log}" + exit 1 + fi +} + +touch_virtualenv() { + if [ -d ${VIRTUALENV_HOME} ]; then + echo "virtualenv exists, skip" + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 + else + mkdir -p ~/.pip + cp pip.conf ~/.pip/ + cp .pydistutils.cfg ~/.pydistutils.cfg + python3 -m venv ${VIRTUALENV_HOME} + if [ "$?" = 0 ]; then + echo "INFO: create virtualenv success" + else + echo "ERROR: create virtualenv failed" + exit 1 + fi + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 + check_requirements + fi +} + +# install_sdk() { +# pushd ${SDK_DIR} +# python setup_cec_base.py develop +# python setup_cec_redis.py develop +# python setup_channel_job.py develop +# python setup_sysom_utils.py develop +# python setup_clogger.py develop +# popd +# } + +deploy() { + check_selinux_status + touch_env_rpms + touch_virtualenv +} + +deploy diff --git a/infrastructure/env/pip.conf b/infrastructure/env/pip.conf new file mode 100644 index 00000000..ccb91ab5 --- /dev/null +++ b/infrastructure/env/pip.conf @@ -0,0 +1,8 @@ +## Note, this file is written by cloud-init on first boot of an instance +## modifications made here will not survive a re-bundle. +### +[global] +index-url=https://mirrors.aliyun.com/pypi/simple/ + +[install] +trusted-host=mirrors.aliyun.com diff --git a/infrastructure/env/requirements.txt b/infrastructure/env/requirements.txt new file mode 100644 index 00000000..517c7d42 --- /dev/null +++ b/infrastructure/env/requirements.txt @@ -0,0 +1,40 @@ +typing-extensions==4.1.1 +attrs==22.2.0 +# cffi==1.15.1 +# aiofiles==0.8.0 +# alembic==1.7.7 +# anyio==3.6.2 +# aiohttp==3.8.3 +# asyncer==0.0.2 +# asyncssh==2.12.0 +# autopep8==2.0.0 +# channels==3.0.4 +# django==3.2.16 +# django-apscheduler==0.6.2 +# django-cors-headers==3.10.1 +# django-filter==21.1 +# django-redis==5.2.0 +# djangorestframework==3.14.0 +# drf-yasg==1.21.4 +# daphne==3.0.2 +# fastapi==0.83.0 +# gevent==22.10.2 +# gunicorn==20.1.0 +# pandas>=1.1.5 +# paramiko==2.12.0 +# prompt-toolkit==3.0.32 +# PyJWT==2.4.0 +# PyMySQL==1.0.2 +# pytest-runner==5.3.2 +# python-multipart==0.0.5 +# rich==12.6.0 +# requests==2.27.1 +# redis==4.3.4 +# schedule==1.1.0 +# uvicorn==0.16.0 +# wheel==0.37.1 +# xlwt==1.3.0 +# xlrd==2.0.1 +# prometheus-client==0.16.0 +# pyyaml==6.0 +# pyyaml-include==1.3 \ No newline at end of file diff --git a/infrastructure/env/start.sh b/infrastructure/env/start.sh new file mode 100755 index 00000000..e69de29b diff --git a/infrastructure/env/stop.sh b/infrastructure/env/stop.sh new file mode 100755 index 00000000..e69de29b diff --git a/infrastructure/local_services/clear.sh b/infrastructure/local_services/clear.sh new file mode 100755 index 00000000..8e8b2f74 --- /dev/null +++ b/infrastructure/local_services/clear.sh @@ -0,0 +1,37 @@ +#!/bin/bash +SERVICE_NAME=sysom-redis +clear_local_redis() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} + +stop_redis() { + ###we need redis version >= 5.0.0, check redis version### + redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` + echo ${redis_version} + if [ $redis_version -lt 5 ] + then + clear_local_redis + else + systemctl stop redis.service + fi +} + +stop_app() { + systemctl stop nginx.service + systemctl stop supervisord + stop_redis +} + +clear_db() { + systemctl start mariadb.service + systemctl stop mariadb.service +} + +clear_app() { + stop_app + clear_db +} + +clear_app diff --git a/infrastructure/local_services/init.sh b/infrastructure/local_services/init.sh new file mode 100755 index 00000000..9784ed56 --- /dev/null +++ b/infrastructure/local_services/init.sh @@ -0,0 +1,115 @@ +#!/bin/bash +#****************************************************************# +# ScriptName: start local service +# Author: huangtuquan +#***************************************************************# +SERVICE_NAME=sysom-redis +OSS_URL=https://sysom.oss-cn-beijing.aliyuncs.com/redis +REDIS_DL_URL=https://download.redis.io/releases +REDIS_DIR=redis-5.0.14 +REDIS_PKG=redis-5.0.14.tar.gz +usr_local_redis=0 + +setup_database() { + echo "INFO: begin create db..." + + systemctl restart mariadb.service + systemctl enable mariadb.service + mysql -uroot -e "create user if not exists 'sysom'@'%' identified by 'sysom_admin';" + mysql -uroot -e "grant usage on *.* to 'sysom'@'localhost' identified by 'sysom_admin'" + mysql -uroot -e "create database sysom character set utf8;" + mysql -uroot -e "create database grafana character set utf8;" + mysql -uroot -e "grant all privileges on sysom.* to 'sysom'@'%';" + mysql -uroot -e "grant all privileges on grafana.* to 'sysom'@'%';" + mysql -uroot -e "flush privileges;" +} +setup_nginx() { + mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak + cp nginx.conf /etc/nginx/ + cp sysom.conf /etc/nginx/conf.d/ + ###change the install dir base on param $1### + sed -i "s;SERVER_PORT;${SERVER_PORT};g" /etc/nginx/conf.d/sysom.conf + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/nginx/conf.d/sysom.conf +} + +init_conf() { + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini +} + +setup_redis() { + ###we need redis version >= 5.0.0, check redis version### + redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` + echo ${redis_version} + if [ $redis_version -lt 5 ] + then + echo "redis version in yum repo is less than 5.0.0, we will compile redis(5.0.14) and install it." + if [ ! -e ${REDIS_PKG} ] + then + wget ${OSS_URL}/${REDIS_PKG} || wget ${REDIS_DL_URL}/${REDIS_PKG} + if [ ! -e ${REDIS_PKG} ] + then + echo "download ${REDIS_PKG} fail" + exit 1 + fi + fi + echo "now uncompress ${REDIS_PKG}, then compile and install it." + tar -zxvf ${REDIS_PKG} + pushd ${REDIS_DIR} + make + mkdir -p ${SERVER_HOME}/redis + cp redis.conf ${SERVER_HOME}/redis/ + cp src/redis-server ${SERVER_HOME}/redis/ + if [ $? -ne 0 ] + then + echo "redis compile or install error, exit 1" + exit 1 + fi + usr_local_redis=1 + popd + fi +} + +start_local_redis() { + init_conf + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +start_app() { + systemctl enable nginx.service + systemctl restart nginx.service + systemctl enable supervisord + systemctl start supervisord + if [ $usr_local_redis == 1 ] + then + ###if redis systemd service has been start, we need stop it first### + systemctl status redis + if [ $? -eq 0 ] + then + systemctl stop redis + fi + start_local_redis + else + systemctl enable redis.service + systemctl restart redis.service + fi +} + +deploy() { + setup_database | tee -a ${LOG_HOME}/setup_database.log 2>&1 + setup_nginx + setup_redis + start_app +} + +deploy diff --git a/infrastructure/local_services/nginx.conf b/infrastructure/local_services/nginx.conf new file mode 100644 index 00000000..959830cd --- /dev/null +++ b/infrastructure/local_services/nginx.conf @@ -0,0 +1,68 @@ +# For more information on configuration, see: +# * Official English Documentation: http://nginx.org/en/docs/ +# * Official Russian Documentation: http://nginx.org/ru/docs/ + +user nginx; +worker_processes auto; +error_log /var/log/nginx/error.log; +pid /run/nginx.pid; + +# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. +include /usr/share/nginx/modules/*.conf; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + server_tokens off; + keepalive_timeout 0; + types_hash_max_size 4096; + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # Load modular configuration files from the /etc/nginx/conf.d directory. + # See http://nginx.org/en/docs/ngx_core_module.html#include + # for more information. + include /etc/nginx/conf.d/*.conf; + + +# Settings for a TLS enabled server. +# +# server { +# listen 443 ssl http2; +# listen [::]:443 ssl http2; +# server_name _; +# root /usr/share/nginx/html; +# +# ssl_certificate "/etc/pki/nginx/server.crt"; +# ssl_certificate_key "/etc/pki/nginx/private/server.key"; +# ssl_session_cache shared:SSL:1m; +# ssl_session_timeout 10m; +# ssl_ciphers HIGH:!aNULL:!MD5; +# ssl_prefer_server_ciphers on; +# +# # Load configuration files for the default server block. +# include /etc/nginx/default.d/*.conf; +# +# error_page 404 /404.html; +# location = /40x.html { +# } +# +# error_page 500 502 503 504 /50x.html; +# location = /50x.html { +# } +# } + +} + diff --git a/infrastructure/local_services/start.sh b/infrastructure/local_services/start.sh new file mode 100755 index 00000000..0f32583c --- /dev/null +++ b/infrastructure/local_services/start.sh @@ -0,0 +1,26 @@ +#!/bin/bash +SERVICE_NAME=sysom-redis +start_local_redis() { + supervisorctl start $SERVICE_NAME +} + +start_redis() { + ###we need redis version >= 5.0.0, check redis version### + redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` + echo ${redis_version} + if [ $redis_version -lt 5 ] + then + start_local_redis + else + systemctl start redis.service + fi +} + +start_app() { + systemctl start mariadb.service + systemctl start nginx.service + systemctl start supervisord + start_redis +} + +start_app diff --git a/infrastructure/local_services/stop.sh b/infrastructure/local_services/stop.sh new file mode 100755 index 00000000..9dda6548 --- /dev/null +++ b/infrastructure/local_services/stop.sh @@ -0,0 +1,25 @@ +#!/bin/bash +SERVICE_NAME=sysom-redis +stop_local_redis() { + supervisorctl stop $SERVICE_NAME +} +stop_redis() { + ###we need redis version >= 5.0.0, check redis version### + redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` + echo ${redis_version} + if [ $redis_version -lt 5 ] + then + stop_local_redis + else + systemctl stop redis.service + fi +} + +stop_app() { + systemctl stop nginx.service + systemctl stop mariadb.service + systemctl stop supervisord + stop_redis +} + +stop_app diff --git a/infrastructure/local_services/sysom-redis.ini b/infrastructure/local_services/sysom-redis.ini new file mode 100644 index 00000000..dcc775ee --- /dev/null +++ b/infrastructure/local_services/sysom-redis.ini @@ -0,0 +1,8 @@ +[program:sysom-redis] +directory = /usr/local/sysom/server/redis +command=/usr/local/sysom/server/redis/redis-server /usr/local/sysom/server/redis/redis.conf +startsecs=3 +autostart=true +autorestart=true +stderr_logfile=/usr/local/sysom/server/logs/sysom-redis-error.log +stdout_logfile=/usr/local/sysom/server/logs/sysom-redis.log diff --git a/infrastructure/local_services/sysom.conf b/infrastructure/local_services/sysom.conf new file mode 100644 index 00000000..94aca073 --- /dev/null +++ b/infrastructure/local_services/sysom.conf @@ -0,0 +1,141 @@ +map $http_upgrade $connection_upgrade { + default upgrade; + '' close; +} + +server { + listen SERVER_PORT; + server_name _; + root /usr/local/sysom/server/target/sysom_web; + index index.html; + client_max_body_size 20m; + + gzip on; + gzip_min_length 1k; + gzip_buffers 4 16k; + gzip_http_version 1.1; + gzip_comp_level 7; + gzip_types text/plain text/css text/javascript application/javascript application/json; + gzip_vary on; + + location /grafana/ { + proxy_pass http://localhost:3000/; + proxy_set_header Host $http_host; + } + + location /ws/ { + proxy_pass http://127.0.0.1:7001; + proxy_read_timeout 180s; + proxy_redirect off; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection $connection_upgrade; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/tasks/ { + proxy_pass http://127.0.0.1:7002; + proxy_read_timeout 180s; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v2/tasks/ { + proxy_pass http://127.0.0.1:7002; + proxy_read_timeout 180s; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/vmcore/ { + proxy_pass http://127.0.0.1:7004; + proxy_read_timeout 180s; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/channel/ { + proxy_pass http://127.0.0.1:7003; + proxy_read_timeout 180s; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + client_max_body_size 500m; + } + + location /api/v1/vul/ { + proxy_pass http://127.0.0.1:7005; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/migration/ { + proxy_pass http://127.0.0.1:7006; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/assessment/ { + proxy_pass http://127.0.0.1:7006; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/implementation/ { + proxy_pass http://127.0.0.1:7006; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/hotfix/ { + proxy_pass http://127.0.0.1:7007; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/demo/ { + proxy_pass http://127.0.0.1:7008; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/v1/monitor/ { + proxy_pass http://127.0.0.1:7009; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location /api/ { + proxy_pass http://127.0.0.1:7001; + proxy_read_timeout 180s; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + + location /checkpreload.htm { + proxy_pass http://127.0.0.1:7001/checkpreload.htm; + proxy_read_timeout 180s; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + + location / { + try_files $uri /index.html; + } + + location /download/ { + alias /usr/local/sysom/server/target/sysom_web/download/; + autoindex on; + autoindex_exact_size off; + autoindex_localtime on; + index index.html index.htm; + limit_rate 20m; + } + +} diff --git a/sysom_server/sdk/.gitignore b/infrastructure/sdk/.gitignore similarity index 100% rename from sysom_server/sdk/.gitignore rename to infrastructure/sdk/.gitignore diff --git a/sysom_server/sdk/cec_base/__init__.py b/infrastructure/sdk/cec_base/__init__.py similarity index 100% rename from sysom_server/sdk/cec_base/__init__.py rename to infrastructure/sdk/cec_base/__init__.py diff --git a/sysom_server/sdk/cec_base/admin.py b/infrastructure/sdk/cec_base/admin.py similarity index 100% rename from sysom_server/sdk/cec_base/admin.py rename to infrastructure/sdk/cec_base/admin.py diff --git a/sysom_server/sdk/cec_base/base.py b/infrastructure/sdk/cec_base/base.py similarity index 100% rename from sysom_server/sdk/cec_base/base.py rename to infrastructure/sdk/cec_base/base.py diff --git a/sysom_server/sdk/cec_base/cec_client.py b/infrastructure/sdk/cec_base/cec_client.py similarity index 100% rename from sysom_server/sdk/cec_base/cec_client.py rename to infrastructure/sdk/cec_base/cec_client.py diff --git a/sysom_server/sdk/cec_base/cli.py b/infrastructure/sdk/cec_base/cli.py similarity index 100% rename from sysom_server/sdk/cec_base/cli.py rename to infrastructure/sdk/cec_base/cli.py diff --git a/sysom_server/sdk/cec_base/consumer.py b/infrastructure/sdk/cec_base/consumer.py similarity index 100% rename from sysom_server/sdk/cec_base/consumer.py rename to infrastructure/sdk/cec_base/consumer.py diff --git a/sysom_server/sdk/cec_base/event.py b/infrastructure/sdk/cec_base/event.py similarity index 100% rename from sysom_server/sdk/cec_base/event.py rename to infrastructure/sdk/cec_base/event.py diff --git a/sysom_server/sdk/cec_base/exceptions.py b/infrastructure/sdk/cec_base/exceptions.py similarity index 100% rename from sysom_server/sdk/cec_base/exceptions.py rename to infrastructure/sdk/cec_base/exceptions.py diff --git a/sysom_server/sdk/cec_base/meta.py b/infrastructure/sdk/cec_base/meta.py similarity index 100% rename from sysom_server/sdk/cec_base/meta.py rename to infrastructure/sdk/cec_base/meta.py diff --git a/sysom_server/sdk/cec_base/producer.py b/infrastructure/sdk/cec_base/producer.py similarity index 100% rename from sysom_server/sdk/cec_base/producer.py rename to infrastructure/sdk/cec_base/producer.py diff --git a/sysom_server/sdk/cec_base/url.py b/infrastructure/sdk/cec_base/url.py similarity index 100% rename from sysom_server/sdk/cec_base/url.py rename to infrastructure/sdk/cec_base/url.py diff --git a/sysom_server/sdk/cec_base/utils.py b/infrastructure/sdk/cec_base/utils.py similarity index 100% rename from sysom_server/sdk/cec_base/utils.py rename to infrastructure/sdk/cec_base/utils.py diff --git a/sysom_server/sdk/cec_redis/__init__.py b/infrastructure/sdk/cec_redis/__init__.py similarity index 100% rename from sysom_server/sdk/cec_redis/__init__.py rename to infrastructure/sdk/cec_redis/__init__.py diff --git a/sysom_server/sdk/cec_redis/admin_static.py b/infrastructure/sdk/cec_redis/admin_static.py similarity index 100% rename from sysom_server/sdk/cec_redis/admin_static.py rename to infrastructure/sdk/cec_redis/admin_static.py diff --git a/sysom_server/sdk/cec_redis/common.py b/infrastructure/sdk/cec_redis/common.py similarity index 100% rename from sysom_server/sdk/cec_redis/common.py rename to infrastructure/sdk/cec_redis/common.py diff --git a/sysom_server/sdk/cec_redis/consume_status_storage.py b/infrastructure/sdk/cec_redis/consume_status_storage.py similarity index 100% rename from sysom_server/sdk/cec_redis/consume_status_storage.py rename to infrastructure/sdk/cec_redis/consume_status_storage.py diff --git a/sysom_server/sdk/cec_redis/heartbeat.py b/infrastructure/sdk/cec_redis/heartbeat.py similarity index 100% rename from sysom_server/sdk/cec_redis/heartbeat.py rename to infrastructure/sdk/cec_redis/heartbeat.py diff --git a/sysom_server/sdk/cec_redis/redis_admin.py b/infrastructure/sdk/cec_redis/redis_admin.py similarity index 100% rename from sysom_server/sdk/cec_redis/redis_admin.py rename to infrastructure/sdk/cec_redis/redis_admin.py diff --git a/sysom_server/sdk/cec_redis/redis_consumer.py b/infrastructure/sdk/cec_redis/redis_consumer.py similarity index 100% rename from sysom_server/sdk/cec_redis/redis_consumer.py rename to infrastructure/sdk/cec_redis/redis_consumer.py diff --git a/sysom_server/sdk/cec_redis/redis_producer.py b/infrastructure/sdk/cec_redis/redis_producer.py similarity index 100% rename from sysom_server/sdk/cec_redis/redis_producer.py rename to infrastructure/sdk/cec_redis/redis_producer.py diff --git a/sysom_server/sdk/cec_redis/utils.py b/infrastructure/sdk/cec_redis/utils.py similarity index 100% rename from sysom_server/sdk/cec_redis/utils.py rename to infrastructure/sdk/cec_redis/utils.py diff --git a/sysom_server/sdk/channel_job/__init__.py b/infrastructure/sdk/channel_job/__init__.py similarity index 100% rename from sysom_server/sdk/channel_job/__init__.py rename to infrastructure/sdk/channel_job/__init__.py diff --git a/sysom_server/sdk/channel_job/exception.py b/infrastructure/sdk/channel_job/exception.py similarity index 100% rename from sysom_server/sdk/channel_job/exception.py rename to infrastructure/sdk/channel_job/exception.py diff --git a/sysom_server/sdk/channel_job/job.py b/infrastructure/sdk/channel_job/job.py similarity index 100% rename from sysom_server/sdk/channel_job/job.py rename to infrastructure/sdk/channel_job/job.py diff --git a/sysom_server/sdk/channel_job/model.py b/infrastructure/sdk/channel_job/model.py similarity index 100% rename from sysom_server/sdk/channel_job/model.py rename to infrastructure/sdk/channel_job/model.py diff --git a/infrastructure/sdk/clear.sh b/infrastructure/sdk/clear.sh new file mode 100755 index 00000000..e69de29b diff --git a/sysom_server/sdk/clogger/__init__.py b/infrastructure/sdk/clogger/__init__.py similarity index 100% rename from sysom_server/sdk/clogger/__init__.py rename to infrastructure/sdk/clogger/__init__.py diff --git a/sysom_server/sdk/clogger/clogger.py b/infrastructure/sdk/clogger/clogger.py similarity index 100% rename from sysom_server/sdk/clogger/clogger.py rename to infrastructure/sdk/clogger/clogger.py diff --git a/sysom_server/sdk/cmg_base/__init__.py b/infrastructure/sdk/cmg_base/__init__.py similarity index 100% rename from sysom_server/sdk/cmg_base/__init__.py rename to infrastructure/sdk/cmg_base/__init__.py diff --git a/sysom_server/sdk/cmg_base/exceptions.py b/infrastructure/sdk/cmg_base/exceptions.py similarity index 100% rename from sysom_server/sdk/cmg_base/exceptions.py rename to infrastructure/sdk/cmg_base/exceptions.py diff --git a/sysom_server/sdk/cmg_base/load_balancing_strategy.py b/infrastructure/sdk/cmg_base/load_balancing_strategy.py similarity index 100% rename from sysom_server/sdk/cmg_base/load_balancing_strategy.py rename to infrastructure/sdk/cmg_base/load_balancing_strategy.py diff --git a/sysom_server/sdk/cmg_base/service_check.py b/infrastructure/sdk/cmg_base/service_check.py similarity index 100% rename from sysom_server/sdk/cmg_base/service_check.py rename to infrastructure/sdk/cmg_base/service_check.py diff --git a/sysom_server/sdk/cmg_base/service_discovery.py b/infrastructure/sdk/cmg_base/service_discovery.py similarity index 100% rename from sysom_server/sdk/cmg_base/service_discovery.py rename to infrastructure/sdk/cmg_base/service_discovery.py diff --git a/sysom_server/sdk/cmg_base/service_instance.py b/infrastructure/sdk/cmg_base/service_instance.py similarity index 100% rename from sysom_server/sdk/cmg_base/service_instance.py rename to infrastructure/sdk/cmg_base/service_instance.py diff --git a/sysom_server/sdk/cmg_base/service_invoker.py b/infrastructure/sdk/cmg_base/service_invoker.py similarity index 100% rename from sysom_server/sdk/cmg_base/service_invoker.py rename to infrastructure/sdk/cmg_base/service_invoker.py diff --git a/sysom_server/sdk/cmg_base/service_registry.py b/infrastructure/sdk/cmg_base/service_registry.py similarity index 100% rename from sysom_server/sdk/cmg_base/service_registry.py rename to infrastructure/sdk/cmg_base/service_registry.py diff --git a/sysom_server/sdk/cmg_base/url.py b/infrastructure/sdk/cmg_base/url.py similarity index 100% rename from sysom_server/sdk/cmg_base/url.py rename to infrastructure/sdk/cmg_base/url.py diff --git a/sysom_server/sdk/cmg_base/utils.py b/infrastructure/sdk/cmg_base/utils.py similarity index 100% rename from sysom_server/sdk/cmg_base/utils.py rename to infrastructure/sdk/cmg_base/utils.py diff --git a/sysom_server/sdk/cmg_redis/__init__.py b/infrastructure/sdk/cmg_redis/__init__.py similarity index 100% rename from sysom_server/sdk/cmg_redis/__init__.py rename to infrastructure/sdk/cmg_redis/__init__.py diff --git a/sysom_server/sdk/cmg_redis/common.py b/infrastructure/sdk/cmg_redis/common.py similarity index 100% rename from sysom_server/sdk/cmg_redis/common.py rename to infrastructure/sdk/cmg_redis/common.py diff --git a/sysom_server/sdk/cmg_redis/redis_service_discovery.py b/infrastructure/sdk/cmg_redis/redis_service_discovery.py similarity index 100% rename from sysom_server/sdk/cmg_redis/redis_service_discovery.py rename to infrastructure/sdk/cmg_redis/redis_service_discovery.py diff --git a/sysom_server/sdk/cmg_redis/redis_service_registry.py b/infrastructure/sdk/cmg_redis/redis_service_registry.py similarity index 100% rename from sysom_server/sdk/cmg_redis/redis_service_registry.py rename to infrastructure/sdk/cmg_redis/redis_service_registry.py diff --git a/sysom_server/sdk/cmg_redis/utils.py b/infrastructure/sdk/cmg_redis/utils.py similarity index 100% rename from sysom_server/sdk/cmg_redis/utils.py rename to infrastructure/sdk/cmg_redis/utils.py diff --git a/infrastructure/sdk/init.sh b/infrastructure/sdk/init.sh new file mode 100755 index 00000000..97e6a0c3 --- /dev/null +++ b/infrastructure/sdk/init.sh @@ -0,0 +1,30 @@ +#!/bin/bash +x +#****************************************************************# +# ScriptName: init sysom env +# Author: huangtuquan +#***************************************************************# + +VIRTUALENV_HOME="${INFRASTRUCTURE_HOME}/virtualenv" +SDK_DIR=$INFRASTRUCTURE_HOME/sdk + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_sdk() { + pushd ${SDK_DIR} + python setup_cec_base.py develop + python setup_cec_redis.py develop + python setup_channel_job.py develop + python setup_sysom_utils.py develop + python setup_clogger.py develop + popd +} + +deploy() { + source_virtualenv + install_sdk +} + +deploy diff --git a/sysom_server/sdk/setup_cec_base.py b/infrastructure/sdk/setup_cec_base.py similarity index 100% rename from sysom_server/sdk/setup_cec_base.py rename to infrastructure/sdk/setup_cec_base.py diff --git a/sysom_server/sdk/setup_cec_redis.py b/infrastructure/sdk/setup_cec_redis.py similarity index 100% rename from sysom_server/sdk/setup_cec_redis.py rename to infrastructure/sdk/setup_cec_redis.py diff --git a/sysom_server/sdk/setup_channel_job.py b/infrastructure/sdk/setup_channel_job.py similarity index 100% rename from sysom_server/sdk/setup_channel_job.py rename to infrastructure/sdk/setup_channel_job.py diff --git a/sysom_server/sdk/setup_clogger.py b/infrastructure/sdk/setup_clogger.py similarity index 100% rename from sysom_server/sdk/setup_clogger.py rename to infrastructure/sdk/setup_clogger.py diff --git a/sysom_server/sdk/setup_cmg_base.py b/infrastructure/sdk/setup_cmg_base.py similarity index 100% rename from sysom_server/sdk/setup_cmg_base.py rename to infrastructure/sdk/setup_cmg_base.py diff --git a/sysom_server/sdk/setup_cmg_redis.py b/infrastructure/sdk/setup_cmg_redis.py similarity index 100% rename from sysom_server/sdk/setup_cmg_redis.py rename to infrastructure/sdk/setup_cmg_redis.py diff --git a/sysom_server/sdk/setup_sysom_utils.py b/infrastructure/sdk/setup_sysom_utils.py similarity index 100% rename from sysom_server/sdk/setup_sysom_utils.py rename to infrastructure/sdk/setup_sysom_utils.py diff --git a/infrastructure/sdk/start.sh b/infrastructure/sdk/start.sh new file mode 100755 index 00000000..e69de29b diff --git a/infrastructure/sdk/stop.sh b/infrastructure/sdk/stop.sh new file mode 100755 index 00000000..e69de29b diff --git a/sysom_server/sdk/sysom_utils/__init__.py b/infrastructure/sdk/sysom_utils/__init__.py similarity index 100% rename from sysom_server/sdk/sysom_utils/__init__.py rename to infrastructure/sdk/sysom_utils/__init__.py diff --git a/sysom_server/sdk/sysom_utils/adddict.py b/infrastructure/sdk/sysom_utils/adddict.py similarity index 100% rename from sysom_server/sdk/sysom_utils/adddict.py rename to infrastructure/sdk/sysom_utils/adddict.py diff --git a/sysom_server/sdk/sysom_utils/config_parser.py b/infrastructure/sdk/sysom_utils/config_parser.py similarity index 100% rename from sysom_server/sdk/sysom_utils/config_parser.py rename to infrastructure/sdk/sysom_utils/config_parser.py diff --git a/sysom_server/sdk/sysom_utils/event_executor.py b/infrastructure/sdk/sysom_utils/event_executor.py similarity index 100% rename from sysom_server/sdk/sysom_utils/event_executor.py rename to infrastructure/sdk/sysom_utils/event_executor.py diff --git a/sysom_server/sdk/sysom_utils/framework_plug_mag.py b/infrastructure/sdk/sysom_utils/framework_plug_mag.py similarity index 100% rename from sysom_server/sdk/sysom_utils/framework_plug_mag.py rename to infrastructure/sdk/sysom_utils/framework_plug_mag.py diff --git a/sysom_server/sdk/sysom_utils/framework_plugins.py b/infrastructure/sdk/sysom_utils/framework_plugins.py similarity index 100% rename from sysom_server/sdk/sysom_utils/framework_plugins.py rename to infrastructure/sdk/sysom_utils/framework_plugins.py diff --git a/sysom_server/sdk/sysom_utils/node_manager.py b/infrastructure/sdk/sysom_utils/node_manager.py similarity index 100% rename from sysom_server/sdk/sysom_utils/node_manager.py rename to infrastructure/sdk/sysom_utils/node_manager.py diff --git a/sysom_server/sdk/sysom_utils/yaml_concat.py b/infrastructure/sdk/sysom_utils/yaml_concat.py similarity index 100% rename from sysom_server/sdk/sysom_utils/yaml_concat.py rename to infrastructure/sdk/sysom_utils/yaml_concat.py diff --git a/sysom_server/sdk/test_cec_redis/__init__.py b/infrastructure/sdk/test_cec_redis/__init__.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/__init__.py rename to infrastructure/sdk/test_cec_redis/__init__.py diff --git a/sysom_server/sdk/test_cec_redis/test_heartbeat.py b/infrastructure/sdk/test_cec_redis/test_heartbeat.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/test_heartbeat.py rename to infrastructure/sdk/test_cec_redis/test_heartbeat.py diff --git a/sysom_server/sdk/test_cec_redis/test_redis_admin.py b/infrastructure/sdk/test_cec_redis/test_redis_admin.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/test_redis_admin.py rename to infrastructure/sdk/test_cec_redis/test_redis_admin.py diff --git a/sysom_server/sdk/test_cec_redis/test_redis_admin_async.py b/infrastructure/sdk/test_cec_redis/test_redis_admin_async.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/test_redis_admin_async.py rename to infrastructure/sdk/test_cec_redis/test_redis_admin_async.py diff --git a/sysom_server/sdk/test_cec_redis/test_redis_consumer.py b/infrastructure/sdk/test_cec_redis/test_redis_consumer.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/test_redis_consumer.py rename to infrastructure/sdk/test_cec_redis/test_redis_consumer.py diff --git a/sysom_server/sdk/test_cec_redis/test_redis_consumer_async.py b/infrastructure/sdk/test_cec_redis/test_redis_consumer_async.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/test_redis_consumer_async.py rename to infrastructure/sdk/test_cec_redis/test_redis_consumer_async.py diff --git a/sysom_server/sdk/test_cec_redis/test_redis_multi_consumer.py b/infrastructure/sdk/test_cec_redis/test_redis_multi_consumer.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/test_redis_multi_consumer.py rename to infrastructure/sdk/test_cec_redis/test_redis_multi_consumer.py diff --git a/sysom_server/sdk/test_cec_redis/test_redis_producer.py b/infrastructure/sdk/test_cec_redis/test_redis_producer.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/test_redis_producer.py rename to infrastructure/sdk/test_cec_redis/test_redis_producer.py diff --git a/sysom_server/sdk/test_cec_redis/test_utils.py b/infrastructure/sdk/test_cec_redis/test_utils.py similarity index 100% rename from sysom_server/sdk/test_cec_redis/test_utils.py rename to infrastructure/sdk/test_cec_redis/test_utils.py diff --git a/sysom_server/sdk/test_cmg_base/__init__.py b/infrastructure/sdk/test_cmg_base/__init__.py similarity index 100% rename from sysom_server/sdk/test_cmg_base/__init__.py rename to infrastructure/sdk/test_cmg_base/__init__.py diff --git a/sysom_server/sdk/test_cmg_base/test_health_check.py b/infrastructure/sdk/test_cmg_base/test_health_check.py similarity index 100% rename from sysom_server/sdk/test_cmg_base/test_health_check.py rename to infrastructure/sdk/test_cmg_base/test_health_check.py diff --git a/sysom_server/sdk/test_cmg_redis/__init__.py b/infrastructure/sdk/test_cmg_redis/__init__.py similarity index 100% rename from sysom_server/sdk/test_cmg_redis/__init__.py rename to infrastructure/sdk/test_cmg_redis/__init__.py diff --git a/sysom_server/sdk/test_cmg_redis/test_service_discovery.py b/infrastructure/sdk/test_cmg_redis/test_service_discovery.py similarity index 100% rename from sysom_server/sdk/test_cmg_redis/test_service_discovery.py rename to infrastructure/sdk/test_cmg_redis/test_service_discovery.py diff --git a/sysom_server/sdk/test_cmg_redis/test_service_registry.py b/infrastructure/sdk/test_cmg_redis/test_service_registry.py similarity index 100% rename from sysom_server/sdk/test_cmg_redis/test_service_registry.py rename to infrastructure/sdk/test_cmg_redis/test_service_registry.py diff --git a/script/deploy.sh b/script/deploy.sh new file mode 100755 index 00000000..728d2786 --- /dev/null +++ b/script/deploy.sh @@ -0,0 +1,227 @@ +#!/bin/bash -x +set -x +ProgName=$(basename $0) +BaseDir=$(dirname $(readlink -f "$0")) + +SYSOM_CONF_DIR=${MICROSERVICE_HOME}/conf +SYSOM_CONF=${SYSOM_CONF_DIR}/config.yml +# SYSOM_DATABASE_HOST=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a host | awk '{print $2}'` +# SYSOM_DATABASE_PORT=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a port | awk '{print $2}'` +# SYSOM_DATABASE_USER=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a user | awk '{print $2}'` +# SYSOM_DATABASE_PASSWORD=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a password | awk '{print $2}'` +SYSOM_DEPLOY_LOG=$LOG_HOME/sysom_deploy.log + +# ###enable the service web menu### +# setup_web_menu_enable() +# { +# service=`echo $1 | grep sysom | awk -F"sysom_" '{print $NF}'` +# if [ "$service" != "" ] +# then +# mysql -h ${SYSOM_DATABASE_HOST} -P ${SYSOM_DATABASE_PORT} -u ${SYSOM_DATABASE_USER} -p${SYSOM_DATABASE_PASSWORD} \ +# -e "use sysom;insert into sys_service_info(service_name, created_at) values ('sysom_${service}', current_timestamp);" +# fi +# } + +#################################################################################################################### +# Helper functions +#################################################################################################################### + +red() { + printf '\33[1;31m%b\n\33[0m' "$1" +} + +green() { + printf '\33[1;32m%b\n\33[0m' "$1" +} + +generate_service_env() { + rm -f ${APP_HOME}/env + cat << EOF > ${APP_HOME}/env +APP_HOME=${APP_HOME} +SERVER_HOME=${APP_HOME}/server +NODE_HOME=${APP_HOME}/node +SERVER_LOCAL_IP=${SERVER_LOCAL_IP} +SERVER_PUBLIC_IP=${SERVER_PUBLIC_IP} +SERVER_PORT=${SERVER_PORT} +EOF +} + +update_global_config() { + sed "s/SERVER_LOCAL_IP: 127.0.0.1/SERVER_LOCAL_IP: $SERVER_LOCAL_IP/g" -i ${SYSOM_CONF} + sed "s/SERVER_PUBLIC_IP: 127.0.0.1/SERVER_PUBLIC_IP: $SERVER_PUBLIC_IP/g" -i ${SYSOM_CONF} + sed "s/SERVER_PORT: 80/SERVER_PORT: $SERVER_PORT/g" -i ${SYSOM_CONF} + sed "s/global_root_path \/usr\/local\/sysom/global_root_path $APP_HOME/g" -i ${SYSOM_CONF} + + # MySQL + sed "/mysql/,/host/s/host: localhost/host: $DB_MYSQL_HOST/g" -i ${SYSOM_CONF} + sed "/mysql/,/port/s/port: 3306/port: $DB_MYSQL_PORT/g" -i ${SYSOM_CONF} + sed "/mysql/,/user/s/user: sysom/user: $DB_MYSQL_USERNAME/g" -i ${SYSOM_CONF} + sed "/mysql/,/password/s/password: sysom_admin/password: $DB_MYSQL_PASSWORD/g" -i ${SYSOM_CONF} + sed "/mysql/,/database/s/database: sysom/database: $DB_MYSQL_DATABASE/g" -i ${SYSOM_CONF} + + # Redis + sed "/redis:/,/host/s/host: localhost/host: $REDIS_HOST/g" -i ${SYSOM_CONF} + sed "/redis:/,/port/s/port: 6379/port: $REDIS_PORT/g" -i ${SYSOM_CONF} + sed "/redis:/,/username/s/username:/username: $REDIS_USERNAME/g" -i ${SYSOM_CONF} + sed "/redis:/,/password/s/password:/password: $REDIS_PASSWORD/g" -i ${SYSOM_CONF} + + ###update local timezone### + local_timezone=`timedatectl status | grep "Time zone" | awk '{print $3}'` + green "current timezone is : $local_timezone" + if [ "$local_timezone" == "" ] + then + echo "get time zone fail, use default time zone Asia/Shanghai" + else + sed "s;Asia/Shanghai;$local_timezone;g" -i ${SYSOM_CONF} + fi +} + +ensure_sysom_conf_exists() { + mkdir -p $SYSOM_CONF_DIR + if [ ! -f "$SYSOM_CONF" ] + then + cp `dirname $BaseDir`/conf/config.yml $SYSOM_CONF + fi +} + +do_deploy_microservices() { + ensure_sysom_conf_exists + + mkdir -p ${MICROSERVICE_HOME} + local_microservice_dir=`dirname $BaseDir`/microservice + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]} + do + if [ "$local_microservice_dir" != "$MICROSERVICE_HOME" ] + then + # Copy microservice dir to deploy path if need + cp -r $local_microservice_dir/${target} $MICROSERVICE_HOME/${target} + fi + + target_dir=${MICROSERVICE_HOME}/${target} + pushd ${target_dir} + + green "$target_dir initialing..................................." + bash -x ${target_dir}/scripts/server_init.sh + if [ $? -ne 0 ];then + red "$target_dir initial fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + # green "setup_web_menu_enable $dir" + # setup_web_menu_enable $dir + popd + done +} + +do_deploy_infrastructure() { + mkdir -p ${INFRASTRUCTURE_HOME} + local_infrastructure_dir=`dirname $BaseDir`/infrastructure + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]} + do + if [ "$local_infrastructure_dir" != "$INFRASTRUCTURE_HOME" ] + then + # Copy microservice dir to deploy path if need + cp -r $local_infrastructure_dir/${target} $INFRASTRUCTURE_HOME/${target} + fi + target_dir=${INFRASTRUCTURE_HOME}/${target} + pushd ${target_dir} + + green "$target_dir initialing..................................." + bash -x ${target_dir}/init.sh + if [ $? -ne 0 ];then + red "$target_dir initial fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + # green "setup_web_menu_enable $dir" + # setup_web_menu_enable $dir + popd + done +} + + +#################################################################################################################### +# Subcommands +#################################################################################################################### + +sub_help(){ + echo "Usage: $ProgName [options]" + echo "Subcommands:" + echo " all Deploy all modules" + echo " infrastructure [ALL | ] Deploy infrastructure components" + echo " Example: $ProgName infrastructure env" + echo " Example: $ProgName infrastructure local_services" + echo " microservice [ALL | ] Deploy all microservices or specific microservice" + echo " Example: $ProgName microservice sysom_diagnosis" + echo "" + echo "For help with each subcommand run:" + echo "$ProgName -h|--help" + echo "" +} + + +sub_all() { + echo "all" +} + +sub_infra() { + sub_infrastructure $@ +} + +# -> env + local_services +sub_infrastructure() { + target=$1 + if [ "$target" == "ALL" ] + then + # Deploy all microservices + for infrastructure in $INFRASTRUCTURE_HOME + do + do_deploy_infrastructure $infrastructure + done + else + # Deploy specific microservices + do_deploy_infrastructure $target + fi +} + +sub_ms() { + sub_microservice $@ +} + +# All microservices +sub_microservice() { + target=$1 + if [ "$target" == "ALL" ] + then + # Deploy all microservices + for microservice in $(ls $MICROSERVICE_HOME) + do + echo ${microservice} + do_deploy_microservices ${microservice} + done + else + # Deploy specific microservices + do_deploy_microservices $target + fi +} + + +subcommand=$1 +case $subcommand in + "" | "-h" | "--help") + sub_help + ;; + *) + shift + sub_${subcommand} $@ + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac \ No newline at end of file diff --git a/script/sysom.sh b/script/sysom.sh new file mode 100755 index 00000000..ed7b5c45 --- /dev/null +++ b/script/sysom.sh @@ -0,0 +1,148 @@ +#!/bin/bash -x + +#################################################################################################################### +# Initialize environment variables +#################################################################################################################### +if [ "$APP_NAME" == "" ] +then + export APP_NAME="sysom" +fi + +if [ "$APP_HOME" == "" ] +then + export APP_HOME=/usr/local/sysom +fi +export SERVER_HOME=${APP_HOME}/server +export NODE_HOME=${APP_HOME}/node +export MICROSERVICE_HOME=${SERVER_HOME}/microservice +export INFRASTRUCTURE_HOME=${SERVER_HOME}/infrastructure + +if [ "$SERVER_LOCAL_IP" == "" ] +then + local_ip=`ip -4 route | grep "link src" | awk -F"link src " '{print $2}' | awk '{print $1}' | head -n 1` + export SERVER_LOCAL_IP=$local_ip +fi + +if [ "$SERVER_PUBLIC_IP" == "" ] +then + export SERVER_PUBLIC_IP=$SERVER_LOCAL_IP +fi + +if [ "$SERVER_PORT" == "" ] +then + export SERVER_PORT=80 +fi + +if [ "$LOG_HOME" == "" ] +then + export LOG_HOME=/var/log/sysom +fi +mkdir -p $LOG_HOME + +if [ "$DB_MYSQL_HOST" == "" ] +then + export DB_MYSQL_HOST=localhost +fi + +if [ "$DB_MYSQL_PORT" == "" ] +then + export DB_MYSQL_PORT=3306 +fi + +if [ "$DB_MYSQL_USERNAME" == "" ] +then + export DB_MYSQL_USERNAME=sysom +fi + +if [ "$DB_MYSQL_PASSWORD" == "" ] +then + export DB_MYSQL_PASSWORD=sysom_admin +fi + +if [ "$DB_MYSQL_DATABASE" == "" ] +then + export DB_MYSQL_DATABASE=sysom +fi + +if [ "$REDIS_HOST" == "" ] +then + export REDIS_HOST=localhost +fi + +if [ "$REDIS_PORT" == "" ] +then + export REDIS_PORT=6379 +fi + +if [ "$REDIS_USERNAME" == "" ] +then + export REDIS_USERNAME="" +fi + +if [ "$REDIS_PASSWORD" == "" ] +then + export REDIS_PASSWORD="" +fi + + +#################################################################################################################### +# Subcommands +#################################################################################################################### + +ProgName=$(basename $0) +BaseDir=`dirname $0` + +sub_help(){ + echo "Usage: $ProgName [options]" + echo "Subcommands:" + echo " deploy Deploy the SysOM module" + echo " update Update SysOM module" + echo " start Start SysOM module" + echo " stop Stop SysOM module" + echo " clear Clear SysOM module" + echo " list List all SysOM module" + echo "" + echo "For help with each subcommand run:" + echo "$ProgName -h|--help" + echo "" +} + +sub_deploy() { + bash +x ${BaseDir}/deploy.sh $@ +} + +sub_update() { + bash +x ${BaseDir}/update.sh $@ +} + +sub_start() { + bash +x ${BaseDir}/start.sh $@ +} + +sub_stop() { + bash +x ${BaseDir}/stop.sh $@ +} + +sub_clear() { + bash +x ${BaseDir}/clear.sh $@ +} + +sub_list() { + echo "list" +} + +subcommand=$1 +case $subcommand in + "" | "-h" | "--help") + sub_help + ;; + *) + shift + sub_${subcommand} $@ + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac \ No newline at end of file -- Gitee From 736c3f16665679445383c6ac8c21e8352b9618c8 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 21 Apr 2023 14:51:43 +0800 Subject: [PATCH 015/196] refactor(sysom_diagnosis): Use sysom.sh Use to unify deployment --- conf/config.yml | 2 + infrastructure/sdk/setup_sysom_utils.py | 5 +- infrastructure/sdk/sysom_utils/adddict.py | 14 ++ .../sdk/sysom_utils/config_parser.py | 20 ++- .../sysom_diagnosis/apps/task/__init__.py | 0 .../sysom_diagnosis/apps/task/admin.py | 0 .../sysom_diagnosis/apps/task/apps.py | 0 .../sysom_diagnosis/apps/task/executor.py | 0 .../sysom_diagnosis/apps/task/filter.py | 0 .../sysom_diagnosis/apps/task/helper.py | 0 .../apps/task/migrations/0001_initial.py | 0 .../migrations/0002_auto_20230110_1738.py | 0 .../apps/task/migrations/__init__.py | 0 .../sysom_diagnosis/apps/task/models.py | 0 .../sysom_diagnosis/apps/task/seriaizer.py | 0 .../sysom_diagnosis/apps/task/tests.py | 0 .../sysom_diagnosis/apps/task/urls.py | 0 .../sysom_diagnosis/apps/task/views.py | 0 .../sysom_diagnosis/conf/common.py | 18 +-- .../sysom_diagnosis/conf/develop.py | 0 .../conf/diagnosis_gunicorn.py | 4 +- .../sysom_diagnosis/conf/product.py | 0 .../sysom_diagnosis/conf/testing.py | 0 .../sysom_diagnosis/config.yml | 0 .../sysom_diagnosis/lib/authentications.py | 0 .../sysom_diagnosis/lib/base_model.py | 0 .../sysom_diagnosis/lib/base_view.py | 0 .../lib/decode/sysom_decode.py | 0 .../sysom_diagnosis/lib/exception.py | 0 .../sysom_diagnosis/lib/paginations.py | 0 .../sysom_diagnosis/lib/renderers.py | 0 .../sysom_diagnosis/lib/response.py | 0 .../sysom_diagnosis/lib/utils.py | 12 +- .../sysom_diagnosis/manage.py | 0 microservice/sysom_diagnosis/requirements.txt | 19 +++ .../sysom_diagnosis/scripts/node_clear.sh | 0 .../sysom_diagnosis/scripts/node_init.sh | 0 .../sysom_diagnosis/scripts/node_update.sh | 0 .../sysom_diagnosis/scripts/server_clear.sh | 8 + .../sysom_diagnosis/scripts/server_deploy.sh | 0 .../sysom_diagnosis/scripts/server_init.sh | 52 +++++++ .../sysom_diagnosis/scripts/server_start.sh | 7 + .../sysom_diagnosis/scripts/server_stop.sh | 7 + .../sysom_diagnosis/service_scripts/command | 0 .../service_scripts/command_post | 0 .../sysom_diagnosis/service_scripts/filecache | 0 .../service_scripts/filecache_post | 0 .../sysom_diagnosis/service_scripts/iofsstat | 0 .../service_scripts/iofsstat_post | 0 .../sysom_diagnosis/service_scripts/iohang | 0 .../service_scripts/iohang_post | 0 .../sysom_diagnosis/service_scripts/iolatency | 0 .../service_scripts/iolatency_post | 0 .../service_scripts/iosdiag_latency | 0 .../service_scripts/iosdiag_latency_post | 0 .../sysom_diagnosis/service_scripts/jitter | 0 .../service_scripts/jitter_post | 0 .../sysom_diagnosis/service_scripts/loadtask | 0 .../service_scripts/loadtask_post | 0 .../sysom_diagnosis/service_scripts/memgraph | 0 .../service_scripts/memgraph_post | 0 .../sysom_diagnosis/service_scripts/oomcheck | 0 .../service_scripts/oomcheck_post | 0 .../sysom_diagnosis/service_scripts/ossre | 0 .../service_scripts/ossre_post | 0 .../service_scripts/packetdrop | 0 .../service_scripts/packetdrop_post | 0 .../sysom_diagnosis/service_scripts/pingtrace | 0 .../service_scripts/pingtrace_post | 0 .../sysom_diagnosis/service_scripts/retran | 0 .../service_scripts/retran_post | 0 .../sysom_diagnosis/service_scripts/schedmoni | 0 .../service_scripts/schedmoni_post | 0 .../service_scripts/taskprofile | 0 .../service_scripts/taskprofile_post | 0 .../sysom_diagnosis/service_scripts/test | 0 .../sysom_diagnosis/service_scripts/test.sh | 0 .../sysom_diagnosis/service_scripts/tpython | 0 .../service_scripts/vmcore_analyse | 0 .../sysom_diagnosis/sysom-diagnosis.ini | 9 ++ .../sysom_diagnosis/__init__.py | 0 .../sysom_diagnosis/sysom_diagnosis/asgi.py | 0 .../sysom_diagnosis/settings.py | 0 .../sysom_diagnosis/sysom_diagnosis/urls.py | 0 .../sysom_diagnosis/sysom_diagnosis/wsgi.py | 0 .../sysom_diagnosis/lib/channels/base.py | 17 --- .../sysom_diagnosis/lib/channels/ssh.py | 140 ------------------ 87 files changed, 144 insertions(+), 190 deletions(-) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/__init__.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/admin.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/apps.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/executor.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/filter.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/helper.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/models.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/seriaizer.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/tests.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/urls.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/apps/task/views.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/conf/common.py (88%) rename {sysom_server => microservice}/sysom_diagnosis/conf/develop.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/conf/diagnosis_gunicorn.py (80%) rename {sysom_server => microservice}/sysom_diagnosis/conf/product.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/conf/testing.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/config.yml (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/authentications.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/base_model.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/base_view.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/decode/sysom_decode.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/exception.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/paginations.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/renderers.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/response.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/lib/utils.py (83%) rename {sysom_server => microservice}/sysom_diagnosis/manage.py (100%) create mode 100644 microservice/sysom_diagnosis/requirements.txt rename {sysom_server => microservice}/sysom_diagnosis/scripts/node_clear.sh (100%) rename {sysom_server => microservice}/sysom_diagnosis/scripts/node_init.sh (100%) rename {sysom_server => microservice}/sysom_diagnosis/scripts/node_update.sh (100%) create mode 100755 microservice/sysom_diagnosis/scripts/server_clear.sh rename sysom_server/sysom_diagnosis/lib/channels/__init__.py => microservice/sysom_diagnosis/scripts/server_deploy.sh (100%) create mode 100755 microservice/sysom_diagnosis/scripts/server_init.sh create mode 100755 microservice/sysom_diagnosis/scripts/server_start.sh create mode 100755 microservice/sysom_diagnosis/scripts/server_stop.sh rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/command (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/command_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/filecache (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/filecache_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/iofsstat (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/iofsstat_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/iohang (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/iohang_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/iolatency (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/iolatency_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/iosdiag_latency (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/iosdiag_latency_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/jitter (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/jitter_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/loadtask (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/loadtask_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/memgraph (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/memgraph_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/oomcheck (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/oomcheck_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/ossre (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/ossre_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/packetdrop (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/packetdrop_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/pingtrace (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/pingtrace_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/retran (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/retran_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/schedmoni (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/schedmoni_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/taskprofile (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/taskprofile_post (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/test (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/test.sh (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/tpython (100%) rename {sysom_server => microservice}/sysom_diagnosis/service_scripts/vmcore_analyse (100%) create mode 100644 microservice/sysom_diagnosis/sysom-diagnosis.ini rename {sysom_server => microservice}/sysom_diagnosis/sysom_diagnosis/__init__.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/sysom_diagnosis/asgi.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/sysom_diagnosis/settings.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/sysom_diagnosis/urls.py (100%) rename {sysom_server => microservice}/sysom_diagnosis/sysom_diagnosis/wsgi.py (100%) delete mode 100644 sysom_server/sysom_diagnosis/lib/channels/base.py delete mode 100644 sysom_server/sysom_diagnosis/lib/channels/ssh.py diff --git a/conf/config.yml b/conf/config.yml index 7c24acb7..de437335 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -51,6 +51,8 @@ sysom_server: producer: cec_auto_mk_topic: true consumer: + cmg: + protocol: redis logger: format: "%(asctime)s | %(levelname)s | %(message)s" level: "INFO" diff --git a/infrastructure/sdk/setup_sysom_utils.py b/infrastructure/sdk/setup_sysom_utils.py index df9eb49a..f30006d1 100644 --- a/infrastructure/sdk/setup_sysom_utils.py +++ b/infrastructure/sdk/setup_sysom_utils.py @@ -21,8 +21,9 @@ setuptools.setup( "channel_job>=0.0.1", "clogger>=0.0.1", "aiohttp>=3.8.3", - "aiofiles>=0.8.0", - "anyio>=3.6.2" + "aiofiles==0.8.0", + "anyio>=3.6.2", + "asyncer==0.0.2" ], classifiers=[ "Programming Language :: Python :: 3", diff --git a/infrastructure/sdk/sysom_utils/adddict.py b/infrastructure/sdk/sysom_utils/adddict.py index f87d7283..1fedea33 100644 --- a/infrastructure/sdk/sysom_utils/adddict.py +++ b/infrastructure/sdk/sysom_utils/adddict.py @@ -94,6 +94,20 @@ class Dict(dict): else: base[key] = value return base + + def get_multi(self, key_str: str): + keys = key_str.split(".") + res = self + for key in keys: + res = res.get(key) + return res + + def set_multi(self, key_str: str, value: any): + keys = key_str.split(".") + res = self + for i in range(len(keys) - 1): + res = res[keys[i]] + res[keys[-1]] = value def copy(self): return copy.copy(self) diff --git a/infrastructure/sdk/sysom_utils/config_parser.py b/infrastructure/sdk/sysom_utils/config_parser.py index adb9eb05..3e7b2dcf 100644 --- a/infrastructure/sdk/sysom_utils/config_parser.py +++ b/infrastructure/sdk/sysom_utils/config_parser.py @@ -8,6 +8,7 @@ Description: """ import yaml from enum import Enum +import os from .yaml_concat import YamlConcatConstructor from .adddict import Dict @@ -37,6 +38,17 @@ SYSOM_CONFIG_SECTION_WEB = "sysom_web" SYSOM_CONFIG_SECTION_NODE = "sysom_node" SYSOM_CONFIG_SECTION_SERVICE = "sysom_service" +ENV_LIST = { + "DB_MYSQL_HOST": "sysom_server.db.mysql.host", + "DB_MYSQL_PORT": "sysom_server.db.mysql.port", + "DB_MYSQL_USERNAME": "sysom_server.db.mysql.user", + "DB_MYSQL_PASSWORD": "sysom_server.db.mysql.password", + "DB_MYSQL_DATABASE": "sysom_server.db.mysql.database", + "REDIS_HOST": "sysom_server.db.redis.host", + "REDIS_PORT": "sysom_server.db.redis.port", + "REDIS_USERNAME": "sysom_server.db.redis.username", + "REDIS_PASSWORD": "sysom_server.db.redis.password" +} class ConfigParserException(Exception): pass @@ -53,7 +65,13 @@ class ConfigParser: self.global_config_path = global_config_path self.service_config_path = service_config_path self._config: Dict = self._load_config() - + self._overwrite_from_env() + + def _overwrite_from_env(self) -> None: + for env, key_str in ENV_LIST.items(): + if os.getenv(env): + self._config.set_multi(key_str, os.getenv(env)) + def _load_config(self) -> Dict: YamlConcatConstructor.add_to_loader_class(loader_class=yaml.FullLoader) global_config: dict = {} diff --git a/sysom_server/sysom_diagnosis/apps/task/__init__.py b/microservice/sysom_diagnosis/apps/task/__init__.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/__init__.py rename to microservice/sysom_diagnosis/apps/task/__init__.py diff --git a/sysom_server/sysom_diagnosis/apps/task/admin.py b/microservice/sysom_diagnosis/apps/task/admin.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/admin.py rename to microservice/sysom_diagnosis/apps/task/admin.py diff --git a/sysom_server/sysom_diagnosis/apps/task/apps.py b/microservice/sysom_diagnosis/apps/task/apps.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/apps.py rename to microservice/sysom_diagnosis/apps/task/apps.py diff --git a/sysom_server/sysom_diagnosis/apps/task/executor.py b/microservice/sysom_diagnosis/apps/task/executor.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/executor.py rename to microservice/sysom_diagnosis/apps/task/executor.py diff --git a/sysom_server/sysom_diagnosis/apps/task/filter.py b/microservice/sysom_diagnosis/apps/task/filter.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/filter.py rename to microservice/sysom_diagnosis/apps/task/filter.py diff --git a/sysom_server/sysom_diagnosis/apps/task/helper.py b/microservice/sysom_diagnosis/apps/task/helper.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/helper.py rename to microservice/sysom_diagnosis/apps/task/helper.py diff --git a/sysom_server/sysom_diagnosis/apps/task/migrations/0001_initial.py b/microservice/sysom_diagnosis/apps/task/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/migrations/0001_initial.py rename to microservice/sysom_diagnosis/apps/task/migrations/0001_initial.py diff --git a/sysom_server/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py b/microservice/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py rename to microservice/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py diff --git a/sysom_server/sysom_diagnosis/apps/task/migrations/__init__.py b/microservice/sysom_diagnosis/apps/task/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/migrations/__init__.py rename to microservice/sysom_diagnosis/apps/task/migrations/__init__.py diff --git a/sysom_server/sysom_diagnosis/apps/task/models.py b/microservice/sysom_diagnosis/apps/task/models.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/models.py rename to microservice/sysom_diagnosis/apps/task/models.py diff --git a/sysom_server/sysom_diagnosis/apps/task/seriaizer.py b/microservice/sysom_diagnosis/apps/task/seriaizer.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/seriaizer.py rename to microservice/sysom_diagnosis/apps/task/seriaizer.py diff --git a/sysom_server/sysom_diagnosis/apps/task/tests.py b/microservice/sysom_diagnosis/apps/task/tests.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/tests.py rename to microservice/sysom_diagnosis/apps/task/tests.py diff --git a/sysom_server/sysom_diagnosis/apps/task/urls.py b/microservice/sysom_diagnosis/apps/task/urls.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/urls.py rename to microservice/sysom_diagnosis/apps/task/urls.py diff --git a/sysom_server/sysom_diagnosis/apps/task/views.py b/microservice/sysom_diagnosis/apps/task/views.py similarity index 100% rename from sysom_server/sysom_diagnosis/apps/task/views.py rename to microservice/sysom_diagnosis/apps/task/views.py diff --git a/sysom_server/sysom_diagnosis/conf/common.py b/microservice/sysom_diagnosis/conf/common.py similarity index 88% rename from sysom_server/sysom_diagnosis/conf/common.py rename to microservice/sysom_diagnosis/conf/common.py index 353379c1..9e52dbe4 100644 --- a/sysom_server/sysom_diagnosis/conf/common.py +++ b/microservice/sysom_diagnosis/conf/common.py @@ -10,7 +10,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) @@ -76,22 +76,6 @@ DATABASES = { } } - -CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": ( - f"redis://{YAML_CONFIG.get_server_config().db.redis.host}" - f":{YAML_CONFIG.get_server_config().db.redis.port}/1" - ), - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "CONNECTION_POOL_KWARGS": {"max_connections": 100}, - "DECODE_RESPONSES": True - } - } -} - ROOT_URLCONF = 'sysom_diagnosis.urls' WSGI_APPLICATION = 'sysom_diagnosis.wsgi.application' diff --git a/sysom_server/sysom_diagnosis/conf/develop.py b/microservice/sysom_diagnosis/conf/develop.py similarity index 100% rename from sysom_server/sysom_diagnosis/conf/develop.py rename to microservice/sysom_diagnosis/conf/develop.py diff --git a/sysom_server/sysom_diagnosis/conf/diagnosis_gunicorn.py b/microservice/sysom_diagnosis/conf/diagnosis_gunicorn.py similarity index 80% rename from sysom_server/sysom_diagnosis/conf/diagnosis_gunicorn.py rename to microservice/sysom_diagnosis/conf/diagnosis_gunicorn.py index 653511d4..70d0098d 100644 --- a/sysom_server/sysom_diagnosis/conf/diagnosis_gunicorn.py +++ b/microservice/sysom_diagnosis/conf/diagnosis_gunicorn.py @@ -5,13 +5,13 @@ workers = 2 # 指定工作进程数 threads = 3 -bind = '127.0.0.1:7002' +bind = '0.0.0.0:7002' # worker_class = 'gevent' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) -accesslog = '/usr/local/sysom/server/logs/sysom-diagnosis-access.log' +accesslog = '/var/log/sysom/sysom-diagnosis-access.log' loglevel = 'error' diff --git a/sysom_server/sysom_diagnosis/conf/product.py b/microservice/sysom_diagnosis/conf/product.py similarity index 100% rename from sysom_server/sysom_diagnosis/conf/product.py rename to microservice/sysom_diagnosis/conf/product.py diff --git a/sysom_server/sysom_diagnosis/conf/testing.py b/microservice/sysom_diagnosis/conf/testing.py similarity index 100% rename from sysom_server/sysom_diagnosis/conf/testing.py rename to microservice/sysom_diagnosis/conf/testing.py diff --git a/sysom_server/sysom_diagnosis/config.yml b/microservice/sysom_diagnosis/config.yml similarity index 100% rename from sysom_server/sysom_diagnosis/config.yml rename to microservice/sysom_diagnosis/config.yml diff --git a/sysom_server/sysom_diagnosis/lib/authentications.py b/microservice/sysom_diagnosis/lib/authentications.py similarity index 100% rename from sysom_server/sysom_diagnosis/lib/authentications.py rename to microservice/sysom_diagnosis/lib/authentications.py diff --git a/sysom_server/sysom_diagnosis/lib/base_model.py b/microservice/sysom_diagnosis/lib/base_model.py similarity index 100% rename from sysom_server/sysom_diagnosis/lib/base_model.py rename to microservice/sysom_diagnosis/lib/base_model.py diff --git a/sysom_server/sysom_diagnosis/lib/base_view.py b/microservice/sysom_diagnosis/lib/base_view.py similarity index 100% rename from sysom_server/sysom_diagnosis/lib/base_view.py rename to microservice/sysom_diagnosis/lib/base_view.py diff --git a/sysom_server/sysom_diagnosis/lib/decode/sysom_decode.py b/microservice/sysom_diagnosis/lib/decode/sysom_decode.py similarity index 100% rename from sysom_server/sysom_diagnosis/lib/decode/sysom_decode.py rename to microservice/sysom_diagnosis/lib/decode/sysom_decode.py diff --git a/sysom_server/sysom_diagnosis/lib/exception.py b/microservice/sysom_diagnosis/lib/exception.py similarity index 100% rename from sysom_server/sysom_diagnosis/lib/exception.py rename to microservice/sysom_diagnosis/lib/exception.py diff --git a/sysom_server/sysom_diagnosis/lib/paginations.py b/microservice/sysom_diagnosis/lib/paginations.py similarity index 100% rename from sysom_server/sysom_diagnosis/lib/paginations.py rename to microservice/sysom_diagnosis/lib/paginations.py diff --git a/sysom_server/sysom_diagnosis/lib/renderers.py b/microservice/sysom_diagnosis/lib/renderers.py similarity index 100% rename from sysom_server/sysom_diagnosis/lib/renderers.py rename to microservice/sysom_diagnosis/lib/renderers.py diff --git a/sysom_server/sysom_diagnosis/lib/response.py b/microservice/sysom_diagnosis/lib/response.py similarity index 100% rename from sysom_server/sysom_diagnosis/lib/response.py rename to microservice/sysom_diagnosis/lib/response.py diff --git a/sysom_server/sysom_diagnosis/lib/utils.py b/microservice/sysom_diagnosis/lib/utils.py similarity index 83% rename from sysom_server/sysom_diagnosis/lib/utils.py rename to microservice/sysom_diagnosis/lib/utils.py index 55ca07c1..3efbb06a 100644 --- a/sysom_server/sysom_diagnosis/lib/utils.py +++ b/microservice/sysom_diagnosis/lib/utils.py @@ -8,9 +8,6 @@ """ import uuid as UUID from datetime import datetime -from paramiko.rsakey import RSAKey -from io import StringIO - CHAR_SET = ("a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", @@ -55,11 +52,4 @@ def uuid_8(): sub = s[i * 4: i * 4 + 4] x = int(sub, 16) result += CHAR_SET[x % 0x3E] - return result - - -def generate_key(): - key_obj = StringIO() - key = RSAKey.generate(2048) - key.write_private_key(key_obj) - return key_obj.getvalue(), 'ssh-rsa ' + key.get_base64() + return result \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/manage.py b/microservice/sysom_diagnosis/manage.py similarity index 100% rename from sysom_server/sysom_diagnosis/manage.py rename to microservice/sysom_diagnosis/manage.py diff --git a/microservice/sysom_diagnosis/requirements.txt b/microservice/sysom_diagnosis/requirements.txt new file mode 100644 index 00000000..f84443b8 --- /dev/null +++ b/microservice/sysom_diagnosis/requirements.txt @@ -0,0 +1,19 @@ +clogger>=0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +sysom_utils>=0.0.1 +django==3.2.16 +django-filter==21.1 +django-cors-headers==3.10.1 +djangorestframework==3.14.0 +paramiko==2.12.0 +PyJWT==2.4.0 +PyMySQL==1.0.2 +typing-extensions==4.1.1 +pyyaml==6.0 +pyyaml-include==1.3 +drf-yasg==1.21.4 +asyncer==0.0.2 +gunicorn==20.1.0 +aiofiles==0.8.0 \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/scripts/node_clear.sh b/microservice/sysom_diagnosis/scripts/node_clear.sh similarity index 100% rename from sysom_server/sysom_diagnosis/scripts/node_clear.sh rename to microservice/sysom_diagnosis/scripts/node_clear.sh diff --git a/sysom_server/sysom_diagnosis/scripts/node_init.sh b/microservice/sysom_diagnosis/scripts/node_init.sh similarity index 100% rename from sysom_server/sysom_diagnosis/scripts/node_init.sh rename to microservice/sysom_diagnosis/scripts/node_init.sh diff --git a/sysom_server/sysom_diagnosis/scripts/node_update.sh b/microservice/sysom_diagnosis/scripts/node_update.sh similarity index 100% rename from sysom_server/sysom_diagnosis/scripts/node_update.sh rename to microservice/sysom_diagnosis/scripts/node_update.sh diff --git a/microservice/sysom_diagnosis/scripts/server_clear.sh b/microservice/sysom_diagnosis/scripts/server_clear.sh new file mode 100755 index 00000000..0545d1a1 --- /dev/null +++ b/microservice/sysom_diagnosis/scripts/server_clear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-diagnosis +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} +clear_app diff --git a/sysom_server/sysom_diagnosis/lib/channels/__init__.py b/microservice/sysom_diagnosis/scripts/server_deploy.sh similarity index 100% rename from sysom_server/sysom_diagnosis/lib/channels/__init__.py rename to microservice/sysom_diagnosis/scripts/server_deploy.sh diff --git a/microservice/sysom_diagnosis/scripts/server_init.sh b/microservice/sysom_diagnosis/scripts/server_init.sh new file mode 100755 index 00000000..03cfd3ee --- /dev/null +++ b/microservice/sysom_diagnosis/scripts/server_init.sh @@ -0,0 +1,52 @@ +#!/bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_diagnosis +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_NAME=sysom-diagnosis + +BASE_DIR=`dirname $0` + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +init_conf() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/diagnosis_gunicorn.py +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl start ${SERVICE_NAME} + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + start_app +} + +deploy diff --git a/microservice/sysom_diagnosis/scripts/server_start.sh b/microservice/sysom_diagnosis/scripts/server_start.sh new file mode 100755 index 00000000..025c5f29 --- /dev/null +++ b/microservice/sysom_diagnosis/scripts/server_start.sh @@ -0,0 +1,7 @@ +#!/bin/bash +SERVICE_NAME=sysom-diagnosis +start_app() { + supervisorctl start $SERVICE_NAME +} + +start_app diff --git a/microservice/sysom_diagnosis/scripts/server_stop.sh b/microservice/sysom_diagnosis/scripts/server_stop.sh new file mode 100755 index 00000000..587cefea --- /dev/null +++ b/microservice/sysom_diagnosis/scripts/server_stop.sh @@ -0,0 +1,7 @@ +#!/bin/bash +SERVICE_NAME=sysom-diagnosis +stop_app() { + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/sysom_server/sysom_diagnosis/service_scripts/command b/microservice/sysom_diagnosis/service_scripts/command similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/command rename to microservice/sysom_diagnosis/service_scripts/command diff --git a/sysom_server/sysom_diagnosis/service_scripts/command_post b/microservice/sysom_diagnosis/service_scripts/command_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/command_post rename to microservice/sysom_diagnosis/service_scripts/command_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/filecache b/microservice/sysom_diagnosis/service_scripts/filecache similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/filecache rename to microservice/sysom_diagnosis/service_scripts/filecache diff --git a/sysom_server/sysom_diagnosis/service_scripts/filecache_post b/microservice/sysom_diagnosis/service_scripts/filecache_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/filecache_post rename to microservice/sysom_diagnosis/service_scripts/filecache_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/iofsstat b/microservice/sysom_diagnosis/service_scripts/iofsstat similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/iofsstat rename to microservice/sysom_diagnosis/service_scripts/iofsstat diff --git a/sysom_server/sysom_diagnosis/service_scripts/iofsstat_post b/microservice/sysom_diagnosis/service_scripts/iofsstat_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/iofsstat_post rename to microservice/sysom_diagnosis/service_scripts/iofsstat_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/iohang b/microservice/sysom_diagnosis/service_scripts/iohang similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/iohang rename to microservice/sysom_diagnosis/service_scripts/iohang diff --git a/sysom_server/sysom_diagnosis/service_scripts/iohang_post b/microservice/sysom_diagnosis/service_scripts/iohang_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/iohang_post rename to microservice/sysom_diagnosis/service_scripts/iohang_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/iolatency b/microservice/sysom_diagnosis/service_scripts/iolatency similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/iolatency rename to microservice/sysom_diagnosis/service_scripts/iolatency diff --git a/sysom_server/sysom_diagnosis/service_scripts/iolatency_post b/microservice/sysom_diagnosis/service_scripts/iolatency_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/iolatency_post rename to microservice/sysom_diagnosis/service_scripts/iolatency_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency b/microservice/sysom_diagnosis/service_scripts/iosdiag_latency similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency rename to microservice/sysom_diagnosis/service_scripts/iosdiag_latency diff --git a/sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency_post b/microservice/sysom_diagnosis/service_scripts/iosdiag_latency_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency_post rename to microservice/sysom_diagnosis/service_scripts/iosdiag_latency_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/jitter b/microservice/sysom_diagnosis/service_scripts/jitter similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/jitter rename to microservice/sysom_diagnosis/service_scripts/jitter diff --git a/sysom_server/sysom_diagnosis/service_scripts/jitter_post b/microservice/sysom_diagnosis/service_scripts/jitter_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/jitter_post rename to microservice/sysom_diagnosis/service_scripts/jitter_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/loadtask b/microservice/sysom_diagnosis/service_scripts/loadtask similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/loadtask rename to microservice/sysom_diagnosis/service_scripts/loadtask diff --git a/sysom_server/sysom_diagnosis/service_scripts/loadtask_post b/microservice/sysom_diagnosis/service_scripts/loadtask_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/loadtask_post rename to microservice/sysom_diagnosis/service_scripts/loadtask_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/memgraph b/microservice/sysom_diagnosis/service_scripts/memgraph similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/memgraph rename to microservice/sysom_diagnosis/service_scripts/memgraph diff --git a/sysom_server/sysom_diagnosis/service_scripts/memgraph_post b/microservice/sysom_diagnosis/service_scripts/memgraph_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/memgraph_post rename to microservice/sysom_diagnosis/service_scripts/memgraph_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/oomcheck b/microservice/sysom_diagnosis/service_scripts/oomcheck similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/oomcheck rename to microservice/sysom_diagnosis/service_scripts/oomcheck diff --git a/sysom_server/sysom_diagnosis/service_scripts/oomcheck_post b/microservice/sysom_diagnosis/service_scripts/oomcheck_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/oomcheck_post rename to microservice/sysom_diagnosis/service_scripts/oomcheck_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/ossre b/microservice/sysom_diagnosis/service_scripts/ossre similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/ossre rename to microservice/sysom_diagnosis/service_scripts/ossre diff --git a/sysom_server/sysom_diagnosis/service_scripts/ossre_post b/microservice/sysom_diagnosis/service_scripts/ossre_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/ossre_post rename to microservice/sysom_diagnosis/service_scripts/ossre_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/packetdrop b/microservice/sysom_diagnosis/service_scripts/packetdrop similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/packetdrop rename to microservice/sysom_diagnosis/service_scripts/packetdrop diff --git a/sysom_server/sysom_diagnosis/service_scripts/packetdrop_post b/microservice/sysom_diagnosis/service_scripts/packetdrop_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/packetdrop_post rename to microservice/sysom_diagnosis/service_scripts/packetdrop_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/pingtrace b/microservice/sysom_diagnosis/service_scripts/pingtrace similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/pingtrace rename to microservice/sysom_diagnosis/service_scripts/pingtrace diff --git a/sysom_server/sysom_diagnosis/service_scripts/pingtrace_post b/microservice/sysom_diagnosis/service_scripts/pingtrace_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/pingtrace_post rename to microservice/sysom_diagnosis/service_scripts/pingtrace_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/retran b/microservice/sysom_diagnosis/service_scripts/retran similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/retran rename to microservice/sysom_diagnosis/service_scripts/retran diff --git a/sysom_server/sysom_diagnosis/service_scripts/retran_post b/microservice/sysom_diagnosis/service_scripts/retran_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/retran_post rename to microservice/sysom_diagnosis/service_scripts/retran_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/schedmoni b/microservice/sysom_diagnosis/service_scripts/schedmoni similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/schedmoni rename to microservice/sysom_diagnosis/service_scripts/schedmoni diff --git a/sysom_server/sysom_diagnosis/service_scripts/schedmoni_post b/microservice/sysom_diagnosis/service_scripts/schedmoni_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/schedmoni_post rename to microservice/sysom_diagnosis/service_scripts/schedmoni_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/taskprofile b/microservice/sysom_diagnosis/service_scripts/taskprofile similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/taskprofile rename to microservice/sysom_diagnosis/service_scripts/taskprofile diff --git a/sysom_server/sysom_diagnosis/service_scripts/taskprofile_post b/microservice/sysom_diagnosis/service_scripts/taskprofile_post similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/taskprofile_post rename to microservice/sysom_diagnosis/service_scripts/taskprofile_post diff --git a/sysom_server/sysom_diagnosis/service_scripts/test b/microservice/sysom_diagnosis/service_scripts/test similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/test rename to microservice/sysom_diagnosis/service_scripts/test diff --git a/sysom_server/sysom_diagnosis/service_scripts/test.sh b/microservice/sysom_diagnosis/service_scripts/test.sh similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/test.sh rename to microservice/sysom_diagnosis/service_scripts/test.sh diff --git a/sysom_server/sysom_diagnosis/service_scripts/tpython b/microservice/sysom_diagnosis/service_scripts/tpython similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/tpython rename to microservice/sysom_diagnosis/service_scripts/tpython diff --git a/sysom_server/sysom_diagnosis/service_scripts/vmcore_analyse b/microservice/sysom_diagnosis/service_scripts/vmcore_analyse similarity index 100% rename from sysom_server/sysom_diagnosis/service_scripts/vmcore_analyse rename to microservice/sysom_diagnosis/service_scripts/vmcore_analyse diff --git a/microservice/sysom_diagnosis/sysom-diagnosis.ini b/microservice/sysom_diagnosis/sysom-diagnosis.ini new file mode 100644 index 00000000..d1f0d000 --- /dev/null +++ b/microservice/sysom_diagnosis/sysom-diagnosis.ini @@ -0,0 +1,9 @@ +[program:sysom-diagnosis] +directory=/usr/local/sysom/server/microservice/sysom_diagnosis +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/diagnosis_gunicorn.py sysom_diagnosis.wsgi:application +startsecs=3 +autostart=true +autorestart=true +stderr_logfile=/var/log/sysom/sysom-diagnosis-error.log +stdout_logfile=/var/log/sysom/sysom-diagnosis.log +environment=PATH=/usr/local/sysom/server/virtualenv/bin:%(ENV_PATH)s \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/__init__.py b/microservice/sysom_diagnosis/sysom_diagnosis/__init__.py similarity index 100% rename from sysom_server/sysom_diagnosis/sysom_diagnosis/__init__.py rename to microservice/sysom_diagnosis/sysom_diagnosis/__init__.py diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/asgi.py b/microservice/sysom_diagnosis/sysom_diagnosis/asgi.py similarity index 100% rename from sysom_server/sysom_diagnosis/sysom_diagnosis/asgi.py rename to microservice/sysom_diagnosis/sysom_diagnosis/asgi.py diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/settings.py b/microservice/sysom_diagnosis/sysom_diagnosis/settings.py similarity index 100% rename from sysom_server/sysom_diagnosis/sysom_diagnosis/settings.py rename to microservice/sysom_diagnosis/sysom_diagnosis/settings.py diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/urls.py b/microservice/sysom_diagnosis/sysom_diagnosis/urls.py similarity index 100% rename from sysom_server/sysom_diagnosis/sysom_diagnosis/urls.py rename to microservice/sysom_diagnosis/sysom_diagnosis/urls.py diff --git a/sysom_server/sysom_diagnosis/sysom_diagnosis/wsgi.py b/microservice/sysom_diagnosis/sysom_diagnosis/wsgi.py similarity index 100% rename from sysom_server/sysom_diagnosis/sysom_diagnosis/wsgi.py rename to microservice/sysom_diagnosis/sysom_diagnosis/wsgi.py diff --git a/sysom_server/sysom_diagnosis/lib/channels/base.py b/sysom_server/sysom_diagnosis/lib/channels/base.py deleted file mode 100644 index 9cea997f..00000000 --- a/sysom_server/sysom_diagnosis/lib/channels/base.py +++ /dev/null @@ -1,17 +0,0 @@ -""" -通道Base - -多通道是以单文件的方式构成,文件名就是通道名称(例如: ssh.py 为ssh通道), 通道 -文件中实现Channel类, 继承BaseChannel类, 必须实现client方法, run_command方法 -""" -from abc import ABCMeta, abstractmethod - -class BaseChannel(metaclass=ABCMeta): - - @abstractmethod - def client(self, **kwargs): - raise NotImplemented - - @abstractmethod - def run_command(self, **kwargs): - raise NotImplemented \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/lib/channels/ssh.py b/sysom_server/sysom_diagnosis/lib/channels/ssh.py deleted file mode 100644 index 732be5c2..00000000 --- a/sysom_server/sysom_diagnosis/lib/channels/ssh.py +++ /dev/null @@ -1,140 +0,0 @@ -''' -@File: ssh.py -@Time: 2022-08-30 11:13:55 -@Author: DM -@Email: wb-msm261421@alibaba-inc.com -@Desc: ssh通道 -''' - -# SSH args eg: -# "channel": "ssh", 选填 (默认ssh) -# "instance": "xxxxxxxx", 必填 -# "cmd": "xxxx" 必填 - - -import json -import paramiko -from clogger import logger -from io import StringIO -from paramiko.client import SSHClient, AutoAddPolicy -from paramiko.rsakey import RSAKey -from sysom_diagnosis.settings import KEY_PATH -from lib.utils import uuid_8 - -from .base import BaseChannel - - -DEFAULT_CONNENT_TIMEOUT = 5 # 默认ssh链接超时时间 5s -DEFAULT_NODE_USER = 'root' # 默认节点用户名 root - - -class SSH: - """ - args: - - hostname 主机IP (必填) - - username 主机用户名, 默认 'root' - - port 主机开放端口, 默认 22 - - connect_timeout 连接超时时间 默认 5s - """ - - def __init__(self, hostname: str, **kwargs) -> None: - self.connect_args = { - 'hostname': hostname, - 'username': kwargs.get('username', DEFAULT_NODE_USER), - 'port': kwargs.get('port', 22), - 'timeout': kwargs.get('timeout', DEFAULT_CONNENT_TIMEOUT), - } - if 'password' in kwargs: - self.connect_args['password'] = kwargs.get('password') - else: - self.connect_args['pkey'] = RSAKey.from_private_key( - StringIO(self.get_ssh_key()['private_key'])) - - self._client: SSHClient = self.client() - - def client(self): - try: - client = SSHClient() - client.set_missing_host_key_policy(AutoAddPolicy) - client.connect(**self.connect_args) - return client - except paramiko.AuthenticationException: - raise Exception('authorization fail, password or pkey error!') - except: - raise Exception('authorization fail!') - - @classmethod - def get_ssh_key(cls) -> dict: - try: - with open(KEY_PATH, 'r') as f: - return json.loads(f.read()) - except Exception as e: - logger.error(e) - return {} - - def run_command(self, command): - if self._client: - ssh_session = self._client.get_transport().open_session() - ssh_session.set_combine_stderr(True) - ssh_session.exec_command(command) - stdout = ssh_session.makefile("rb", -1) - statue = ssh_session.recv_exit_status() - output = stdout.read().decode() - return statue, output - else: - raise Exception('No client!') - - def add_public_key(self): - public_key = self.get_ssh_key()['public_key'] - command = f'mkdir -p -m 700 ~/.ssh && \ - echo {public_key!r} >> ~/.ssh/authorized_keys && \ - chmod 600 ~/.ssh/authorized_keys' - statue, _ = self.run_command(command) - if statue != 0: - raise Exception('add public key faild!') - - @staticmethod - def validate_ssh_host(ip: str, password: str, port: int = 22, username: str = 'root'): - try: - ssh = SSH(hostname=ip, password=password, - port=port, username=username, timeout=2) - ssh.add_public_key() - return True, 'authorization success' - except Exception as e: - return False, f'error: {e}' - - -class Channel(BaseChannel): - FIELDS = ('instance', 'cmd') - - def __init__(self, *args, **kwargs) -> None: - self.kwargs = kwargs - self.ssh = None - self.shell_script = None - - self.validate_kwargs() - - def validate_kwargs(self): - for item in filter( - lambda x: not x[1], [(field, self.kwargs.get(field, None)) - for field in self.FIELDS] - ): - raise Exception(f'parameter: {item[0]} not found!') - - if not self.ssh: - self.ssh = SSH(hostname=self.kwargs['instance']) - self.shell_script = self.kwargs['cmd'] - - def client(self): - return self.ssh if self.ssh else SSH(hostname=self.kwargs['instance']) - - def run_command(self): - kwargs = dict() - invoke_id = uuid_8() - kwargs['invoke_id'] = invoke_id - status, res = self.ssh.run_command(self.shell_script) - return { - 'invoke_id': invoke_id, - 'state': status, - 'result': {'state': status, 'result': res}, - } -- Gitee From 6b3b68ca4911951deeca4e596dac7a072822d958 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 21 Apr 2023 16:12:59 +0800 Subject: [PATCH 016/196] feat(sysom_diagnosis): Support docker deploment --- sysom_diagnosis_dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 sysom_diagnosis_dockerfile diff --git a/sysom_diagnosis_dockerfile b/sysom_diagnosis_dockerfile new file mode 100644 index 00000000..51972064 --- /dev/null +++ b/sysom_diagnosis_dockerfile @@ -0,0 +1,31 @@ +FROM openanolis/anolisos:8.8 + +# Add epel +RUN yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm +RUN bash -c "sed -i 's|^#baseurl=https://download.example/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel*" +RUN bash -c "sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel*" + +# Add required yum packages +RUN yum makecache +RUN yum install -y supervisor cronie net-tools +RUN systemctl enable crond +# RUN systemctl enable supervisord + +# Init sysom-diagnosis +ARG SYSOM_HOME=/usr/local/sysom +ARG SYSOM_SERVER_HOME=${SYSOM_HOME}/server + +RUN mkdir /root/sysom +COPY conf /root/sysom/conf +COPY script /root/sysom/script +COPY infrastructure /root/sysom/infrastructure +COPY microservice /root/sysom/microservice + +RUN bash -x /root/sysom/script/sysom.sh deploy infrastructure env,sdk +RUN bash -x /root/sysom/script/sysom.sh deploy microservice sysom_diagnosis +RUN sed "s/nodaemon=false/nodaemon=true/g" -i /etc/supervisord.conf + +RUN yum clean all + +# # 环境准备 +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file -- Gitee From d24a7fc95a1ca8800496eb448bf8bcb3d2b3a5d4 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 21 Apr 2023 16:49:49 +0800 Subject: [PATCH 017/196] refactor(sysom_channel): Use sysom.sh to unify deployment --- .../sysom_channel/alembic.ini | 0 .../sysom_channel/alembic/README | 0 .../sysom_channel/alembic/env.py | 4 +- .../sysom_channel/alembic/script.py.mako | 0 .../alembic/versions/1663292d0bd8_init_db.py | 0 .../3f54250e3fca_add_channel_params_table.py | 0 .../sysom_channel/app/__init__.py | 0 .../sysom_channel/app/crud.py | 0 .../sysom_channel/app/database.py | 0 .../sysom_channel/app/executor.py | 0 .../sysom_channel/app/models.py | 0 .../sysom_channel/app/routers/cec_status.py | 0 .../sysom_channel/app/routers/config.py | 0 .../sysom_channel/app/routers/file.py | 0 .../sysom_channel/app/routers/health.py | 0 .../sysom_channel/app/schemas.py | 0 .../sysom_channel/conf/channel_gunicorn.py | 18 ++ .../sysom_channel/conf/common.py | 2 +- .../sysom_channel/conf/develop.py | 0 .../sysom_channel/conf/product.py | 0 .../sysom_channel/conf/settings.py | 0 .../sysom_channel/conf/testing.py | 0 .../sysom_channel/config.yml | 0 .../sysom_channel/lib/channels/__init__.py | 0 .../sysom_channel/lib/channels/base.py | 0 .../sysom_channel/lib/channels/ssh.py | 0 .../sysom_channel/lib/ssh.py | 0 .../sysom_channel/lib/utils.py | 0 .../sysom_channel/main.py | 0 microservice/sysom_channel/requirements.txt | 19 ++ .../sysom_channel/scripts/server_clear.sh | 8 + .../sysom_channel/scripts/server_init.sh | 52 ++++++ .../sysom_channel/scripts/server_start.sh | 7 + .../sysom_channel/scripts/server_stop.sh | 7 + microservice/sysom_channel/sysom-channel.ini | 9 + script/clear.sh | 162 ++++++++++++++++++ 36 files changed, 285 insertions(+), 3 deletions(-) rename {sysom_server => microservice}/sysom_channel/alembic.ini (100%) rename {sysom_server => microservice}/sysom_channel/alembic/README (100%) rename {sysom_server => microservice}/sysom_channel/alembic/env.py (96%) rename {sysom_server => microservice}/sysom_channel/alembic/script.py.mako (100%) rename {sysom_server => microservice}/sysom_channel/alembic/versions/1663292d0bd8_init_db.py (100%) rename {sysom_server => microservice}/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py (100%) rename {sysom_server => microservice}/sysom_channel/app/__init__.py (100%) rename {sysom_server => microservice}/sysom_channel/app/crud.py (100%) rename {sysom_server => microservice}/sysom_channel/app/database.py (100%) rename {sysom_server => microservice}/sysom_channel/app/executor.py (100%) rename {sysom_server => microservice}/sysom_channel/app/models.py (100%) rename {sysom_server => microservice}/sysom_channel/app/routers/cec_status.py (100%) rename {sysom_server => microservice}/sysom_channel/app/routers/config.py (100%) rename {sysom_server => microservice}/sysom_channel/app/routers/file.py (100%) rename {sysom_server => microservice}/sysom_channel/app/routers/health.py (100%) rename {sysom_server => microservice}/sysom_channel/app/schemas.py (100%) create mode 100644 microservice/sysom_channel/conf/channel_gunicorn.py rename {sysom_server => microservice}/sysom_channel/conf/common.py (97%) rename {sysom_server => microservice}/sysom_channel/conf/develop.py (100%) rename {sysom_server => microservice}/sysom_channel/conf/product.py (100%) rename {sysom_server => microservice}/sysom_channel/conf/settings.py (100%) rename {sysom_server => microservice}/sysom_channel/conf/testing.py (100%) rename {sysom_server => microservice}/sysom_channel/config.yml (100%) rename {sysom_server => microservice}/sysom_channel/lib/channels/__init__.py (100%) rename {sysom_server => microservice}/sysom_channel/lib/channels/base.py (100%) rename {sysom_server => microservice}/sysom_channel/lib/channels/ssh.py (100%) rename {sysom_server => microservice}/sysom_channel/lib/ssh.py (100%) rename {sysom_server => microservice}/sysom_channel/lib/utils.py (100%) rename {sysom_server => microservice}/sysom_channel/main.py (100%) create mode 100644 microservice/sysom_channel/requirements.txt create mode 100644 microservice/sysom_channel/scripts/server_clear.sh create mode 100644 microservice/sysom_channel/scripts/server_init.sh create mode 100644 microservice/sysom_channel/scripts/server_start.sh create mode 100644 microservice/sysom_channel/scripts/server_stop.sh create mode 100644 microservice/sysom_channel/sysom-channel.ini create mode 100644 script/clear.sh diff --git a/sysom_server/sysom_channel/alembic.ini b/microservice/sysom_channel/alembic.ini similarity index 100% rename from sysom_server/sysom_channel/alembic.ini rename to microservice/sysom_channel/alembic.ini diff --git a/sysom_server/sysom_channel/alembic/README b/microservice/sysom_channel/alembic/README similarity index 100% rename from sysom_server/sysom_channel/alembic/README rename to microservice/sysom_channel/alembic/README diff --git a/sysom_server/sysom_channel/alembic/env.py b/microservice/sysom_channel/alembic/env.py similarity index 96% rename from sysom_server/sysom_channel/alembic/env.py rename to microservice/sysom_channel/alembic/env.py index ab547501..177ac6e3 100644 --- a/sysom_server/sysom_channel/alembic/env.py +++ b/microservice/sysom_channel/alembic/env.py @@ -10,9 +10,9 @@ from sysom_utils import ConfigParser ################################################################## # Load yaml config first ################################################################## -BASE_DIR = Path(__file__).resolve().parent.parent +BASE_DIR = Path(__file__).resolve().parent YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" -YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" +YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR.parent}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) mysql_config = YAML_CONFIG.get_server_config().db.mysql diff --git a/sysom_server/sysom_channel/alembic/script.py.mako b/microservice/sysom_channel/alembic/script.py.mako similarity index 100% rename from sysom_server/sysom_channel/alembic/script.py.mako rename to microservice/sysom_channel/alembic/script.py.mako diff --git a/sysom_server/sysom_channel/alembic/versions/1663292d0bd8_init_db.py b/microservice/sysom_channel/alembic/versions/1663292d0bd8_init_db.py similarity index 100% rename from sysom_server/sysom_channel/alembic/versions/1663292d0bd8_init_db.py rename to microservice/sysom_channel/alembic/versions/1663292d0bd8_init_db.py diff --git a/sysom_server/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py b/microservice/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py similarity index 100% rename from sysom_server/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py rename to microservice/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py diff --git a/sysom_server/sysom_channel/app/__init__.py b/microservice/sysom_channel/app/__init__.py similarity index 100% rename from sysom_server/sysom_channel/app/__init__.py rename to microservice/sysom_channel/app/__init__.py diff --git a/sysom_server/sysom_channel/app/crud.py b/microservice/sysom_channel/app/crud.py similarity index 100% rename from sysom_server/sysom_channel/app/crud.py rename to microservice/sysom_channel/app/crud.py diff --git a/sysom_server/sysom_channel/app/database.py b/microservice/sysom_channel/app/database.py similarity index 100% rename from sysom_server/sysom_channel/app/database.py rename to microservice/sysom_channel/app/database.py diff --git a/sysom_server/sysom_channel/app/executor.py b/microservice/sysom_channel/app/executor.py similarity index 100% rename from sysom_server/sysom_channel/app/executor.py rename to microservice/sysom_channel/app/executor.py diff --git a/sysom_server/sysom_channel/app/models.py b/microservice/sysom_channel/app/models.py similarity index 100% rename from sysom_server/sysom_channel/app/models.py rename to microservice/sysom_channel/app/models.py diff --git a/sysom_server/sysom_channel/app/routers/cec_status.py b/microservice/sysom_channel/app/routers/cec_status.py similarity index 100% rename from sysom_server/sysom_channel/app/routers/cec_status.py rename to microservice/sysom_channel/app/routers/cec_status.py diff --git a/sysom_server/sysom_channel/app/routers/config.py b/microservice/sysom_channel/app/routers/config.py similarity index 100% rename from sysom_server/sysom_channel/app/routers/config.py rename to microservice/sysom_channel/app/routers/config.py diff --git a/sysom_server/sysom_channel/app/routers/file.py b/microservice/sysom_channel/app/routers/file.py similarity index 100% rename from sysom_server/sysom_channel/app/routers/file.py rename to microservice/sysom_channel/app/routers/file.py diff --git a/sysom_server/sysom_channel/app/routers/health.py b/microservice/sysom_channel/app/routers/health.py similarity index 100% rename from sysom_server/sysom_channel/app/routers/health.py rename to microservice/sysom_channel/app/routers/health.py diff --git a/sysom_server/sysom_channel/app/schemas.py b/microservice/sysom_channel/app/schemas.py similarity index 100% rename from sysom_server/sysom_channel/app/schemas.py rename to microservice/sysom_channel/app/schemas.py diff --git a/microservice/sysom_channel/conf/channel_gunicorn.py b/microservice/sysom_channel/conf/channel_gunicorn.py new file mode 100644 index 00000000..41957101 --- /dev/null +++ b/microservice/sysom_channel/conf/channel_gunicorn.py @@ -0,0 +1,18 @@ +''' +Channel Service Gunicorn Settings +''' +workers = 2 # 指定工作进程数 + +threads = 3 + +bind = '0.0.0.0:7003' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/sysom-channel-access.log' + +loglevel = 'error' + +proc_name = 'channel_service' diff --git a/sysom_server/sysom_channel/conf/common.py b/microservice/sysom_channel/conf/common.py similarity index 97% rename from sysom_server/sysom_channel/conf/common.py rename to microservice/sysom_channel/conf/common.py index 3011d3c7..0fa64060 100644 --- a/sysom_server/sysom_channel/conf/common.py +++ b/microservice/sysom_channel/conf/common.py @@ -16,7 +16,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/sysom_server/sysom_channel/conf/develop.py b/microservice/sysom_channel/conf/develop.py similarity index 100% rename from sysom_server/sysom_channel/conf/develop.py rename to microservice/sysom_channel/conf/develop.py diff --git a/sysom_server/sysom_channel/conf/product.py b/microservice/sysom_channel/conf/product.py similarity index 100% rename from sysom_server/sysom_channel/conf/product.py rename to microservice/sysom_channel/conf/product.py diff --git a/sysom_server/sysom_channel/conf/settings.py b/microservice/sysom_channel/conf/settings.py similarity index 100% rename from sysom_server/sysom_channel/conf/settings.py rename to microservice/sysom_channel/conf/settings.py diff --git a/sysom_server/sysom_channel/conf/testing.py b/microservice/sysom_channel/conf/testing.py similarity index 100% rename from sysom_server/sysom_channel/conf/testing.py rename to microservice/sysom_channel/conf/testing.py diff --git a/sysom_server/sysom_channel/config.yml b/microservice/sysom_channel/config.yml similarity index 100% rename from sysom_server/sysom_channel/config.yml rename to microservice/sysom_channel/config.yml diff --git a/sysom_server/sysom_channel/lib/channels/__init__.py b/microservice/sysom_channel/lib/channels/__init__.py similarity index 100% rename from sysom_server/sysom_channel/lib/channels/__init__.py rename to microservice/sysom_channel/lib/channels/__init__.py diff --git a/sysom_server/sysom_channel/lib/channels/base.py b/microservice/sysom_channel/lib/channels/base.py similarity index 100% rename from sysom_server/sysom_channel/lib/channels/base.py rename to microservice/sysom_channel/lib/channels/base.py diff --git a/sysom_server/sysom_channel/lib/channels/ssh.py b/microservice/sysom_channel/lib/channels/ssh.py similarity index 100% rename from sysom_server/sysom_channel/lib/channels/ssh.py rename to microservice/sysom_channel/lib/channels/ssh.py diff --git a/sysom_server/sysom_channel/lib/ssh.py b/microservice/sysom_channel/lib/ssh.py similarity index 100% rename from sysom_server/sysom_channel/lib/ssh.py rename to microservice/sysom_channel/lib/ssh.py diff --git a/sysom_server/sysom_channel/lib/utils.py b/microservice/sysom_channel/lib/utils.py similarity index 100% rename from sysom_server/sysom_channel/lib/utils.py rename to microservice/sysom_channel/lib/utils.py diff --git a/sysom_server/sysom_channel/main.py b/microservice/sysom_channel/main.py similarity index 100% rename from sysom_server/sysom_channel/main.py rename to microservice/sysom_channel/main.py diff --git a/microservice/sysom_channel/requirements.txt b/microservice/sysom_channel/requirements.txt new file mode 100644 index 00000000..b34b6ab5 --- /dev/null +++ b/microservice/sysom_channel/requirements.txt @@ -0,0 +1,19 @@ +clogger>=0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +sysom_utils>=0.0.1 +aiofiles==0.8.0 +alembic==1.7.7 +anyio==3.6.2 +asyncer==0.0.2 +asyncssh==2.12.0 +fastapi==0.83.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +paramiko==2.12.0 +python-multipart==0.0.5 +prometheus_client==0.16.0 +uvicorn==0.16.0 +gunicorn==20.1.0 \ No newline at end of file diff --git a/microservice/sysom_channel/scripts/server_clear.sh b/microservice/sysom_channel/scripts/server_clear.sh new file mode 100644 index 00000000..63e6c7c8 --- /dev/null +++ b/microservice/sysom_channel/scripts/server_clear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-channel +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} +clear_app diff --git a/microservice/sysom_channel/scripts/server_init.sh b/microservice/sysom_channel/scripts/server_init.sh new file mode 100644 index 00000000..c8cc69b3 --- /dev/null +++ b/microservice/sysom_channel/scripts/server_init.sh @@ -0,0 +1,52 @@ +#!/bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_channel +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_NAME=sysom-channel + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +init_conf() { + pushd ${SERVICE_HOME} + alembic upgrade head + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + start_app +} + +deploy diff --git a/microservice/sysom_channel/scripts/server_start.sh b/microservice/sysom_channel/scripts/server_start.sh new file mode 100644 index 00000000..080f5f53 --- /dev/null +++ b/microservice/sysom_channel/scripts/server_start.sh @@ -0,0 +1,7 @@ +#!/bin/bash +SERVICE_NAME=sysom-channel +start_app() { + supervisorctl start $SERVICE_NAME +} + +start_app diff --git a/microservice/sysom_channel/scripts/server_stop.sh b/microservice/sysom_channel/scripts/server_stop.sh new file mode 100644 index 00000000..f58c6440 --- /dev/null +++ b/microservice/sysom_channel/scripts/server_stop.sh @@ -0,0 +1,7 @@ +#!/bin/bash +SERVICE_NAME=sysom-channel +stop_app() { + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/microservice/sysom_channel/sysom-channel.ini b/microservice/sysom_channel/sysom-channel.ini new file mode 100644 index 00000000..28e973b2 --- /dev/null +++ b/microservice/sysom_channel/sysom-channel.ini @@ -0,0 +1,9 @@ +[program:sysom-channel] +directory = /usr/local/sysom/server/microservice/sysom_channel +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/channel_gunicorn.py main:app +startsecs=3 +autostart=true +autorestart=true +environment=PATH="/usr/local/sysom/server/virtualenv/bin/" +stderr_logfile=/var/log/sysom/sysom-channel-error.log +stdout_logfile=/var/log/sysom/sysom-channel.log diff --git a/script/clear.sh b/script/clear.sh new file mode 100644 index 00000000..26e5994f --- /dev/null +++ b/script/clear.sh @@ -0,0 +1,162 @@ +#!/bin/bash -x + +#################################################################################################################### +# Helper functions +#################################################################################################################### + +red() { + printf '\33[1;31m%b\n\33[0m' "$1" +} + +green() { + printf '\33[1;32m%b\n\33[0m' "$1" +} + + +do_clear_infrastructure() { + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]} + do + target_dir=${INFRASTRUCTURE_HOME}/${target} + pushd ${target_dir} + + green "$target_dir clear..................................." + bash -x ${target_dir}/clear.sh + if [ $? -ne 0 ];then + red "$target_dir clear fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + popd + + # rm dir + rm -rf ${target_dir} + done + + # rm infrastructure if the folder is empty + if [ "`ls ${INFRASTRUCTURE_HOME} | wc -l`" == "0" ] + then + rm -rf ${INFRASTRUCTURE_HOME} + fi +} + +do_clear_microservice() { + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]} + do + target_dir=${MICROSERVICE_HOME}/${target} + if [ ! -d "$target_dir" ] + then + continue + fi + pushd ${target_dir} + + green "$target_dir clear..................................." + bash -x ${target_dir}/scripts/server_clear.sh + if [ $? -ne 0 ];then + red "$target_dir clear fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + popd + + # rm dir + rm -rf ${target_dir} + done + + dircount=`ls -l ${MICROSERVICE_HOME}| grep "^d" | wc -l` + # rm infrastructure if the folder is empty + if [ "$dircount" == "0" ] + then + rm -rf ${MICROSERVICE_HOME} + else + if [ "$dircount" == "1" ] && [ -d "${MICROSERVICE_HOME}/conf" ] + then + rm -rf ${MICROSERVICE_HOME} + fi + fi +} + +#################################################################################################################### +# Subcommands +#################################################################################################################### + +sub_help(){ + echo "Usage: $ProgName [options]" + echo "Subcommands:" + echo " all Clear all modules" + echo " infrastructure [ALL | ] Clear all infrastructure or specific infrastructure" + echo " Example: $ProgName infrastructure env" + echo " Example: $ProgName infrastructure local_services" + echo " microservice [ALL | ] Clear all microservices or specific microservice" + echo " Example: $ProgName microservice sysom_diagnosis" + echo "" + echo "For help with each subcommand run:" + echo "$ProgName -h|--help" + echo "" +} + + +sub_all() { + echo "all" +} + +sub_infra() { + sub_infrastructure $@ +} + +# -> env + local_services +sub_infrastructure() { + target=$1 + if [ "$target" == "ALL" ] + then + # Deploy all microservices + for infrastructure in $INFRASTRUCTURE_HOME + do + do_clear_infrastructure $infrastructure + done + else + # Deploy specific microservices + do_clear_infrastructure $target + fi +} + +sub_ms() { + sub_microservice $@ +} + +# All microservices +sub_microservice() { + target=$1 + if [ "$target" == "ALL" ] + then + # Deploy all microservices + for microservice in $(ls $MICROSERVICE_HOME) + do + echo ${microservice} + do_clear_microservice ${microservice} + done + else + # Deploy specific microservices + do_clear_microservice $target + fi +} + + +subcommand=$1 +case $subcommand in + "" | "-h" | "--help") + sub_help + ;; + *) + shift + sub_${subcommand} $@ + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac \ No newline at end of file -- Gitee From 6688df0ec77680879418f0eef35961b3ee7b5dfe Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 21 Apr 2023 17:43:18 +0800 Subject: [PATCH 018/196] feat(sysom_channel): Support docker deploment --- sysom_channel_dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 sysom_channel_dockerfile diff --git a/sysom_channel_dockerfile b/sysom_channel_dockerfile new file mode 100644 index 00000000..70e34b9f --- /dev/null +++ b/sysom_channel_dockerfile @@ -0,0 +1,31 @@ +FROM openanolis/anolisos:8.8 + +# Add epel +RUN yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm +RUN bash -c "sed -i 's|^#baseurl=https://download.example/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel*" +RUN bash -c "sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel*" + +# Add required yum packages +RUN yum makecache +RUN yum install -y supervisor cronie net-tools +RUN systemctl enable crond +# RUN systemctl enable supervisord + +# Init sysom-diagnosis +ARG SYSOM_HOME=/usr/local/sysom +ARG SYSOM_SERVER_HOME=${SYSOM_HOME}/server + +RUN mkdir /root/sysom +COPY conf /root/sysom/conf +COPY script /root/sysom/script +COPY infrastructure /root/sysom/infrastructure +COPY microservice /root/sysom/microservice + +RUN bash -x /root/sysom/script/sysom.sh deploy infrastructure env,sdk +RUN bash -x /root/sysom/script/sysom.sh deploy microservice sysom_channel +RUN sed "s/nodaemon=false/nodaemon=true/g" -i /etc/supervisord.conf + +RUN yum clean all + +# # 环境准备 +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file -- Gitee From 134e0c2c80131794ff0f9707a6aceea6bc11beaa Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 12:54:57 +0800 Subject: [PATCH 019/196] refactor(sysom_api): Use sysom.sh to unify deployment --- infrastructure/env/requirements.txt | 24 ++++----- infrastructure/sdk/cec_redis/redis_admin.py | 3 -- .../sdk/cec_redis/redis_consumer.py | 4 +- .../sdk/cec_redis/redis_producer.py | 4 +- .../sysom_api/.gitignore | 0 .../sysom_api/apps/__init__.py | 0 .../sysom_api/apps/accounts/__init__.py | 0 .../sysom_api/apps/accounts/admin.py | 0 .../sysom_api/apps/accounts/apps.py | 0 .../sysom_api/apps/accounts/authentication.py | 0 .../apps/accounts/migrations/0001_initial.py | 0 .../migrations/0002_user_allow_login.py | 0 .../apps/accounts/migrations/__init__.py | 0 .../sysom_api/apps/accounts/models.py | 0 .../sysom_api/apps/accounts/permissions.py | 0 .../sysom_api/apps/accounts/serializer.py | 0 .../sysom_api/apps/accounts/tests.py | 0 .../sysom_api/apps/accounts/urls.py | 0 .../sysom_api/apps/accounts/views.py | 0 .../sysom_api/apps/alarm/__init__.py | 0 .../sysom_api/apps/alarm/admin.py | 0 .../sysom_api/apps/alarm/apps.py | 0 .../apps/alarm/migrations/0001_initial.py | 0 .../apps/alarm/migrations/__init__.py | 0 .../sysom_api/apps/alarm/models.py | 0 .../sysom_api/apps/alarm/serializer.py | 0 .../sysom_api/apps/alarm/subscribe.json | 0 .../sysom_api/apps/alarm/urls.py | 0 .../sysom_api/apps/alarm/views.py | 0 .../sysom_api/apps/common/__init__.py | 0 .../apps/common/common_model_viewset.py | 0 .../sysom_api/apps/host/__init__.py | 0 .../sysom_api/apps/host/admin.py | 0 .../sysom_api/apps/host/apps.py | 0 .../sysom_api/apps/host/cec_api.py | 0 .../sysom_api/apps/host/heartbeat.py | 0 .../apps/host/migrations/0001_initial.py | 0 .../migrations/0002_hostmodel_host_info.py | 0 .../migrations/0003_alter_hostmodel_status.py | 0 .../migrations/0004_auto_20221227_1705.py | 0 .../apps/host/migrations/__init__.py | 0 .../sysom_api/apps/host/models.py | 0 .../sysom_api/apps/host/serializer.py | 0 .../sysom_api/apps/host/tests.py | 0 .../sysom_api/apps/host/urls.py | 0 .../sysom_api/apps/host/views.py | 0 .../sysom_api/apps/services/__init__.py | 0 .../sysom_api/apps/services/admin.py | 0 .../sysom_api/apps/services/apps.py | 0 .../apps/services/migrations/0001_initial.py | 0 .../apps/services/migrations/__init__.py | 0 .../sysom_api/apps/services/models.py | 0 .../sysom_api/apps/services/serializer.py | 0 .../sysom_api/apps/services/tests.py | 0 .../sysom_api/apps/services/urls.py | 0 .../sysom_api/apps/services/views.py | 0 .../sysom_api/conf/__init__.py | 0 .../sysom_api/conf/common.py | 2 +- .../sysom_api/conf/develop.py | 0 .../sysom_api/conf/product.py | 0 .../sysom_api/conf/testing.py | 0 .../sysom_api/config.yml | 0 .../sysom_api/consumer/__init__.py | 0 .../sysom_api/consumer/consumers.py | 0 .../sysom_api/consumer/middleware.py | 0 .../sysom_api/consumer/routing.py | 0 .../sysom_api/lib/__init__.py | 0 .../sysom_api/lib/authentications.py | 0 .../sysom_api/lib/base_model.py | 0 .../sysom_api/lib/decode/sysom_decode.py | 0 .../sysom_api/lib/excel.py | 0 .../sysom_api/lib/exception.py | 0 .../sysom_api/lib/paginations.py | 0 .../sysom_api/lib/renderers.py | 0 .../sysom_api/lib/response.py | 0 .../sysom_api/lib/ssh.py | 0 .../sysom_api/lib/utils.py | 0 .../sysom_api/manage.py | 0 microservice/sysom_api/requirements.txt | 22 ++++++++ .../sysom_api/scripts/server_clear.sh | 8 +++ microservice/sysom_api/scripts/server_init.sh | 53 +++++++++++++++++++ .../sysom_api/scripts/server_start.sh | 10 ++++ microservice/sysom_api/scripts/server_stop.sh | 10 ++++ microservice/sysom_api/sysom-api.ini | 13 +++++ .../sysom_api/sysom/__init__.py | 0 .../sysom_api/sysom/asgi.py | 0 .../sysom_api/sysom/settings.py | 0 .../sysom_api/sysom/urls.py | 0 .../sysom_api/sysom/wsgi.py | 0 89 files changed, 131 insertions(+), 22 deletions(-) rename {sysom_server => microservice}/sysom_api/.gitignore (100%) rename {sysom_server => microservice}/sysom_api/apps/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/admin.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/apps.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/authentication.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/migrations/0002_user_allow_login.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/models.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/permissions.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/serializer.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/tests.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/urls.py (100%) rename {sysom_server => microservice}/sysom_api/apps/accounts/views.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/admin.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/apps.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/models.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/serializer.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/subscribe.json (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/urls.py (100%) rename {sysom_server => microservice}/sysom_api/apps/alarm/views.py (100%) rename {sysom_server => microservice}/sysom_api/apps/common/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/common/common_model_viewset.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/admin.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/apps.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/cec_api.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/heartbeat.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/models.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/serializer.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/tests.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/urls.py (100%) rename {sysom_server => microservice}/sysom_api/apps/host/views.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/admin.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/apps.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/models.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/serializer.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/tests.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/urls.py (100%) rename {sysom_server => microservice}/sysom_api/apps/services/views.py (100%) rename {sysom_server => microservice}/sysom_api/conf/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/conf/common.py (98%) rename {sysom_server => microservice}/sysom_api/conf/develop.py (100%) rename {sysom_server => microservice}/sysom_api/conf/product.py (100%) rename {sysom_server => microservice}/sysom_api/conf/testing.py (100%) rename {sysom_server => microservice}/sysom_api/config.yml (100%) rename {sysom_server => microservice}/sysom_api/consumer/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/consumer/consumers.py (100%) rename {sysom_server => microservice}/sysom_api/consumer/middleware.py (100%) rename {sysom_server => microservice}/sysom_api/consumer/routing.py (100%) rename {sysom_server => microservice}/sysom_api/lib/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/lib/authentications.py (100%) rename {sysom_server => microservice}/sysom_api/lib/base_model.py (100%) rename {sysom_server => microservice}/sysom_api/lib/decode/sysom_decode.py (100%) rename {sysom_server => microservice}/sysom_api/lib/excel.py (100%) rename {sysom_server => microservice}/sysom_api/lib/exception.py (100%) rename {sysom_server => microservice}/sysom_api/lib/paginations.py (100%) rename {sysom_server => microservice}/sysom_api/lib/renderers.py (100%) rename {sysom_server => microservice}/sysom_api/lib/response.py (100%) rename {sysom_server => microservice}/sysom_api/lib/ssh.py (100%) rename {sysom_server => microservice}/sysom_api/lib/utils.py (100%) rename {sysom_server => microservice}/sysom_api/manage.py (100%) create mode 100644 microservice/sysom_api/requirements.txt create mode 100644 microservice/sysom_api/scripts/server_clear.sh create mode 100644 microservice/sysom_api/scripts/server_init.sh create mode 100644 microservice/sysom_api/scripts/server_start.sh create mode 100644 microservice/sysom_api/scripts/server_stop.sh create mode 100644 microservice/sysom_api/sysom-api.ini rename {sysom_server => microservice}/sysom_api/sysom/__init__.py (100%) rename {sysom_server => microservice}/sysom_api/sysom/asgi.py (100%) rename {sysom_server => microservice}/sysom_api/sysom/settings.py (100%) rename {sysom_server => microservice}/sysom_api/sysom/urls.py (100%) rename {sysom_server => microservice}/sysom_api/sysom/wsgi.py (100%) diff --git a/infrastructure/env/requirements.txt b/infrastructure/env/requirements.txt index 517c7d42..012fd9a0 100644 --- a/infrastructure/env/requirements.txt +++ b/infrastructure/env/requirements.txt @@ -1,11 +1,11 @@ typing-extensions==4.1.1 attrs==22.2.0 -# cffi==1.15.1 -# aiofiles==0.8.0 +cffi==1.15.1 +aiofiles==0.8.0 # alembic==1.7.7 -# anyio==3.6.2 -# aiohttp==3.8.3 -# asyncer==0.0.2 +anyio==3.6.2 +aiohttp==3.8.4 +asyncer==0.0.2 # asyncssh==2.12.0 # autopep8==2.0.0 # channels==3.0.4 @@ -23,18 +23,18 @@ attrs==22.2.0 # pandas>=1.1.5 # paramiko==2.12.0 # prompt-toolkit==3.0.32 -# PyJWT==2.4.0 -# PyMySQL==1.0.2 +PyJWT==2.4.0 +PyMySQL==1.0.2 # pytest-runner==5.3.2 # python-multipart==0.0.5 # rich==12.6.0 -# requests==2.27.1 -# redis==4.3.4 -# schedule==1.1.0 +requests==2.27.1 +redis==4.3.4 +schedule==1.1.0 # uvicorn==0.16.0 # wheel==0.37.1 # xlwt==1.3.0 # xlrd==2.0.1 # prometheus-client==0.16.0 -# pyyaml==6.0 -# pyyaml-include==1.3 \ No newline at end of file +pyyaml==6.0 +pyyaml-include==1.3 \ No newline at end of file diff --git a/infrastructure/sdk/cec_redis/redis_admin.py b/infrastructure/sdk/cec_redis/redis_admin.py index bbe7aead..35280488 100644 --- a/infrastructure/sdk/cec_redis/redis_admin.py +++ b/infrastructure/sdk/cec_redis/redis_admin.py @@ -603,9 +603,6 @@ class RedisAdmin(Admin, ClientBase): logger.info( f"{self} disconnect from '{self._current_url}' successfully.") - def __del__(self): - self.disconnect() - def client(self): """Get inner redis client diff --git a/infrastructure/sdk/cec_redis/redis_consumer.py b/infrastructure/sdk/cec_redis/redis_consumer.py index 0de5681a..8ef0dce0 100644 --- a/infrastructure/sdk/cec_redis/redis_consumer.py +++ b/infrastructure/sdk/cec_redis/redis_consumer.py @@ -7,6 +7,7 @@ File redis_consumer.py Description: """ import json +import atexit from queue import Queue from typing import List, Optional @@ -442,9 +443,6 @@ class RedisConsumer(Consumer, ClientBase): cec_url = CecUrl.parse(url) return self.connect_by_cec_url(cec_url) - def __del__(self): - self.disconnect() - def disconnect(self): """Disconnect from redis server diff --git a/infrastructure/sdk/cec_redis/redis_producer.py b/infrastructure/sdk/cec_redis/redis_producer.py index 755bdb67..adc2dbe4 100644 --- a/infrastructure/sdk/cec_redis/redis_producer.py +++ b/infrastructure/sdk/cec_redis/redis_producer.py @@ -7,6 +7,7 @@ File redis_producer.py Description: """ import json +import atexit from typing import Callable, Union, Optional from redis import Redis @@ -205,9 +206,6 @@ class RedisProducer(Producer, ClientBase): cec_url = CecUrl.parse(url) return self.connect_by_cec_url(cec_url) - def __del__(self): - self.disconnect() - def disconnect(self): """Disconnect from redis server diff --git a/sysom_server/sysom_api/.gitignore b/microservice/sysom_api/.gitignore similarity index 100% rename from sysom_server/sysom_api/.gitignore rename to microservice/sysom_api/.gitignore diff --git a/sysom_server/sysom_api/apps/__init__.py b/microservice/sysom_api/apps/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/__init__.py rename to microservice/sysom_api/apps/__init__.py diff --git a/sysom_server/sysom_api/apps/accounts/__init__.py b/microservice/sysom_api/apps/accounts/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/__init__.py rename to microservice/sysom_api/apps/accounts/__init__.py diff --git a/sysom_server/sysom_api/apps/accounts/admin.py b/microservice/sysom_api/apps/accounts/admin.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/admin.py rename to microservice/sysom_api/apps/accounts/admin.py diff --git a/sysom_server/sysom_api/apps/accounts/apps.py b/microservice/sysom_api/apps/accounts/apps.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/apps.py rename to microservice/sysom_api/apps/accounts/apps.py diff --git a/sysom_server/sysom_api/apps/accounts/authentication.py b/microservice/sysom_api/apps/accounts/authentication.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/authentication.py rename to microservice/sysom_api/apps/accounts/authentication.py diff --git a/sysom_server/sysom_api/apps/accounts/migrations/0001_initial.py b/microservice/sysom_api/apps/accounts/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/migrations/0001_initial.py rename to microservice/sysom_api/apps/accounts/migrations/0001_initial.py diff --git a/sysom_server/sysom_api/apps/accounts/migrations/0002_user_allow_login.py b/microservice/sysom_api/apps/accounts/migrations/0002_user_allow_login.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/migrations/0002_user_allow_login.py rename to microservice/sysom_api/apps/accounts/migrations/0002_user_allow_login.py diff --git a/sysom_server/sysom_api/apps/accounts/migrations/__init__.py b/microservice/sysom_api/apps/accounts/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/migrations/__init__.py rename to microservice/sysom_api/apps/accounts/migrations/__init__.py diff --git a/sysom_server/sysom_api/apps/accounts/models.py b/microservice/sysom_api/apps/accounts/models.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/models.py rename to microservice/sysom_api/apps/accounts/models.py diff --git a/sysom_server/sysom_api/apps/accounts/permissions.py b/microservice/sysom_api/apps/accounts/permissions.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/permissions.py rename to microservice/sysom_api/apps/accounts/permissions.py diff --git a/sysom_server/sysom_api/apps/accounts/serializer.py b/microservice/sysom_api/apps/accounts/serializer.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/serializer.py rename to microservice/sysom_api/apps/accounts/serializer.py diff --git a/sysom_server/sysom_api/apps/accounts/tests.py b/microservice/sysom_api/apps/accounts/tests.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/tests.py rename to microservice/sysom_api/apps/accounts/tests.py diff --git a/sysom_server/sysom_api/apps/accounts/urls.py b/microservice/sysom_api/apps/accounts/urls.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/urls.py rename to microservice/sysom_api/apps/accounts/urls.py diff --git a/sysom_server/sysom_api/apps/accounts/views.py b/microservice/sysom_api/apps/accounts/views.py similarity index 100% rename from sysom_server/sysom_api/apps/accounts/views.py rename to microservice/sysom_api/apps/accounts/views.py diff --git a/sysom_server/sysom_api/apps/alarm/__init__.py b/microservice/sysom_api/apps/alarm/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/__init__.py rename to microservice/sysom_api/apps/alarm/__init__.py diff --git a/sysom_server/sysom_api/apps/alarm/admin.py b/microservice/sysom_api/apps/alarm/admin.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/admin.py rename to microservice/sysom_api/apps/alarm/admin.py diff --git a/sysom_server/sysom_api/apps/alarm/apps.py b/microservice/sysom_api/apps/alarm/apps.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/apps.py rename to microservice/sysom_api/apps/alarm/apps.py diff --git a/sysom_server/sysom_api/apps/alarm/migrations/0001_initial.py b/microservice/sysom_api/apps/alarm/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/migrations/0001_initial.py rename to microservice/sysom_api/apps/alarm/migrations/0001_initial.py diff --git a/sysom_server/sysom_api/apps/alarm/migrations/__init__.py b/microservice/sysom_api/apps/alarm/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/migrations/__init__.py rename to microservice/sysom_api/apps/alarm/migrations/__init__.py diff --git a/sysom_server/sysom_api/apps/alarm/models.py b/microservice/sysom_api/apps/alarm/models.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/models.py rename to microservice/sysom_api/apps/alarm/models.py diff --git a/sysom_server/sysom_api/apps/alarm/serializer.py b/microservice/sysom_api/apps/alarm/serializer.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/serializer.py rename to microservice/sysom_api/apps/alarm/serializer.py diff --git a/sysom_server/sysom_api/apps/alarm/subscribe.json b/microservice/sysom_api/apps/alarm/subscribe.json similarity index 100% rename from sysom_server/sysom_api/apps/alarm/subscribe.json rename to microservice/sysom_api/apps/alarm/subscribe.json diff --git a/sysom_server/sysom_api/apps/alarm/urls.py b/microservice/sysom_api/apps/alarm/urls.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/urls.py rename to microservice/sysom_api/apps/alarm/urls.py diff --git a/sysom_server/sysom_api/apps/alarm/views.py b/microservice/sysom_api/apps/alarm/views.py similarity index 100% rename from sysom_server/sysom_api/apps/alarm/views.py rename to microservice/sysom_api/apps/alarm/views.py diff --git a/sysom_server/sysom_api/apps/common/__init__.py b/microservice/sysom_api/apps/common/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/common/__init__.py rename to microservice/sysom_api/apps/common/__init__.py diff --git a/sysom_server/sysom_api/apps/common/common_model_viewset.py b/microservice/sysom_api/apps/common/common_model_viewset.py similarity index 100% rename from sysom_server/sysom_api/apps/common/common_model_viewset.py rename to microservice/sysom_api/apps/common/common_model_viewset.py diff --git a/sysom_server/sysom_api/apps/host/__init__.py b/microservice/sysom_api/apps/host/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/host/__init__.py rename to microservice/sysom_api/apps/host/__init__.py diff --git a/sysom_server/sysom_api/apps/host/admin.py b/microservice/sysom_api/apps/host/admin.py similarity index 100% rename from sysom_server/sysom_api/apps/host/admin.py rename to microservice/sysom_api/apps/host/admin.py diff --git a/sysom_server/sysom_api/apps/host/apps.py b/microservice/sysom_api/apps/host/apps.py similarity index 100% rename from sysom_server/sysom_api/apps/host/apps.py rename to microservice/sysom_api/apps/host/apps.py diff --git a/sysom_server/sysom_api/apps/host/cec_api.py b/microservice/sysom_api/apps/host/cec_api.py similarity index 100% rename from sysom_server/sysom_api/apps/host/cec_api.py rename to microservice/sysom_api/apps/host/cec_api.py diff --git a/sysom_server/sysom_api/apps/host/heartbeat.py b/microservice/sysom_api/apps/host/heartbeat.py similarity index 100% rename from sysom_server/sysom_api/apps/host/heartbeat.py rename to microservice/sysom_api/apps/host/heartbeat.py diff --git a/sysom_server/sysom_api/apps/host/migrations/0001_initial.py b/microservice/sysom_api/apps/host/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_api/apps/host/migrations/0001_initial.py rename to microservice/sysom_api/apps/host/migrations/0001_initial.py diff --git a/sysom_server/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py b/microservice/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py similarity index 100% rename from sysom_server/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py rename to microservice/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py diff --git a/sysom_server/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py b/microservice/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py similarity index 100% rename from sysom_server/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py rename to microservice/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py diff --git a/sysom_server/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py b/microservice/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py similarity index 100% rename from sysom_server/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py rename to microservice/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py diff --git a/sysom_server/sysom_api/apps/host/migrations/__init__.py b/microservice/sysom_api/apps/host/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/host/migrations/__init__.py rename to microservice/sysom_api/apps/host/migrations/__init__.py diff --git a/sysom_server/sysom_api/apps/host/models.py b/microservice/sysom_api/apps/host/models.py similarity index 100% rename from sysom_server/sysom_api/apps/host/models.py rename to microservice/sysom_api/apps/host/models.py diff --git a/sysom_server/sysom_api/apps/host/serializer.py b/microservice/sysom_api/apps/host/serializer.py similarity index 100% rename from sysom_server/sysom_api/apps/host/serializer.py rename to microservice/sysom_api/apps/host/serializer.py diff --git a/sysom_server/sysom_api/apps/host/tests.py b/microservice/sysom_api/apps/host/tests.py similarity index 100% rename from sysom_server/sysom_api/apps/host/tests.py rename to microservice/sysom_api/apps/host/tests.py diff --git a/sysom_server/sysom_api/apps/host/urls.py b/microservice/sysom_api/apps/host/urls.py similarity index 100% rename from sysom_server/sysom_api/apps/host/urls.py rename to microservice/sysom_api/apps/host/urls.py diff --git a/sysom_server/sysom_api/apps/host/views.py b/microservice/sysom_api/apps/host/views.py similarity index 100% rename from sysom_server/sysom_api/apps/host/views.py rename to microservice/sysom_api/apps/host/views.py diff --git a/sysom_server/sysom_api/apps/services/__init__.py b/microservice/sysom_api/apps/services/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/services/__init__.py rename to microservice/sysom_api/apps/services/__init__.py diff --git a/sysom_server/sysom_api/apps/services/admin.py b/microservice/sysom_api/apps/services/admin.py similarity index 100% rename from sysom_server/sysom_api/apps/services/admin.py rename to microservice/sysom_api/apps/services/admin.py diff --git a/sysom_server/sysom_api/apps/services/apps.py b/microservice/sysom_api/apps/services/apps.py similarity index 100% rename from sysom_server/sysom_api/apps/services/apps.py rename to microservice/sysom_api/apps/services/apps.py diff --git a/sysom_server/sysom_api/apps/services/migrations/0001_initial.py b/microservice/sysom_api/apps/services/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_api/apps/services/migrations/0001_initial.py rename to microservice/sysom_api/apps/services/migrations/0001_initial.py diff --git a/sysom_server/sysom_api/apps/services/migrations/__init__.py b/microservice/sysom_api/apps/services/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_api/apps/services/migrations/__init__.py rename to microservice/sysom_api/apps/services/migrations/__init__.py diff --git a/sysom_server/sysom_api/apps/services/models.py b/microservice/sysom_api/apps/services/models.py similarity index 100% rename from sysom_server/sysom_api/apps/services/models.py rename to microservice/sysom_api/apps/services/models.py diff --git a/sysom_server/sysom_api/apps/services/serializer.py b/microservice/sysom_api/apps/services/serializer.py similarity index 100% rename from sysom_server/sysom_api/apps/services/serializer.py rename to microservice/sysom_api/apps/services/serializer.py diff --git a/sysom_server/sysom_api/apps/services/tests.py b/microservice/sysom_api/apps/services/tests.py similarity index 100% rename from sysom_server/sysom_api/apps/services/tests.py rename to microservice/sysom_api/apps/services/tests.py diff --git a/sysom_server/sysom_api/apps/services/urls.py b/microservice/sysom_api/apps/services/urls.py similarity index 100% rename from sysom_server/sysom_api/apps/services/urls.py rename to microservice/sysom_api/apps/services/urls.py diff --git a/sysom_server/sysom_api/apps/services/views.py b/microservice/sysom_api/apps/services/views.py similarity index 100% rename from sysom_server/sysom_api/apps/services/views.py rename to microservice/sysom_api/apps/services/views.py diff --git a/sysom_server/sysom_api/conf/__init__.py b/microservice/sysom_api/conf/__init__.py similarity index 100% rename from sysom_server/sysom_api/conf/__init__.py rename to microservice/sysom_api/conf/__init__.py diff --git a/sysom_server/sysom_api/conf/common.py b/microservice/sysom_api/conf/common.py similarity index 98% rename from sysom_server/sysom_api/conf/common.py rename to microservice/sysom_api/conf/common.py index 29f75b96..447023c1 100644 --- a/sysom_server/sysom_api/conf/common.py +++ b/microservice/sysom_api/conf/common.py @@ -11,7 +11,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/sysom_server/sysom_api/conf/develop.py b/microservice/sysom_api/conf/develop.py similarity index 100% rename from sysom_server/sysom_api/conf/develop.py rename to microservice/sysom_api/conf/develop.py diff --git a/sysom_server/sysom_api/conf/product.py b/microservice/sysom_api/conf/product.py similarity index 100% rename from sysom_server/sysom_api/conf/product.py rename to microservice/sysom_api/conf/product.py diff --git a/sysom_server/sysom_api/conf/testing.py b/microservice/sysom_api/conf/testing.py similarity index 100% rename from sysom_server/sysom_api/conf/testing.py rename to microservice/sysom_api/conf/testing.py diff --git a/sysom_server/sysom_api/config.yml b/microservice/sysom_api/config.yml similarity index 100% rename from sysom_server/sysom_api/config.yml rename to microservice/sysom_api/config.yml diff --git a/sysom_server/sysom_api/consumer/__init__.py b/microservice/sysom_api/consumer/__init__.py similarity index 100% rename from sysom_server/sysom_api/consumer/__init__.py rename to microservice/sysom_api/consumer/__init__.py diff --git a/sysom_server/sysom_api/consumer/consumers.py b/microservice/sysom_api/consumer/consumers.py similarity index 100% rename from sysom_server/sysom_api/consumer/consumers.py rename to microservice/sysom_api/consumer/consumers.py diff --git a/sysom_server/sysom_api/consumer/middleware.py b/microservice/sysom_api/consumer/middleware.py similarity index 100% rename from sysom_server/sysom_api/consumer/middleware.py rename to microservice/sysom_api/consumer/middleware.py diff --git a/sysom_server/sysom_api/consumer/routing.py b/microservice/sysom_api/consumer/routing.py similarity index 100% rename from sysom_server/sysom_api/consumer/routing.py rename to microservice/sysom_api/consumer/routing.py diff --git a/sysom_server/sysom_api/lib/__init__.py b/microservice/sysom_api/lib/__init__.py similarity index 100% rename from sysom_server/sysom_api/lib/__init__.py rename to microservice/sysom_api/lib/__init__.py diff --git a/sysom_server/sysom_api/lib/authentications.py b/microservice/sysom_api/lib/authentications.py similarity index 100% rename from sysom_server/sysom_api/lib/authentications.py rename to microservice/sysom_api/lib/authentications.py diff --git a/sysom_server/sysom_api/lib/base_model.py b/microservice/sysom_api/lib/base_model.py similarity index 100% rename from sysom_server/sysom_api/lib/base_model.py rename to microservice/sysom_api/lib/base_model.py diff --git a/sysom_server/sysom_api/lib/decode/sysom_decode.py b/microservice/sysom_api/lib/decode/sysom_decode.py similarity index 100% rename from sysom_server/sysom_api/lib/decode/sysom_decode.py rename to microservice/sysom_api/lib/decode/sysom_decode.py diff --git a/sysom_server/sysom_api/lib/excel.py b/microservice/sysom_api/lib/excel.py similarity index 100% rename from sysom_server/sysom_api/lib/excel.py rename to microservice/sysom_api/lib/excel.py diff --git a/sysom_server/sysom_api/lib/exception.py b/microservice/sysom_api/lib/exception.py similarity index 100% rename from sysom_server/sysom_api/lib/exception.py rename to microservice/sysom_api/lib/exception.py diff --git a/sysom_server/sysom_api/lib/paginations.py b/microservice/sysom_api/lib/paginations.py similarity index 100% rename from sysom_server/sysom_api/lib/paginations.py rename to microservice/sysom_api/lib/paginations.py diff --git a/sysom_server/sysom_api/lib/renderers.py b/microservice/sysom_api/lib/renderers.py similarity index 100% rename from sysom_server/sysom_api/lib/renderers.py rename to microservice/sysom_api/lib/renderers.py diff --git a/sysom_server/sysom_api/lib/response.py b/microservice/sysom_api/lib/response.py similarity index 100% rename from sysom_server/sysom_api/lib/response.py rename to microservice/sysom_api/lib/response.py diff --git a/sysom_server/sysom_api/lib/ssh.py b/microservice/sysom_api/lib/ssh.py similarity index 100% rename from sysom_server/sysom_api/lib/ssh.py rename to microservice/sysom_api/lib/ssh.py diff --git a/sysom_server/sysom_api/lib/utils.py b/microservice/sysom_api/lib/utils.py similarity index 100% rename from sysom_server/sysom_api/lib/utils.py rename to microservice/sysom_api/lib/utils.py diff --git a/sysom_server/sysom_api/manage.py b/microservice/sysom_api/manage.py similarity index 100% rename from sysom_server/sysom_api/manage.py rename to microservice/sysom_api/manage.py diff --git a/microservice/sysom_api/requirements.txt b/microservice/sysom_api/requirements.txt new file mode 100644 index 00000000..f57e6c2e --- /dev/null +++ b/microservice/sysom_api/requirements.txt @@ -0,0 +1,22 @@ +channels==3.0.4 +django==3.2.16 +django-filter==21.1 +django-cors-headers==3.10.1 +django-apscheduler==0.6.2 +django-redis==5.2.0 +djangorestframework==3.14.0 +drf-yasg==1.21.4 +daphne==3.0.2 +paramiko==2.12.0 +pandas>=1.1.5 +python-multipart==0.0.5 +prometheus_client==0.16.0 +PyJWT==2.4.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +requests==2.27.1 +schedule==1.1.0 +gunicorn==20.1.0 +xlwt==1.3.0 +xlrd==2.0.1 \ No newline at end of file diff --git a/microservice/sysom_api/scripts/server_clear.sh b/microservice/sysom_api/scripts/server_clear.sh new file mode 100644 index 00000000..07b92432 --- /dev/null +++ b/microservice/sysom_api/scripts/server_clear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-api +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} +clear_app diff --git a/microservice/sysom_api/scripts/server_init.sh b/microservice/sysom_api/scripts/server_init.sh new file mode 100644 index 00000000..f5731753 --- /dev/null +++ b/microservice/sysom_api/scripts/server_init.sh @@ -0,0 +1,53 @@ +#!/bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_api +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_NAME=sysom-api + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +init_conf() { + mkdir -p /run/daphne + pushd ${SERVICE_HOME} + python manage.py migrate + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME}:0 + if [ $? -eq 0 ] + then + echo "${SERVICE_NAME} service start success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + start_app +} + +deploy diff --git a/microservice/sysom_api/scripts/server_start.sh b/microservice/sysom_api/scripts/server_start.sh new file mode 100644 index 00000000..953f2cb9 --- /dev/null +++ b/microservice/sysom_api/scripts/server_start.sh @@ -0,0 +1,10 @@ +#!/bin/bash +SERVICE_NAME=sysom-api +stop_app() { + for service in `supervisorctl status | grep ${SERVICE_NAME} | awk '{print $1}'` + do + supervisorctl start $service + done +} + +stop_app diff --git a/microservice/sysom_api/scripts/server_stop.sh b/microservice/sysom_api/scripts/server_stop.sh new file mode 100644 index 00000000..eb4150ad --- /dev/null +++ b/microservice/sysom_api/scripts/server_stop.sh @@ -0,0 +1,10 @@ +#!/bin/bash +SERVICE_NAME=sysom-api +stop_app() { + for service in `supervisorctl status | grep ${SERVICE_NAME} | awk '{print $1}'` + do + supervisorctl stop $service + done +} + +stop_app diff --git a/microservice/sysom_api/sysom-api.ini b/microservice/sysom_api/sysom-api.ini new file mode 100644 index 00000000..120a106b --- /dev/null +++ b/microservice/sysom_api/sysom-api.ini @@ -0,0 +1,13 @@ +[fcgi-program:sysom-api] +socket=tcp://0.0.0.0:7001 +directory=/usr/local/sysom/server/microservice/sysom_api +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/daphne -u /run/daphne%(process_num)d.sock --fd 0 --access-log /var/log/sysom/sysom-api-access.log --proxy-headers sysom.asgi:application +numprocs=4 +process_name=%(process_num)d +autostart=true +autorestart=true +redirect_stderr=true +stopasgroup=true +stdout_logfile=/var/log/sysom/sysom-api.log +stderr_logfile=/var/log/sysom/sysom-api-error.log +environment=PATH=/usr/local/sysom/server/virtualenv/bin:%(ENV_PATH)s diff --git a/sysom_server/sysom_api/sysom/__init__.py b/microservice/sysom_api/sysom/__init__.py similarity index 100% rename from sysom_server/sysom_api/sysom/__init__.py rename to microservice/sysom_api/sysom/__init__.py diff --git a/sysom_server/sysom_api/sysom/asgi.py b/microservice/sysom_api/sysom/asgi.py similarity index 100% rename from sysom_server/sysom_api/sysom/asgi.py rename to microservice/sysom_api/sysom/asgi.py diff --git a/sysom_server/sysom_api/sysom/settings.py b/microservice/sysom_api/sysom/settings.py similarity index 100% rename from sysom_server/sysom_api/sysom/settings.py rename to microservice/sysom_api/sysom/settings.py diff --git a/sysom_server/sysom_api/sysom/urls.py b/microservice/sysom_api/sysom/urls.py similarity index 100% rename from sysom_server/sysom_api/sysom/urls.py rename to microservice/sysom_api/sysom/urls.py diff --git a/sysom_server/sysom_api/sysom/wsgi.py b/microservice/sysom_api/sysom/wsgi.py similarity index 100% rename from sysom_server/sysom_api/sysom/wsgi.py rename to microservice/sysom_api/sysom/wsgi.py -- Gitee From 5f5125002447a4de0b156b9c636a9792ed1448d5 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 12:55:27 +0800 Subject: [PATCH 020/196] feat(sysom_api): Support docker deploment --- sysom_api_dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 sysom_api_dockerfile diff --git a/sysom_api_dockerfile b/sysom_api_dockerfile new file mode 100644 index 00000000..4eaa0b9c --- /dev/null +++ b/sysom_api_dockerfile @@ -0,0 +1,31 @@ +FROM openanolis/anolisos:8.8 + +# Add epel +RUN yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm +RUN bash -c "sed -i 's|^#baseurl=https://download.example/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel*" +RUN bash -c "sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel*" + +# Add required yum packages +RUN yum makecache +RUN yum install -y supervisor cronie net-tools +RUN systemctl enable crond +# RUN systemctl enable supervisord + +# Init sysom-diagnosis +ARG SYSOM_HOME=/usr/local/sysom +ARG SYSOM_SERVER_HOME=${SYSOM_HOME}/server + +RUN mkdir /root/sysom +COPY conf /root/sysom/conf +COPY script /root/sysom/script +COPY infrastructure /root/sysom/infrastructure +COPY microservice /root/sysom/microservice + +RUN bash -x /root/sysom/script/sysom.sh deploy infrastructure env,sdk +RUN bash -x /root/sysom/script/sysom.sh deploy microservice sysom_api +RUN sed "s/nodaemon=false/nodaemon=true/g" -i /etc/supervisord.conf + +RUN yum clean all + +# # 环境准备 +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file -- Gitee From 3fed012bc5ef1857dd133ff3bf24997a87c82c18 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 14:16:54 +0800 Subject: [PATCH 021/196] refactor(sysom_monitor_server): Use sysom.sh to unify deployment --- microservice/sysom_channel/sysom-channel.ini | 4 +- .../sysom_monitor_server/.gitignore | 0 .../sysom_monitor_server/app/routeres/home.py | 0 .../sysom_monitor_server/conf/common.py | 2 +- .../sysom_monitor_server/conf/develop.py | 0 .../sysom_monitor_server/conf/gunicorn.py | 18 +++++++ .../sysom_monitor_server/conf/product.py | 0 .../sysom_monitor_server/conf/settings.py | 0 .../sysom_monitor_server/conf/testing.py | 0 .../sysom_monitor_server/config.yml | 0 .../sysom_monitor_server/main.py | 0 .../sysom_monitor_server/requirements.txt | 12 +++++ .../scripts/node_clear.sh | 0 .../sysom_monitor_server/scripts/node_init.sh | 0 .../scripts/node_update.sh | 0 .../scripts/server_clear.sh | 8 +++ .../scripts/server_init.sh | 52 +++++++++++++++++++ .../scripts/server_start.sh | 7 +++ .../scripts/server_stop.sh | 7 +++ .../sysom-monitor-server.ini | 9 ++++ 20 files changed, 116 insertions(+), 3 deletions(-) rename {sysom_server => microservice}/sysom_monitor_server/.gitignore (100%) rename {sysom_server => microservice}/sysom_monitor_server/app/routeres/home.py (100%) rename {sysom_server => microservice}/sysom_monitor_server/conf/common.py (94%) rename {sysom_server => microservice}/sysom_monitor_server/conf/develop.py (100%) create mode 100644 microservice/sysom_monitor_server/conf/gunicorn.py rename {sysom_server => microservice}/sysom_monitor_server/conf/product.py (100%) rename {sysom_server => microservice}/sysom_monitor_server/conf/settings.py (100%) rename {sysom_server => microservice}/sysom_monitor_server/conf/testing.py (100%) rename {sysom_server => microservice}/sysom_monitor_server/config.yml (100%) rename {sysom_server => microservice}/sysom_monitor_server/main.py (100%) create mode 100644 microservice/sysom_monitor_server/requirements.txt rename {sysom_server => microservice}/sysom_monitor_server/scripts/node_clear.sh (100%) rename {sysom_server => microservice}/sysom_monitor_server/scripts/node_init.sh (100%) rename {sysom_server => microservice}/sysom_monitor_server/scripts/node_update.sh (100%) create mode 100644 microservice/sysom_monitor_server/scripts/server_clear.sh create mode 100644 microservice/sysom_monitor_server/scripts/server_init.sh create mode 100644 microservice/sysom_monitor_server/scripts/server_start.sh create mode 100644 microservice/sysom_monitor_server/scripts/server_stop.sh create mode 100644 microservice/sysom_monitor_server/sysom-monitor-server.ini diff --git a/microservice/sysom_channel/sysom-channel.ini b/microservice/sysom_channel/sysom-channel.ini index 28e973b2..7e6ed3ec 100644 --- a/microservice/sysom_channel/sysom-channel.ini +++ b/microservice/sysom_channel/sysom-channel.ini @@ -1,9 +1,9 @@ [program:sysom-channel] -directory = /usr/local/sysom/server/microservice/sysom_channel +directory=/usr/local/sysom/server/microservice/sysom_channel command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/channel_gunicorn.py main:app startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" +environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-channel-error.log stdout_logfile=/var/log/sysom/sysom-channel.log diff --git a/sysom_server/sysom_monitor_server/.gitignore b/microservice/sysom_monitor_server/.gitignore similarity index 100% rename from sysom_server/sysom_monitor_server/.gitignore rename to microservice/sysom_monitor_server/.gitignore diff --git a/sysom_server/sysom_monitor_server/app/routeres/home.py b/microservice/sysom_monitor_server/app/routeres/home.py similarity index 100% rename from sysom_server/sysom_monitor_server/app/routeres/home.py rename to microservice/sysom_monitor_server/app/routeres/home.py diff --git a/sysom_server/sysom_monitor_server/conf/common.py b/microservice/sysom_monitor_server/conf/common.py similarity index 94% rename from sysom_server/sysom_monitor_server/conf/common.py rename to microservice/sysom_monitor_server/conf/common.py index 19e9b46c..03d8c698 100644 --- a/sysom_server/sysom_monitor_server/conf/common.py +++ b/microservice/sysom_monitor_server/conf/common.py @@ -17,7 +17,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/sysom_server/sysom_monitor_server/conf/develop.py b/microservice/sysom_monitor_server/conf/develop.py similarity index 100% rename from sysom_server/sysom_monitor_server/conf/develop.py rename to microservice/sysom_monitor_server/conf/develop.py diff --git a/microservice/sysom_monitor_server/conf/gunicorn.py b/microservice/sysom_monitor_server/conf/gunicorn.py new file mode 100644 index 00000000..f846eba5 --- /dev/null +++ b/microservice/sysom_monitor_server/conf/gunicorn.py @@ -0,0 +1,18 @@ +''' +Monitor server Service Gunicorn Settings +''' +workers = 2 # 指定工作进程数 + +threads = 3 + +bind = '0.0.0.0:7009' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/sysom-monitor-server-access.log' + +loglevel = 'error' + +proc_name = 'monitor_server_service' diff --git a/sysom_server/sysom_monitor_server/conf/product.py b/microservice/sysom_monitor_server/conf/product.py similarity index 100% rename from sysom_server/sysom_monitor_server/conf/product.py rename to microservice/sysom_monitor_server/conf/product.py diff --git a/sysom_server/sysom_monitor_server/conf/settings.py b/microservice/sysom_monitor_server/conf/settings.py similarity index 100% rename from sysom_server/sysom_monitor_server/conf/settings.py rename to microservice/sysom_monitor_server/conf/settings.py diff --git a/sysom_server/sysom_monitor_server/conf/testing.py b/microservice/sysom_monitor_server/conf/testing.py similarity index 100% rename from sysom_server/sysom_monitor_server/conf/testing.py rename to microservice/sysom_monitor_server/conf/testing.py diff --git a/sysom_server/sysom_monitor_server/config.yml b/microservice/sysom_monitor_server/config.yml similarity index 100% rename from sysom_server/sysom_monitor_server/config.yml rename to microservice/sysom_monitor_server/config.yml diff --git a/sysom_server/sysom_monitor_server/main.py b/microservice/sysom_monitor_server/main.py similarity index 100% rename from sysom_server/sysom_monitor_server/main.py rename to microservice/sysom_monitor_server/main.py diff --git a/microservice/sysom_monitor_server/requirements.txt b/microservice/sysom_monitor_server/requirements.txt new file mode 100644 index 00000000..ee35d62b --- /dev/null +++ b/microservice/sysom_monitor_server/requirements.txt @@ -0,0 +1,12 @@ +clogger>=0.0.1 +sysom_utils>=0.0.1 +alembic==1.7.7 +anyio==3.6.2 +asyncer==0.0.2 +fastapi==0.83.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +python-multipart==0.0.5 +uvicorn==0.16.0 +gunicorn==20.1.0 \ No newline at end of file diff --git a/sysom_server/sysom_monitor_server/scripts/node_clear.sh b/microservice/sysom_monitor_server/scripts/node_clear.sh similarity index 100% rename from sysom_server/sysom_monitor_server/scripts/node_clear.sh rename to microservice/sysom_monitor_server/scripts/node_clear.sh diff --git a/sysom_server/sysom_monitor_server/scripts/node_init.sh b/microservice/sysom_monitor_server/scripts/node_init.sh similarity index 100% rename from sysom_server/sysom_monitor_server/scripts/node_init.sh rename to microservice/sysom_monitor_server/scripts/node_init.sh diff --git a/sysom_server/sysom_monitor_server/scripts/node_update.sh b/microservice/sysom_monitor_server/scripts/node_update.sh similarity index 100% rename from sysom_server/sysom_monitor_server/scripts/node_update.sh rename to microservice/sysom_monitor_server/scripts/node_update.sh diff --git a/microservice/sysom_monitor_server/scripts/server_clear.sh b/microservice/sysom_monitor_server/scripts/server_clear.sh new file mode 100644 index 00000000..ca965e5c --- /dev/null +++ b/microservice/sysom_monitor_server/scripts/server_clear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-monitor-server +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} +clear_app diff --git a/microservice/sysom_monitor_server/scripts/server_init.sh b/microservice/sysom_monitor_server/scripts/server_init.sh new file mode 100644 index 00000000..4399b882 --- /dev/null +++ b/microservice/sysom_monitor_server/scripts/server_init.sh @@ -0,0 +1,52 @@ +#!/bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_monitor_server +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_NAME=sysom-monitor-server + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +init_conf() { + pushd ${SERVICE_HOME} + alembic upgrade head + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + start_app +} + +deploy diff --git a/microservice/sysom_monitor_server/scripts/server_start.sh b/microservice/sysom_monitor_server/scripts/server_start.sh new file mode 100644 index 00000000..e18fa9c5 --- /dev/null +++ b/microservice/sysom_monitor_server/scripts/server_start.sh @@ -0,0 +1,7 @@ +#!/bin/bash +SERVICE_NAME=sysom-monitor-server +start_app() { + supervisorctl start $SERVICE_NAME +} + +start_app diff --git a/microservice/sysom_monitor_server/scripts/server_stop.sh b/microservice/sysom_monitor_server/scripts/server_stop.sh new file mode 100644 index 00000000..1391f149 --- /dev/null +++ b/microservice/sysom_monitor_server/scripts/server_stop.sh @@ -0,0 +1,7 @@ +#!/bin/bash +SERVICE_NAME=sysom-monitor-server +stop_app() { + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/microservice/sysom_monitor_server/sysom-monitor-server.ini b/microservice/sysom_monitor_server/sysom-monitor-server.ini new file mode 100644 index 00000000..b60da419 --- /dev/null +++ b/microservice/sysom_monitor_server/sysom-monitor-server.ini @@ -0,0 +1,9 @@ +[program:sysom-monitor-server] +directory=/usr/local/sysom/server/microservice/sysom_monitor_server +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +startsecs=3 +autostart=true +autorestart=true +environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +stderr_logfile=/var/log/sysom/sysom-monitor-server-error.log +stdout_logfile=/var/log/sysom/sysom-monitor-server.log -- Gitee From e2b0920ab5d6ebccd81000b6eeb6835b7bcafdec Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 14:17:17 +0800 Subject: [PATCH 022/196] feat(sysom_monitor_server): Support docker deploment --- sysom_monitor_server_dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 sysom_monitor_server_dockerfile diff --git a/sysom_monitor_server_dockerfile b/sysom_monitor_server_dockerfile new file mode 100644 index 00000000..a930cc60 --- /dev/null +++ b/sysom_monitor_server_dockerfile @@ -0,0 +1,31 @@ +FROM openanolis/anolisos:8.8 + +# Add epel +RUN yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm +RUN bash -c "sed -i 's|^#baseurl=https://download.example/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel*" +RUN bash -c "sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel*" + +# Add required yum packages +RUN yum makecache +RUN yum install -y supervisor cronie net-tools +RUN systemctl enable crond +# RUN systemctl enable supervisord + +# Init sysom-diagnosis +ARG SYSOM_HOME=/usr/local/sysom +ARG SYSOM_SERVER_HOME=${SYSOM_HOME}/server + +RUN mkdir /root/sysom +COPY conf /root/sysom/conf +COPY script /root/sysom/script +COPY infrastructure /root/sysom/infrastructure +COPY microservice /root/sysom/microservice + +RUN bash -x /root/sysom/script/sysom.sh deploy infrastructure env,sdk +RUN bash -x /root/sysom/script/sysom.sh deploy microservice sysom_monitor_server +RUN sed "s/nodaemon=false/nodaemon=true/g" -i /etc/supervisord.conf + +RUN yum clean all + +# # 环境准备 +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file -- Gitee From baaf67fe42a6fbf5a372fc7d080310eb8ccb1af0 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 15:27:23 +0800 Subject: [PATCH 023/196] refactor(sysom_vmcore): Use sysom.sh to unify deployment --- .../conf/{channel_gunicorn.py => gunicorn.py} | 0 microservice/sysom_channel/sysom-channel.ini | 2 +- .../sysom_vmcore/apps/vmcore/__init__.py | 0 .../sysom_vmcore/apps/vmcore/admin.py | 0 .../sysom_vmcore/apps/vmcore/apps.py | 0 .../apps/vmcore/migrations/0001_initial.py | 0 .../apps/vmcore/migrations/__init__.py | 0 .../sysom_vmcore/apps/vmcore/models.py | 0 .../sysom_vmcore/apps/vmcore/serializer.py | 0 .../sysom_vmcore/apps/vmcore/tests.py | 0 .../sysom_vmcore/apps/vmcore/urls.py | 0 .../sysom_vmcore/apps/vmcore/views.py | 0 .../sysom_vmcore/apps/vmcore/vmcore.json | 0 .../sysom_vmcore/conf/common.py | 2 +- .../sysom_vmcore/conf/develop.py | 0 .../sysom_vmcore/conf/gunicorn.py | 4 +- .../sysom_vmcore/conf/product.py | 0 .../sysom_vmcore/conf/testing.py | 0 .../sysom_vmcore/config.yml | 0 .../sysom_vmcore/lib/authentications.py | 0 .../sysom_vmcore/lib/base_model.py | 0 .../sysom_vmcore/lib/decode/sysom_decode.py | 0 .../sysom_vmcore/lib/exception.py | 0 .../sysom_vmcore/lib/paginations.py | 0 .../sysom_vmcore/lib/renderers.py | 0 .../sysom_vmcore/lib/response.py | 0 .../sysom_vmcore/lib/utils.py | 0 .../sysom_vmcore/manage.py | 0 microservice/sysom_vmcore/requirements.txt | 15 + .../sysom_vmcore/scripts/node_clear.sh | 0 .../sysom_vmcore/scripts/node_init.sh | 0 .../sysom_vmcore/scripts/node_update.sh | 0 .../sysom_vmcore/scripts/parse_panic.py | 565 ++++++++++++++++++ .../sysom_vmcore/scripts/server_clear.sh | 15 + .../sysom_vmcore/scripts/server_init.sh | 77 +++ .../sysom_vmcore/scripts/server_start.sh | 13 + .../sysom_vmcore/scripts/server_stop.sh | 13 + .../sysom_vmcore/scripts/vmcore_collect.py | 0 .../sysom_vmcore/scripts/vmcore_const.py | 25 + microservice/sysom_vmcore/sysom-vmcore.ini | 9 + .../sysom_vmcore/sysom_vmcore/__init__.py | 0 .../sysom_vmcore/sysom_vmcore/asgi.py | 0 .../sysom_vmcore/sysom_vmcore/settings.py | 0 .../sysom_vmcore/sysom_vmcore/urls.py | 0 .../sysom_vmcore/sysom_vmcore/wsgi.py | 0 45 files changed, 736 insertions(+), 4 deletions(-) rename microservice/sysom_channel/conf/{channel_gunicorn.py => gunicorn.py} (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/__init__.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/admin.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/apps.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/models.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/serializer.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/tests.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/urls.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/views.py (100%) rename {sysom_server => microservice}/sysom_vmcore/apps/vmcore/vmcore.json (100%) rename {sysom_server => microservice}/sysom_vmcore/conf/common.py (98%) rename {sysom_server => microservice}/sysom_vmcore/conf/develop.py (100%) rename sysom_server/sysom_vmcore/conf/vmcore_gunicorn.py => microservice/sysom_vmcore/conf/gunicorn.py (80%) rename {sysom_server => microservice}/sysom_vmcore/conf/product.py (100%) rename {sysom_server => microservice}/sysom_vmcore/conf/testing.py (100%) rename {sysom_server => microservice}/sysom_vmcore/config.yml (100%) rename {sysom_server => microservice}/sysom_vmcore/lib/authentications.py (100%) rename {sysom_server => microservice}/sysom_vmcore/lib/base_model.py (100%) rename {sysom_server => microservice}/sysom_vmcore/lib/decode/sysom_decode.py (100%) rename {sysom_server => microservice}/sysom_vmcore/lib/exception.py (100%) rename {sysom_server => microservice}/sysom_vmcore/lib/paginations.py (100%) rename {sysom_server => microservice}/sysom_vmcore/lib/renderers.py (100%) rename {sysom_server => microservice}/sysom_vmcore/lib/response.py (100%) rename {sysom_server => microservice}/sysom_vmcore/lib/utils.py (100%) rename {sysom_server => microservice}/sysom_vmcore/manage.py (100%) create mode 100644 microservice/sysom_vmcore/requirements.txt rename {sysom_server => microservice}/sysom_vmcore/scripts/node_clear.sh (100%) rename {sysom_server => microservice}/sysom_vmcore/scripts/node_init.sh (100%) rename {sysom_server => microservice}/sysom_vmcore/scripts/node_update.sh (100%) create mode 100644 microservice/sysom_vmcore/scripts/parse_panic.py create mode 100644 microservice/sysom_vmcore/scripts/server_clear.sh create mode 100644 microservice/sysom_vmcore/scripts/server_init.sh create mode 100644 microservice/sysom_vmcore/scripts/server_start.sh create mode 100644 microservice/sysom_vmcore/scripts/server_stop.sh rename {sysom_server => microservice}/sysom_vmcore/scripts/vmcore_collect.py (100%) create mode 100644 microservice/sysom_vmcore/scripts/vmcore_const.py create mode 100644 microservice/sysom_vmcore/sysom-vmcore.ini rename {sysom_server => microservice}/sysom_vmcore/sysom_vmcore/__init__.py (100%) rename {sysom_server => microservice}/sysom_vmcore/sysom_vmcore/asgi.py (100%) rename {sysom_server => microservice}/sysom_vmcore/sysom_vmcore/settings.py (100%) rename {sysom_server => microservice}/sysom_vmcore/sysom_vmcore/urls.py (100%) rename {sysom_server => microservice}/sysom_vmcore/sysom_vmcore/wsgi.py (100%) diff --git a/microservice/sysom_channel/conf/channel_gunicorn.py b/microservice/sysom_channel/conf/gunicorn.py similarity index 100% rename from microservice/sysom_channel/conf/channel_gunicorn.py rename to microservice/sysom_channel/conf/gunicorn.py diff --git a/microservice/sysom_channel/sysom-channel.ini b/microservice/sysom_channel/sysom-channel.ini index 7e6ed3ec..bd174b39 100644 --- a/microservice/sysom_channel/sysom-channel.ini +++ b/microservice/sysom_channel/sysom-channel.ini @@ -1,6 +1,6 @@ [program:sysom-channel] directory=/usr/local/sysom/server/microservice/sysom_channel -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/channel_gunicorn.py main:app +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app startsecs=3 autostart=true autorestart=true diff --git a/sysom_server/sysom_vmcore/apps/vmcore/__init__.py b/microservice/sysom_vmcore/apps/vmcore/__init__.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/__init__.py rename to microservice/sysom_vmcore/apps/vmcore/__init__.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/admin.py b/microservice/sysom_vmcore/apps/vmcore/admin.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/admin.py rename to microservice/sysom_vmcore/apps/vmcore/admin.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/apps.py b/microservice/sysom_vmcore/apps/vmcore/apps.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/apps.py rename to microservice/sysom_vmcore/apps/vmcore/apps.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/migrations/0001_initial.py b/microservice/sysom_vmcore/apps/vmcore/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/migrations/0001_initial.py rename to microservice/sysom_vmcore/apps/vmcore/migrations/0001_initial.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/migrations/__init__.py b/microservice/sysom_vmcore/apps/vmcore/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/migrations/__init__.py rename to microservice/sysom_vmcore/apps/vmcore/migrations/__init__.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/models.py b/microservice/sysom_vmcore/apps/vmcore/models.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/models.py rename to microservice/sysom_vmcore/apps/vmcore/models.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/serializer.py b/microservice/sysom_vmcore/apps/vmcore/serializer.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/serializer.py rename to microservice/sysom_vmcore/apps/vmcore/serializer.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/tests.py b/microservice/sysom_vmcore/apps/vmcore/tests.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/tests.py rename to microservice/sysom_vmcore/apps/vmcore/tests.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/urls.py b/microservice/sysom_vmcore/apps/vmcore/urls.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/urls.py rename to microservice/sysom_vmcore/apps/vmcore/urls.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/views.py b/microservice/sysom_vmcore/apps/vmcore/views.py similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/views.py rename to microservice/sysom_vmcore/apps/vmcore/views.py diff --git a/sysom_server/sysom_vmcore/apps/vmcore/vmcore.json b/microservice/sysom_vmcore/apps/vmcore/vmcore.json similarity index 100% rename from sysom_server/sysom_vmcore/apps/vmcore/vmcore.json rename to microservice/sysom_vmcore/apps/vmcore/vmcore.json diff --git a/sysom_server/sysom_vmcore/conf/common.py b/microservice/sysom_vmcore/conf/common.py similarity index 98% rename from sysom_server/sysom_vmcore/conf/common.py rename to microservice/sysom_vmcore/conf/common.py index 73be5e25..454c7c9f 100644 --- a/sysom_server/sysom_vmcore/conf/common.py +++ b/microservice/sysom_vmcore/conf/common.py @@ -21,7 +21,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/sysom_server/sysom_vmcore/conf/develop.py b/microservice/sysom_vmcore/conf/develop.py similarity index 100% rename from sysom_server/sysom_vmcore/conf/develop.py rename to microservice/sysom_vmcore/conf/develop.py diff --git a/sysom_server/sysom_vmcore/conf/vmcore_gunicorn.py b/microservice/sysom_vmcore/conf/gunicorn.py similarity index 80% rename from sysom_server/sysom_vmcore/conf/vmcore_gunicorn.py rename to microservice/sysom_vmcore/conf/gunicorn.py index c45abfce..3274a317 100644 --- a/sysom_server/sysom_vmcore/conf/vmcore_gunicorn.py +++ b/microservice/sysom_vmcore/conf/gunicorn.py @@ -5,13 +5,13 @@ workers = 2 # 指定工作进程数 threads = 3 -bind = '127.0.0.1:7004' +bind = '0.0.0.0:7004' # worker_class = 'gevent' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) -accesslog = '/usr/local/sysom/server/logs/sysom-vmcore-access.log' +accesslog = '/var/log/sysom/sysom-vmcore-access.log' loglevel = 'error' diff --git a/sysom_server/sysom_vmcore/conf/product.py b/microservice/sysom_vmcore/conf/product.py similarity index 100% rename from sysom_server/sysom_vmcore/conf/product.py rename to microservice/sysom_vmcore/conf/product.py diff --git a/sysom_server/sysom_vmcore/conf/testing.py b/microservice/sysom_vmcore/conf/testing.py similarity index 100% rename from sysom_server/sysom_vmcore/conf/testing.py rename to microservice/sysom_vmcore/conf/testing.py diff --git a/sysom_server/sysom_vmcore/config.yml b/microservice/sysom_vmcore/config.yml similarity index 100% rename from sysom_server/sysom_vmcore/config.yml rename to microservice/sysom_vmcore/config.yml diff --git a/sysom_server/sysom_vmcore/lib/authentications.py b/microservice/sysom_vmcore/lib/authentications.py similarity index 100% rename from sysom_server/sysom_vmcore/lib/authentications.py rename to microservice/sysom_vmcore/lib/authentications.py diff --git a/sysom_server/sysom_vmcore/lib/base_model.py b/microservice/sysom_vmcore/lib/base_model.py similarity index 100% rename from sysom_server/sysom_vmcore/lib/base_model.py rename to microservice/sysom_vmcore/lib/base_model.py diff --git a/sysom_server/sysom_vmcore/lib/decode/sysom_decode.py b/microservice/sysom_vmcore/lib/decode/sysom_decode.py similarity index 100% rename from sysom_server/sysom_vmcore/lib/decode/sysom_decode.py rename to microservice/sysom_vmcore/lib/decode/sysom_decode.py diff --git a/sysom_server/sysom_vmcore/lib/exception.py b/microservice/sysom_vmcore/lib/exception.py similarity index 100% rename from sysom_server/sysom_vmcore/lib/exception.py rename to microservice/sysom_vmcore/lib/exception.py diff --git a/sysom_server/sysom_vmcore/lib/paginations.py b/microservice/sysom_vmcore/lib/paginations.py similarity index 100% rename from sysom_server/sysom_vmcore/lib/paginations.py rename to microservice/sysom_vmcore/lib/paginations.py diff --git a/sysom_server/sysom_vmcore/lib/renderers.py b/microservice/sysom_vmcore/lib/renderers.py similarity index 100% rename from sysom_server/sysom_vmcore/lib/renderers.py rename to microservice/sysom_vmcore/lib/renderers.py diff --git a/sysom_server/sysom_vmcore/lib/response.py b/microservice/sysom_vmcore/lib/response.py similarity index 100% rename from sysom_server/sysom_vmcore/lib/response.py rename to microservice/sysom_vmcore/lib/response.py diff --git a/sysom_server/sysom_vmcore/lib/utils.py b/microservice/sysom_vmcore/lib/utils.py similarity index 100% rename from sysom_server/sysom_vmcore/lib/utils.py rename to microservice/sysom_vmcore/lib/utils.py diff --git a/sysom_server/sysom_vmcore/manage.py b/microservice/sysom_vmcore/manage.py similarity index 100% rename from sysom_server/sysom_vmcore/manage.py rename to microservice/sysom_vmcore/manage.py diff --git a/microservice/sysom_vmcore/requirements.txt b/microservice/sysom_vmcore/requirements.txt new file mode 100644 index 00000000..c114ec30 --- /dev/null +++ b/microservice/sysom_vmcore/requirements.txt @@ -0,0 +1,15 @@ +clogger>=0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +drf-yasg==1.21.4 +sysom_utils>=0.0.1 +django==3.2.16 +django-filter==21.1 +django-cors-headers==3.10.1 +djangorestframework==3.14.0 +paramiko==2.12.0 +PyJWT==2.4.0 +PyMySQL==1.0.2 +requests==2.27.1 +gunicorn==20.1.0 \ No newline at end of file diff --git a/sysom_server/sysom_vmcore/scripts/node_clear.sh b/microservice/sysom_vmcore/scripts/node_clear.sh similarity index 100% rename from sysom_server/sysom_vmcore/scripts/node_clear.sh rename to microservice/sysom_vmcore/scripts/node_clear.sh diff --git a/sysom_server/sysom_vmcore/scripts/node_init.sh b/microservice/sysom_vmcore/scripts/node_init.sh similarity index 100% rename from sysom_server/sysom_vmcore/scripts/node_init.sh rename to microservice/sysom_vmcore/scripts/node_init.sh diff --git a/sysom_server/sysom_vmcore/scripts/node_update.sh b/microservice/sysom_vmcore/scripts/node_update.sh similarity index 100% rename from sysom_server/sysom_vmcore/scripts/node_update.sh rename to microservice/sysom_vmcore/scripts/node_update.sh diff --git a/microservice/sysom_vmcore/scripts/parse_panic.py b/microservice/sysom_vmcore/scripts/parse_panic.py new file mode 100644 index 00000000..14ce4b68 --- /dev/null +++ b/microservice/sysom_vmcore/scripts/parse_panic.py @@ -0,0 +1,565 @@ +# -*- coding: utf-8 -*- +# @Author: lichen/zhilan + +import os +import sys +import time +import subprocess +import re +import sqlite3 +import json +import traceback +import importlib +import argparse +import requests +import vmcore_const +import time +from datetime import datetime +import threading +import queue +queue = queue.Queue() + +if sys.version[0] == '2': + reload(sys) + sys.setdefaultencoding('utf8') + +# crashkey_type={ +# 0:func_name +# 1:calltrace +# 2:crashkey +# 3:bugon_file +#} +nfs_root = '/usr/vmcore-nfs' +root_url = 'http://localhost' + +ltime_pattern = re.compile(r'^\[\s*([0-9]+)\..*\]') +rip_pattern = re.compile(r'\[\s*\S+\] RIP: 0010:.*\[<([0-9a-f]+)>\] (.+)') +rip_pattern_1 = re.compile(r'\[\s*\S+\] RIP: 0010:(\S+)') +rip_pattern_2 = re.compile(r'\[\s*\S+\] RIP .*\[<([0-9a-f]+)>\] (.+)') +ripmod_pattern = re.compile(r'\[\s*\S+\] RIP.* \[(\S+)\]$') +bugat_pattern = re.compile(r'.+\] kernel BUG at (\S+)!') +ver_pattern = re.compile(r'Comm: (\S*).*(Tainted:|Not tainted).* (\S+) #') +unload_pattern = re.compile(r'\[last unloaded: (\S+)\]') +title_pattern = re.compile(r'\[\s*\S+\] ((BUG:|Kernel panic|Bad pagetable:|divide error:|kernel BUG at|general protection fault:) .+)') +vertype_pattern = re.compile(r'(\d+)\.(\d+)\.') +last_strhost = '' + +ignore_funcs = ["schedule","schedule_timeout","ret_from_fork","kthread", + "do_syscall_64","entry_SYSCALL_64_after_swapgs","system_call_fastpath","fastpath", + "entry_SYSCALL_64_after_hwframe", + "start_secondary","cpu_startup_entry","arch_cpu_idle","default_idle", + "do_IRQ","common_interrupt","irq_exit","do_softirq", + "__schedule","io_schedule_timeout","io_schedule","dump_stack", + "exit_to_usermode_loop","stub_clone","schedule_preempt_disabled","oom_kill_process", + "unwind_backtrace","dump_header","show_stack","dump_backtrace","panic","watchdog_timer_fn", + "nmi_panic","watchdog_overflow_callback","__perf_event_overflow","perf_event_overflow","intel_pmu_handle_irq", + "perf_event_nmi_handler","nmi_handle","do_nmi","end_repeat_nmi","watchdog", + "__hrtimer_run_queues","hrtimer_interrupt","local_apic_timer_interrupt","smp_apic_timer_interrupt","apic_timer_interrupt", + "__pv_queued_spin_lock_slowpath","queued_spin_lock_slowpath" +] + +def get_column_value(column, line): + match = rip_pattern.match(line) + if match is None: + match = rip_pattern_2.match(line) + + if column['func_name']=='NA' and match: + column['rip']=match.group(1) + column['func_name']=match.group(2).split('+')[0] + column['func_name']=column['func_name'].split('.')[0] + ripmod_match = ripmod_pattern.match(line.strip()) + if ripmod_match: + column['ripmod']=ripmod_match.group(1) + match = rip_pattern_1.match(line) + if column['func_name']=='NA' and column.get('func_name_1','') =='NA' and match: + column['func_name_1']=match.group(1).split('+')[0] + column['func_name_1']=column['func_name_1'].split('.')[0] + + match = bugat_pattern.match(line) + if match: + column['bugat'] = match.group(1) + idx = line.find('Comm:') + if idx > 0: + match = ver_pattern.match(line, idx) + if match: + column['comm'] = match.group(1) + column['ver'] = match.group(3) + idx = line.find('[last unloaded:') + if idx > 0: + match = unload_pattern.match(line, idx) + if match: + column['unload'] = match.group(1) + match = title_pattern.match(line) + if match and column['title'] == 'NA': + column['title'] = match.group(1) + if column['func_name'] != 'NA': + column['tmp_func_name'] = column['func_name'] + column['tmp_rip'] = column['rip'] + column['tmp_ripmod'] = column['ripmod'] + column['func_name'] = '' + column['rip'] = '' + column['ripmod'] = '' + +def get_stamp(line): + match = ltime_pattern.match(line) + if match: + return int(match.group(1)) + return 0 + +def get_last_time(f): + ret = 10 + try: + f.seek(-512, os.SEEK_END) + except: + pass + for line in f.readlines(): + ret = get_stamp(line) + if ret > 0: + break + f.seek(0, os.SEEK_SET) + return ret-10 + +def fix_func_name(column): + if column['dmesg'].find('SysRq : Trigger a crash') > 0: + column['func_name'] = 'sysrq_handle_crash' + column['title'] = 'sysrq: SysRq : Trigger a crash' + column['status'] = vmcore_const.STATUS_SYSRQ + column['crashkey_type'] = 2 + column['crashkey'] = 'sysrq_handle_crash' + if column['dmesg'].find('Kernel panic - not syncing: Fatal machine check') > 0: + column['func_name'] = 'fatal_machine_check' + column['title'] = 'Kernel panic - not syncing: Fatal machine check' + column['status'] = vmcore_const.STATUS_HWERROR + column['crashkey_type'] = 2 + column['crashkey'] = 'fatal_machine_check' + column['panic_class'] = 'HardwareError' + if column['dmesg'].find('Kernel panic - not syncing: Fatal hardware error') > 0: + column['func_name'] = 'fatal_hardware_error' + column['title'] = 'Kernel panic - not syncing: Fatal machine check' + column['status'] = vmcore_const.STATUS_HWERROR + column['crashkey_type'] = 2 + column['crashkey'] = 'fatal_hardware_error' + column['panic_class'] = 'HardwareError' + if column['dmesg'].find('Fatal local machine check') > 0: + column['func_name'] = 'fatal_machine_check' + column['title'] = 'Kernel panic - not syncing: Fatal local machine check' + column['status'] = vmcore_const.STATUS_HWERROR + column['crashkey_type'] = 2 + column['crashkey'] = 'fatal_machine_check' + column['panic_class'] = 'HardwareError' + if 'bugat' in column: + column['bugon_file'] = column['bugat'].split(':')[0] + column['crashkey_type'] = 3 + + +def parse_rawdmesg(column): + dmesgs = column['rawdmesg'].splitlines() + column['rawdmesg'] = '' + result = '' + for line in dmesgs: + if line.find('Modules linked in') >= 0: + column['modules'] = line[line.find(':')+1:] + result += line+'\n' + get_column_value(column, line) + column['dmesg'] = result + fix_func_name(column) + +def parse_file(name, column): + f = open(name, 'r') + result = '' + for line in f.readlines(): + if line.find('Modules linked in') >= 0: + column['modules'] = line[line.find(':')+1:] + if len(column['modules']) >= 512: + column['modules'] = column['modules'][:-512] + result += line + get_column_value(column, line) + f.close() + column['dmesg'] = result + column['dmesg_file'] = name + if 'tmp_func_name' in column and column['func_name'] == 'NA' and column['tmp_func_name'] != 'NA': + column['func_name'] = column['tmp_func_name'] + column['rip'] = column['tmp_rip'] + column['ripmod'] = column['ripmod'] + fix_func_name(column) + if column['ripmod'] != 'NA': + if column['ripmod'] not in vmcore_const.BASEMODS: + column['panic_class'] = 'Module(%s)'%(column['ripmod']) + +line_pattern = re.compile(r'.+[0-9]+\].+\[.*\][? ]* (\S+)\+0x(\S+)/0x(\S+)') +line_pattern_1 = re.compile(r'.+[0-9]+\][? ]*(\S+)\+0x(\S+)/0x(\S+)') +def get_calltrace(column): + meettitle = 0 + list1 = [] + lines = column['dmesg'].split('\n') + modname = [] + tmplist = [] + workqueue = '' + nocalltrace = True + hung_flag = False + if column['title'].find('unrecovered softlockup') >= 0: + hung_flag = True + + invalidrip = False + if (column['rip'] == 'NA'and column['func_name'] == 'NA') or column['func_name'].startswith('0x'): + invalidrip = True + + badrip = False + if column['dmesg'].find('Code: Bad RIP value.') >= 0: + badrip = True + + question_continue = True + question_count = 0 + + for r in lines: + if r.find(column['title']) >= 0: + nocalltrace = True + meettitle = 1 + tmplist.extend(list1) + del list1[:] + question_count = 0 + question_continue = True + continue + + if r.find('Workqueue: events ') >= 0: + idx = r.find('Workqueue: events ') + workqueue = r[idx+18:] + + if r.find('EFLAGS: ') >= 0: + idx = r.find('EFLAGS: ') + eflags = r[idx+8:] + #print 'eflags',eflags + try: + eflags = int(eflags,16) + if (eflags >> 9) % 2 == 0: + badrip = True + except: + pass + if r.find("<>") >= 0: + if column['func_name'] == 'NA': + tmpline = lines[lines.index(r)-1] + m = line_pattern.match(tmpline) + if m: + column['func_name'] = m.group(1) + else: + m = line_pattern_1.match(tmpline) + if m: + column['func_name'] = m.group(1) + + if r.find('') >= 0: + badrip = True + + if hung_flag and r.find('') >= 0: + try: + if r.find('> ') >= 0 and r.find(' <') >= 0: + idx = r.find(' <') + idx2 = r.rfind('> ',0) + r = r[0:idx] + r[idx2+1:] + except: + import traceback + traceback.print_exc() + del list1[:] + question_count = 0 + question_continue = True + + if r.find("Call Trace:") > 0 or r.find("<>") > 0 or r.find("") > 0 or r.find("") >= 0: + try: + if r.find('> ') >= 0 and r.find(' <') >= 0: + idx = r.find(' <') + idx2 = r.rfind('> ',0) + r = r[0:idx] + r[idx2+1:] + except: + import traceback + traceback.print_exc() + del list1[:] + question_count = 0 + question_continue = True + modname = [] + + if r.find('?') >= 0: + if workqueue != '' and r.find(workqueue) >= 0: + list1.append(workqueue) + #print r + #print invalidrip,badrip,question_continue + if invalidrip and badrip and question_continue: + m2 = line_pattern.match(r) + if m2: + #print m2.group(1),m2.group(2),m2.group(3) + if m2.group(1).split('.')[0] == column['func_name'] or m2.group(1) in ignore_funcs: + continue + nocalltrace = False + if m2.group(2) != m2.group(3): + tmp = m2.group(1) + tmp = tmp.split('.')[0] + list1.append(tmp) + #print 'append: ',m2.group(1) + #print list1 + question_count += 1 + else: + m2 = line_pattern_1.match(r) + if m2: + #print m2.group(1),m2.group(2),m2.group(3) + if m2.group(1).split('.')[0] == column['func_name'] or m2.group(1) in ignore_funcs: + continue + nocalltrace = False + if m2.group(2) != m2.group(3): + tmp = m2.group(1) + tmp = tmp.split('.')[0] + list1.append(tmp) + #print 'append: ',m2.group(1) + #print list1 + question_count += 1 + continue + if question_count > 0: + question_continue = False + + m = line_pattern.match(r) + if m: + nocalltrace = False + if m.group(1).split('.')[0] == column['func_name'] or m.group(1) in ignore_funcs: + continue + if m.group(1) == 'panic': + del list1[:] + question_count = 0 + question_continue = True + modname = [] + continue + if len(list1) == 0 and m.group(1) in ignore_funcs: + continue + if len(modname) < 2: + modname.append(r.strip()) + tmp = m.group(1) + tmp = tmp.split('.')[0] + list1.append(tmp) + #print 'append: ',m.group(1) + #print list1 + else: + m = line_pattern_1.match(r) + if m: + nocalltrace = False + if m.group(1).split('.')[0] == column['func_name'] or m.group(1) in ignore_funcs: + continue + if m.group(1) == 'panic': + del list1[:] + question_count = 0 + question_continue = True + modname = [] + continue + if len(list1) == 0 and m.group(1) in ignore_funcs: + continue + if len(modname) < 2: + modname.append(r.strip()) + tmp = m.group(1) + tmp = tmp.split('.')[0] + list1.append(tmp) + #print 'append: ',m.group(1) + #print list1 + else: + if len(list1) > 0 and meettitle == 1: + break + if len(list1) == 0 and nocalltrace: + list1 = tmplist + + if column['func_name'] == 'NA' and len(list1) > 0: + column['func_name'] = list1[0] + del list1[0] + + column['calltrace_list'] = [] + column['calltrace_list'].extend(list1) + + calltrace = column['func_name'] + if calltrace != '': + calltrace = calltrace.split('+')[0] + if len(list1) > 2: + list1 = list1[0:2] + for i in list1: + calltrace = ''.join([calltrace,'$',i]) + column['calltrace'] = calltrace + + +def clarify_panic_type(column): + column['panic_type'] = 0 + if column['title'].find('divide error') >= 0: + column['panic_type'] = vmcore_const.PANIC_DIVIDEERROR + elif column['bugon_file'] != 'NA': + column['panic_type'] = vmcore_const.PANIC_BUGON + elif column['title'].find('NULL pointer dereference') >= 0: + column['panic_type'] = vmcore_const.PANIC_NULLPOINTER + elif column['title'].find('Kernel stack is corrupted') >= 0: + column['panic_type'] = vmcore_const.PANIC_STACKCORRUPTION + elif column['title'].find('hard LOCKUP') >= 0: + column['panic_type'] = vmcore_const.PANIC_HARDLOCKUP + elif column['title'].find('hung_task') >= 0: + column['panic_type'] = vmcore_const.PANIC_HUNGTASK + elif column['title'].find('RCU Stall') >= 0: + column['panic_type'] = vmcore_const.PANIC_RCUSTALL + elif (column['title'].find('soft lockup') >= 0 or column['title'].find('softlockup') >= 0): + column['panic_type'] = vmcore_const.PANIC_SOFTLOCKUP + +def check_panic(column): + if 'rawdmesg' not in column and os.path.isfile(column['dmesg_file']) == False: + return False + + matched = False + if 'rawdmesg' in column: + parse_rawdmesg(column) + else: + parse_file(column['dmesg_file'], column) + + m = vertype_pattern.match(column['ver']) + if m: + column['vertype'] = int(m.group(1)) * 100 + int(m.group(2)) + + get_calltrace(column) + if column['calltrace'] == 'NA': + column['crashkey_type'] = 0 + if column['crashkey_type'] == 0 and column['func_name'] != 'NA': + column['crashkey'] = '%d$%s'%(column['vertype'],column['func_name']) + elif column['crashkey_type'] == 1 and column['calltrace'] != 'NA': + column['crashkey'] = '%d$%s'%(column['vertype'],column['calltrace']) + elif column['crashkey_type'] == 2 and column['crashkey'] != 'NA': + column['crashkey'] = '%d$%s'%(column['vertype'],column['crashkey']) + elif column['crashkey_type'] == 3 and column['bugon_file'] != 'NA': + column['crashkey'] = '%d$%s$%s'%(column['vertype'],column['bugon_file'],column['calltrace']) + + clarify_panic_type(column) + #return False + + ip={'ip':column['ip']} + host_url = root_url+"/api/v1/host/" + res = requests.get(host_url,params=ip) + if res.status_code != 200 or res.text == '[]': + print("查询主机名失败") + return False + + column['hostname'] = res.json()['data'][0]['hostname'] + vmcore_url = root_url+"/api/v1/vmcore/" + data = json.dumps(column) + headers = {'content-type': 'application/json'} + res = requests.post(url=vmcore_url, data=data, headers=headers) + print(res.json()) + if res.status_code == 200: + print(f"add {column['name']} to db") + else: + print("插入失败") + return False + + idx = 0 + column['calltrace_list'].insert(0,column['func_name']) + for line in column['calltrace_list']: + calltrace_info = {'name':column['name'], 'line':line, 'idx':idx} + calltrace_url = root_url+"/api/v1/vmcore/" + data = json.dumps(calltrace_info) + headers = {'content-type': 'application/json'} + res = requests.post(url=calltrace_url, data=data, headers=headers) + if res.status_code != 200: + print(f"{column['name']} 插入calltrace line失败") + return False + idx += 1 + print(f"add {column['name']} calltrace line to db") + return True +def do_cmd(cmd): + output = os.popen(cmd) + ret = output.read().strip() + output.close() + return ret + +def init_column(column): + column['upload_time'] = int(time.time()) + column['vmcore_file'] = 'NA' + column['dmesg_file'] = 'NA' + column['rip'] = 'NA' + column['ripmod'] = 'NA' + column['comm'] = 'NA' + column['ver'] = 'NA' + column['vertype'] = 0 + column['func_name'] = 'NA' + column['title'] = 'NA' + column['status'] = 0 + column['calltrace'] = 'NA' + column['bugon_file'] = 'NA' + column['crashkey_type'] = 1 + column['crashkey'] = 'NA' + column['modules'] = 'NA' + column['panic_type'] = 0 + column['panic_class'] = 'BaseKernel' + column['issue_id'] = 0 + +def parse_new_crash(crash_dir): + try: + column = {} + column['name'] = crash_dir.split('/')[-1] + core_time = column['name'].split('_')[0] + core_time = datetime.strptime(core_time, "%Y%m%d%H%M%S") + column['core_time'] = core_time.strftime("%Y-%m-%d %H:%M:%S") + column['ip'] = column['name'].split('_')[1] + column['hostname'] = column['name'].split('_')[1] + init_column(column) + column['dmesg_file'] = '%s/vmcore-dmesg.txt' % crash_dir + column['vmcore_file'] = '%s/vmcore' % crash_dir + ret = check_panic(column) + if ret: + with open('%s/.upload'%crash_dir,'w'): + pass + except: + import traceback + traceback.print_exc() + +def mount_nfs(): + global nfs_root + get_config={'get_config':'1'} + host_url = root_url+"/api/v1/vmcore/" + res = requests.get(host_url,params=get_config) + if res.status_code != 200 or res.text == '[]': + print("无法查询nfs配置") + return False + server_host = res.json()['data']['server_host'] + mount_point = res.json()['data']['mount_point'] + if not server_host : + return False + + cmd = 'mount -t nfs %s:%s %s' % (server_host,mount_point,nfs_root) + ret = os.system(cmd) + if ret != 0: + print('failed to mount to nfs %s' % nfs_root) + return False + return True + +def unmount_nfs(): + global nfs_root + cmd = 'umount %s' % nfs_root + ret = os.system(cmd) + if ret != 0: + print(f'failed to unmount nfs at {nfs_root}') + +def main(): + global nfs_root + global root_url + server_port = 80 + if len(sys.argv) > 1: + nfs_root = sys.argv[1] + if len(sys.argv) > 2: + server_port = sys.argv[2] + root_url = root_url + ":" + server_port + dirs_list = [] + #while True: + hasnfs = mount_nfs() + files = os.listdir(nfs_root) + files_path = [f'{nfs_root}/{file}' for file in files] + for file in files_path: + if os.path.isfile(file): + continue + dirs_list.append(file) + dirs_list.sort(key=lambda fp: os.path.getmtime(fp),reverse=True) + for dir in dirs_list: + tmp = '%s/.upload' % dir + if os.path.exists(tmp): + break + parse_new_crash(dir) + #time.sleep(20) + if hasnfs: + unmount_nfs() + + +if __name__ == "__main__": + main() diff --git a/microservice/sysom_vmcore/scripts/server_clear.sh b/microservice/sysom_vmcore/scripts/server_clear.sh new file mode 100644 index 00000000..06921d22 --- /dev/null +++ b/microservice/sysom_vmcore/scripts/server_clear.sh @@ -0,0 +1,15 @@ +#!/bin/bash +SERVICE_NAME=sysom-vmcore +clear_app() { + sed -i '/vmcore/d' /var/spool/cron/root + sed -i '/vmcore/d' /etc/exports + exportfs -rv + systemctl stop nfs-server + systemctl stop rpcbind + systemctl stop nfs + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} + +clear_app diff --git a/microservice/sysom_vmcore/scripts/server_init.sh b/microservice/sysom_vmcore/scripts/server_init.sh new file mode 100644 index 00000000..70318fc7 --- /dev/null +++ b/microservice/sysom_vmcore/scripts/server_init.sh @@ -0,0 +1,77 @@ +#! /bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_vmcore +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +VIRTUALENV_PYTHON3=${SERVER_HOME}/virtualenv/bin/python3 +SERVICE_NAME=sysom-vmcore + +BASE_DIR=$(dirname $0) + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +init_conf() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +start_nfs() +{ + systemctl start rpcbind && systemctl enable rpcbind + systemctl start nfs && systemctl enable nfs + if [ $? -ne 0 ];then + systemctl start nfs-server && systemctl enable nfs-server + fi + + nfs_mask=`ip -4 route | grep "link src" | grep $SERVER_LOCAL_IP | awk '{print $1}' | head -n 1` + file_path=${SERVER_HOME}/vmcore/vmcore-nfs + mkdir -p ${file_path} + echo "${file_path} ${nfs_mask}(rw,async)" >> /etc/exports + exportfs -rv + chmod -R 777 ${file_path} +} + +start_cron() +{ + pushd $BASE_DIR + cp parse_panic.py ${SERVER_HOME}/vmcore + cp vmcore_const.py ${SERVER_HOME}/vmcore + echo "* * * * * pushd ${SERVER_HOME}/vmcore;${VIRTUALENV_PYTHON3} parse_panic.py ${file_path} ${SERVER_PORT};popd" >> /var/spool/cron/root + popd +} + +deploy() { + source_virtualenv + install_requirement + init_conf + start_app + start_nfs + start_cron +} + +deploy diff --git a/microservice/sysom_vmcore/scripts/server_start.sh b/microservice/sysom_vmcore/scripts/server_start.sh new file mode 100644 index 00000000..1bb89fb1 --- /dev/null +++ b/microservice/sysom_vmcore/scripts/server_start.sh @@ -0,0 +1,13 @@ +#!/bin/bash +SERVICE_NAME=sysom-vmcore +start_app() { + systemctl start nfs-server + systemctl start rpcbind + systemctl start nfs + sed -i '/vmcore/s;^#;;g' /var/spool/cron/root + sed -i '/vmcore/s;^#;;g' /etc/exports + exportfs -rv + supervisorctl start $SERVICE_NAME +} + +start_app diff --git a/microservice/sysom_vmcore/scripts/server_stop.sh b/microservice/sysom_vmcore/scripts/server_stop.sh new file mode 100644 index 00000000..aadecadc --- /dev/null +++ b/microservice/sysom_vmcore/scripts/server_stop.sh @@ -0,0 +1,13 @@ +#!/bin/bash +SERVICE_NAME=sysom-vmcore +stop_app() { + sed -i '/vmcore/s;^;#;g' /var/spool/cron/root + sed -i '/vmcore/s;^;#;g' /etc/exports + exportfs -rv + systemctl stop nfs-server + systemctl stop rpcbind + systemctl stop nfs + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/sysom_server/sysom_vmcore/scripts/vmcore_collect.py b/microservice/sysom_vmcore/scripts/vmcore_collect.py similarity index 100% rename from sysom_server/sysom_vmcore/scripts/vmcore_collect.py rename to microservice/sysom_vmcore/scripts/vmcore_collect.py diff --git a/microservice/sysom_vmcore/scripts/vmcore_const.py b/microservice/sysom_vmcore/scripts/vmcore_const.py new file mode 100644 index 00000000..8f9ac84a --- /dev/null +++ b/microservice/sysom_vmcore/scripts/vmcore_const.py @@ -0,0 +1,25 @@ +STATUS_HWERROR=5 +STATUS_SYSRQ=6 +# panic type +PANIC_UAF=1 +PANIC_DOUBLEFREE=2 +PANIC_OOBREAD=3 +PANIC_OOBWRITE=4 +PANIC_NULLPOINTER=5 +PANIC_UNINITVAR=6 +PANIC_STACKCORRUPTION=7 +PANIC_INVALIDIPPTR=8 +PANIC_INVALIDDATAPTR=9 +PANIC_BUGON=10 +PANIC_DIVIDEERROR=11 +PANIC_HARDLOCKUP=12 +PANIC_SOFTLOCKUP=13 +PANIC_HUNGTASK=14 +PANIC_RCUSTALL=15 + +BASEMODS = ['ext4','jbd2','overlay','libata','libiscsi','bridge','nf_conntrack','nf_conntrack_ipv4', + 'nf_nat','nf_nat_ipv4','iptable_nat','tun','binfmt_misc','xt_CHECKSUM','iptable_mangle', + 'nf_defrag_ipv4','xt_conntrack','ipt_REJECT','nf_reject_ipv4','stp','llc','ebtable_filter', + 'ebtables','ip6_tables','iptable_filter','iscsi_tcp','libiscsi_tcp','scsi_transport_iscsi', + 'bonding','dm_mod','sg','ip_tables','mbcache','sd_mod','mpt3sas','raid_class','scsi_transport_sas', + 'ahci','libahci','btrfs','zram','numa_balancer'] diff --git a/microservice/sysom_vmcore/sysom-vmcore.ini b/microservice/sysom_vmcore/sysom-vmcore.ini new file mode 100644 index 00000000..c436b818 --- /dev/null +++ b/microservice/sysom_vmcore/sysom-vmcore.ini @@ -0,0 +1,9 @@ +[program:sysom-vmcore] +directory = /usr/local/sysom/server/microservice/sysom_vmcore +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vmcore.wsgi:application +startsecs=3 +autostart=true +autorestart=true +environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +stderr_logfile=/var/log/sysom/sysom-vmcore-error.log +stdout_logfile=/var/log/sysom/sysom-vmcore.log diff --git a/sysom_server/sysom_vmcore/sysom_vmcore/__init__.py b/microservice/sysom_vmcore/sysom_vmcore/__init__.py similarity index 100% rename from sysom_server/sysom_vmcore/sysom_vmcore/__init__.py rename to microservice/sysom_vmcore/sysom_vmcore/__init__.py diff --git a/sysom_server/sysom_vmcore/sysom_vmcore/asgi.py b/microservice/sysom_vmcore/sysom_vmcore/asgi.py similarity index 100% rename from sysom_server/sysom_vmcore/sysom_vmcore/asgi.py rename to microservice/sysom_vmcore/sysom_vmcore/asgi.py diff --git a/sysom_server/sysom_vmcore/sysom_vmcore/settings.py b/microservice/sysom_vmcore/sysom_vmcore/settings.py similarity index 100% rename from sysom_server/sysom_vmcore/sysom_vmcore/settings.py rename to microservice/sysom_vmcore/sysom_vmcore/settings.py diff --git a/sysom_server/sysom_vmcore/sysom_vmcore/urls.py b/microservice/sysom_vmcore/sysom_vmcore/urls.py similarity index 100% rename from sysom_server/sysom_vmcore/sysom_vmcore/urls.py rename to microservice/sysom_vmcore/sysom_vmcore/urls.py diff --git a/sysom_server/sysom_vmcore/sysom_vmcore/wsgi.py b/microservice/sysom_vmcore/sysom_vmcore/wsgi.py similarity index 100% rename from sysom_server/sysom_vmcore/sysom_vmcore/wsgi.py rename to microservice/sysom_vmcore/sysom_vmcore/wsgi.py -- Gitee From 340ec76301cbdb58418a0cc4bf1b23bbfadcefcc Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 17:00:10 +0800 Subject: [PATCH 024/196] refactor(sysom_migration): Use sysom.sh to unify deployment --- .../apps/migration/__init__.py | 0 .../sysom_migration/apps/migration/admin.py | 0 .../sysom_migration/apps/migration/apps.py | 0 .../apps/migration/migrations/0001_initial.py | 0 .../apps/migration/migrations/__init__.py | 0 .../sysom_migration/apps/migration/models.py | 0 .../sysom_migration/apps/migration/tests.py | 0 .../sysom_migration/apps/migration/urls.py | 0 .../sysom_migration/apps/migration/views.py | 0 .../sysom_migration/conf/common.py | 2 +- .../sysom_migration/conf/develop.py | 0 .../sysom_migration/conf/gunicorn.py | 4 +- .../sysom_migration/conf/product.py | 0 .../sysom_migration/conf/testing.py | 0 .../sysom_migration/config.yml | 0 .../sysom_migration/lib/authentications.py | 0 .../sysom_migration/lib/base_model.py | 0 .../sysom_migration/lib/base_view.py | 0 .../sysom_migration/lib/channel.py | 0 .../lib/decode/sysom_decode.py | 0 .../sysom_migration/lib/exception.py | 0 .../sysom_migration/lib/paginations.py | 0 .../sysom_migration/lib/renderers.py | 0 .../sysom_migration/lib/response.py | 0 .../sysom_migration/lib/script.py | 0 .../sysom_migration/lib/utils.py | 0 .../sysom_migration/manage.py | 0 microservice/sysom_migration/requirements.txt | 15 ++++ .../sysom_migration/scripts/server_clear.sh | 0 .../sysom_migration/scripts/server_init.sh | 85 +++++++++++++++++++ .../sysom_migration/scripts/server_start.sh | 0 .../sysom_migration/scripts/server_stop.sh | 0 .../sysom_migration/sysom-migration.ini | 9 ++ .../sysom_migration/__init__.py | 0 .../sysom_migration/sysom_migration/asgi.py | 0 .../sysom_migration/settings.py | 0 .../sysom_migration/sysom_migration/urls.py | 0 .../sysom_migration/sysom_migration/wsgi.py | 0 38 files changed, 112 insertions(+), 3 deletions(-) rename {sysom_server => microservice}/sysom_migration/apps/migration/__init__.py (100%) rename {sysom_server => microservice}/sysom_migration/apps/migration/admin.py (100%) rename {sysom_server => microservice}/sysom_migration/apps/migration/apps.py (100%) rename {sysom_server => microservice}/sysom_migration/apps/migration/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_migration/apps/migration/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_migration/apps/migration/models.py (100%) rename {sysom_server => microservice}/sysom_migration/apps/migration/tests.py (100%) rename {sysom_server => microservice}/sysom_migration/apps/migration/urls.py (100%) rename {sysom_server => microservice}/sysom_migration/apps/migration/views.py (100%) rename {sysom_server => microservice}/sysom_migration/conf/common.py (98%) rename {sysom_server => microservice}/sysom_migration/conf/develop.py (100%) rename sysom_server/sysom_migration/conf/migration_gunicorn.py => microservice/sysom_migration/conf/gunicorn.py (80%) rename {sysom_server => microservice}/sysom_migration/conf/product.py (100%) rename {sysom_server => microservice}/sysom_migration/conf/testing.py (100%) rename {sysom_server => microservice}/sysom_migration/config.yml (100%) rename {sysom_server => microservice}/sysom_migration/lib/authentications.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/base_model.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/base_view.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/channel.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/decode/sysom_decode.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/exception.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/paginations.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/renderers.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/response.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/script.py (100%) rename {sysom_server => microservice}/sysom_migration/lib/utils.py (100%) rename {sysom_server => microservice}/sysom_migration/manage.py (100%) create mode 100644 microservice/sysom_migration/requirements.txt create mode 100644 microservice/sysom_migration/scripts/server_clear.sh create mode 100644 microservice/sysom_migration/scripts/server_init.sh create mode 100644 microservice/sysom_migration/scripts/server_start.sh create mode 100644 microservice/sysom_migration/scripts/server_stop.sh create mode 100644 microservice/sysom_migration/sysom-migration.ini rename {sysom_server => microservice}/sysom_migration/sysom_migration/__init__.py (100%) rename {sysom_server => microservice}/sysom_migration/sysom_migration/asgi.py (100%) rename {sysom_server => microservice}/sysom_migration/sysom_migration/settings.py (100%) rename {sysom_server => microservice}/sysom_migration/sysom_migration/urls.py (100%) rename {sysom_server => microservice}/sysom_migration/sysom_migration/wsgi.py (100%) diff --git a/sysom_server/sysom_migration/apps/migration/__init__.py b/microservice/sysom_migration/apps/migration/__init__.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/__init__.py rename to microservice/sysom_migration/apps/migration/__init__.py diff --git a/sysom_server/sysom_migration/apps/migration/admin.py b/microservice/sysom_migration/apps/migration/admin.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/admin.py rename to microservice/sysom_migration/apps/migration/admin.py diff --git a/sysom_server/sysom_migration/apps/migration/apps.py b/microservice/sysom_migration/apps/migration/apps.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/apps.py rename to microservice/sysom_migration/apps/migration/apps.py diff --git a/sysom_server/sysom_migration/apps/migration/migrations/0001_initial.py b/microservice/sysom_migration/apps/migration/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/migrations/0001_initial.py rename to microservice/sysom_migration/apps/migration/migrations/0001_initial.py diff --git a/sysom_server/sysom_migration/apps/migration/migrations/__init__.py b/microservice/sysom_migration/apps/migration/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/migrations/__init__.py rename to microservice/sysom_migration/apps/migration/migrations/__init__.py diff --git a/sysom_server/sysom_migration/apps/migration/models.py b/microservice/sysom_migration/apps/migration/models.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/models.py rename to microservice/sysom_migration/apps/migration/models.py diff --git a/sysom_server/sysom_migration/apps/migration/tests.py b/microservice/sysom_migration/apps/migration/tests.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/tests.py rename to microservice/sysom_migration/apps/migration/tests.py diff --git a/sysom_server/sysom_migration/apps/migration/urls.py b/microservice/sysom_migration/apps/migration/urls.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/urls.py rename to microservice/sysom_migration/apps/migration/urls.py diff --git a/sysom_server/sysom_migration/apps/migration/views.py b/microservice/sysom_migration/apps/migration/views.py similarity index 100% rename from sysom_server/sysom_migration/apps/migration/views.py rename to microservice/sysom_migration/apps/migration/views.py diff --git a/sysom_server/sysom_migration/conf/common.py b/microservice/sysom_migration/conf/common.py similarity index 98% rename from sysom_server/sysom_migration/conf/common.py rename to microservice/sysom_migration/conf/common.py index c805ae13..981ad5e1 100644 --- a/sysom_server/sysom_migration/conf/common.py +++ b/microservice/sysom_migration/conf/common.py @@ -20,7 +20,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/sysom_server/sysom_migration/conf/develop.py b/microservice/sysom_migration/conf/develop.py similarity index 100% rename from sysom_server/sysom_migration/conf/develop.py rename to microservice/sysom_migration/conf/develop.py diff --git a/sysom_server/sysom_migration/conf/migration_gunicorn.py b/microservice/sysom_migration/conf/gunicorn.py similarity index 80% rename from sysom_server/sysom_migration/conf/migration_gunicorn.py rename to microservice/sysom_migration/conf/gunicorn.py index 8d80fda1..52639f43 100644 --- a/sysom_server/sysom_migration/conf/migration_gunicorn.py +++ b/microservice/sysom_migration/conf/gunicorn.py @@ -5,13 +5,13 @@ workers = 4 # 指定工作进程数 threads = 8 -bind = '127.0.0.1:7006' +bind = '0.0.0.0:7006' # worker_class = 'gevent' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) -accesslog = '/usr/local/sysom/server/logs/sysom-migration-access.log' +accesslog = '/var/log/sysom/sysom-migration-access.log' loglevel = 'error' diff --git a/sysom_server/sysom_migration/conf/product.py b/microservice/sysom_migration/conf/product.py similarity index 100% rename from sysom_server/sysom_migration/conf/product.py rename to microservice/sysom_migration/conf/product.py diff --git a/sysom_server/sysom_migration/conf/testing.py b/microservice/sysom_migration/conf/testing.py similarity index 100% rename from sysom_server/sysom_migration/conf/testing.py rename to microservice/sysom_migration/conf/testing.py diff --git a/sysom_server/sysom_migration/config.yml b/microservice/sysom_migration/config.yml similarity index 100% rename from sysom_server/sysom_migration/config.yml rename to microservice/sysom_migration/config.yml diff --git a/sysom_server/sysom_migration/lib/authentications.py b/microservice/sysom_migration/lib/authentications.py similarity index 100% rename from sysom_server/sysom_migration/lib/authentications.py rename to microservice/sysom_migration/lib/authentications.py diff --git a/sysom_server/sysom_migration/lib/base_model.py b/microservice/sysom_migration/lib/base_model.py similarity index 100% rename from sysom_server/sysom_migration/lib/base_model.py rename to microservice/sysom_migration/lib/base_model.py diff --git a/sysom_server/sysom_migration/lib/base_view.py b/microservice/sysom_migration/lib/base_view.py similarity index 100% rename from sysom_server/sysom_migration/lib/base_view.py rename to microservice/sysom_migration/lib/base_view.py diff --git a/sysom_server/sysom_migration/lib/channel.py b/microservice/sysom_migration/lib/channel.py similarity index 100% rename from sysom_server/sysom_migration/lib/channel.py rename to microservice/sysom_migration/lib/channel.py diff --git a/sysom_server/sysom_migration/lib/decode/sysom_decode.py b/microservice/sysom_migration/lib/decode/sysom_decode.py similarity index 100% rename from sysom_server/sysom_migration/lib/decode/sysom_decode.py rename to microservice/sysom_migration/lib/decode/sysom_decode.py diff --git a/sysom_server/sysom_migration/lib/exception.py b/microservice/sysom_migration/lib/exception.py similarity index 100% rename from sysom_server/sysom_migration/lib/exception.py rename to microservice/sysom_migration/lib/exception.py diff --git a/sysom_server/sysom_migration/lib/paginations.py b/microservice/sysom_migration/lib/paginations.py similarity index 100% rename from sysom_server/sysom_migration/lib/paginations.py rename to microservice/sysom_migration/lib/paginations.py diff --git a/sysom_server/sysom_migration/lib/renderers.py b/microservice/sysom_migration/lib/renderers.py similarity index 100% rename from sysom_server/sysom_migration/lib/renderers.py rename to microservice/sysom_migration/lib/renderers.py diff --git a/sysom_server/sysom_migration/lib/response.py b/microservice/sysom_migration/lib/response.py similarity index 100% rename from sysom_server/sysom_migration/lib/response.py rename to microservice/sysom_migration/lib/response.py diff --git a/sysom_server/sysom_migration/lib/script.py b/microservice/sysom_migration/lib/script.py similarity index 100% rename from sysom_server/sysom_migration/lib/script.py rename to microservice/sysom_migration/lib/script.py diff --git a/sysom_server/sysom_migration/lib/utils.py b/microservice/sysom_migration/lib/utils.py similarity index 100% rename from sysom_server/sysom_migration/lib/utils.py rename to microservice/sysom_migration/lib/utils.py diff --git a/sysom_server/sysom_migration/manage.py b/microservice/sysom_migration/manage.py similarity index 100% rename from sysom_server/sysom_migration/manage.py rename to microservice/sysom_migration/manage.py diff --git a/microservice/sysom_migration/requirements.txt b/microservice/sysom_migration/requirements.txt new file mode 100644 index 00000000..c114ec30 --- /dev/null +++ b/microservice/sysom_migration/requirements.txt @@ -0,0 +1,15 @@ +clogger>=0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +drf-yasg==1.21.4 +sysom_utils>=0.0.1 +django==3.2.16 +django-filter==21.1 +django-cors-headers==3.10.1 +djangorestframework==3.14.0 +paramiko==2.12.0 +PyJWT==2.4.0 +PyMySQL==1.0.2 +requests==2.27.1 +gunicorn==20.1.0 \ No newline at end of file diff --git a/microservice/sysom_migration/scripts/server_clear.sh b/microservice/sysom_migration/scripts/server_clear.sh new file mode 100644 index 00000000..e69de29b diff --git a/microservice/sysom_migration/scripts/server_init.sh b/microservice/sysom_migration/scripts/server_init.sh new file mode 100644 index 00000000..1df902f8 --- /dev/null +++ b/microservice/sysom_migration/scripts/server_init.sh @@ -0,0 +1,85 @@ +#!/bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_migration +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_NAME=sysom-migration +ANCE_X86_PKG=ance-0.1.1-1.x86_64.rpm +ANCE_ARM_PKG=ance-0.1.1-1.aarch64.rpm +ANOLIS_X86_SQLITE=Anolis_OS-8.6.x86_64.sqlite +ANOLIS_ARM_SQLITE=Anolis_OS-8.6.aarch64.sqlite +ANOLIS_MIGRATION_PKGS_X86=anolis_migration_pkgs_x86_64.tar.gz +ANOLIS_MIGRATION_PKGS_ARM=anolis_migration_pkgs_aarch64.tar.gz + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +init_conf() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/gunicorn.py +} + +check_or_download_ance() { + mkdir -p ${SERVICE_HOME}/ance + pushd ${SERVICE_HOME}/ance + if [ ! -f "${ANCE_X86_PKG}" ]; then + wget "https://ance.oss-cn-hangzhou.aliyuncs.com/release/x86_64/${ANCE_X86_PKG}" + fi + if [ ! -f "${ANCE_ARM_PKG}" ]; then + wget "https://ance.oss-cn-hangzhou.aliyuncs.com/release/aarch64/${ANCE_ARM_PKG}" + fi + if [ ! -f "${ANOLIS_X86_SQLITE}" ]; then + wget "https://ance.oss-cn-hangzhou.aliyuncs.com/databases/${ANOLIS_X86_SQLITE}" + fi + if [ ! -f "${ANOLIS_ARM_SQLITE}" ]; then + wget "https://ance.oss-cn-hangzhou.aliyuncs.com/databases/${ANOLIS_ARM_SQLITE}" + fi + if [ ! -f "${ANOLIS_MIGRATION_PKGS_X86}" ]; then + wget "https://gitee.com/src-anolis-sig/leapp/releases/download/v1.0.2-all-in-one/${ANOLIS_MIGRATION_PKGS_X86}" + fi + if [ ! -f "${ANOLIS_MIGRATION_PKGS_ARM}" ]; then + wget "https://gitee.com/src-anolis-sig/leapp/releases/download/v1.0.2-all-in-one/${ANOLIS_MIGRATION_PKGS_ARM}" + fi + popd +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + check_or_download_ance + start_app +} + +deploy diff --git a/microservice/sysom_migration/scripts/server_start.sh b/microservice/sysom_migration/scripts/server_start.sh new file mode 100644 index 00000000..e69de29b diff --git a/microservice/sysom_migration/scripts/server_stop.sh b/microservice/sysom_migration/scripts/server_stop.sh new file mode 100644 index 00000000..e69de29b diff --git a/microservice/sysom_migration/sysom-migration.ini b/microservice/sysom_migration/sysom-migration.ini new file mode 100644 index 00000000..46cb2244 --- /dev/null +++ b/microservice/sysom_migration/sysom-migration.ini @@ -0,0 +1,9 @@ +[program:sysom-migration] +directory = /usr/local/sysom/server/microservice/sysom_migration +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_migration.wsgi:application +startsecs=3 +autostart=true +autorestart=true +environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +stderr_logfile=/var/log/sysom/sysom-migration-error.log +stdout_logfile=/var/log/sysom/sysom-migration.log diff --git a/sysom_server/sysom_migration/sysom_migration/__init__.py b/microservice/sysom_migration/sysom_migration/__init__.py similarity index 100% rename from sysom_server/sysom_migration/sysom_migration/__init__.py rename to microservice/sysom_migration/sysom_migration/__init__.py diff --git a/sysom_server/sysom_migration/sysom_migration/asgi.py b/microservice/sysom_migration/sysom_migration/asgi.py similarity index 100% rename from sysom_server/sysom_migration/sysom_migration/asgi.py rename to microservice/sysom_migration/sysom_migration/asgi.py diff --git a/sysom_server/sysom_migration/sysom_migration/settings.py b/microservice/sysom_migration/sysom_migration/settings.py similarity index 100% rename from sysom_server/sysom_migration/sysom_migration/settings.py rename to microservice/sysom_migration/sysom_migration/settings.py diff --git a/sysom_server/sysom_migration/sysom_migration/urls.py b/microservice/sysom_migration/sysom_migration/urls.py similarity index 100% rename from sysom_server/sysom_migration/sysom_migration/urls.py rename to microservice/sysom_migration/sysom_migration/urls.py diff --git a/sysom_server/sysom_migration/sysom_migration/wsgi.py b/microservice/sysom_migration/sysom_migration/wsgi.py similarity index 100% rename from sysom_server/sysom_migration/sysom_migration/wsgi.py rename to microservice/sysom_migration/sysom_migration/wsgi.py -- Gitee From 92203bc5889de4ad35ab76878c41734db1054023 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 17:00:41 +0800 Subject: [PATCH 025/196] feat(sysom_vmcore): Support docker deploment --- sysom_migration_dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 sysom_migration_dockerfile diff --git a/sysom_migration_dockerfile b/sysom_migration_dockerfile new file mode 100644 index 00000000..e34ec3d2 --- /dev/null +++ b/sysom_migration_dockerfile @@ -0,0 +1,31 @@ +FROM openanolis/anolisos:8.8 + +# Add epel +RUN yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm +RUN bash -c "sed -i 's|^#baseurl=https://download.example/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel*" +RUN bash -c "sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel*" + +# Add required yum packages +RUN yum makecache +RUN yum install -y supervisor cronie net-tools wget +RUN systemctl enable crond +# RUN systemctl enable supervisord + +# Init sysom-diagnosis +ARG SYSOM_HOME=/usr/local/sysom +ARG SYSOM_SERVER_HOME=${SYSOM_HOME}/server + +RUN mkdir /root/sysom +COPY conf /root/sysom/conf +COPY script /root/sysom/script +COPY infrastructure /root/sysom/infrastructure +COPY microservice /root/sysom/microservice + +RUN bash -x /root/sysom/script/sysom.sh deploy infrastructure env,sdk +RUN bash -x /root/sysom/script/sysom.sh deploy microservice sysom_migration +RUN sed "s/nodaemon=false/nodaemon=true/g" -i /etc/supervisord.conf + +RUN yum clean all + +# # 环境准备 +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file -- Gitee From 8d50cf6ddb28de96a6a81be6ed29cde407863f43 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 17:14:46 +0800 Subject: [PATCH 026/196] refactor(sysom_vul): Use sysom.sh to unify deployment --- .../sysom_vul/apps/host/__init__.py | 0 .../sysom_vul/apps/host/app.py | 0 .../sysom_vul/apps/host/models.py | 0 .../sysom_vul/apps/host/urls.py | 0 .../sysom_vul/apps/vul/__init__.py | 0 .../sysom_vul/apps/vul/admin.py | 0 .../sysom_vul/apps/vul/apps.py | 0 .../sysom_vul/apps/vul/async_fetch.py | 0 .../sysom_vul/apps/vul/executor.py | 0 .../apps/vul/migrations/0001_initial.py | 0 .../vul/migrations/0002_auto_20230324_1840.py | 0 .../sysom_vul/apps/vul/migrations/__init__.py | 0 .../sysom_vul/apps/vul/models.py | 0 .../sysom_vul/apps/vul/serializer.py | 0 .../sysom_vul/apps/vul/ssh_pool.py | 0 .../sysom_vul/apps/vul/tests.py | 0 .../sysom_vul/apps/vul/urls.py | 0 .../sysom_vul/apps/vul/views.py | 0 .../sysom_vul/apps/vul/vul.py | 0 .../sysom_vul/conf/__init__.py | 0 .../sysom_vul/conf/common.py | 2 +- .../sysom_vul/conf/develop.py | 0 .../sysom_vul/conf/gunicorn.py | 4 +- .../sysom_vul/conf/product.py | 0 .../sysom_vul/conf/testing.py | 0 .../sysom_vul/config.yml | 0 .../sysom_vul/lib/authentications.py | 0 .../sysom_vul/lib/base_model.py | 0 .../sysom_vul/lib/decode/sysom_decode.py | 0 .../sysom_vul/lib/exception.py | 0 .../sysom_vul/lib/paginations.py | 0 .../sysom_vul/lib/response.py | 0 .../sysom_vul/lib/utils.py | 0 .../sysom_vul/manage.py | 0 microservice/sysom_vul/requirements.txt | 17 ++++++ .../sysom_vul/scripts/server_clear.sh | 8 +++ microservice/sysom_vul/scripts/server_init.sh | 54 +++++++++++++++++++ .../sysom_vul/scripts/server_start.sh | 10 ++++ microservice/sysom_vul/scripts/server_stop.sh | 9 ++++ microservice/sysom_vul/sysom-vul.ini | 9 ++++ .../sysom_vul/sysom_vul/__init__.py | 0 .../sysom_vul/sysom_vul/asgi.py | 0 .../sysom_vul/sysom_vul/settings.py | 0 .../sysom_vul/sysom_vul/urls.py | 0 .../sysom_vul/sysom_vul/wsgi.py | 0 45 files changed, 110 insertions(+), 3 deletions(-) rename {sysom_server => microservice}/sysom_vul/apps/host/__init__.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/host/app.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/host/models.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/host/urls.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/__init__.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/admin.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/apps.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/async_fetch.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/executor.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/models.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/serializer.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/ssh_pool.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/tests.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/urls.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/views.py (100%) rename {sysom_server => microservice}/sysom_vul/apps/vul/vul.py (100%) rename {sysom_server => microservice}/sysom_vul/conf/__init__.py (100%) rename {sysom_server => microservice}/sysom_vul/conf/common.py (98%) rename {sysom_server => microservice}/sysom_vul/conf/develop.py (100%) rename sysom_server/sysom_vul/conf/vul_gunicorn.py => microservice/sysom_vul/conf/gunicorn.py (81%) rename {sysom_server => microservice}/sysom_vul/conf/product.py (100%) rename {sysom_server => microservice}/sysom_vul/conf/testing.py (100%) rename {sysom_server => microservice}/sysom_vul/config.yml (100%) rename {sysom_server => microservice}/sysom_vul/lib/authentications.py (100%) rename {sysom_server => microservice}/sysom_vul/lib/base_model.py (100%) rename {sysom_server => microservice}/sysom_vul/lib/decode/sysom_decode.py (100%) rename {sysom_server => microservice}/sysom_vul/lib/exception.py (100%) rename {sysom_server => microservice}/sysom_vul/lib/paginations.py (100%) rename {sysom_server => microservice}/sysom_vul/lib/response.py (100%) rename {sysom_server => microservice}/sysom_vul/lib/utils.py (100%) rename {sysom_server => microservice}/sysom_vul/manage.py (100%) create mode 100644 microservice/sysom_vul/requirements.txt create mode 100644 microservice/sysom_vul/scripts/server_clear.sh create mode 100644 microservice/sysom_vul/scripts/server_init.sh create mode 100644 microservice/sysom_vul/scripts/server_start.sh create mode 100644 microservice/sysom_vul/scripts/server_stop.sh create mode 100644 microservice/sysom_vul/sysom-vul.ini rename {sysom_server => microservice}/sysom_vul/sysom_vul/__init__.py (100%) rename {sysom_server => microservice}/sysom_vul/sysom_vul/asgi.py (100%) rename {sysom_server => microservice}/sysom_vul/sysom_vul/settings.py (100%) rename {sysom_server => microservice}/sysom_vul/sysom_vul/urls.py (100%) rename {sysom_server => microservice}/sysom_vul/sysom_vul/wsgi.py (100%) diff --git a/sysom_server/sysom_vul/apps/host/__init__.py b/microservice/sysom_vul/apps/host/__init__.py similarity index 100% rename from sysom_server/sysom_vul/apps/host/__init__.py rename to microservice/sysom_vul/apps/host/__init__.py diff --git a/sysom_server/sysom_vul/apps/host/app.py b/microservice/sysom_vul/apps/host/app.py similarity index 100% rename from sysom_server/sysom_vul/apps/host/app.py rename to microservice/sysom_vul/apps/host/app.py diff --git a/sysom_server/sysom_vul/apps/host/models.py b/microservice/sysom_vul/apps/host/models.py similarity index 100% rename from sysom_server/sysom_vul/apps/host/models.py rename to microservice/sysom_vul/apps/host/models.py diff --git a/sysom_server/sysom_vul/apps/host/urls.py b/microservice/sysom_vul/apps/host/urls.py similarity index 100% rename from sysom_server/sysom_vul/apps/host/urls.py rename to microservice/sysom_vul/apps/host/urls.py diff --git a/sysom_server/sysom_vul/apps/vul/__init__.py b/microservice/sysom_vul/apps/vul/__init__.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/__init__.py rename to microservice/sysom_vul/apps/vul/__init__.py diff --git a/sysom_server/sysom_vul/apps/vul/admin.py b/microservice/sysom_vul/apps/vul/admin.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/admin.py rename to microservice/sysom_vul/apps/vul/admin.py diff --git a/sysom_server/sysom_vul/apps/vul/apps.py b/microservice/sysom_vul/apps/vul/apps.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/apps.py rename to microservice/sysom_vul/apps/vul/apps.py diff --git a/sysom_server/sysom_vul/apps/vul/async_fetch.py b/microservice/sysom_vul/apps/vul/async_fetch.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/async_fetch.py rename to microservice/sysom_vul/apps/vul/async_fetch.py diff --git a/sysom_server/sysom_vul/apps/vul/executor.py b/microservice/sysom_vul/apps/vul/executor.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/executor.py rename to microservice/sysom_vul/apps/vul/executor.py diff --git a/sysom_server/sysom_vul/apps/vul/migrations/0001_initial.py b/microservice/sysom_vul/apps/vul/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/migrations/0001_initial.py rename to microservice/sysom_vul/apps/vul/migrations/0001_initial.py diff --git a/sysom_server/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py b/microservice/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py rename to microservice/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py diff --git a/sysom_server/sysom_vul/apps/vul/migrations/__init__.py b/microservice/sysom_vul/apps/vul/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/migrations/__init__.py rename to microservice/sysom_vul/apps/vul/migrations/__init__.py diff --git a/sysom_server/sysom_vul/apps/vul/models.py b/microservice/sysom_vul/apps/vul/models.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/models.py rename to microservice/sysom_vul/apps/vul/models.py diff --git a/sysom_server/sysom_vul/apps/vul/serializer.py b/microservice/sysom_vul/apps/vul/serializer.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/serializer.py rename to microservice/sysom_vul/apps/vul/serializer.py diff --git a/sysom_server/sysom_vul/apps/vul/ssh_pool.py b/microservice/sysom_vul/apps/vul/ssh_pool.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/ssh_pool.py rename to microservice/sysom_vul/apps/vul/ssh_pool.py diff --git a/sysom_server/sysom_vul/apps/vul/tests.py b/microservice/sysom_vul/apps/vul/tests.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/tests.py rename to microservice/sysom_vul/apps/vul/tests.py diff --git a/sysom_server/sysom_vul/apps/vul/urls.py b/microservice/sysom_vul/apps/vul/urls.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/urls.py rename to microservice/sysom_vul/apps/vul/urls.py diff --git a/sysom_server/sysom_vul/apps/vul/views.py b/microservice/sysom_vul/apps/vul/views.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/views.py rename to microservice/sysom_vul/apps/vul/views.py diff --git a/sysom_server/sysom_vul/apps/vul/vul.py b/microservice/sysom_vul/apps/vul/vul.py similarity index 100% rename from sysom_server/sysom_vul/apps/vul/vul.py rename to microservice/sysom_vul/apps/vul/vul.py diff --git a/sysom_server/sysom_vul/conf/__init__.py b/microservice/sysom_vul/conf/__init__.py similarity index 100% rename from sysom_server/sysom_vul/conf/__init__.py rename to microservice/sysom_vul/conf/__init__.py diff --git a/sysom_server/sysom_vul/conf/common.py b/microservice/sysom_vul/conf/common.py similarity index 98% rename from sysom_server/sysom_vul/conf/common.py rename to microservice/sysom_vul/conf/common.py index 9a1d96a0..111eb9b1 100644 --- a/sysom_server/sysom_vul/conf/common.py +++ b/microservice/sysom_vul/conf/common.py @@ -21,7 +21,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/sysom_server/sysom_vul/conf/develop.py b/microservice/sysom_vul/conf/develop.py similarity index 100% rename from sysom_server/sysom_vul/conf/develop.py rename to microservice/sysom_vul/conf/develop.py diff --git a/sysom_server/sysom_vul/conf/vul_gunicorn.py b/microservice/sysom_vul/conf/gunicorn.py similarity index 81% rename from sysom_server/sysom_vul/conf/vul_gunicorn.py rename to microservice/sysom_vul/conf/gunicorn.py index 23ff0e35..15162c5c 100644 --- a/sysom_server/sysom_vul/conf/vul_gunicorn.py +++ b/microservice/sysom_vul/conf/gunicorn.py @@ -5,7 +5,7 @@ workers = 2 # 指定工作进程数 threads = 3 -bind = '127.0.0.1:7005' +bind = '0.0.0.0:7005' timeout = 90 @@ -13,7 +13,7 @@ timeout = 90 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) -accesslog = '/usr/local/sysom/server/logs/sysom-vul-access.log' +accesslog = '/var/log/sysom/sysom-vul-access.log' loglevel = 'error' diff --git a/sysom_server/sysom_vul/conf/product.py b/microservice/sysom_vul/conf/product.py similarity index 100% rename from sysom_server/sysom_vul/conf/product.py rename to microservice/sysom_vul/conf/product.py diff --git a/sysom_server/sysom_vul/conf/testing.py b/microservice/sysom_vul/conf/testing.py similarity index 100% rename from sysom_server/sysom_vul/conf/testing.py rename to microservice/sysom_vul/conf/testing.py diff --git a/sysom_server/sysom_vul/config.yml b/microservice/sysom_vul/config.yml similarity index 100% rename from sysom_server/sysom_vul/config.yml rename to microservice/sysom_vul/config.yml diff --git a/sysom_server/sysom_vul/lib/authentications.py b/microservice/sysom_vul/lib/authentications.py similarity index 100% rename from sysom_server/sysom_vul/lib/authentications.py rename to microservice/sysom_vul/lib/authentications.py diff --git a/sysom_server/sysom_vul/lib/base_model.py b/microservice/sysom_vul/lib/base_model.py similarity index 100% rename from sysom_server/sysom_vul/lib/base_model.py rename to microservice/sysom_vul/lib/base_model.py diff --git a/sysom_server/sysom_vul/lib/decode/sysom_decode.py b/microservice/sysom_vul/lib/decode/sysom_decode.py similarity index 100% rename from sysom_server/sysom_vul/lib/decode/sysom_decode.py rename to microservice/sysom_vul/lib/decode/sysom_decode.py diff --git a/sysom_server/sysom_vul/lib/exception.py b/microservice/sysom_vul/lib/exception.py similarity index 100% rename from sysom_server/sysom_vul/lib/exception.py rename to microservice/sysom_vul/lib/exception.py diff --git a/sysom_server/sysom_vul/lib/paginations.py b/microservice/sysom_vul/lib/paginations.py similarity index 100% rename from sysom_server/sysom_vul/lib/paginations.py rename to microservice/sysom_vul/lib/paginations.py diff --git a/sysom_server/sysom_vul/lib/response.py b/microservice/sysom_vul/lib/response.py similarity index 100% rename from sysom_server/sysom_vul/lib/response.py rename to microservice/sysom_vul/lib/response.py diff --git a/sysom_server/sysom_vul/lib/utils.py b/microservice/sysom_vul/lib/utils.py similarity index 100% rename from sysom_server/sysom_vul/lib/utils.py rename to microservice/sysom_vul/lib/utils.py diff --git a/sysom_server/sysom_vul/manage.py b/microservice/sysom_vul/manage.py similarity index 100% rename from sysom_server/sysom_vul/manage.py rename to microservice/sysom_vul/manage.py diff --git a/microservice/sysom_vul/requirements.txt b/microservice/sysom_vul/requirements.txt new file mode 100644 index 00000000..ed045a76 --- /dev/null +++ b/microservice/sysom_vul/requirements.txt @@ -0,0 +1,17 @@ +aiohttp==3.8.4 +clogger>=0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +drf-yasg==1.21.4 +sysom_utils>=0.0.1 +django==3.2.16 +django-apscheduler==0.6.2 +django-filter==21.1 +django-cors-headers==3.10.1 +djangorestframework==3.14.0 +paramiko==2.12.0 +PyJWT==2.4.0 +PyMySQL==1.0.2 +requests==2.27.1 +gunicorn==20.1.0 diff --git a/microservice/sysom_vul/scripts/server_clear.sh b/microservice/sysom_vul/scripts/server_clear.sh new file mode 100644 index 00000000..9f3c3130 --- /dev/null +++ b/microservice/sysom_vul/scripts/server_clear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-vul +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} +clear_app diff --git a/microservice/sysom_vul/scripts/server_init.sh b/microservice/sysom_vul/scripts/server_init.sh new file mode 100644 index 00000000..4ede0d19 --- /dev/null +++ b/microservice/sysom_vul/scripts/server_init.sh @@ -0,0 +1,54 @@ +#!/bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_vul +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_NAME=sysom-vul + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +init_conf() { + pushd ${SERVER_HOME} + python manage.py migrate + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVER_HOME}/conf/vul_gunicorn.py +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + start_app +} + +deploy diff --git a/microservice/sysom_vul/scripts/server_start.sh b/microservice/sysom_vul/scripts/server_start.sh new file mode 100644 index 00000000..eeb0d1de --- /dev/null +++ b/microservice/sysom_vul/scripts/server_start.sh @@ -0,0 +1,10 @@ +#!/bin/bash + + +SERVICE_NAME=sysom-vul + +start_app() { + supervisorctl start $SERVICE_NAME +} + +start_app diff --git a/microservice/sysom_vul/scripts/server_stop.sh b/microservice/sysom_vul/scripts/server_stop.sh new file mode 100644 index 00000000..594c064f --- /dev/null +++ b/microservice/sysom_vul/scripts/server_stop.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +SERVICE_NAME=sysom-vul + +stop_app() { + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/microservice/sysom_vul/sysom-vul.ini b/microservice/sysom_vul/sysom-vul.ini new file mode 100644 index 00000000..4efe27c5 --- /dev/null +++ b/microservice/sysom_vul/sysom-vul.ini @@ -0,0 +1,9 @@ +[program:sysom-vul] +directory = /usr/local/sysom/server/microservice/sysom_vul +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vul.wsgi:application +startsecs=3 +autostart=true +autorestart=true +environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +stderr_logfile=/var/log/sysom/sysom-vul-error.log +stdout_logfile=/var/log/sysom/sysom-vul.log diff --git a/sysom_server/sysom_vul/sysom_vul/__init__.py b/microservice/sysom_vul/sysom_vul/__init__.py similarity index 100% rename from sysom_server/sysom_vul/sysom_vul/__init__.py rename to microservice/sysom_vul/sysom_vul/__init__.py diff --git a/sysom_server/sysom_vul/sysom_vul/asgi.py b/microservice/sysom_vul/sysom_vul/asgi.py similarity index 100% rename from sysom_server/sysom_vul/sysom_vul/asgi.py rename to microservice/sysom_vul/sysom_vul/asgi.py diff --git a/sysom_server/sysom_vul/sysom_vul/settings.py b/microservice/sysom_vul/sysom_vul/settings.py similarity index 100% rename from sysom_server/sysom_vul/sysom_vul/settings.py rename to microservice/sysom_vul/sysom_vul/settings.py diff --git a/sysom_server/sysom_vul/sysom_vul/urls.py b/microservice/sysom_vul/sysom_vul/urls.py similarity index 100% rename from sysom_server/sysom_vul/sysom_vul/urls.py rename to microservice/sysom_vul/sysom_vul/urls.py diff --git a/sysom_server/sysom_vul/sysom_vul/wsgi.py b/microservice/sysom_vul/sysom_vul/wsgi.py similarity index 100% rename from sysom_server/sysom_vul/sysom_vul/wsgi.py rename to microservice/sysom_vul/sysom_vul/wsgi.py -- Gitee From fd7fb5149be570c55069ff9b62f3b9add263450d Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 17:15:15 +0800 Subject: [PATCH 027/196] feat(sysom_vul): Support docker deploment --- sysom_vul_dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 sysom_vul_dockerfile diff --git a/sysom_vul_dockerfile b/sysom_vul_dockerfile new file mode 100644 index 00000000..378fb79b --- /dev/null +++ b/sysom_vul_dockerfile @@ -0,0 +1,31 @@ +FROM openanolis/anolisos:8.8 + +# Add epel +RUN yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm +RUN bash -c "sed -i 's|^#baseurl=https://download.example/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel*" +RUN bash -c "sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel*" + +# Add required yum packages +RUN yum makecache +RUN yum install -y supervisor cronie net-tools rpcbind nfs-utils +RUN systemctl enable crond +# RUN systemctl enable supervisord + +# Init sysom-diagnosis +ARG SYSOM_HOME=/usr/local/sysom +ARG SYSOM_SERVER_HOME=${SYSOM_HOME}/server + +RUN mkdir /root/sysom +COPY conf /root/sysom/conf +COPY script /root/sysom/script +COPY infrastructure /root/sysom/infrastructure +COPY microservice /root/sysom/microservice + +RUN bash -x /root/sysom/script/sysom.sh deploy infrastructure env,sdk +RUN bash -x /root/sysom/script/sysom.sh deploy microservice sysom_vul +RUN sed "s/nodaemon=false/nodaemon=true/g" -i /etc/supervisord.conf + +RUN yum clean all + +# # 环境准备 +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file -- Gitee From 710999afb61207dd3b2f14f4b4914d81e29d3808 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 17:38:48 +0800 Subject: [PATCH 028/196] refactor(sysom_hotfix): Use sysom.sh to unify deployment --- microservice/.gitignore | 3 + .../sysom_hotfix/apps/hotfix/__init__.py | 0 .../sysom_hotfix/apps/hotfix/admin.py | 0 .../sysom_hotfix/apps/hotfix/apps.py | 0 .../sysom_hotfix/apps/hotfix/function.py | 0 .../apps/hotfix/migrations/0001_initial.py | 0 .../migrations/0002_auto_20230303_1601.py | 0 .../apps/hotfix/migrations/__init__.py | 0 .../sysom_hotfix/apps/hotfix/models.py | 0 .../sysom_hotfix/apps/hotfix/serializer.py | 0 .../sysom_hotfix/apps/hotfix/urls.py | 0 .../sysom_hotfix/apps/hotfix/views.py | 0 .../sysom_hotfix/conf/__init__.py | 0 .../sysom_hotfix/conf/common.py | 3 +- .../sysom_hotfix/conf/develop.py | 0 .../sysom_hotfix/conf/gunicorn.py | 6 +- .../sysom_hotfix/conf/product.py | 0 .../sysom_hotfix/conf/testing.py | 0 .../sysom_hotfix/config.yml | 0 .../sysom_hotfix/lib/__init__.py | 0 .../sysom_hotfix/lib/authentications.py | 0 .../sysom_hotfix/lib/base_model.py | 0 .../sysom_hotfix/lib/base_view.py | 0 .../sysom_hotfix/lib/decode/sysom_decode.py | 0 .../sysom_hotfix/lib/excel.py | 0 .../sysom_hotfix/lib/exception.py | 0 .../sysom_hotfix/lib/paginations.py | 0 .../sysom_hotfix/lib/renderers.py | 0 .../sysom_hotfix/lib/response.py | 0 .../sysom_hotfix/lib/utils.py | 0 .../sysom_hotfix/manage.py | 0 microservice/sysom_hotfix/requirements.txt | 21 ++++++ .../sysom_hotfix/scripts/server_clear.sh | 8 +++ .../sysom_hotfix/scripts/server_init.sh | 66 +++++++++++++++++++ .../sysom_hotfix/scripts/server_start.sh | 8 +++ .../sysom_hotfix/scripts/server_stop.sh | 9 +++ microservice/sysom_hotfix/sysom-hotfix.ini | 9 +++ .../sysom_hotfix/sysom_hotfix/__init__.py | 0 .../sysom_hotfix/sysom_hotfix/asgi.py | 0 .../sysom_hotfix/sysom_hotfix/settings.py | 0 .../sysom_hotfix/sysom_hotfix/urls.py | 0 .../sysom_hotfix/sysom_hotfix/wsgi.py | 0 42 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 microservice/.gitignore rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/__init__.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/admin.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/apps.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/function.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/migrations/0001_initial.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/migrations/__init__.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/models.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/serializer.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/urls.py (100%) rename {sysom_server => microservice}/sysom_hotfix/apps/hotfix/views.py (100%) rename {sysom_server => microservice}/sysom_hotfix/conf/__init__.py (100%) rename {sysom_server => microservice}/sysom_hotfix/conf/common.py (97%) rename {sysom_server => microservice}/sysom_hotfix/conf/develop.py (100%) rename sysom_server/sysom_hotfix/conf/hotfix_gunicorn.py => microservice/sysom_hotfix/conf/gunicorn.py (65%) rename {sysom_server => microservice}/sysom_hotfix/conf/product.py (100%) rename {sysom_server => microservice}/sysom_hotfix/conf/testing.py (100%) rename {sysom_server => microservice}/sysom_hotfix/config.yml (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/__init__.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/authentications.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/base_model.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/base_view.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/decode/sysom_decode.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/excel.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/exception.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/paginations.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/renderers.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/response.py (100%) rename {sysom_server => microservice}/sysom_hotfix/lib/utils.py (100%) rename {sysom_server => microservice}/sysom_hotfix/manage.py (100%) create mode 100644 microservice/sysom_hotfix/requirements.txt create mode 100644 microservice/sysom_hotfix/scripts/server_clear.sh create mode 100644 microservice/sysom_hotfix/scripts/server_init.sh create mode 100644 microservice/sysom_hotfix/scripts/server_start.sh create mode 100644 microservice/sysom_hotfix/scripts/server_stop.sh create mode 100644 microservice/sysom_hotfix/sysom-hotfix.ini rename {sysom_server => microservice}/sysom_hotfix/sysom_hotfix/__init__.py (100%) rename {sysom_server => microservice}/sysom_hotfix/sysom_hotfix/asgi.py (100%) rename {sysom_server => microservice}/sysom_hotfix/sysom_hotfix/settings.py (100%) rename {sysom_server => microservice}/sysom_hotfix/sysom_hotfix/urls.py (100%) rename {sysom_server => microservice}/sysom_hotfix/sysom_hotfix/wsgi.py (100%) diff --git a/microservice/.gitignore b/microservice/.gitignore new file mode 100644 index 00000000..98c4b8d7 --- /dev/null +++ b/microservice/.gitignore @@ -0,0 +1,3 @@ +/conf/ +__pycache__ +logs \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/__init__.py b/microservice/sysom_hotfix/apps/hotfix/__init__.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/__init__.py rename to microservice/sysom_hotfix/apps/hotfix/__init__.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/admin.py b/microservice/sysom_hotfix/apps/hotfix/admin.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/admin.py rename to microservice/sysom_hotfix/apps/hotfix/admin.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/apps.py b/microservice/sysom_hotfix/apps/hotfix/apps.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/apps.py rename to microservice/sysom_hotfix/apps/hotfix/apps.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/function.py b/microservice/sysom_hotfix/apps/hotfix/function.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/function.py rename to microservice/sysom_hotfix/apps/hotfix/function.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/migrations/0001_initial.py b/microservice/sysom_hotfix/apps/hotfix/migrations/0001_initial.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/migrations/0001_initial.py rename to microservice/sysom_hotfix/apps/hotfix/migrations/0001_initial.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py b/microservice/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py rename to microservice/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/migrations/__init__.py b/microservice/sysom_hotfix/apps/hotfix/migrations/__init__.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/migrations/__init__.py rename to microservice/sysom_hotfix/apps/hotfix/migrations/__init__.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/models.py b/microservice/sysom_hotfix/apps/hotfix/models.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/models.py rename to microservice/sysom_hotfix/apps/hotfix/models.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/serializer.py b/microservice/sysom_hotfix/apps/hotfix/serializer.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/serializer.py rename to microservice/sysom_hotfix/apps/hotfix/serializer.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/urls.py b/microservice/sysom_hotfix/apps/hotfix/urls.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/urls.py rename to microservice/sysom_hotfix/apps/hotfix/urls.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/microservice/sysom_hotfix/apps/hotfix/views.py similarity index 100% rename from sysom_server/sysom_hotfix/apps/hotfix/views.py rename to microservice/sysom_hotfix/apps/hotfix/views.py diff --git a/sysom_server/sysom_hotfix/conf/__init__.py b/microservice/sysom_hotfix/conf/__init__.py similarity index 100% rename from sysom_server/sysom_hotfix/conf/__init__.py rename to microservice/sysom_hotfix/conf/__init__.py diff --git a/sysom_server/sysom_hotfix/conf/common.py b/microservice/sysom_hotfix/conf/common.py similarity index 97% rename from sysom_server/sysom_hotfix/conf/common.py rename to microservice/sysom_hotfix/conf/common.py index 8bb3de2c..81c4c989 100644 --- a/sysom_server/sysom_hotfix/conf/common.py +++ b/microservice/sysom_hotfix/conf/common.py @@ -11,7 +11,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) @@ -40,7 +40,6 @@ INSTALLED_APPS = [ 'rest_framework', 'corsheaders', 'drf_yasg', # 在线API文档 - 'channels', 'django_filters', 'django_apscheduler', ] diff --git a/sysom_server/sysom_hotfix/conf/develop.py b/microservice/sysom_hotfix/conf/develop.py similarity index 100% rename from sysom_server/sysom_hotfix/conf/develop.py rename to microservice/sysom_hotfix/conf/develop.py diff --git a/sysom_server/sysom_hotfix/conf/hotfix_gunicorn.py b/microservice/sysom_hotfix/conf/gunicorn.py similarity index 65% rename from sysom_server/sysom_hotfix/conf/hotfix_gunicorn.py rename to microservice/sysom_hotfix/conf/gunicorn.py index cb587a19..07db94d5 100644 --- a/sysom_server/sysom_hotfix/conf/hotfix_gunicorn.py +++ b/microservice/sysom_hotfix/conf/gunicorn.py @@ -5,13 +5,13 @@ workers = 2 # 指定工作进程数 threads = 3 -bind = '127.0.0.1:7007' +bind = '0.0.0.0:7007' -worker_class = 'gevent' # 工作模式线程, 默认为sync模式 +# worker_class = 'gevent' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) -accesslog = '/usr/local/sysom/server/logs/sysom-hotfix-access.log' +accesslog = '/var/log/sysom/sysom-hotfix-access.log' loglevel = 'error' diff --git a/sysom_server/sysom_hotfix/conf/product.py b/microservice/sysom_hotfix/conf/product.py similarity index 100% rename from sysom_server/sysom_hotfix/conf/product.py rename to microservice/sysom_hotfix/conf/product.py diff --git a/sysom_server/sysom_hotfix/conf/testing.py b/microservice/sysom_hotfix/conf/testing.py similarity index 100% rename from sysom_server/sysom_hotfix/conf/testing.py rename to microservice/sysom_hotfix/conf/testing.py diff --git a/sysom_server/sysom_hotfix/config.yml b/microservice/sysom_hotfix/config.yml similarity index 100% rename from sysom_server/sysom_hotfix/config.yml rename to microservice/sysom_hotfix/config.yml diff --git a/sysom_server/sysom_hotfix/lib/__init__.py b/microservice/sysom_hotfix/lib/__init__.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/__init__.py rename to microservice/sysom_hotfix/lib/__init__.py diff --git a/sysom_server/sysom_hotfix/lib/authentications.py b/microservice/sysom_hotfix/lib/authentications.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/authentications.py rename to microservice/sysom_hotfix/lib/authentications.py diff --git a/sysom_server/sysom_hotfix/lib/base_model.py b/microservice/sysom_hotfix/lib/base_model.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/base_model.py rename to microservice/sysom_hotfix/lib/base_model.py diff --git a/sysom_server/sysom_hotfix/lib/base_view.py b/microservice/sysom_hotfix/lib/base_view.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/base_view.py rename to microservice/sysom_hotfix/lib/base_view.py diff --git a/sysom_server/sysom_hotfix/lib/decode/sysom_decode.py b/microservice/sysom_hotfix/lib/decode/sysom_decode.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/decode/sysom_decode.py rename to microservice/sysom_hotfix/lib/decode/sysom_decode.py diff --git a/sysom_server/sysom_hotfix/lib/excel.py b/microservice/sysom_hotfix/lib/excel.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/excel.py rename to microservice/sysom_hotfix/lib/excel.py diff --git a/sysom_server/sysom_hotfix/lib/exception.py b/microservice/sysom_hotfix/lib/exception.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/exception.py rename to microservice/sysom_hotfix/lib/exception.py diff --git a/sysom_server/sysom_hotfix/lib/paginations.py b/microservice/sysom_hotfix/lib/paginations.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/paginations.py rename to microservice/sysom_hotfix/lib/paginations.py diff --git a/sysom_server/sysom_hotfix/lib/renderers.py b/microservice/sysom_hotfix/lib/renderers.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/renderers.py rename to microservice/sysom_hotfix/lib/renderers.py diff --git a/sysom_server/sysom_hotfix/lib/response.py b/microservice/sysom_hotfix/lib/response.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/response.py rename to microservice/sysom_hotfix/lib/response.py diff --git a/sysom_server/sysom_hotfix/lib/utils.py b/microservice/sysom_hotfix/lib/utils.py similarity index 100% rename from sysom_server/sysom_hotfix/lib/utils.py rename to microservice/sysom_hotfix/lib/utils.py diff --git a/sysom_server/sysom_hotfix/manage.py b/microservice/sysom_hotfix/manage.py similarity index 100% rename from sysom_server/sysom_hotfix/manage.py rename to microservice/sysom_hotfix/manage.py diff --git a/microservice/sysom_hotfix/requirements.txt b/microservice/sysom_hotfix/requirements.txt new file mode 100644 index 00000000..3952b08c --- /dev/null +++ b/microservice/sysom_hotfix/requirements.txt @@ -0,0 +1,21 @@ +clogger>=0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +drf_yasg==1.21.4 +sysom_utils>=0.0.1 +django==3.2.16 +django-apscheduler==0.6.2 +django-filter==21.1 +django-cors-headers==3.10.1 +djangorestframework==3.14.0 +paramiko==2.12.0 +PyJWT==2.4.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +pandas>=1.1.5 +requests==2.27.1 +gunicorn==20.1.0 +xlwt==1.3.0 +xlrd==2.0.1 diff --git a/microservice/sysom_hotfix/scripts/server_clear.sh b/microservice/sysom_hotfix/scripts/server_clear.sh new file mode 100644 index 00000000..5103464e --- /dev/null +++ b/microservice/sysom_hotfix/scripts/server_clear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-hotfix +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} +clear_app diff --git a/microservice/sysom_hotfix/scripts/server_init.sh b/microservice/sysom_hotfix/scripts/server_init.sh new file mode 100644 index 00000000..fb610ead --- /dev/null +++ b/microservice/sysom_hotfix/scripts/server_init.sh @@ -0,0 +1,66 @@ +#! /bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_hotfix +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_NAME=sysom-hotfix + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +init_conf() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/hotfix_gunicorn.py +} + +start_nfs() +{ + systemctl start rpcbind && systemctl enable rpcbind + systemctl start nfs && systemctl enable nfs + if [ $? -ne 0 ];then + systemctl start nfs-server && systemctl enable nfs-server + fi + + nfs_mask=`ip -4 route | grep "link src" | grep $SERVER_LOCAL_IP | awk '{print $1}' | head -n 1` + file_path=${SERVER_HOME}/hotfix_builder/hotfix-nfs + mkdir -p ${file_path} + echo "${file_path} ${nfs_mask}(rw,async)" >> /etc/exports + exportfs -rv + chmod -R 777 ${file_path} +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + start_app + start_nfs +} + +deploy diff --git a/microservice/sysom_hotfix/scripts/server_start.sh b/microservice/sysom_hotfix/scripts/server_start.sh new file mode 100644 index 00000000..f967321b --- /dev/null +++ b/microservice/sysom_hotfix/scripts/server_start.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-hotfix + +start_app() { + supervisorctl start $SERVICE_NAME +} + +start_app diff --git a/microservice/sysom_hotfix/scripts/server_stop.sh b/microservice/sysom_hotfix/scripts/server_stop.sh new file mode 100644 index 00000000..88041451 --- /dev/null +++ b/microservice/sysom_hotfix/scripts/server_stop.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +SERVICE_NAME=sysom-hotfix + +stop_app() { + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/microservice/sysom_hotfix/sysom-hotfix.ini b/microservice/sysom_hotfix/sysom-hotfix.ini new file mode 100644 index 00000000..e25d68db --- /dev/null +++ b/microservice/sysom_hotfix/sysom-hotfix.ini @@ -0,0 +1,9 @@ +[program:sysom-hotfix] +directory = /usr/local/sysom/server/microservice/sysom_hotfix +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_hotfix.wsgi:application +startsecs=3 +autostart=true +autorestart=true +environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +stderr_logfile=/var/log/sysom/sysom-hotfix-error.log +stdout_logfile=/var/log/sysom/sysom-hotfix.log diff --git a/sysom_server/sysom_hotfix/sysom_hotfix/__init__.py b/microservice/sysom_hotfix/sysom_hotfix/__init__.py similarity index 100% rename from sysom_server/sysom_hotfix/sysom_hotfix/__init__.py rename to microservice/sysom_hotfix/sysom_hotfix/__init__.py diff --git a/sysom_server/sysom_hotfix/sysom_hotfix/asgi.py b/microservice/sysom_hotfix/sysom_hotfix/asgi.py similarity index 100% rename from sysom_server/sysom_hotfix/sysom_hotfix/asgi.py rename to microservice/sysom_hotfix/sysom_hotfix/asgi.py diff --git a/sysom_server/sysom_hotfix/sysom_hotfix/settings.py b/microservice/sysom_hotfix/sysom_hotfix/settings.py similarity index 100% rename from sysom_server/sysom_hotfix/sysom_hotfix/settings.py rename to microservice/sysom_hotfix/sysom_hotfix/settings.py diff --git a/sysom_server/sysom_hotfix/sysom_hotfix/urls.py b/microservice/sysom_hotfix/sysom_hotfix/urls.py similarity index 100% rename from sysom_server/sysom_hotfix/sysom_hotfix/urls.py rename to microservice/sysom_hotfix/sysom_hotfix/urls.py diff --git a/sysom_server/sysom_hotfix/sysom_hotfix/wsgi.py b/microservice/sysom_hotfix/sysom_hotfix/wsgi.py similarity index 100% rename from sysom_server/sysom_hotfix/sysom_hotfix/wsgi.py rename to microservice/sysom_hotfix/sysom_hotfix/wsgi.py -- Gitee From a29fb3ca6c0039ca67336d072810f717b159a0ee Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 17:39:12 +0800 Subject: [PATCH 029/196] feat(sysom_hotfix): Support docker deploment --- sysom_hotfix_dockerfile | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 sysom_hotfix_dockerfile diff --git a/sysom_hotfix_dockerfile b/sysom_hotfix_dockerfile new file mode 100644 index 00000000..ffc3106a --- /dev/null +++ b/sysom_hotfix_dockerfile @@ -0,0 +1,31 @@ +FROM openanolis/anolisos:8.8 + +# Add epel +RUN yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm +RUN bash -c "sed -i 's|^#baseurl=https://download.example/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel*" +RUN bash -c "sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel*" + +# Add required yum packages +RUN yum makecache +RUN yum install -y supervisor cronie net-tools rpcbind nfs-utils +RUN systemctl enable crond +# RUN systemctl enable supervisord + +# Init sysom-diagnosis +ARG SYSOM_HOME=/usr/local/sysom +ARG SYSOM_SERVER_HOME=${SYSOM_HOME}/server + +RUN mkdir /root/sysom +COPY conf /root/sysom/conf +COPY script /root/sysom/script +COPY infrastructure /root/sysom/infrastructure +COPY microservice /root/sysom/microservice + +RUN bash -x /root/sysom/script/sysom.sh deploy infrastructure env,sdk +RUN bash -x /root/sysom/script/sysom.sh deploy microservice sysom_hotfix +RUN sed "s/nodaemon=false/nodaemon=true/g" -i /etc/supervisord.conf + +RUN yum clean all + +# # 环境准备 +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file -- Gitee From 44c1c44ce96372bd9ba5e1a58ae8b3f3533510dd Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 25 Apr 2023 17:49:33 +0800 Subject: [PATCH 030/196] refactor(sysom_hotfix_builder): Use sysom.sh to unify deployment --- .../sysom_hotfix_builder/build_hotfix.sh | 0 .../sysom_hotfix_builder/build_rpm.sh | 0 .../sysom_hotfix_builder/builder.ini | 0 .../sysom_hotfix_builder/builder.py | 0 .../sysom_hotfix_builder/check_env.sh | 0 .../sysom_hotfix_builder/img_list.json | 0 .../sysom_hotfix_builder/requirements.txt | 5 ++ .../scripts/server_clear.sh | 25 ++++++++ .../scripts/server_init.sh | 61 +++++++++++++++++++ .../scripts/server_start.sh | 8 +++ .../scripts/server_stop.sh | 9 +++ .../sysom-hotfix-builder.ini | 9 +++ 12 files changed, 117 insertions(+) rename {sysom_server => microservice}/sysom_hotfix_builder/build_hotfix.sh (100%) rename {sysom_server => microservice}/sysom_hotfix_builder/build_rpm.sh (100%) rename {sysom_server => microservice}/sysom_hotfix_builder/builder.ini (100%) rename {sysom_server => microservice}/sysom_hotfix_builder/builder.py (100%) rename {sysom_server => microservice}/sysom_hotfix_builder/check_env.sh (100%) rename {sysom_server => microservice}/sysom_hotfix_builder/img_list.json (100%) create mode 100644 microservice/sysom_hotfix_builder/requirements.txt create mode 100644 microservice/sysom_hotfix_builder/scripts/server_clear.sh create mode 100644 microservice/sysom_hotfix_builder/scripts/server_init.sh create mode 100644 microservice/sysom_hotfix_builder/scripts/server_start.sh create mode 100644 microservice/sysom_hotfix_builder/scripts/server_stop.sh create mode 100644 microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini diff --git a/sysom_server/sysom_hotfix_builder/build_hotfix.sh b/microservice/sysom_hotfix_builder/build_hotfix.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/build_hotfix.sh rename to microservice/sysom_hotfix_builder/build_hotfix.sh diff --git a/sysom_server/sysom_hotfix_builder/build_rpm.sh b/microservice/sysom_hotfix_builder/build_rpm.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/build_rpm.sh rename to microservice/sysom_hotfix_builder/build_rpm.sh diff --git a/sysom_server/sysom_hotfix_builder/builder.ini b/microservice/sysom_hotfix_builder/builder.ini similarity index 100% rename from sysom_server/sysom_hotfix_builder/builder.ini rename to microservice/sysom_hotfix_builder/builder.ini diff --git a/sysom_server/sysom_hotfix_builder/builder.py b/microservice/sysom_hotfix_builder/builder.py similarity index 100% rename from sysom_server/sysom_hotfix_builder/builder.py rename to microservice/sysom_hotfix_builder/builder.py diff --git a/sysom_server/sysom_hotfix_builder/check_env.sh b/microservice/sysom_hotfix_builder/check_env.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/check_env.sh rename to microservice/sysom_hotfix_builder/check_env.sh diff --git a/sysom_server/sysom_hotfix_builder/img_list.json b/microservice/sysom_hotfix_builder/img_list.json similarity index 100% rename from sysom_server/sysom_hotfix_builder/img_list.json rename to microservice/sysom_hotfix_builder/img_list.json diff --git a/microservice/sysom_hotfix_builder/requirements.txt b/microservice/sysom_hotfix_builder/requirements.txt new file mode 100644 index 00000000..f3e2139b --- /dev/null +++ b/microservice/sysom_hotfix_builder/requirements.txt @@ -0,0 +1,5 @@ +clogger>=0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +requests==2.27.1 \ No newline at end of file diff --git a/microservice/sysom_hotfix_builder/scripts/server_clear.sh b/microservice/sysom_hotfix_builder/scripts/server_clear.sh new file mode 100644 index 00000000..ea55d38b --- /dev/null +++ b/microservice/sysom_hotfix_builder/scripts/server_clear.sh @@ -0,0 +1,25 @@ +#!/bin/bash +SERVICE_NAME=sysom-hotfix-builder + +clear_app() { + LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix + sed -i '/hotfix_builder/d' /etc/exports + + # kill all kpatch-build process + echo "find : `ps -eo comm,pid | grep "kpatch-build"`" + for each_line in `ps -eo comm,pid | grep "kpatch-build"`; do + echo $each_line + if [[ $each_line =~ "kpatch-build" ]]; then + echo "find kpatch-build" + else + kill -9 $each_line + fi + done + + umount $LOCAL_NFS_HOME + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} + +clear_app diff --git a/microservice/sysom_hotfix_builder/scripts/server_init.sh b/microservice/sysom_hotfix_builder/scripts/server_init.sh new file mode 100644 index 00000000..2e6b34d5 --- /dev/null +++ b/microservice/sysom_hotfix_builder/scripts/server_init.sh @@ -0,0 +1,61 @@ +#! /bin/bash +SERVICE_HOME=${MICROSERVICE_HOME}/sysom_hotfix +VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_NAME=sysom-hotfix-builder +NFS_SERVER_IP=${SERVER_LOCAL_IP} + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_requirement() { + pushd ${SERVICE_HOME} + pip install -r requirements.txt + popd +} + +init_conf() { + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ +} + +install_package() { + rpm -q --quiet make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || yum install -y make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || exit 1 + + rpm -q --quiet docker git || yum install -y docker git || echo "Warngin : Docker is not installed in this machine!" +} + +mount_nfs() +{ + HOTFIX_NFS_HOME=${SERVER_HOME}/hotfix_builder/hotfix-nfs + LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix + + mkdir -p ${LOCAL_NFS_HOME} + + sudo umount $LOCAL_NFS_HOME # Remove the mounted directory in case it mounted before + sudo mount -t nfs ${NFS_SERVER_IP}:${HOTFIX_NFS_HOME} ${LOCAL_NFS_HOME} || exit 1 +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + install_package + mount_nfs + start_app +} + +deploy diff --git a/microservice/sysom_hotfix_builder/scripts/server_start.sh b/microservice/sysom_hotfix_builder/scripts/server_start.sh new file mode 100644 index 00000000..ead33613 --- /dev/null +++ b/microservice/sysom_hotfix_builder/scripts/server_start.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-hotfix-builder + +start_app() { + supervisorctl start $SERVICE_NAME +} + +start_app diff --git a/microservice/sysom_hotfix_builder/scripts/server_stop.sh b/microservice/sysom_hotfix_builder/scripts/server_stop.sh new file mode 100644 index 00000000..6817fbbc --- /dev/null +++ b/microservice/sysom_hotfix_builder/scripts/server_stop.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +SERVICE_NAME=sysom-hotfix-builder + +stop_app() { + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini b/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini new file mode 100644 index 00000000..0e1c2560 --- /dev/null +++ b/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini @@ -0,0 +1,9 @@ +[program:sysom-hotfix-builder] +directory = /usr/local/sysom/server/microservice/sysom_hotfix_builder +command=/usr/local/sysom/server/infrastructure/virtualenv/bin/python3 builder.py +startsecs=3 +autostart=true +autorestart=true +environment=PATH=/usr/local/sysom/server/infrastructure/virtualenv/bin:%(ENV_PATH)s +stderr_logfile=/var/log/sysom/sysom-hotfix-builder-error.log +stdout_logfile=/var/log/sysom/sysom-hotfix-builder.log -- Gitee From a5ecab756289f23de2d9c362291443273380dfcd Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 26 Apr 2023 17:05:52 +0800 Subject: [PATCH 031/196] reafactor(deploy): Use sysom.sh to deploy whole project --- .../sysom_api_dockerfile | 0 .../sysom_channel_dockerfile | 0 .../sysom_diagnosis_dockerfile | 0 .../sysom_hotfix_dockerfile | 0 .../sysom_migration_dockerfile | 0 .../sysom_monitor_server_dockerfile | 0 docker/sysom_vmcore_dockerfile | 31 + .../sysom_vul_dockerfile | 0 .../{env => 0_env}/.pydistutils.cfg | 0 infrastructure/{env => 0_env}/clear.sh | 2 +- infrastructure/{env => 0_env}/init.sh | 8 +- infrastructure/{env => 0_env}/pip.conf | 0 .../{env => 0_env}/requirements.txt | 0 infrastructure/{env => 0_env}/start.sh | 0 infrastructure/{env => 0_env}/stop.sh | 0 infrastructure/{sdk => 1_sdk}/.gitignore | 0 .../{sdk => 1_sdk}/cec_base/__init__.py | 0 .../{sdk => 1_sdk}/cec_base/admin.py | 0 .../{sdk => 1_sdk}/cec_base/base.py | 0 .../{sdk => 1_sdk}/cec_base/cec_client.py | 0 infrastructure/{sdk => 1_sdk}/cec_base/cli.py | 0 .../{sdk => 1_sdk}/cec_base/consumer.py | 0 .../{sdk => 1_sdk}/cec_base/event.py | 0 .../{sdk => 1_sdk}/cec_base/exceptions.py | 0 .../{sdk => 1_sdk}/cec_base/meta.py | 0 .../{sdk => 1_sdk}/cec_base/producer.py | 0 infrastructure/{sdk => 1_sdk}/cec_base/url.py | 0 .../{sdk => 1_sdk}/cec_base/utils.py | 0 .../{sdk => 1_sdk}/cec_redis/__init__.py | 0 .../{sdk => 1_sdk}/cec_redis/admin_static.py | 0 .../{sdk => 1_sdk}/cec_redis/common.py | 0 .../cec_redis/consume_status_storage.py | 0 .../{sdk => 1_sdk}/cec_redis/heartbeat.py | 0 .../{sdk => 1_sdk}/cec_redis/redis_admin.py | 0 .../cec_redis/redis_consumer.py | 0 .../cec_redis/redis_producer.py | 0 .../{sdk => 1_sdk}/cec_redis/utils.py | 0 .../{sdk => 1_sdk}/channel_job/__init__.py | 0 .../{sdk => 1_sdk}/channel_job/exception.py | 0 .../{sdk => 1_sdk}/channel_job/job.py | 0 .../{sdk => 1_sdk}/channel_job/model.py | 0 infrastructure/{sdk => 1_sdk}/clear.sh | 0 .../{sdk => 1_sdk}/clogger/__init__.py | 0 .../{sdk => 1_sdk}/clogger/clogger.py | 0 .../{sdk => 1_sdk}/cmg_base/__init__.py | 0 .../{sdk => 1_sdk}/cmg_base/exceptions.py | 0 .../cmg_base/load_balancing_strategy.py | 0 .../{sdk => 1_sdk}/cmg_base/service_check.py | 0 .../cmg_base/service_discovery.py | 0 .../cmg_base/service_instance.py | 0 .../cmg_base/service_invoker.py | 0 .../cmg_base/service_registry.py | 0 infrastructure/{sdk => 1_sdk}/cmg_base/url.py | 0 .../{sdk => 1_sdk}/cmg_base/utils.py | 0 .../{sdk => 1_sdk}/cmg_redis/__init__.py | 0 .../{sdk => 1_sdk}/cmg_redis/common.py | 0 .../cmg_redis/redis_service_discovery.py | 0 .../cmg_redis/redis_service_registry.py | 0 .../{sdk => 1_sdk}/cmg_redis/utils.py | 0 infrastructure/{sdk => 1_sdk}/init.sh | 4 +- .../{sdk => 1_sdk}/setup_cec_base.py | 0 .../{sdk => 1_sdk}/setup_cec_redis.py | 0 .../{sdk => 1_sdk}/setup_channel_job.py | 0 .../{sdk => 1_sdk}/setup_clogger.py | 0 .../{sdk => 1_sdk}/setup_cmg_base.py | 0 .../{sdk => 1_sdk}/setup_cmg_redis.py | 0 .../{sdk => 1_sdk}/setup_sysom_utils.py | 0 infrastructure/{sdk => 1_sdk}/start.sh | 0 infrastructure/{sdk => 1_sdk}/stop.sh | 0 .../{sdk => 1_sdk}/sysom_utils/__init__.py | 0 .../{sdk => 1_sdk}/sysom_utils/adddict.py | 0 .../sysom_utils/config_parser.py | 0 .../sysom_utils/event_executor.py | 0 .../sysom_utils/framework_plug_mag.py | 0 .../sysom_utils/framework_plugins.py | 0 .../sysom_utils/node_manager.py | 0 .../{sdk => 1_sdk}/sysom_utils/yaml_concat.py | 0 .../{sdk => 1_sdk}/test_cec_redis/__init__.py | 0 .../test_cec_redis/test_heartbeat.py | 0 .../test_cec_redis/test_redis_admin.py | 0 .../test_cec_redis/test_redis_admin_async.py | 0 .../test_cec_redis/test_redis_consumer.py | 0 .../test_redis_consumer_async.py | 0 .../test_redis_multi_consumer.py | 0 .../test_cec_redis/test_redis_producer.py | 0 .../test_cec_redis/test_utils.py | 0 .../{sdk => 1_sdk}/test_cmg_base/__init__.py | 0 .../test_cmg_base/test_health_check.py | 0 .../{sdk => 1_sdk}/test_cmg_redis/__init__.py | 0 .../test_cmg_redis/test_service_discovery.py | 0 .../test_cmg_redis/test_service_registry.py | 0 .../clear.sh | 0 .../init.sh | 6 +- .../nginx.conf | 0 .../start.sh | 0 .../stop.sh | 0 .../2_local_services}/sysom-redis.ini | 4 +- .../sysom.conf | 2 +- .../3_monitor}/Readme.txt | 0 .../3_monitor}/clear.sh | 0 .../3_monitor}/grafana_api_set.sh | 2 +- .../3_monitor}/grafana_recover.sh | 0 .../3_monitor}/init.sh | 5 +- .../3_monitor}/local_copy_pkg.sh | 0 .../3_monitor}/prometheus_get_node.py | 0 .../3_monitor}/start.sh | 0 .../3_monitor}/stop.sh | 0 .../sysom-cec-status-dashboard.json | 0 .../3_monitor}/sysom-dashboard.json | 0 .../3_monitor}/sysom-migration-dashboard.json | 0 .../3_monitor}/sysom-netinfo-dashboard.json | 0 infrastructure/3_monitor/sysom-prometheus.ini | 8 + infrastructure/clear_exclude | 1 + .../deploy_exclude | 0 infrastructure/local_services/sysom-redis.ini | 8 - .../{sysom_api => 0_sysom_api}/.gitignore | 0 .../apps/__init__.py | 0 .../apps/accounts}/__init__.py | 0 .../apps/accounts/admin.py | 0 .../apps/accounts/apps.py | 0 .../apps/accounts/authentication.py | 0 .../apps/accounts/migrations/0001_initial.py | 0 .../migrations/0002_user_allow_login.py | 0 .../apps/accounts}/migrations/__init__.py | 0 .../apps/accounts/models.py | 0 .../apps/accounts/permissions.py | 0 .../apps/accounts/serializer.py | 0 .../apps/accounts/tests.py | 0 .../apps/accounts/urls.py | 0 .../apps/accounts/views.py | 0 .../apps/alarm/__init__.py | 0 .../apps/alarm/admin.py | 0 .../apps/alarm/apps.py | 0 .../apps/alarm/migrations/0001_initial.py | 0 .../apps/alarm/migrations}/__init__.py | 0 .../apps/alarm/models.py | 0 .../apps/alarm/serializer.py | 0 .../apps/alarm/subscribe.json | 0 .../apps/alarm/urls.py | 0 .../apps/alarm/views.py | 0 .../apps/common}/__init__.py | 0 .../apps/common/common_model_viewset.py | 0 .../apps/host}/__init__.py | 0 .../apps/host/admin.py | 0 .../apps/host/apps.py | 0 .../apps/host/cec_api.py | 0 .../apps/host/heartbeat.py | 0 .../apps/host/migrations/0001_initial.py | 0 .../migrations/0002_hostmodel_host_info.py | 0 .../migrations/0003_alter_hostmodel_status.py | 0 .../migrations/0004_auto_20221227_1705.py | 0 .../apps/host}/migrations/__init__.py | 0 .../apps/host/models.py | 0 .../apps/host/serializer.py | 0 .../apps/host/tests.py | 0 .../apps/host/urls.py | 0 .../apps/host/views.py | 0 .../apps/services/__init__.py | 0 .../apps/services/admin.py | 0 .../apps/services/apps.py | 0 .../apps/services/migrations/0001_initial.py | 0 .../apps/services/migrations}/__init__.py | 0 .../apps/services/models.py | 0 .../apps/services/serializer.py | 0 .../apps/services/tests.py | 0 .../apps/services/urls.py | 0 .../apps/services/views.py | 0 microservice/0_sysom_api/conf/__init__.py | 0 .../{sysom_api => 0_sysom_api}/conf/common.py | 2 +- .../conf/develop.py | 0 .../conf/product.py | 0 .../conf/testing.py | 0 .../{sysom_api => 0_sysom_api}/config.yml | 0 .../consumer/__init__.py | 0 .../consumer/consumers.py | 0 .../consumer/middleware.py | 0 .../consumer/routing.py | 0 .../lib/__init__.py | 0 .../lib/authentications.py | 0 .../lib/base_model.py | 0 .../lib/decode/sysom_decode.py | 0 .../{sysom_api => 0_sysom_api}/lib/excel.py | 0 .../lib/exception.py | 0 .../lib/paginations.py | 0 .../lib/renderers.py | 0 .../lib/response.py | 0 .../{sysom_api => 0_sysom_api}/lib/ssh.py | 0 .../{sysom_api => 0_sysom_api}/lib/utils.py | 0 .../{sysom_api => 0_sysom_api}/manage.py | 0 .../requirements.txt | 0 .../scripts/server_clear.sh | 0 .../scripts/server_init.sh | 4 +- .../scripts/server_start.sh | 0 .../scripts/server_stop.sh | 0 .../{sysom_api => 0_sysom_api}/sysom-api.ini | 4 +- .../sysom/__init__.py | 0 .../{sysom_api => 0_sysom_api}/sysom/asgi.py | 0 .../sysom/settings.py | 0 .../{sysom_api => 0_sysom_api}/sysom/urls.py | 0 .../{sysom_api => 0_sysom_api}/sysom/wsgi.py | 0 .../alembic.ini | 0 .../alembic/README | 0 .../alembic/env.py | 0 .../alembic/script.py.mako | 0 .../alembic/versions/1663292d0bd8_init_db.py | 0 .../3f54250e3fca_add_channel_params_table.py | 0 .../app/__init__.py | 0 .../app/crud.py | 0 .../app/database.py | 0 .../app/executor.py | 0 .../app/models.py | 0 .../app/routers/cec_status.py | 0 .../app/routers/config.py | 0 .../app/routers/file.py | 0 .../app/routers/health.py | 0 .../app/schemas.py | 0 .../conf/common.py | 2 +- .../conf/develop.py | 0 .../conf/gunicorn.py | 0 .../conf/product.py | 0 .../conf/settings.py | 0 .../conf/testing.py | 0 .../config.yml | 0 .../lib/channels/__init__.py | 0 .../lib/channels/base.py | 0 .../lib/channels/ssh.py | 0 .../lib/ssh.py | 0 .../lib/utils.py | 0 .../main.py | 0 .../requirements.txt | 0 .../scripts/server_clear.sh | 0 .../scripts/server_init.sh | 4 +- .../scripts/server_start.sh | 0 .../scripts/server_stop.sh | 0 .../sysom-channel.ini | 6 +- microservice/clear_exclude | 0 microservice/deploy_exclude | 1 + microservice/sysom_diagnosis/conf/common.py | 2 +- .../{diagnosis_gunicorn.py => gunicorn.py} | 0 .../sysom_diagnosis/scripts/server_init.sh | 8 +- .../sysom_diagnosis/sysom-diagnosis.ini | 2 +- microservice/sysom_hotfix/conf/common.py | 2 +- .../sysom_hotfix/scripts/server_init.sh | 8 +- microservice/sysom_hotfix/sysom-hotfix.ini | 4 +- .../scripts/server_init.sh | 4 +- .../sysom-hotfix-builder.ini | 4 +- microservice/sysom_migration/conf/common.py | 2 +- .../sysom_migration/scripts/server_clear.sh | 8 + .../sysom_migration/scripts/server_init.sh | 4 +- .../sysom_migration/sysom-migration.ini | 4 +- .../sysom_monitor_server/conf/common.py | 2 +- .../scripts/server_init.sh | 4 +- .../sysom-monitor-server.ini | 4 +- microservice/sysom_vmcore/conf/common.py | 2 +- .../sysom_vmcore/scripts/server_init.sh | 6 +- microservice/sysom_vmcore/sysom-vmcore.ini | 4 +- microservice/sysom_vul/conf/common.py | 2 +- microservice/sysom_vul/scripts/server_init.sh | 6 +- microservice/sysom_vul/sysom-vul.ini | 4 +- package.sh | 9 +- package_rpm_offline.sh | 0 package_rpm_online.sh | 0 script/clear.sh | 100 ++-- script/deploy.sh | 141 +++-- script/deploy/clear.sh | 61 ++ script/deploy/deploy.sh | 94 +++ script/node/clear.sh | 20 - script/node/diagnosis/clear.sh | 8 - script/node/diagnosis/init.sh | 21 - script/node/init.sh | 22 - script/node/monitor/clear.sh | 14 - script/node/monitor/init.sh | 43 -- script/node/monitor/raptor_profiling_clear.sh | 12 - .../node/monitor/raptor_profiling_deploy.sh | 36 -- script/node/vmcore/clear.sh | 1 - script/node/vmcore/clear_vmcore.sh | 6 - script/node/vmcore/init.sh | 1 - script/node/vmcore/init_vmcore.sh | 37 -- script/node/vmcore/vmcore_collect.py | 160 ----- script/server/0_env/init.sh | 119 ---- script/server/0_env/pip.conf | 8 - script/server/0_env/requirements.txt | 38 -- script/server/0_local_services/clear.sh | 37 -- script/server/0_local_services/init.sh | 115 ---- script/server/0_local_services/nginx.conf | 68 --- script/server/0_local_services/start.sh | 26 - script/server/0_local_services/stop.sh | 25 - script/server/0_local_services/sysom.conf | 141 ----- script/server/0_sysom_api/clear.sh | 8 - script/server/0_sysom_api/init.sh | 57 -- script/server/0_sysom_api/start.sh | 10 - script/server/0_sysom_api/stop.sh | 10 - script/server/0_sysom_api/sysom-api.ini | 13 - script/server/0_sysom_channel/clear.sh | 8 - script/server/0_sysom_channel/init.sh | 47 -- script/server/0_sysom_channel/start.sh | 7 - script/server/0_sysom_channel/stop.sh | 7 - .../server/0_sysom_channel/sysom-channel.ini | 9 - .../1_sysom_monitor/sysom-prometheus.ini | 8 - script/server/1_sysom_monitor_server/clear.sh | 8 - script/server/1_sysom_monitor_server/init.sh | 47 -- script/server/1_sysom_monitor_server/start.sh | 7 - script/server/1_sysom_monitor_server/stop.sh | 7 - .../sysom-monitor-server.ini | 9 - script/server/2_sysom_vmcore/clear.sh | 15 - script/server/2_sysom_vmcore/init.sh | 71 --- script/server/2_sysom_vmcore/parse_panic.py | 565 ------------------ script/server/2_sysom_vmcore/start.sh | 13 - script/server/2_sysom_vmcore/stop.sh | 13 - script/server/2_sysom_vmcore/sysom-vmcore.ini | 9 - script/server/2_sysom_vmcore/vmcore_const.py | 25 - script/server/3_sysom_diagnosis/clear.sh | 8 - script/server/3_sysom_diagnosis/init.sh | 49 -- script/server/3_sysom_diagnosis/start.sh | 7 - script/server/3_sysom_diagnosis/stop.sh | 7 - .../3_sysom_diagnosis/sysom-diagnosis.ini | 9 - script/server/4_sysom_vul/clear.sh | 8 - script/server/4_sysom_vul/init.sh | 49 -- script/server/4_sysom_vul/start.sh | 10 - script/server/4_sysom_vul/stop.sh | 9 - script/server/4_sysom_vul/sysom-vul.ini | 9 - script/server/5_sysom_migration/clear.sh | 8 - script/server/5_sysom_migration/init.sh | 80 --- script/server/5_sysom_migration/start.sh | 10 - script/server/5_sysom_migration/stop.sh | 9 - .../5_sysom_migration/sysom-migration.ini | 9 - script/server/6_sysom_hotfix/clear.sh | 8 - script/server/6_sysom_hotfix/init.sh | 61 -- script/server/6_sysom_hotfix/start.sh | 8 - script/server/6_sysom_hotfix/stop.sh | 9 - script/server/6_sysom_hotfix/sysom-hotfix.ini | 9 - script/server/6_sysom_hotfix_builder/clear.sh | 25 - script/server/6_sysom_hotfix_builder/init.sh | 56 -- script/server/6_sysom_hotfix_builder/start.sh | 8 - script/server/6_sysom_hotfix_builder/stop.sh | 9 - .../sysom-hotfix-builder.ini | 9 - script/server/7_sysom_demo/clear.sh | 8 - script/server/7_sysom_demo/init.sh | 43 -- script/server/7_sysom_demo/start.sh | 7 - script/server/7_sysom_demo/stop.sh | 7 - script/server/7_sysom_demo/sysom-demo.ini | 9 - script/server/clear.sh | 31 +- script/server/conf | 32 - script/server/init.sh | 152 +---- script/server/stop.sh | 21 - script/sysom.sh | 11 +- sysom_server/.gitignore | 3 - sysom_server/sysom_demo/main.py | 29 - 348 files changed, 460 insertions(+), 2902 deletions(-) rename sysom_api_dockerfile => docker/sysom_api_dockerfile (100%) rename sysom_channel_dockerfile => docker/sysom_channel_dockerfile (100%) rename sysom_diagnosis_dockerfile => docker/sysom_diagnosis_dockerfile (100%) rename sysom_hotfix_dockerfile => docker/sysom_hotfix_dockerfile (100%) rename sysom_migration_dockerfile => docker/sysom_migration_dockerfile (100%) rename sysom_monitor_server_dockerfile => docker/sysom_monitor_server_dockerfile (100%) create mode 100644 docker/sysom_vmcore_dockerfile rename sysom_vul_dockerfile => docker/sysom_vul_dockerfile (100%) rename infrastructure/{env => 0_env}/.pydistutils.cfg (100%) rename infrastructure/{env => 0_env}/clear.sh (63%) rename infrastructure/{env => 0_env}/init.sh (94%) rename infrastructure/{env => 0_env}/pip.conf (100%) rename infrastructure/{env => 0_env}/requirements.txt (100%) rename infrastructure/{env => 0_env}/start.sh (100%) rename infrastructure/{env => 0_env}/stop.sh (100%) rename infrastructure/{sdk => 1_sdk}/.gitignore (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/admin.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/base.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/cec_client.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/cli.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/consumer.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/event.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/exceptions.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/meta.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/producer.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/url.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_base/utils.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/admin_static.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/common.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/consume_status_storage.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/heartbeat.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/redis_admin.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/redis_consumer.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/redis_producer.py (100%) rename infrastructure/{sdk => 1_sdk}/cec_redis/utils.py (100%) rename infrastructure/{sdk => 1_sdk}/channel_job/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/channel_job/exception.py (100%) rename infrastructure/{sdk => 1_sdk}/channel_job/job.py (100%) rename infrastructure/{sdk => 1_sdk}/channel_job/model.py (100%) rename infrastructure/{sdk => 1_sdk}/clear.sh (100%) rename infrastructure/{sdk => 1_sdk}/clogger/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/clogger/clogger.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/exceptions.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/load_balancing_strategy.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/service_check.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/service_discovery.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/service_instance.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/service_invoker.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/service_registry.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/url.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_base/utils.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_redis/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_redis/common.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_redis/redis_service_discovery.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_redis/redis_service_registry.py (100%) rename infrastructure/{sdk => 1_sdk}/cmg_redis/utils.py (100%) rename infrastructure/{sdk => 1_sdk}/init.sh (87%) rename infrastructure/{sdk => 1_sdk}/setup_cec_base.py (100%) rename infrastructure/{sdk => 1_sdk}/setup_cec_redis.py (100%) rename infrastructure/{sdk => 1_sdk}/setup_channel_job.py (100%) rename infrastructure/{sdk => 1_sdk}/setup_clogger.py (100%) rename infrastructure/{sdk => 1_sdk}/setup_cmg_base.py (100%) rename infrastructure/{sdk => 1_sdk}/setup_cmg_redis.py (100%) rename infrastructure/{sdk => 1_sdk}/setup_sysom_utils.py (100%) rename infrastructure/{sdk => 1_sdk}/start.sh (100%) rename infrastructure/{sdk => 1_sdk}/stop.sh (100%) rename infrastructure/{sdk => 1_sdk}/sysom_utils/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/sysom_utils/adddict.py (100%) rename infrastructure/{sdk => 1_sdk}/sysom_utils/config_parser.py (100%) rename infrastructure/{sdk => 1_sdk}/sysom_utils/event_executor.py (100%) rename infrastructure/{sdk => 1_sdk}/sysom_utils/framework_plug_mag.py (100%) rename infrastructure/{sdk => 1_sdk}/sysom_utils/framework_plugins.py (100%) rename infrastructure/{sdk => 1_sdk}/sysom_utils/node_manager.py (100%) rename infrastructure/{sdk => 1_sdk}/sysom_utils/yaml_concat.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/test_heartbeat.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/test_redis_admin.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/test_redis_admin_async.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/test_redis_consumer.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/test_redis_consumer_async.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/test_redis_multi_consumer.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/test_redis_producer.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cec_redis/test_utils.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cmg_base/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cmg_base/test_health_check.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cmg_redis/__init__.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cmg_redis/test_service_discovery.py (100%) rename infrastructure/{sdk => 1_sdk}/test_cmg_redis/test_service_registry.py (100%) rename infrastructure/{local_services => 2_local_services}/clear.sh (100%) rename infrastructure/{local_services => 2_local_services}/init.sh (96%) rename infrastructure/{local_services => 2_local_services}/nginx.conf (100%) rename infrastructure/{local_services => 2_local_services}/start.sh (100%) rename infrastructure/{local_services => 2_local_services}/stop.sh (100%) rename {script/server/0_local_services => infrastructure/2_local_services}/sysom-redis.ini (61%) rename infrastructure/{local_services => 2_local_services}/sysom.conf (98%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/Readme.txt (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/clear.sh (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/grafana_api_set.sh (99%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/grafana_recover.sh (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/init.sh (96%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/local_copy_pkg.sh (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/prometheus_get_node.py (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/start.sh (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/stop.sh (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/sysom-cec-status-dashboard.json (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/sysom-dashboard.json (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/sysom-migration-dashboard.json (100%) rename {script/server/1_sysom_monitor => infrastructure/3_monitor}/sysom-netinfo-dashboard.json (100%) create mode 100644 infrastructure/3_monitor/sysom-prometheus.ini create mode 100644 infrastructure/clear_exclude rename microservice/sysom_api/apps/accounts/__init__.py => infrastructure/deploy_exclude (100%) delete mode 100644 infrastructure/local_services/sysom-redis.ini rename microservice/{sysom_api => 0_sysom_api}/.gitignore (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/__init__.py (100%) rename microservice/{sysom_api/apps/accounts/migrations => 0_sysom_api/apps/accounts}/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/admin.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/apps.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/authentication.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/migrations/0001_initial.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/migrations/0002_user_allow_login.py (100%) rename microservice/{sysom_api/apps/alarm => 0_sysom_api/apps/accounts}/migrations/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/models.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/permissions.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/serializer.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/tests.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/urls.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/accounts/views.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/admin.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/apps.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/migrations/0001_initial.py (100%) rename microservice/{sysom_api/apps/common => 0_sysom_api/apps/alarm/migrations}/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/models.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/serializer.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/subscribe.json (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/urls.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/alarm/views.py (100%) rename microservice/{sysom_api/apps/host => 0_sysom_api/apps/common}/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/common/common_model_viewset.py (100%) rename microservice/{sysom_api/apps/host/migrations => 0_sysom_api/apps/host}/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/admin.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/apps.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/cec_api.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/heartbeat.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/migrations/0001_initial.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/migrations/0002_hostmodel_host_info.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/migrations/0003_alter_hostmodel_status.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/migrations/0004_auto_20221227_1705.py (100%) rename microservice/{sysom_api/apps/services => 0_sysom_api/apps/host}/migrations/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/models.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/serializer.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/tests.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/urls.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/host/views.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/admin.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/apps.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/migrations/0001_initial.py (100%) rename microservice/{sysom_api/conf => 0_sysom_api/apps/services/migrations}/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/models.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/serializer.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/tests.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/urls.py (100%) rename microservice/{sysom_api => 0_sysom_api}/apps/services/views.py (100%) create mode 100644 microservice/0_sysom_api/conf/__init__.py rename microservice/{sysom_api => 0_sysom_api}/conf/common.py (98%) rename microservice/{sysom_api => 0_sysom_api}/conf/develop.py (100%) rename microservice/{sysom_api => 0_sysom_api}/conf/product.py (100%) rename microservice/{sysom_api => 0_sysom_api}/conf/testing.py (100%) rename microservice/{sysom_api => 0_sysom_api}/config.yml (100%) rename microservice/{sysom_api => 0_sysom_api}/consumer/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/consumer/consumers.py (100%) rename microservice/{sysom_api => 0_sysom_api}/consumer/middleware.py (100%) rename microservice/{sysom_api => 0_sysom_api}/consumer/routing.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/authentications.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/base_model.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/decode/sysom_decode.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/excel.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/exception.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/paginations.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/renderers.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/response.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/ssh.py (100%) rename microservice/{sysom_api => 0_sysom_api}/lib/utils.py (100%) rename microservice/{sysom_api => 0_sysom_api}/manage.py (100%) rename microservice/{sysom_api => 0_sysom_api}/requirements.txt (100%) rename microservice/{sysom_api => 0_sysom_api}/scripts/server_clear.sh (100%) rename microservice/{sysom_api => 0_sysom_api}/scripts/server_init.sh (90%) rename microservice/{sysom_api => 0_sysom_api}/scripts/server_start.sh (100%) rename microservice/{sysom_api => 0_sysom_api}/scripts/server_stop.sh (100%) rename microservice/{sysom_api => 0_sysom_api}/sysom-api.ini (55%) rename microservice/{sysom_api => 0_sysom_api}/sysom/__init__.py (100%) rename microservice/{sysom_api => 0_sysom_api}/sysom/asgi.py (100%) rename microservice/{sysom_api => 0_sysom_api}/sysom/settings.py (100%) rename microservice/{sysom_api => 0_sysom_api}/sysom/urls.py (100%) rename microservice/{sysom_api => 0_sysom_api}/sysom/wsgi.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/alembic.ini (100%) rename microservice/{sysom_channel => 0_sysom_channel}/alembic/README (100%) rename microservice/{sysom_channel => 0_sysom_channel}/alembic/env.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/alembic/script.py.mako (100%) rename microservice/{sysom_channel => 0_sysom_channel}/alembic/versions/1663292d0bd8_init_db.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/alembic/versions/3f54250e3fca_add_channel_params_table.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/__init__.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/crud.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/database.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/executor.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/models.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/routers/cec_status.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/routers/config.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/routers/file.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/routers/health.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/app/schemas.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/conf/common.py (97%) rename microservice/{sysom_channel => 0_sysom_channel}/conf/develop.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/conf/gunicorn.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/conf/product.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/conf/settings.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/conf/testing.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/config.yml (100%) rename microservice/{sysom_channel => 0_sysom_channel}/lib/channels/__init__.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/lib/channels/base.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/lib/channels/ssh.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/lib/ssh.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/lib/utils.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/main.py (100%) rename microservice/{sysom_channel => 0_sysom_channel}/requirements.txt (100%) rename microservice/{sysom_channel => 0_sysom_channel}/scripts/server_clear.sh (100%) rename microservice/{sysom_channel => 0_sysom_channel}/scripts/server_init.sh (90%) rename microservice/{sysom_channel => 0_sysom_channel}/scripts/server_start.sh (100%) rename microservice/{sysom_channel => 0_sysom_channel}/scripts/server_stop.sh (100%) rename microservice/{sysom_channel => 0_sysom_channel}/sysom-channel.ini (41%) create mode 100644 microservice/clear_exclude create mode 100644 microservice/deploy_exclude rename microservice/sysom_diagnosis/conf/{diagnosis_gunicorn.py => gunicorn.py} (100%) mode change 100644 => 100755 package.sh mode change 100644 => 100755 package_rpm_offline.sh mode change 100644 => 100755 package_rpm_online.sh mode change 100644 => 100755 script/clear.sh create mode 100755 script/deploy/clear.sh create mode 100755 script/deploy/deploy.sh delete mode 100644 script/node/clear.sh delete mode 100755 script/node/diagnosis/clear.sh delete mode 100755 script/node/diagnosis/init.sh delete mode 100644 script/node/init.sh delete mode 100755 script/node/monitor/clear.sh delete mode 100755 script/node/monitor/init.sh delete mode 100755 script/node/monitor/raptor_profiling_clear.sh delete mode 100755 script/node/monitor/raptor_profiling_deploy.sh delete mode 120000 script/node/vmcore/clear.sh delete mode 100644 script/node/vmcore/clear_vmcore.sh delete mode 120000 script/node/vmcore/init.sh delete mode 100644 script/node/vmcore/init_vmcore.sh delete mode 100644 script/node/vmcore/vmcore_collect.py delete mode 100755 script/server/0_env/init.sh delete mode 100644 script/server/0_env/pip.conf delete mode 100644 script/server/0_env/requirements.txt delete mode 100755 script/server/0_local_services/clear.sh delete mode 100755 script/server/0_local_services/init.sh delete mode 100644 script/server/0_local_services/nginx.conf delete mode 100755 script/server/0_local_services/start.sh delete mode 100755 script/server/0_local_services/stop.sh delete mode 100644 script/server/0_local_services/sysom.conf delete mode 100755 script/server/0_sysom_api/clear.sh delete mode 100755 script/server/0_sysom_api/init.sh delete mode 100755 script/server/0_sysom_api/start.sh delete mode 100755 script/server/0_sysom_api/stop.sh delete mode 100644 script/server/0_sysom_api/sysom-api.ini delete mode 100755 script/server/0_sysom_channel/clear.sh delete mode 100755 script/server/0_sysom_channel/init.sh delete mode 100755 script/server/0_sysom_channel/start.sh delete mode 100755 script/server/0_sysom_channel/stop.sh delete mode 100644 script/server/0_sysom_channel/sysom-channel.ini delete mode 100644 script/server/1_sysom_monitor/sysom-prometheus.ini delete mode 100755 script/server/1_sysom_monitor_server/clear.sh delete mode 100755 script/server/1_sysom_monitor_server/init.sh delete mode 100755 script/server/1_sysom_monitor_server/start.sh delete mode 100755 script/server/1_sysom_monitor_server/stop.sh delete mode 100644 script/server/1_sysom_monitor_server/sysom-monitor-server.ini delete mode 100644 script/server/2_sysom_vmcore/clear.sh delete mode 100644 script/server/2_sysom_vmcore/init.sh delete mode 100644 script/server/2_sysom_vmcore/parse_panic.py delete mode 100644 script/server/2_sysom_vmcore/start.sh delete mode 100644 script/server/2_sysom_vmcore/stop.sh delete mode 100644 script/server/2_sysom_vmcore/sysom-vmcore.ini delete mode 100644 script/server/2_sysom_vmcore/vmcore_const.py delete mode 100755 script/server/3_sysom_diagnosis/clear.sh delete mode 100755 script/server/3_sysom_diagnosis/init.sh delete mode 100755 script/server/3_sysom_diagnosis/start.sh delete mode 100755 script/server/3_sysom_diagnosis/stop.sh delete mode 100644 script/server/3_sysom_diagnosis/sysom-diagnosis.ini delete mode 100644 script/server/4_sysom_vul/clear.sh delete mode 100644 script/server/4_sysom_vul/init.sh delete mode 100644 script/server/4_sysom_vul/start.sh delete mode 100644 script/server/4_sysom_vul/stop.sh delete mode 100644 script/server/4_sysom_vul/sysom-vul.ini delete mode 100644 script/server/5_sysom_migration/clear.sh delete mode 100644 script/server/5_sysom_migration/init.sh delete mode 100644 script/server/5_sysom_migration/start.sh delete mode 100644 script/server/5_sysom_migration/stop.sh delete mode 100644 script/server/5_sysom_migration/sysom-migration.ini delete mode 100644 script/server/6_sysom_hotfix/clear.sh delete mode 100644 script/server/6_sysom_hotfix/init.sh delete mode 100644 script/server/6_sysom_hotfix/start.sh delete mode 100644 script/server/6_sysom_hotfix/stop.sh delete mode 100644 script/server/6_sysom_hotfix/sysom-hotfix.ini delete mode 100644 script/server/6_sysom_hotfix_builder/clear.sh delete mode 100644 script/server/6_sysom_hotfix_builder/init.sh delete mode 100644 script/server/6_sysom_hotfix_builder/start.sh delete mode 100644 script/server/6_sysom_hotfix_builder/stop.sh delete mode 100644 script/server/6_sysom_hotfix_builder/sysom-hotfix-builder.ini delete mode 100755 script/server/7_sysom_demo/clear.sh delete mode 100755 script/server/7_sysom_demo/init.sh delete mode 100755 script/server/7_sysom_demo/start.sh delete mode 100755 script/server/7_sysom_demo/stop.sh delete mode 100644 script/server/7_sysom_demo/sysom-demo.ini delete mode 100644 script/server/conf delete mode 100644 script/server/stop.sh mode change 100755 => 100644 script/sysom.sh delete mode 100644 sysom_server/.gitignore delete mode 100644 sysom_server/sysom_demo/main.py diff --git a/sysom_api_dockerfile b/docker/sysom_api_dockerfile similarity index 100% rename from sysom_api_dockerfile rename to docker/sysom_api_dockerfile diff --git a/sysom_channel_dockerfile b/docker/sysom_channel_dockerfile similarity index 100% rename from sysom_channel_dockerfile rename to docker/sysom_channel_dockerfile diff --git a/sysom_diagnosis_dockerfile b/docker/sysom_diagnosis_dockerfile similarity index 100% rename from sysom_diagnosis_dockerfile rename to docker/sysom_diagnosis_dockerfile diff --git a/sysom_hotfix_dockerfile b/docker/sysom_hotfix_dockerfile similarity index 100% rename from sysom_hotfix_dockerfile rename to docker/sysom_hotfix_dockerfile diff --git a/sysom_migration_dockerfile b/docker/sysom_migration_dockerfile similarity index 100% rename from sysom_migration_dockerfile rename to docker/sysom_migration_dockerfile diff --git a/sysom_monitor_server_dockerfile b/docker/sysom_monitor_server_dockerfile similarity index 100% rename from sysom_monitor_server_dockerfile rename to docker/sysom_monitor_server_dockerfile diff --git a/docker/sysom_vmcore_dockerfile b/docker/sysom_vmcore_dockerfile new file mode 100644 index 00000000..6bbe9548 --- /dev/null +++ b/docker/sysom_vmcore_dockerfile @@ -0,0 +1,31 @@ +FROM openanolis/anolisos:8.8 + +# Add epel +RUN yum install -y https://mirrors.aliyun.com/epel/epel-release-latest-8.noarch.rpm +RUN bash -c "sed -i 's|^#baseurl=https://download.example/pub|baseurl=https://mirrors.aliyun.com|' /etc/yum.repos.d/epel*" +RUN bash -c "sed -i 's|^metalink|#metalink|' /etc/yum.repos.d/epel*" + +# Add required yum packages +RUN yum makecache +RUN yum install -y supervisor cronie net-tools rpcbind nfs-utils +RUN systemctl enable crond +# RUN systemctl enable supervisord + +# Init sysom-diagnosis +ARG SYSOM_HOME=/usr/local/sysom +ARG SYSOM_SERVER_HOME=${SYSOM_HOME}/server + +RUN mkdir /root/sysom +COPY conf /root/sysom/conf +COPY script /root/sysom/script +COPY infrastructure /root/sysom/infrastructure +COPY microservice /root/sysom/microservice + +RUN bash -x /root/sysom/script/sysom.sh deploy infrastructure env,sdk +RUN bash -x /root/sysom/script/sysom.sh deploy microservice sysom_vmcore +RUN sed "s/nodaemon=false/nodaemon=true/g" -i /etc/supervisord.conf + +RUN yum clean all + +# # 环境准备 +ENTRYPOINT ["/usr/bin/supervisord", "-c", "/etc/supervisord.conf"] \ No newline at end of file diff --git a/sysom_vul_dockerfile b/docker/sysom_vul_dockerfile similarity index 100% rename from sysom_vul_dockerfile rename to docker/sysom_vul_dockerfile diff --git a/infrastructure/env/.pydistutils.cfg b/infrastructure/0_env/.pydistutils.cfg similarity index 100% rename from infrastructure/env/.pydistutils.cfg rename to infrastructure/0_env/.pydistutils.cfg diff --git a/infrastructure/env/clear.sh b/infrastructure/0_env/clear.sh similarity index 63% rename from infrastructure/env/clear.sh rename to infrastructure/0_env/clear.sh index 1663cd0b..6f9baa9c 100755 --- a/infrastructure/env/clear.sh +++ b/infrastructure/0_env/clear.sh @@ -1,6 +1,6 @@ #!/bin/bash +x -VIRTUALENV_HOME="${INFRASTRUCTURE_HOME}/virtualenv" +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME remove_virtualenv() { rm -rf ${VIRTUALENV_HOME} diff --git a/infrastructure/env/init.sh b/infrastructure/0_env/init.sh similarity index 94% rename from infrastructure/env/init.sh rename to infrastructure/0_env/init.sh index ed371cc3..946cd247 100755 --- a/infrastructure/env/init.sh +++ b/infrastructure/0_env/init.sh @@ -5,7 +5,7 @@ #***************************************************************# ALIYUN_MIRROR="https://mirrors.aliyun.com/pypi/simple/" -VIRTUALENV_HOME="${INFRASTRUCTURE_HOME}/virtualenv" +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME if [ "$UID" -ne 0 ]; then echo "Please run as root" @@ -44,9 +44,9 @@ touch_env_rpms() { check_requirements() { echo "INFO: begin install requirements..." - if ! [ -d ${SERVER_HOME}/logs/ ]; then - mkdir -p ${SERVER_HOME}/logs/ || exit 1 - fi + # if ! [ -d ${SERVER_HOME}/logs/ ]; then + # mkdir -p ${SERVER_HOME}/logs/ || exit 1 + # fi local requirements_log="${LOG_HOME}/requirements.log" touch "$requirements_log" || exit diff --git a/infrastructure/env/pip.conf b/infrastructure/0_env/pip.conf similarity index 100% rename from infrastructure/env/pip.conf rename to infrastructure/0_env/pip.conf diff --git a/infrastructure/env/requirements.txt b/infrastructure/0_env/requirements.txt similarity index 100% rename from infrastructure/env/requirements.txt rename to infrastructure/0_env/requirements.txt diff --git a/infrastructure/env/start.sh b/infrastructure/0_env/start.sh similarity index 100% rename from infrastructure/env/start.sh rename to infrastructure/0_env/start.sh diff --git a/infrastructure/env/stop.sh b/infrastructure/0_env/stop.sh similarity index 100% rename from infrastructure/env/stop.sh rename to infrastructure/0_env/stop.sh diff --git a/infrastructure/sdk/.gitignore b/infrastructure/1_sdk/.gitignore similarity index 100% rename from infrastructure/sdk/.gitignore rename to infrastructure/1_sdk/.gitignore diff --git a/infrastructure/sdk/cec_base/__init__.py b/infrastructure/1_sdk/cec_base/__init__.py similarity index 100% rename from infrastructure/sdk/cec_base/__init__.py rename to infrastructure/1_sdk/cec_base/__init__.py diff --git a/infrastructure/sdk/cec_base/admin.py b/infrastructure/1_sdk/cec_base/admin.py similarity index 100% rename from infrastructure/sdk/cec_base/admin.py rename to infrastructure/1_sdk/cec_base/admin.py diff --git a/infrastructure/sdk/cec_base/base.py b/infrastructure/1_sdk/cec_base/base.py similarity index 100% rename from infrastructure/sdk/cec_base/base.py rename to infrastructure/1_sdk/cec_base/base.py diff --git a/infrastructure/sdk/cec_base/cec_client.py b/infrastructure/1_sdk/cec_base/cec_client.py similarity index 100% rename from infrastructure/sdk/cec_base/cec_client.py rename to infrastructure/1_sdk/cec_base/cec_client.py diff --git a/infrastructure/sdk/cec_base/cli.py b/infrastructure/1_sdk/cec_base/cli.py similarity index 100% rename from infrastructure/sdk/cec_base/cli.py rename to infrastructure/1_sdk/cec_base/cli.py diff --git a/infrastructure/sdk/cec_base/consumer.py b/infrastructure/1_sdk/cec_base/consumer.py similarity index 100% rename from infrastructure/sdk/cec_base/consumer.py rename to infrastructure/1_sdk/cec_base/consumer.py diff --git a/infrastructure/sdk/cec_base/event.py b/infrastructure/1_sdk/cec_base/event.py similarity index 100% rename from infrastructure/sdk/cec_base/event.py rename to infrastructure/1_sdk/cec_base/event.py diff --git a/infrastructure/sdk/cec_base/exceptions.py b/infrastructure/1_sdk/cec_base/exceptions.py similarity index 100% rename from infrastructure/sdk/cec_base/exceptions.py rename to infrastructure/1_sdk/cec_base/exceptions.py diff --git a/infrastructure/sdk/cec_base/meta.py b/infrastructure/1_sdk/cec_base/meta.py similarity index 100% rename from infrastructure/sdk/cec_base/meta.py rename to infrastructure/1_sdk/cec_base/meta.py diff --git a/infrastructure/sdk/cec_base/producer.py b/infrastructure/1_sdk/cec_base/producer.py similarity index 100% rename from infrastructure/sdk/cec_base/producer.py rename to infrastructure/1_sdk/cec_base/producer.py diff --git a/infrastructure/sdk/cec_base/url.py b/infrastructure/1_sdk/cec_base/url.py similarity index 100% rename from infrastructure/sdk/cec_base/url.py rename to infrastructure/1_sdk/cec_base/url.py diff --git a/infrastructure/sdk/cec_base/utils.py b/infrastructure/1_sdk/cec_base/utils.py similarity index 100% rename from infrastructure/sdk/cec_base/utils.py rename to infrastructure/1_sdk/cec_base/utils.py diff --git a/infrastructure/sdk/cec_redis/__init__.py b/infrastructure/1_sdk/cec_redis/__init__.py similarity index 100% rename from infrastructure/sdk/cec_redis/__init__.py rename to infrastructure/1_sdk/cec_redis/__init__.py diff --git a/infrastructure/sdk/cec_redis/admin_static.py b/infrastructure/1_sdk/cec_redis/admin_static.py similarity index 100% rename from infrastructure/sdk/cec_redis/admin_static.py rename to infrastructure/1_sdk/cec_redis/admin_static.py diff --git a/infrastructure/sdk/cec_redis/common.py b/infrastructure/1_sdk/cec_redis/common.py similarity index 100% rename from infrastructure/sdk/cec_redis/common.py rename to infrastructure/1_sdk/cec_redis/common.py diff --git a/infrastructure/sdk/cec_redis/consume_status_storage.py b/infrastructure/1_sdk/cec_redis/consume_status_storage.py similarity index 100% rename from infrastructure/sdk/cec_redis/consume_status_storage.py rename to infrastructure/1_sdk/cec_redis/consume_status_storage.py diff --git a/infrastructure/sdk/cec_redis/heartbeat.py b/infrastructure/1_sdk/cec_redis/heartbeat.py similarity index 100% rename from infrastructure/sdk/cec_redis/heartbeat.py rename to infrastructure/1_sdk/cec_redis/heartbeat.py diff --git a/infrastructure/sdk/cec_redis/redis_admin.py b/infrastructure/1_sdk/cec_redis/redis_admin.py similarity index 100% rename from infrastructure/sdk/cec_redis/redis_admin.py rename to infrastructure/1_sdk/cec_redis/redis_admin.py diff --git a/infrastructure/sdk/cec_redis/redis_consumer.py b/infrastructure/1_sdk/cec_redis/redis_consumer.py similarity index 100% rename from infrastructure/sdk/cec_redis/redis_consumer.py rename to infrastructure/1_sdk/cec_redis/redis_consumer.py diff --git a/infrastructure/sdk/cec_redis/redis_producer.py b/infrastructure/1_sdk/cec_redis/redis_producer.py similarity index 100% rename from infrastructure/sdk/cec_redis/redis_producer.py rename to infrastructure/1_sdk/cec_redis/redis_producer.py diff --git a/infrastructure/sdk/cec_redis/utils.py b/infrastructure/1_sdk/cec_redis/utils.py similarity index 100% rename from infrastructure/sdk/cec_redis/utils.py rename to infrastructure/1_sdk/cec_redis/utils.py diff --git a/infrastructure/sdk/channel_job/__init__.py b/infrastructure/1_sdk/channel_job/__init__.py similarity index 100% rename from infrastructure/sdk/channel_job/__init__.py rename to infrastructure/1_sdk/channel_job/__init__.py diff --git a/infrastructure/sdk/channel_job/exception.py b/infrastructure/1_sdk/channel_job/exception.py similarity index 100% rename from infrastructure/sdk/channel_job/exception.py rename to infrastructure/1_sdk/channel_job/exception.py diff --git a/infrastructure/sdk/channel_job/job.py b/infrastructure/1_sdk/channel_job/job.py similarity index 100% rename from infrastructure/sdk/channel_job/job.py rename to infrastructure/1_sdk/channel_job/job.py diff --git a/infrastructure/sdk/channel_job/model.py b/infrastructure/1_sdk/channel_job/model.py similarity index 100% rename from infrastructure/sdk/channel_job/model.py rename to infrastructure/1_sdk/channel_job/model.py diff --git a/infrastructure/sdk/clear.sh b/infrastructure/1_sdk/clear.sh similarity index 100% rename from infrastructure/sdk/clear.sh rename to infrastructure/1_sdk/clear.sh diff --git a/infrastructure/sdk/clogger/__init__.py b/infrastructure/1_sdk/clogger/__init__.py similarity index 100% rename from infrastructure/sdk/clogger/__init__.py rename to infrastructure/1_sdk/clogger/__init__.py diff --git a/infrastructure/sdk/clogger/clogger.py b/infrastructure/1_sdk/clogger/clogger.py similarity index 100% rename from infrastructure/sdk/clogger/clogger.py rename to infrastructure/1_sdk/clogger/clogger.py diff --git a/infrastructure/sdk/cmg_base/__init__.py b/infrastructure/1_sdk/cmg_base/__init__.py similarity index 100% rename from infrastructure/sdk/cmg_base/__init__.py rename to infrastructure/1_sdk/cmg_base/__init__.py diff --git a/infrastructure/sdk/cmg_base/exceptions.py b/infrastructure/1_sdk/cmg_base/exceptions.py similarity index 100% rename from infrastructure/sdk/cmg_base/exceptions.py rename to infrastructure/1_sdk/cmg_base/exceptions.py diff --git a/infrastructure/sdk/cmg_base/load_balancing_strategy.py b/infrastructure/1_sdk/cmg_base/load_balancing_strategy.py similarity index 100% rename from infrastructure/sdk/cmg_base/load_balancing_strategy.py rename to infrastructure/1_sdk/cmg_base/load_balancing_strategy.py diff --git a/infrastructure/sdk/cmg_base/service_check.py b/infrastructure/1_sdk/cmg_base/service_check.py similarity index 100% rename from infrastructure/sdk/cmg_base/service_check.py rename to infrastructure/1_sdk/cmg_base/service_check.py diff --git a/infrastructure/sdk/cmg_base/service_discovery.py b/infrastructure/1_sdk/cmg_base/service_discovery.py similarity index 100% rename from infrastructure/sdk/cmg_base/service_discovery.py rename to infrastructure/1_sdk/cmg_base/service_discovery.py diff --git a/infrastructure/sdk/cmg_base/service_instance.py b/infrastructure/1_sdk/cmg_base/service_instance.py similarity index 100% rename from infrastructure/sdk/cmg_base/service_instance.py rename to infrastructure/1_sdk/cmg_base/service_instance.py diff --git a/infrastructure/sdk/cmg_base/service_invoker.py b/infrastructure/1_sdk/cmg_base/service_invoker.py similarity index 100% rename from infrastructure/sdk/cmg_base/service_invoker.py rename to infrastructure/1_sdk/cmg_base/service_invoker.py diff --git a/infrastructure/sdk/cmg_base/service_registry.py b/infrastructure/1_sdk/cmg_base/service_registry.py similarity index 100% rename from infrastructure/sdk/cmg_base/service_registry.py rename to infrastructure/1_sdk/cmg_base/service_registry.py diff --git a/infrastructure/sdk/cmg_base/url.py b/infrastructure/1_sdk/cmg_base/url.py similarity index 100% rename from infrastructure/sdk/cmg_base/url.py rename to infrastructure/1_sdk/cmg_base/url.py diff --git a/infrastructure/sdk/cmg_base/utils.py b/infrastructure/1_sdk/cmg_base/utils.py similarity index 100% rename from infrastructure/sdk/cmg_base/utils.py rename to infrastructure/1_sdk/cmg_base/utils.py diff --git a/infrastructure/sdk/cmg_redis/__init__.py b/infrastructure/1_sdk/cmg_redis/__init__.py similarity index 100% rename from infrastructure/sdk/cmg_redis/__init__.py rename to infrastructure/1_sdk/cmg_redis/__init__.py diff --git a/infrastructure/sdk/cmg_redis/common.py b/infrastructure/1_sdk/cmg_redis/common.py similarity index 100% rename from infrastructure/sdk/cmg_redis/common.py rename to infrastructure/1_sdk/cmg_redis/common.py diff --git a/infrastructure/sdk/cmg_redis/redis_service_discovery.py b/infrastructure/1_sdk/cmg_redis/redis_service_discovery.py similarity index 100% rename from infrastructure/sdk/cmg_redis/redis_service_discovery.py rename to infrastructure/1_sdk/cmg_redis/redis_service_discovery.py diff --git a/infrastructure/sdk/cmg_redis/redis_service_registry.py b/infrastructure/1_sdk/cmg_redis/redis_service_registry.py similarity index 100% rename from infrastructure/sdk/cmg_redis/redis_service_registry.py rename to infrastructure/1_sdk/cmg_redis/redis_service_registry.py diff --git a/infrastructure/sdk/cmg_redis/utils.py b/infrastructure/1_sdk/cmg_redis/utils.py similarity index 100% rename from infrastructure/sdk/cmg_redis/utils.py rename to infrastructure/1_sdk/cmg_redis/utils.py diff --git a/infrastructure/sdk/init.sh b/infrastructure/1_sdk/init.sh similarity index 87% rename from infrastructure/sdk/init.sh rename to infrastructure/1_sdk/init.sh index 97e6a0c3..e8465846 100755 --- a/infrastructure/sdk/init.sh +++ b/infrastructure/1_sdk/init.sh @@ -4,8 +4,8 @@ # Author: huangtuquan #***************************************************************# -VIRTUALENV_HOME="${INFRASTRUCTURE_HOME}/virtualenv" -SDK_DIR=$INFRASTRUCTURE_HOME/sdk +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME +SDK_DIR=$(cd $(dirname ${BASH_SOURCE[0]}) && pwd) source_virtualenv() { echo "INFO: activate virtualenv..." diff --git a/infrastructure/sdk/setup_cec_base.py b/infrastructure/1_sdk/setup_cec_base.py similarity index 100% rename from infrastructure/sdk/setup_cec_base.py rename to infrastructure/1_sdk/setup_cec_base.py diff --git a/infrastructure/sdk/setup_cec_redis.py b/infrastructure/1_sdk/setup_cec_redis.py similarity index 100% rename from infrastructure/sdk/setup_cec_redis.py rename to infrastructure/1_sdk/setup_cec_redis.py diff --git a/infrastructure/sdk/setup_channel_job.py b/infrastructure/1_sdk/setup_channel_job.py similarity index 100% rename from infrastructure/sdk/setup_channel_job.py rename to infrastructure/1_sdk/setup_channel_job.py diff --git a/infrastructure/sdk/setup_clogger.py b/infrastructure/1_sdk/setup_clogger.py similarity index 100% rename from infrastructure/sdk/setup_clogger.py rename to infrastructure/1_sdk/setup_clogger.py diff --git a/infrastructure/sdk/setup_cmg_base.py b/infrastructure/1_sdk/setup_cmg_base.py similarity index 100% rename from infrastructure/sdk/setup_cmg_base.py rename to infrastructure/1_sdk/setup_cmg_base.py diff --git a/infrastructure/sdk/setup_cmg_redis.py b/infrastructure/1_sdk/setup_cmg_redis.py similarity index 100% rename from infrastructure/sdk/setup_cmg_redis.py rename to infrastructure/1_sdk/setup_cmg_redis.py diff --git a/infrastructure/sdk/setup_sysom_utils.py b/infrastructure/1_sdk/setup_sysom_utils.py similarity index 100% rename from infrastructure/sdk/setup_sysom_utils.py rename to infrastructure/1_sdk/setup_sysom_utils.py diff --git a/infrastructure/sdk/start.sh b/infrastructure/1_sdk/start.sh similarity index 100% rename from infrastructure/sdk/start.sh rename to infrastructure/1_sdk/start.sh diff --git a/infrastructure/sdk/stop.sh b/infrastructure/1_sdk/stop.sh similarity index 100% rename from infrastructure/sdk/stop.sh rename to infrastructure/1_sdk/stop.sh diff --git a/infrastructure/sdk/sysom_utils/__init__.py b/infrastructure/1_sdk/sysom_utils/__init__.py similarity index 100% rename from infrastructure/sdk/sysom_utils/__init__.py rename to infrastructure/1_sdk/sysom_utils/__init__.py diff --git a/infrastructure/sdk/sysom_utils/adddict.py b/infrastructure/1_sdk/sysom_utils/adddict.py similarity index 100% rename from infrastructure/sdk/sysom_utils/adddict.py rename to infrastructure/1_sdk/sysom_utils/adddict.py diff --git a/infrastructure/sdk/sysom_utils/config_parser.py b/infrastructure/1_sdk/sysom_utils/config_parser.py similarity index 100% rename from infrastructure/sdk/sysom_utils/config_parser.py rename to infrastructure/1_sdk/sysom_utils/config_parser.py diff --git a/infrastructure/sdk/sysom_utils/event_executor.py b/infrastructure/1_sdk/sysom_utils/event_executor.py similarity index 100% rename from infrastructure/sdk/sysom_utils/event_executor.py rename to infrastructure/1_sdk/sysom_utils/event_executor.py diff --git a/infrastructure/sdk/sysom_utils/framework_plug_mag.py b/infrastructure/1_sdk/sysom_utils/framework_plug_mag.py similarity index 100% rename from infrastructure/sdk/sysom_utils/framework_plug_mag.py rename to infrastructure/1_sdk/sysom_utils/framework_plug_mag.py diff --git a/infrastructure/sdk/sysom_utils/framework_plugins.py b/infrastructure/1_sdk/sysom_utils/framework_plugins.py similarity index 100% rename from infrastructure/sdk/sysom_utils/framework_plugins.py rename to infrastructure/1_sdk/sysom_utils/framework_plugins.py diff --git a/infrastructure/sdk/sysom_utils/node_manager.py b/infrastructure/1_sdk/sysom_utils/node_manager.py similarity index 100% rename from infrastructure/sdk/sysom_utils/node_manager.py rename to infrastructure/1_sdk/sysom_utils/node_manager.py diff --git a/infrastructure/sdk/sysom_utils/yaml_concat.py b/infrastructure/1_sdk/sysom_utils/yaml_concat.py similarity index 100% rename from infrastructure/sdk/sysom_utils/yaml_concat.py rename to infrastructure/1_sdk/sysom_utils/yaml_concat.py diff --git a/infrastructure/sdk/test_cec_redis/__init__.py b/infrastructure/1_sdk/test_cec_redis/__init__.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/__init__.py rename to infrastructure/1_sdk/test_cec_redis/__init__.py diff --git a/infrastructure/sdk/test_cec_redis/test_heartbeat.py b/infrastructure/1_sdk/test_cec_redis/test_heartbeat.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/test_heartbeat.py rename to infrastructure/1_sdk/test_cec_redis/test_heartbeat.py diff --git a/infrastructure/sdk/test_cec_redis/test_redis_admin.py b/infrastructure/1_sdk/test_cec_redis/test_redis_admin.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/test_redis_admin.py rename to infrastructure/1_sdk/test_cec_redis/test_redis_admin.py diff --git a/infrastructure/sdk/test_cec_redis/test_redis_admin_async.py b/infrastructure/1_sdk/test_cec_redis/test_redis_admin_async.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/test_redis_admin_async.py rename to infrastructure/1_sdk/test_cec_redis/test_redis_admin_async.py diff --git a/infrastructure/sdk/test_cec_redis/test_redis_consumer.py b/infrastructure/1_sdk/test_cec_redis/test_redis_consumer.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/test_redis_consumer.py rename to infrastructure/1_sdk/test_cec_redis/test_redis_consumer.py diff --git a/infrastructure/sdk/test_cec_redis/test_redis_consumer_async.py b/infrastructure/1_sdk/test_cec_redis/test_redis_consumer_async.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/test_redis_consumer_async.py rename to infrastructure/1_sdk/test_cec_redis/test_redis_consumer_async.py diff --git a/infrastructure/sdk/test_cec_redis/test_redis_multi_consumer.py b/infrastructure/1_sdk/test_cec_redis/test_redis_multi_consumer.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/test_redis_multi_consumer.py rename to infrastructure/1_sdk/test_cec_redis/test_redis_multi_consumer.py diff --git a/infrastructure/sdk/test_cec_redis/test_redis_producer.py b/infrastructure/1_sdk/test_cec_redis/test_redis_producer.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/test_redis_producer.py rename to infrastructure/1_sdk/test_cec_redis/test_redis_producer.py diff --git a/infrastructure/sdk/test_cec_redis/test_utils.py b/infrastructure/1_sdk/test_cec_redis/test_utils.py similarity index 100% rename from infrastructure/sdk/test_cec_redis/test_utils.py rename to infrastructure/1_sdk/test_cec_redis/test_utils.py diff --git a/infrastructure/sdk/test_cmg_base/__init__.py b/infrastructure/1_sdk/test_cmg_base/__init__.py similarity index 100% rename from infrastructure/sdk/test_cmg_base/__init__.py rename to infrastructure/1_sdk/test_cmg_base/__init__.py diff --git a/infrastructure/sdk/test_cmg_base/test_health_check.py b/infrastructure/1_sdk/test_cmg_base/test_health_check.py similarity index 100% rename from infrastructure/sdk/test_cmg_base/test_health_check.py rename to infrastructure/1_sdk/test_cmg_base/test_health_check.py diff --git a/infrastructure/sdk/test_cmg_redis/__init__.py b/infrastructure/1_sdk/test_cmg_redis/__init__.py similarity index 100% rename from infrastructure/sdk/test_cmg_redis/__init__.py rename to infrastructure/1_sdk/test_cmg_redis/__init__.py diff --git a/infrastructure/sdk/test_cmg_redis/test_service_discovery.py b/infrastructure/1_sdk/test_cmg_redis/test_service_discovery.py similarity index 100% rename from infrastructure/sdk/test_cmg_redis/test_service_discovery.py rename to infrastructure/1_sdk/test_cmg_redis/test_service_discovery.py diff --git a/infrastructure/sdk/test_cmg_redis/test_service_registry.py b/infrastructure/1_sdk/test_cmg_redis/test_service_registry.py similarity index 100% rename from infrastructure/sdk/test_cmg_redis/test_service_registry.py rename to infrastructure/1_sdk/test_cmg_redis/test_service_registry.py diff --git a/infrastructure/local_services/clear.sh b/infrastructure/2_local_services/clear.sh similarity index 100% rename from infrastructure/local_services/clear.sh rename to infrastructure/2_local_services/clear.sh diff --git a/infrastructure/local_services/init.sh b/infrastructure/2_local_services/init.sh similarity index 96% rename from infrastructure/local_services/init.sh rename to infrastructure/2_local_services/init.sh index 9784ed56..a06901ca 100755 --- a/infrastructure/local_services/init.sh +++ b/infrastructure/2_local_services/init.sh @@ -58,9 +58,9 @@ setup_redis() { tar -zxvf ${REDIS_PKG} pushd ${REDIS_DIR} make - mkdir -p ${SERVER_HOME}/redis - cp redis.conf ${SERVER_HOME}/redis/ - cp src/redis-server ${SERVER_HOME}/redis/ + mkdir -p ${ENVIRONMENT_HOME}/redis + cp redis.conf ${ENVIRONMENT_HOME}/redis/ + cp src/redis-server ${ENVIRONMENT_HOME}/redis/ if [ $? -ne 0 ] then echo "redis compile or install error, exit 1" diff --git a/infrastructure/local_services/nginx.conf b/infrastructure/2_local_services/nginx.conf similarity index 100% rename from infrastructure/local_services/nginx.conf rename to infrastructure/2_local_services/nginx.conf diff --git a/infrastructure/local_services/start.sh b/infrastructure/2_local_services/start.sh similarity index 100% rename from infrastructure/local_services/start.sh rename to infrastructure/2_local_services/start.sh diff --git a/infrastructure/local_services/stop.sh b/infrastructure/2_local_services/stop.sh similarity index 100% rename from infrastructure/local_services/stop.sh rename to infrastructure/2_local_services/stop.sh diff --git a/script/server/0_local_services/sysom-redis.ini b/infrastructure/2_local_services/sysom-redis.ini similarity index 61% rename from script/server/0_local_services/sysom-redis.ini rename to infrastructure/2_local_services/sysom-redis.ini index dcc775ee..b0f35b7e 100644 --- a/script/server/0_local_services/sysom-redis.ini +++ b/infrastructure/2_local_services/sysom-redis.ini @@ -4,5 +4,5 @@ command=/usr/local/sysom/server/redis/redis-server /usr/local/sysom/server/redis startsecs=3 autostart=true autorestart=true -stderr_logfile=/usr/local/sysom/server/logs/sysom-redis-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-redis.log +stderr_logfile=/var/log/sysom/sysom-redis-error.log +stdout_logfile=/var/log/sysom/sysom-redis.log diff --git a/infrastructure/local_services/sysom.conf b/infrastructure/2_local_services/sysom.conf similarity index 98% rename from infrastructure/local_services/sysom.conf rename to infrastructure/2_local_services/sysom.conf index 94aca073..864ac5d2 100644 --- a/infrastructure/local_services/sysom.conf +++ b/infrastructure/2_local_services/sysom.conf @@ -6,7 +6,7 @@ map $http_upgrade $connection_upgrade { server { listen SERVER_PORT; server_name _; - root /usr/local/sysom/server/target/sysom_web; + root /usr/local/sysom/web; index index.html; client_max_body_size 20m; diff --git a/script/server/1_sysom_monitor/Readme.txt b/infrastructure/3_monitor/Readme.txt similarity index 100% rename from script/server/1_sysom_monitor/Readme.txt rename to infrastructure/3_monitor/Readme.txt diff --git a/script/server/1_sysom_monitor/clear.sh b/infrastructure/3_monitor/clear.sh similarity index 100% rename from script/server/1_sysom_monitor/clear.sh rename to infrastructure/3_monitor/clear.sh diff --git a/script/server/1_sysom_monitor/grafana_api_set.sh b/infrastructure/3_monitor/grafana_api_set.sh similarity index 99% rename from script/server/1_sysom_monitor/grafana_api_set.sh rename to infrastructure/3_monitor/grafana_api_set.sh index 63064591..88ff504e 100755 --- a/script/server/1_sysom_monitor/grafana_api_set.sh +++ b/infrastructure/3_monitor/grafana_api_set.sh @@ -1,7 +1,6 @@ #!/bin/bash -x GRAFANA_CONFIG=/etc/grafana/grafana.ini GRAFANA_SAMPLE_CONFIG=/usr/share/grafana/conf/sample.ini -SYSOM_CONF=${SERVER_HOME}/target/conf/config.yml GRAFANA_SERVER=grafana-server ##fix sometime grafana.ini not found## @@ -134,3 +133,4 @@ sed 's/;root_url = %(protocol)s:\/\/%(domain)s:%(http_port)s\//root_url = %(prot systemctl restart $GRAFANA_SERVER grafana-cli admin reset-admin-password "sysom@openanolis.cn" + diff --git a/script/server/1_sysom_monitor/grafana_recover.sh b/infrastructure/3_monitor/grafana_recover.sh similarity index 100% rename from script/server/1_sysom_monitor/grafana_recover.sh rename to infrastructure/3_monitor/grafana_recover.sh diff --git a/script/server/1_sysom_monitor/init.sh b/infrastructure/3_monitor/init.sh similarity index 96% rename from script/server/1_sysom_monitor/init.sh rename to infrastructure/3_monitor/init.sh index 3a3ee7f6..888ef7f8 100755 --- a/script/server/1_sysom_monitor/init.sh +++ b/infrastructure/3_monitor/init.sh @@ -1,6 +1,5 @@ #!/bin/bash -x -NODE_INIT_DIR=${SERVER_HOME}/target/sysom_web/download/sysom_node_init -RESOURCE_DIR=${SERVER_HOME}/monitor +RESOURCE_DIR=${ENVIRONMENT_HOME}/monitor ARCH=`uname -m` GRAFANA_PKG=grafana-9.2.2-1.${ARCH}.rpm PROMETHEUS_VER=2.29.1 @@ -21,7 +20,7 @@ NODE_DL_URL=https://github.com/prometheus/node_exporter/releases/download/v${NOD NODE_INIT_SCRIPT=${SERVER_HOME}/target/sysom_server/sysom_diagnosis/service_scripts/node_init NODE_DELETE_SCRIPT=${SERVER_HOME}/target/sysom_server/sysom_diagnosis/service_scripts/node_delete SERVICE_NAME=sysom-prometheus -VIRTUALENV_PYTHON3=${SERVER_HOME}/virtualenv/bin/python3 +VIRTUALENV_PYTHON3=${GLOBAL_VIRTUALENV_HOME}/bin/python3 BASE_DIR=`dirname $0` diff --git a/script/server/1_sysom_monitor/local_copy_pkg.sh b/infrastructure/3_monitor/local_copy_pkg.sh similarity index 100% rename from script/server/1_sysom_monitor/local_copy_pkg.sh rename to infrastructure/3_monitor/local_copy_pkg.sh diff --git a/script/server/1_sysom_monitor/prometheus_get_node.py b/infrastructure/3_monitor/prometheus_get_node.py similarity index 100% rename from script/server/1_sysom_monitor/prometheus_get_node.py rename to infrastructure/3_monitor/prometheus_get_node.py diff --git a/script/server/1_sysom_monitor/start.sh b/infrastructure/3_monitor/start.sh similarity index 100% rename from script/server/1_sysom_monitor/start.sh rename to infrastructure/3_monitor/start.sh diff --git a/script/server/1_sysom_monitor/stop.sh b/infrastructure/3_monitor/stop.sh similarity index 100% rename from script/server/1_sysom_monitor/stop.sh rename to infrastructure/3_monitor/stop.sh diff --git a/script/server/1_sysom_monitor/sysom-cec-status-dashboard.json b/infrastructure/3_monitor/sysom-cec-status-dashboard.json similarity index 100% rename from script/server/1_sysom_monitor/sysom-cec-status-dashboard.json rename to infrastructure/3_monitor/sysom-cec-status-dashboard.json diff --git a/script/server/1_sysom_monitor/sysom-dashboard.json b/infrastructure/3_monitor/sysom-dashboard.json similarity index 100% rename from script/server/1_sysom_monitor/sysom-dashboard.json rename to infrastructure/3_monitor/sysom-dashboard.json diff --git a/script/server/1_sysom_monitor/sysom-migration-dashboard.json b/infrastructure/3_monitor/sysom-migration-dashboard.json similarity index 100% rename from script/server/1_sysom_monitor/sysom-migration-dashboard.json rename to infrastructure/3_monitor/sysom-migration-dashboard.json diff --git a/script/server/1_sysom_monitor/sysom-netinfo-dashboard.json b/infrastructure/3_monitor/sysom-netinfo-dashboard.json similarity index 100% rename from script/server/1_sysom_monitor/sysom-netinfo-dashboard.json rename to infrastructure/3_monitor/sysom-netinfo-dashboard.json diff --git a/infrastructure/3_monitor/sysom-prometheus.ini b/infrastructure/3_monitor/sysom-prometheus.ini new file mode 100644 index 00000000..62c0cec5 --- /dev/null +++ b/infrastructure/3_monitor/sysom-prometheus.ini @@ -0,0 +1,8 @@ +[program:sysom-prometheus] +directory = /usr/local/sysom/server/environment/monitor/prometheus +command=/usr/local/sysom/server/environment/monitor/prometheus/prometheus +startsecs=3 +autostart=true +autorestart=true +stderr_logfile=/var/log/sysom/sysom-prometheus-error.log +stdout_logfile=/var/log/sysom/sysom-prometheus.log diff --git a/infrastructure/clear_exclude b/infrastructure/clear_exclude new file mode 100644 index 00000000..e1994367 --- /dev/null +++ b/infrastructure/clear_exclude @@ -0,0 +1 @@ +0_env \ No newline at end of file diff --git a/microservice/sysom_api/apps/accounts/__init__.py b/infrastructure/deploy_exclude similarity index 100% rename from microservice/sysom_api/apps/accounts/__init__.py rename to infrastructure/deploy_exclude diff --git a/infrastructure/local_services/sysom-redis.ini b/infrastructure/local_services/sysom-redis.ini deleted file mode 100644 index dcc775ee..00000000 --- a/infrastructure/local_services/sysom-redis.ini +++ /dev/null @@ -1,8 +0,0 @@ -[program:sysom-redis] -directory = /usr/local/sysom/server/redis -command=/usr/local/sysom/server/redis/redis-server /usr/local/sysom/server/redis/redis.conf -startsecs=3 -autostart=true -autorestart=true -stderr_logfile=/usr/local/sysom/server/logs/sysom-redis-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-redis.log diff --git a/microservice/sysom_api/.gitignore b/microservice/0_sysom_api/.gitignore similarity index 100% rename from microservice/sysom_api/.gitignore rename to microservice/0_sysom_api/.gitignore diff --git a/microservice/sysom_api/apps/__init__.py b/microservice/0_sysom_api/apps/__init__.py similarity index 100% rename from microservice/sysom_api/apps/__init__.py rename to microservice/0_sysom_api/apps/__init__.py diff --git a/microservice/sysom_api/apps/accounts/migrations/__init__.py b/microservice/0_sysom_api/apps/accounts/__init__.py similarity index 100% rename from microservice/sysom_api/apps/accounts/migrations/__init__.py rename to microservice/0_sysom_api/apps/accounts/__init__.py diff --git a/microservice/sysom_api/apps/accounts/admin.py b/microservice/0_sysom_api/apps/accounts/admin.py similarity index 100% rename from microservice/sysom_api/apps/accounts/admin.py rename to microservice/0_sysom_api/apps/accounts/admin.py diff --git a/microservice/sysom_api/apps/accounts/apps.py b/microservice/0_sysom_api/apps/accounts/apps.py similarity index 100% rename from microservice/sysom_api/apps/accounts/apps.py rename to microservice/0_sysom_api/apps/accounts/apps.py diff --git a/microservice/sysom_api/apps/accounts/authentication.py b/microservice/0_sysom_api/apps/accounts/authentication.py similarity index 100% rename from microservice/sysom_api/apps/accounts/authentication.py rename to microservice/0_sysom_api/apps/accounts/authentication.py diff --git a/microservice/sysom_api/apps/accounts/migrations/0001_initial.py b/microservice/0_sysom_api/apps/accounts/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_api/apps/accounts/migrations/0001_initial.py rename to microservice/0_sysom_api/apps/accounts/migrations/0001_initial.py diff --git a/microservice/sysom_api/apps/accounts/migrations/0002_user_allow_login.py b/microservice/0_sysom_api/apps/accounts/migrations/0002_user_allow_login.py similarity index 100% rename from microservice/sysom_api/apps/accounts/migrations/0002_user_allow_login.py rename to microservice/0_sysom_api/apps/accounts/migrations/0002_user_allow_login.py diff --git a/microservice/sysom_api/apps/alarm/migrations/__init__.py b/microservice/0_sysom_api/apps/accounts/migrations/__init__.py similarity index 100% rename from microservice/sysom_api/apps/alarm/migrations/__init__.py rename to microservice/0_sysom_api/apps/accounts/migrations/__init__.py diff --git a/microservice/sysom_api/apps/accounts/models.py b/microservice/0_sysom_api/apps/accounts/models.py similarity index 100% rename from microservice/sysom_api/apps/accounts/models.py rename to microservice/0_sysom_api/apps/accounts/models.py diff --git a/microservice/sysom_api/apps/accounts/permissions.py b/microservice/0_sysom_api/apps/accounts/permissions.py similarity index 100% rename from microservice/sysom_api/apps/accounts/permissions.py rename to microservice/0_sysom_api/apps/accounts/permissions.py diff --git a/microservice/sysom_api/apps/accounts/serializer.py b/microservice/0_sysom_api/apps/accounts/serializer.py similarity index 100% rename from microservice/sysom_api/apps/accounts/serializer.py rename to microservice/0_sysom_api/apps/accounts/serializer.py diff --git a/microservice/sysom_api/apps/accounts/tests.py b/microservice/0_sysom_api/apps/accounts/tests.py similarity index 100% rename from microservice/sysom_api/apps/accounts/tests.py rename to microservice/0_sysom_api/apps/accounts/tests.py diff --git a/microservice/sysom_api/apps/accounts/urls.py b/microservice/0_sysom_api/apps/accounts/urls.py similarity index 100% rename from microservice/sysom_api/apps/accounts/urls.py rename to microservice/0_sysom_api/apps/accounts/urls.py diff --git a/microservice/sysom_api/apps/accounts/views.py b/microservice/0_sysom_api/apps/accounts/views.py similarity index 100% rename from microservice/sysom_api/apps/accounts/views.py rename to microservice/0_sysom_api/apps/accounts/views.py diff --git a/microservice/sysom_api/apps/alarm/__init__.py b/microservice/0_sysom_api/apps/alarm/__init__.py similarity index 100% rename from microservice/sysom_api/apps/alarm/__init__.py rename to microservice/0_sysom_api/apps/alarm/__init__.py diff --git a/microservice/sysom_api/apps/alarm/admin.py b/microservice/0_sysom_api/apps/alarm/admin.py similarity index 100% rename from microservice/sysom_api/apps/alarm/admin.py rename to microservice/0_sysom_api/apps/alarm/admin.py diff --git a/microservice/sysom_api/apps/alarm/apps.py b/microservice/0_sysom_api/apps/alarm/apps.py similarity index 100% rename from microservice/sysom_api/apps/alarm/apps.py rename to microservice/0_sysom_api/apps/alarm/apps.py diff --git a/microservice/sysom_api/apps/alarm/migrations/0001_initial.py b/microservice/0_sysom_api/apps/alarm/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_api/apps/alarm/migrations/0001_initial.py rename to microservice/0_sysom_api/apps/alarm/migrations/0001_initial.py diff --git a/microservice/sysom_api/apps/common/__init__.py b/microservice/0_sysom_api/apps/alarm/migrations/__init__.py similarity index 100% rename from microservice/sysom_api/apps/common/__init__.py rename to microservice/0_sysom_api/apps/alarm/migrations/__init__.py diff --git a/microservice/sysom_api/apps/alarm/models.py b/microservice/0_sysom_api/apps/alarm/models.py similarity index 100% rename from microservice/sysom_api/apps/alarm/models.py rename to microservice/0_sysom_api/apps/alarm/models.py diff --git a/microservice/sysom_api/apps/alarm/serializer.py b/microservice/0_sysom_api/apps/alarm/serializer.py similarity index 100% rename from microservice/sysom_api/apps/alarm/serializer.py rename to microservice/0_sysom_api/apps/alarm/serializer.py diff --git a/microservice/sysom_api/apps/alarm/subscribe.json b/microservice/0_sysom_api/apps/alarm/subscribe.json similarity index 100% rename from microservice/sysom_api/apps/alarm/subscribe.json rename to microservice/0_sysom_api/apps/alarm/subscribe.json diff --git a/microservice/sysom_api/apps/alarm/urls.py b/microservice/0_sysom_api/apps/alarm/urls.py similarity index 100% rename from microservice/sysom_api/apps/alarm/urls.py rename to microservice/0_sysom_api/apps/alarm/urls.py diff --git a/microservice/sysom_api/apps/alarm/views.py b/microservice/0_sysom_api/apps/alarm/views.py similarity index 100% rename from microservice/sysom_api/apps/alarm/views.py rename to microservice/0_sysom_api/apps/alarm/views.py diff --git a/microservice/sysom_api/apps/host/__init__.py b/microservice/0_sysom_api/apps/common/__init__.py similarity index 100% rename from microservice/sysom_api/apps/host/__init__.py rename to microservice/0_sysom_api/apps/common/__init__.py diff --git a/microservice/sysom_api/apps/common/common_model_viewset.py b/microservice/0_sysom_api/apps/common/common_model_viewset.py similarity index 100% rename from microservice/sysom_api/apps/common/common_model_viewset.py rename to microservice/0_sysom_api/apps/common/common_model_viewset.py diff --git a/microservice/sysom_api/apps/host/migrations/__init__.py b/microservice/0_sysom_api/apps/host/__init__.py similarity index 100% rename from microservice/sysom_api/apps/host/migrations/__init__.py rename to microservice/0_sysom_api/apps/host/__init__.py diff --git a/microservice/sysom_api/apps/host/admin.py b/microservice/0_sysom_api/apps/host/admin.py similarity index 100% rename from microservice/sysom_api/apps/host/admin.py rename to microservice/0_sysom_api/apps/host/admin.py diff --git a/microservice/sysom_api/apps/host/apps.py b/microservice/0_sysom_api/apps/host/apps.py similarity index 100% rename from microservice/sysom_api/apps/host/apps.py rename to microservice/0_sysom_api/apps/host/apps.py diff --git a/microservice/sysom_api/apps/host/cec_api.py b/microservice/0_sysom_api/apps/host/cec_api.py similarity index 100% rename from microservice/sysom_api/apps/host/cec_api.py rename to microservice/0_sysom_api/apps/host/cec_api.py diff --git a/microservice/sysom_api/apps/host/heartbeat.py b/microservice/0_sysom_api/apps/host/heartbeat.py similarity index 100% rename from microservice/sysom_api/apps/host/heartbeat.py rename to microservice/0_sysom_api/apps/host/heartbeat.py diff --git a/microservice/sysom_api/apps/host/migrations/0001_initial.py b/microservice/0_sysom_api/apps/host/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_api/apps/host/migrations/0001_initial.py rename to microservice/0_sysom_api/apps/host/migrations/0001_initial.py diff --git a/microservice/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py b/microservice/0_sysom_api/apps/host/migrations/0002_hostmodel_host_info.py similarity index 100% rename from microservice/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py rename to microservice/0_sysom_api/apps/host/migrations/0002_hostmodel_host_info.py diff --git a/microservice/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py b/microservice/0_sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py similarity index 100% rename from microservice/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py rename to microservice/0_sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py diff --git a/microservice/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py b/microservice/0_sysom_api/apps/host/migrations/0004_auto_20221227_1705.py similarity index 100% rename from microservice/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py rename to microservice/0_sysom_api/apps/host/migrations/0004_auto_20221227_1705.py diff --git a/microservice/sysom_api/apps/services/migrations/__init__.py b/microservice/0_sysom_api/apps/host/migrations/__init__.py similarity index 100% rename from microservice/sysom_api/apps/services/migrations/__init__.py rename to microservice/0_sysom_api/apps/host/migrations/__init__.py diff --git a/microservice/sysom_api/apps/host/models.py b/microservice/0_sysom_api/apps/host/models.py similarity index 100% rename from microservice/sysom_api/apps/host/models.py rename to microservice/0_sysom_api/apps/host/models.py diff --git a/microservice/sysom_api/apps/host/serializer.py b/microservice/0_sysom_api/apps/host/serializer.py similarity index 100% rename from microservice/sysom_api/apps/host/serializer.py rename to microservice/0_sysom_api/apps/host/serializer.py diff --git a/microservice/sysom_api/apps/host/tests.py b/microservice/0_sysom_api/apps/host/tests.py similarity index 100% rename from microservice/sysom_api/apps/host/tests.py rename to microservice/0_sysom_api/apps/host/tests.py diff --git a/microservice/sysom_api/apps/host/urls.py b/microservice/0_sysom_api/apps/host/urls.py similarity index 100% rename from microservice/sysom_api/apps/host/urls.py rename to microservice/0_sysom_api/apps/host/urls.py diff --git a/microservice/sysom_api/apps/host/views.py b/microservice/0_sysom_api/apps/host/views.py similarity index 100% rename from microservice/sysom_api/apps/host/views.py rename to microservice/0_sysom_api/apps/host/views.py diff --git a/microservice/sysom_api/apps/services/__init__.py b/microservice/0_sysom_api/apps/services/__init__.py similarity index 100% rename from microservice/sysom_api/apps/services/__init__.py rename to microservice/0_sysom_api/apps/services/__init__.py diff --git a/microservice/sysom_api/apps/services/admin.py b/microservice/0_sysom_api/apps/services/admin.py similarity index 100% rename from microservice/sysom_api/apps/services/admin.py rename to microservice/0_sysom_api/apps/services/admin.py diff --git a/microservice/sysom_api/apps/services/apps.py b/microservice/0_sysom_api/apps/services/apps.py similarity index 100% rename from microservice/sysom_api/apps/services/apps.py rename to microservice/0_sysom_api/apps/services/apps.py diff --git a/microservice/sysom_api/apps/services/migrations/0001_initial.py b/microservice/0_sysom_api/apps/services/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_api/apps/services/migrations/0001_initial.py rename to microservice/0_sysom_api/apps/services/migrations/0001_initial.py diff --git a/microservice/sysom_api/conf/__init__.py b/microservice/0_sysom_api/apps/services/migrations/__init__.py similarity index 100% rename from microservice/sysom_api/conf/__init__.py rename to microservice/0_sysom_api/apps/services/migrations/__init__.py diff --git a/microservice/sysom_api/apps/services/models.py b/microservice/0_sysom_api/apps/services/models.py similarity index 100% rename from microservice/sysom_api/apps/services/models.py rename to microservice/0_sysom_api/apps/services/models.py diff --git a/microservice/sysom_api/apps/services/serializer.py b/microservice/0_sysom_api/apps/services/serializer.py similarity index 100% rename from microservice/sysom_api/apps/services/serializer.py rename to microservice/0_sysom_api/apps/services/serializer.py diff --git a/microservice/sysom_api/apps/services/tests.py b/microservice/0_sysom_api/apps/services/tests.py similarity index 100% rename from microservice/sysom_api/apps/services/tests.py rename to microservice/0_sysom_api/apps/services/tests.py diff --git a/microservice/sysom_api/apps/services/urls.py b/microservice/0_sysom_api/apps/services/urls.py similarity index 100% rename from microservice/sysom_api/apps/services/urls.py rename to microservice/0_sysom_api/apps/services/urls.py diff --git a/microservice/sysom_api/apps/services/views.py b/microservice/0_sysom_api/apps/services/views.py similarity index 100% rename from microservice/sysom_api/apps/services/views.py rename to microservice/0_sysom_api/apps/services/views.py diff --git a/microservice/0_sysom_api/conf/__init__.py b/microservice/0_sysom_api/conf/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/microservice/sysom_api/conf/common.py b/microservice/0_sysom_api/conf/common.py similarity index 98% rename from microservice/sysom_api/conf/common.py rename to microservice/0_sysom_api/conf/common.py index 447023c1..29f75b96 100644 --- a/microservice/sysom_api/conf/common.py +++ b/microservice/0_sysom_api/conf/common.py @@ -11,7 +11,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/microservice/sysom_api/conf/develop.py b/microservice/0_sysom_api/conf/develop.py similarity index 100% rename from microservice/sysom_api/conf/develop.py rename to microservice/0_sysom_api/conf/develop.py diff --git a/microservice/sysom_api/conf/product.py b/microservice/0_sysom_api/conf/product.py similarity index 100% rename from microservice/sysom_api/conf/product.py rename to microservice/0_sysom_api/conf/product.py diff --git a/microservice/sysom_api/conf/testing.py b/microservice/0_sysom_api/conf/testing.py similarity index 100% rename from microservice/sysom_api/conf/testing.py rename to microservice/0_sysom_api/conf/testing.py diff --git a/microservice/sysom_api/config.yml b/microservice/0_sysom_api/config.yml similarity index 100% rename from microservice/sysom_api/config.yml rename to microservice/0_sysom_api/config.yml diff --git a/microservice/sysom_api/consumer/__init__.py b/microservice/0_sysom_api/consumer/__init__.py similarity index 100% rename from microservice/sysom_api/consumer/__init__.py rename to microservice/0_sysom_api/consumer/__init__.py diff --git a/microservice/sysom_api/consumer/consumers.py b/microservice/0_sysom_api/consumer/consumers.py similarity index 100% rename from microservice/sysom_api/consumer/consumers.py rename to microservice/0_sysom_api/consumer/consumers.py diff --git a/microservice/sysom_api/consumer/middleware.py b/microservice/0_sysom_api/consumer/middleware.py similarity index 100% rename from microservice/sysom_api/consumer/middleware.py rename to microservice/0_sysom_api/consumer/middleware.py diff --git a/microservice/sysom_api/consumer/routing.py b/microservice/0_sysom_api/consumer/routing.py similarity index 100% rename from microservice/sysom_api/consumer/routing.py rename to microservice/0_sysom_api/consumer/routing.py diff --git a/microservice/sysom_api/lib/__init__.py b/microservice/0_sysom_api/lib/__init__.py similarity index 100% rename from microservice/sysom_api/lib/__init__.py rename to microservice/0_sysom_api/lib/__init__.py diff --git a/microservice/sysom_api/lib/authentications.py b/microservice/0_sysom_api/lib/authentications.py similarity index 100% rename from microservice/sysom_api/lib/authentications.py rename to microservice/0_sysom_api/lib/authentications.py diff --git a/microservice/sysom_api/lib/base_model.py b/microservice/0_sysom_api/lib/base_model.py similarity index 100% rename from microservice/sysom_api/lib/base_model.py rename to microservice/0_sysom_api/lib/base_model.py diff --git a/microservice/sysom_api/lib/decode/sysom_decode.py b/microservice/0_sysom_api/lib/decode/sysom_decode.py similarity index 100% rename from microservice/sysom_api/lib/decode/sysom_decode.py rename to microservice/0_sysom_api/lib/decode/sysom_decode.py diff --git a/microservice/sysom_api/lib/excel.py b/microservice/0_sysom_api/lib/excel.py similarity index 100% rename from microservice/sysom_api/lib/excel.py rename to microservice/0_sysom_api/lib/excel.py diff --git a/microservice/sysom_api/lib/exception.py b/microservice/0_sysom_api/lib/exception.py similarity index 100% rename from microservice/sysom_api/lib/exception.py rename to microservice/0_sysom_api/lib/exception.py diff --git a/microservice/sysom_api/lib/paginations.py b/microservice/0_sysom_api/lib/paginations.py similarity index 100% rename from microservice/sysom_api/lib/paginations.py rename to microservice/0_sysom_api/lib/paginations.py diff --git a/microservice/sysom_api/lib/renderers.py b/microservice/0_sysom_api/lib/renderers.py similarity index 100% rename from microservice/sysom_api/lib/renderers.py rename to microservice/0_sysom_api/lib/renderers.py diff --git a/microservice/sysom_api/lib/response.py b/microservice/0_sysom_api/lib/response.py similarity index 100% rename from microservice/sysom_api/lib/response.py rename to microservice/0_sysom_api/lib/response.py diff --git a/microservice/sysom_api/lib/ssh.py b/microservice/0_sysom_api/lib/ssh.py similarity index 100% rename from microservice/sysom_api/lib/ssh.py rename to microservice/0_sysom_api/lib/ssh.py diff --git a/microservice/sysom_api/lib/utils.py b/microservice/0_sysom_api/lib/utils.py similarity index 100% rename from microservice/sysom_api/lib/utils.py rename to microservice/0_sysom_api/lib/utils.py diff --git a/microservice/sysom_api/manage.py b/microservice/0_sysom_api/manage.py similarity index 100% rename from microservice/sysom_api/manage.py rename to microservice/0_sysom_api/manage.py diff --git a/microservice/sysom_api/requirements.txt b/microservice/0_sysom_api/requirements.txt similarity index 100% rename from microservice/sysom_api/requirements.txt rename to microservice/0_sysom_api/requirements.txt diff --git a/microservice/sysom_api/scripts/server_clear.sh b/microservice/0_sysom_api/scripts/server_clear.sh similarity index 100% rename from microservice/sysom_api/scripts/server_clear.sh rename to microservice/0_sysom_api/scripts/server_clear.sh diff --git a/microservice/sysom_api/scripts/server_init.sh b/microservice/0_sysom_api/scripts/server_init.sh similarity index 90% rename from microservice/sysom_api/scripts/server_init.sh rename to microservice/0_sysom_api/scripts/server_init.sh index f5731753..6347540b 100644 --- a/microservice/sysom_api/scripts/server_init.sh +++ b/microservice/0_sysom_api/scripts/server_init.sh @@ -1,6 +1,6 @@ #!/bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_api -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-api if [ "$UID" -ne 0 ]; then diff --git a/microservice/sysom_api/scripts/server_start.sh b/microservice/0_sysom_api/scripts/server_start.sh similarity index 100% rename from microservice/sysom_api/scripts/server_start.sh rename to microservice/0_sysom_api/scripts/server_start.sh diff --git a/microservice/sysom_api/scripts/server_stop.sh b/microservice/0_sysom_api/scripts/server_stop.sh similarity index 100% rename from microservice/sysom_api/scripts/server_stop.sh rename to microservice/0_sysom_api/scripts/server_stop.sh diff --git a/microservice/sysom_api/sysom-api.ini b/microservice/0_sysom_api/sysom-api.ini similarity index 55% rename from microservice/sysom_api/sysom-api.ini rename to microservice/0_sysom_api/sysom-api.ini index 120a106b..bddc95da 100644 --- a/microservice/sysom_api/sysom-api.ini +++ b/microservice/0_sysom_api/sysom-api.ini @@ -1,7 +1,7 @@ [fcgi-program:sysom-api] socket=tcp://0.0.0.0:7001 -directory=/usr/local/sysom/server/microservice/sysom_api -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/daphne -u /run/daphne%(process_num)d.sock --fd 0 --access-log /var/log/sysom/sysom-api-access.log --proxy-headers sysom.asgi:application +directory=/usr/local/sysom/server/microservice/0_sysom_api +command=/usr/local/sysom/server/environment/virtualenv/bin/daphne -u /run/daphne%(process_num)d.sock --fd 0 --access-log /var/log/sysom/sysom-api-access.log --proxy-headers sysom.asgi:application numprocs=4 process_name=%(process_num)d autostart=true diff --git a/microservice/sysom_api/sysom/__init__.py b/microservice/0_sysom_api/sysom/__init__.py similarity index 100% rename from microservice/sysom_api/sysom/__init__.py rename to microservice/0_sysom_api/sysom/__init__.py diff --git a/microservice/sysom_api/sysom/asgi.py b/microservice/0_sysom_api/sysom/asgi.py similarity index 100% rename from microservice/sysom_api/sysom/asgi.py rename to microservice/0_sysom_api/sysom/asgi.py diff --git a/microservice/sysom_api/sysom/settings.py b/microservice/0_sysom_api/sysom/settings.py similarity index 100% rename from microservice/sysom_api/sysom/settings.py rename to microservice/0_sysom_api/sysom/settings.py diff --git a/microservice/sysom_api/sysom/urls.py b/microservice/0_sysom_api/sysom/urls.py similarity index 100% rename from microservice/sysom_api/sysom/urls.py rename to microservice/0_sysom_api/sysom/urls.py diff --git a/microservice/sysom_api/sysom/wsgi.py b/microservice/0_sysom_api/sysom/wsgi.py similarity index 100% rename from microservice/sysom_api/sysom/wsgi.py rename to microservice/0_sysom_api/sysom/wsgi.py diff --git a/microservice/sysom_channel/alembic.ini b/microservice/0_sysom_channel/alembic.ini similarity index 100% rename from microservice/sysom_channel/alembic.ini rename to microservice/0_sysom_channel/alembic.ini diff --git a/microservice/sysom_channel/alembic/README b/microservice/0_sysom_channel/alembic/README similarity index 100% rename from microservice/sysom_channel/alembic/README rename to microservice/0_sysom_channel/alembic/README diff --git a/microservice/sysom_channel/alembic/env.py b/microservice/0_sysom_channel/alembic/env.py similarity index 100% rename from microservice/sysom_channel/alembic/env.py rename to microservice/0_sysom_channel/alembic/env.py diff --git a/microservice/sysom_channel/alembic/script.py.mako b/microservice/0_sysom_channel/alembic/script.py.mako similarity index 100% rename from microservice/sysom_channel/alembic/script.py.mako rename to microservice/0_sysom_channel/alembic/script.py.mako diff --git a/microservice/sysom_channel/alembic/versions/1663292d0bd8_init_db.py b/microservice/0_sysom_channel/alembic/versions/1663292d0bd8_init_db.py similarity index 100% rename from microservice/sysom_channel/alembic/versions/1663292d0bd8_init_db.py rename to microservice/0_sysom_channel/alembic/versions/1663292d0bd8_init_db.py diff --git a/microservice/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py b/microservice/0_sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py similarity index 100% rename from microservice/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py rename to microservice/0_sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py diff --git a/microservice/sysom_channel/app/__init__.py b/microservice/0_sysom_channel/app/__init__.py similarity index 100% rename from microservice/sysom_channel/app/__init__.py rename to microservice/0_sysom_channel/app/__init__.py diff --git a/microservice/sysom_channel/app/crud.py b/microservice/0_sysom_channel/app/crud.py similarity index 100% rename from microservice/sysom_channel/app/crud.py rename to microservice/0_sysom_channel/app/crud.py diff --git a/microservice/sysom_channel/app/database.py b/microservice/0_sysom_channel/app/database.py similarity index 100% rename from microservice/sysom_channel/app/database.py rename to microservice/0_sysom_channel/app/database.py diff --git a/microservice/sysom_channel/app/executor.py b/microservice/0_sysom_channel/app/executor.py similarity index 100% rename from microservice/sysom_channel/app/executor.py rename to microservice/0_sysom_channel/app/executor.py diff --git a/microservice/sysom_channel/app/models.py b/microservice/0_sysom_channel/app/models.py similarity index 100% rename from microservice/sysom_channel/app/models.py rename to microservice/0_sysom_channel/app/models.py diff --git a/microservice/sysom_channel/app/routers/cec_status.py b/microservice/0_sysom_channel/app/routers/cec_status.py similarity index 100% rename from microservice/sysom_channel/app/routers/cec_status.py rename to microservice/0_sysom_channel/app/routers/cec_status.py diff --git a/microservice/sysom_channel/app/routers/config.py b/microservice/0_sysom_channel/app/routers/config.py similarity index 100% rename from microservice/sysom_channel/app/routers/config.py rename to microservice/0_sysom_channel/app/routers/config.py diff --git a/microservice/sysom_channel/app/routers/file.py b/microservice/0_sysom_channel/app/routers/file.py similarity index 100% rename from microservice/sysom_channel/app/routers/file.py rename to microservice/0_sysom_channel/app/routers/file.py diff --git a/microservice/sysom_channel/app/routers/health.py b/microservice/0_sysom_channel/app/routers/health.py similarity index 100% rename from microservice/sysom_channel/app/routers/health.py rename to microservice/0_sysom_channel/app/routers/health.py diff --git a/microservice/sysom_channel/app/schemas.py b/microservice/0_sysom_channel/app/schemas.py similarity index 100% rename from microservice/sysom_channel/app/schemas.py rename to microservice/0_sysom_channel/app/schemas.py diff --git a/microservice/sysom_channel/conf/common.py b/microservice/0_sysom_channel/conf/common.py similarity index 97% rename from microservice/sysom_channel/conf/common.py rename to microservice/0_sysom_channel/conf/common.py index 0fa64060..c495992a 100644 --- a/microservice/sysom_channel/conf/common.py +++ b/microservice/0_sysom_channel/conf/common.py @@ -16,7 +16,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"/etc/sysom/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/microservice/sysom_channel/conf/develop.py b/microservice/0_sysom_channel/conf/develop.py similarity index 100% rename from microservice/sysom_channel/conf/develop.py rename to microservice/0_sysom_channel/conf/develop.py diff --git a/microservice/sysom_channel/conf/gunicorn.py b/microservice/0_sysom_channel/conf/gunicorn.py similarity index 100% rename from microservice/sysom_channel/conf/gunicorn.py rename to microservice/0_sysom_channel/conf/gunicorn.py diff --git a/microservice/sysom_channel/conf/product.py b/microservice/0_sysom_channel/conf/product.py similarity index 100% rename from microservice/sysom_channel/conf/product.py rename to microservice/0_sysom_channel/conf/product.py diff --git a/microservice/sysom_channel/conf/settings.py b/microservice/0_sysom_channel/conf/settings.py similarity index 100% rename from microservice/sysom_channel/conf/settings.py rename to microservice/0_sysom_channel/conf/settings.py diff --git a/microservice/sysom_channel/conf/testing.py b/microservice/0_sysom_channel/conf/testing.py similarity index 100% rename from microservice/sysom_channel/conf/testing.py rename to microservice/0_sysom_channel/conf/testing.py diff --git a/microservice/sysom_channel/config.yml b/microservice/0_sysom_channel/config.yml similarity index 100% rename from microservice/sysom_channel/config.yml rename to microservice/0_sysom_channel/config.yml diff --git a/microservice/sysom_channel/lib/channels/__init__.py b/microservice/0_sysom_channel/lib/channels/__init__.py similarity index 100% rename from microservice/sysom_channel/lib/channels/__init__.py rename to microservice/0_sysom_channel/lib/channels/__init__.py diff --git a/microservice/sysom_channel/lib/channels/base.py b/microservice/0_sysom_channel/lib/channels/base.py similarity index 100% rename from microservice/sysom_channel/lib/channels/base.py rename to microservice/0_sysom_channel/lib/channels/base.py diff --git a/microservice/sysom_channel/lib/channels/ssh.py b/microservice/0_sysom_channel/lib/channels/ssh.py similarity index 100% rename from microservice/sysom_channel/lib/channels/ssh.py rename to microservice/0_sysom_channel/lib/channels/ssh.py diff --git a/microservice/sysom_channel/lib/ssh.py b/microservice/0_sysom_channel/lib/ssh.py similarity index 100% rename from microservice/sysom_channel/lib/ssh.py rename to microservice/0_sysom_channel/lib/ssh.py diff --git a/microservice/sysom_channel/lib/utils.py b/microservice/0_sysom_channel/lib/utils.py similarity index 100% rename from microservice/sysom_channel/lib/utils.py rename to microservice/0_sysom_channel/lib/utils.py diff --git a/microservice/sysom_channel/main.py b/microservice/0_sysom_channel/main.py similarity index 100% rename from microservice/sysom_channel/main.py rename to microservice/0_sysom_channel/main.py diff --git a/microservice/sysom_channel/requirements.txt b/microservice/0_sysom_channel/requirements.txt similarity index 100% rename from microservice/sysom_channel/requirements.txt rename to microservice/0_sysom_channel/requirements.txt diff --git a/microservice/sysom_channel/scripts/server_clear.sh b/microservice/0_sysom_channel/scripts/server_clear.sh similarity index 100% rename from microservice/sysom_channel/scripts/server_clear.sh rename to microservice/0_sysom_channel/scripts/server_clear.sh diff --git a/microservice/sysom_channel/scripts/server_init.sh b/microservice/0_sysom_channel/scripts/server_init.sh similarity index 90% rename from microservice/sysom_channel/scripts/server_init.sh rename to microservice/0_sysom_channel/scripts/server_init.sh index c8cc69b3..99f6a395 100644 --- a/microservice/sysom_channel/scripts/server_init.sh +++ b/microservice/0_sysom_channel/scripts/server_init.sh @@ -1,6 +1,6 @@ #!/bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_channel -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-channel if [ "$UID" -ne 0 ]; then diff --git a/microservice/sysom_channel/scripts/server_start.sh b/microservice/0_sysom_channel/scripts/server_start.sh similarity index 100% rename from microservice/sysom_channel/scripts/server_start.sh rename to microservice/0_sysom_channel/scripts/server_start.sh diff --git a/microservice/sysom_channel/scripts/server_stop.sh b/microservice/0_sysom_channel/scripts/server_stop.sh similarity index 100% rename from microservice/sysom_channel/scripts/server_stop.sh rename to microservice/0_sysom_channel/scripts/server_stop.sh diff --git a/microservice/sysom_channel/sysom-channel.ini b/microservice/0_sysom_channel/sysom-channel.ini similarity index 41% rename from microservice/sysom_channel/sysom-channel.ini rename to microservice/0_sysom_channel/sysom-channel.ini index bd174b39..5de0b95b 100644 --- a/microservice/sysom_channel/sysom-channel.ini +++ b/microservice/0_sysom_channel/sysom-channel.ini @@ -1,9 +1,9 @@ [program:sysom-channel] -directory=/usr/local/sysom/server/microservice/sysom_channel -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +directory=/usr/local/sysom/server/microservice/0_sysom_channel +command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-channel-error.log stdout_logfile=/var/log/sysom/sysom-channel.log diff --git a/microservice/clear_exclude b/microservice/clear_exclude new file mode 100644 index 00000000..e69de29b diff --git a/microservice/deploy_exclude b/microservice/deploy_exclude new file mode 100644 index 00000000..5e69798f --- /dev/null +++ b/microservice/deploy_exclude @@ -0,0 +1 @@ +sysom_hotfix_builder \ No newline at end of file diff --git a/microservice/sysom_diagnosis/conf/common.py b/microservice/sysom_diagnosis/conf/common.py index 9e52dbe4..9fa34009 100644 --- a/microservice/sysom_diagnosis/conf/common.py +++ b/microservice/sysom_diagnosis/conf/common.py @@ -10,7 +10,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/microservice/sysom_diagnosis/conf/diagnosis_gunicorn.py b/microservice/sysom_diagnosis/conf/gunicorn.py similarity index 100% rename from microservice/sysom_diagnosis/conf/diagnosis_gunicorn.py rename to microservice/sysom_diagnosis/conf/gunicorn.py diff --git a/microservice/sysom_diagnosis/scripts/server_init.sh b/microservice/sysom_diagnosis/scripts/server_init.sh index 03cfd3ee..2f0f210b 100755 --- a/microservice/sysom_diagnosis/scripts/server_init.sh +++ b/microservice/sysom_diagnosis/scripts/server_init.sh @@ -1,10 +1,8 @@ #!/bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_diagnosis -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-diagnosis -BASE_DIR=`dirname $0` - source_virtualenv() { echo "INFO: activate virtualenv..." source ${VIRTUALENV_HOME}/bin/activate || exit 1 @@ -25,7 +23,7 @@ init_conf() { ###change the install dir base on param $1### sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` - sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/diagnosis_gunicorn.py + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/gunicorn.py } start_app() { diff --git a/microservice/sysom_diagnosis/sysom-diagnosis.ini b/microservice/sysom_diagnosis/sysom-diagnosis.ini index d1f0d000..6a04f364 100644 --- a/microservice/sysom_diagnosis/sysom-diagnosis.ini +++ b/microservice/sysom_diagnosis/sysom-diagnosis.ini @@ -1,6 +1,6 @@ [program:sysom-diagnosis] directory=/usr/local/sysom/server/microservice/sysom_diagnosis -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/diagnosis_gunicorn.py sysom_diagnosis.wsgi:application +command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_diagnosis.wsgi:application startsecs=3 autostart=true autorestart=true diff --git a/microservice/sysom_hotfix/conf/common.py b/microservice/sysom_hotfix/conf/common.py index 81c4c989..7b9539c4 100644 --- a/microservice/sysom_hotfix/conf/common.py +++ b/microservice/sysom_hotfix/conf/common.py @@ -11,7 +11,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/microservice/sysom_hotfix/scripts/server_init.sh b/microservice/sysom_hotfix/scripts/server_init.sh index fb610ead..f4d8c17e 100644 --- a/microservice/sysom_hotfix/scripts/server_init.sh +++ b/microservice/sysom_hotfix/scripts/server_init.sh @@ -1,6 +1,6 @@ #! /bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_hotfix -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-hotfix source_virtualenv() { @@ -56,6 +56,10 @@ start_app() { } deploy() { + rpm -q --quiet nfs-utils || yum install -y nfs-utils + rpm -q --quiet rpcbind || yum install -y rpcbind + systemctl start rpcbind && systemctl enable rpcbind + systemctl start nfs && systemctl enable nfs source_virtualenv install_requirement init_conf diff --git a/microservice/sysom_hotfix/sysom-hotfix.ini b/microservice/sysom_hotfix/sysom-hotfix.ini index e25d68db..e2c211ee 100644 --- a/microservice/sysom_hotfix/sysom-hotfix.ini +++ b/microservice/sysom_hotfix/sysom-hotfix.ini @@ -1,9 +1,9 @@ [program:sysom-hotfix] directory = /usr/local/sysom/server/microservice/sysom_hotfix -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_hotfix.wsgi:application +command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_hotfix.wsgi:application startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-hotfix-error.log stdout_logfile=/var/log/sysom/sysom-hotfix.log diff --git a/microservice/sysom_hotfix_builder/scripts/server_init.sh b/microservice/sysom_hotfix_builder/scripts/server_init.sh index 2e6b34d5..685e2dcd 100644 --- a/microservice/sysom_hotfix_builder/scripts/server_init.sh +++ b/microservice/sysom_hotfix_builder/scripts/server_init.sh @@ -1,6 +1,6 @@ #! /bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_hotfix -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-hotfix-builder NFS_SERVER_IP=${SERVER_LOCAL_IP} diff --git a/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini b/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini index 0e1c2560..bd7427d0 100644 --- a/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini +++ b/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini @@ -1,9 +1,9 @@ [program:sysom-hotfix-builder] directory = /usr/local/sysom/server/microservice/sysom_hotfix_builder -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/python3 builder.py +command=/usr/local/sysom/server/environment/virtualenv/bin/python3 builder.py startsecs=3 autostart=true autorestart=true -environment=PATH=/usr/local/sysom/server/infrastructure/virtualenv/bin:%(ENV_PATH)s +environment=PATH=/usr/local/sysom/server/environment/virtualenv/bin:%(ENV_PATH)s stderr_logfile=/var/log/sysom/sysom-hotfix-builder-error.log stdout_logfile=/var/log/sysom/sysom-hotfix-builder.log diff --git a/microservice/sysom_migration/conf/common.py b/microservice/sysom_migration/conf/common.py index 981ad5e1..c805ae13 100644 --- a/microservice/sysom_migration/conf/common.py +++ b/microservice/sysom_migration/conf/common.py @@ -20,7 +20,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/microservice/sysom_migration/scripts/server_clear.sh b/microservice/sysom_migration/scripts/server_clear.sh index e69de29b..a958999b 100644 --- a/microservice/sysom_migration/scripts/server_clear.sh +++ b/microservice/sysom_migration/scripts/server_clear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-migration +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} +clear_app diff --git a/microservice/sysom_migration/scripts/server_init.sh b/microservice/sysom_migration/scripts/server_init.sh index 1df902f8..d6a543a7 100644 --- a/microservice/sysom_migration/scripts/server_init.sh +++ b/microservice/sysom_migration/scripts/server_init.sh @@ -1,6 +1,6 @@ #!/bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_migration -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-migration ANCE_X86_PKG=ance-0.1.1-1.x86_64.rpm ANCE_ARM_PKG=ance-0.1.1-1.aarch64.rpm diff --git a/microservice/sysom_migration/sysom-migration.ini b/microservice/sysom_migration/sysom-migration.ini index 46cb2244..1a08106f 100644 --- a/microservice/sysom_migration/sysom-migration.ini +++ b/microservice/sysom_migration/sysom-migration.ini @@ -1,9 +1,9 @@ [program:sysom-migration] directory = /usr/local/sysom/server/microservice/sysom_migration -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_migration.wsgi:application +command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_migration.wsgi:application startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-migration-error.log stdout_logfile=/var/log/sysom/sysom-migration.log diff --git a/microservice/sysom_monitor_server/conf/common.py b/microservice/sysom_monitor_server/conf/common.py index 03d8c698..19e9b46c 100644 --- a/microservice/sysom_monitor_server/conf/common.py +++ b/microservice/sysom_monitor_server/conf/common.py @@ -17,7 +17,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/microservice/sysom_monitor_server/scripts/server_init.sh b/microservice/sysom_monitor_server/scripts/server_init.sh index 4399b882..f732901b 100644 --- a/microservice/sysom_monitor_server/scripts/server_init.sh +++ b/microservice/sysom_monitor_server/scripts/server_init.sh @@ -1,6 +1,6 @@ #!/bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_monitor_server -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-monitor-server if [ "$UID" -ne 0 ]; then diff --git a/microservice/sysom_monitor_server/sysom-monitor-server.ini b/microservice/sysom_monitor_server/sysom-monitor-server.ini index b60da419..3eeb4f5d 100644 --- a/microservice/sysom_monitor_server/sysom-monitor-server.ini +++ b/microservice/sysom_monitor_server/sysom-monitor-server.ini @@ -1,9 +1,9 @@ [program:sysom-monitor-server] directory=/usr/local/sysom/server/microservice/sysom_monitor_server -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-monitor-server-error.log stdout_logfile=/var/log/sysom/sysom-monitor-server.log diff --git a/microservice/sysom_vmcore/conf/common.py b/microservice/sysom_vmcore/conf/common.py index 454c7c9f..73be5e25 100644 --- a/microservice/sysom_vmcore/conf/common.py +++ b/microservice/sysom_vmcore/conf/common.py @@ -21,7 +21,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/microservice/sysom_vmcore/scripts/server_init.sh b/microservice/sysom_vmcore/scripts/server_init.sh index 70318fc7..f8e2ab70 100644 --- a/microservice/sysom_vmcore/scripts/server_init.sh +++ b/microservice/sysom_vmcore/scripts/server_init.sh @@ -1,7 +1,7 @@ #! /bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_vmcore -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv -VIRTUALENV_PYTHON3=${SERVER_HOME}/virtualenv/bin/python3 +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME +VIRTUALENV_PYTHON3=${VIRTUALENV_HOME}/bin/python3 SERVICE_NAME=sysom-vmcore BASE_DIR=$(dirname $0) diff --git a/microservice/sysom_vmcore/sysom-vmcore.ini b/microservice/sysom_vmcore/sysom-vmcore.ini index c436b818..e07d1943 100644 --- a/microservice/sysom_vmcore/sysom-vmcore.ini +++ b/microservice/sysom_vmcore/sysom-vmcore.ini @@ -1,9 +1,9 @@ [program:sysom-vmcore] directory = /usr/local/sysom/server/microservice/sysom_vmcore -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vmcore.wsgi:application +command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vmcore.wsgi:application startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-vmcore-error.log stdout_logfile=/var/log/sysom/sysom-vmcore.log diff --git a/microservice/sysom_vul/conf/common.py b/microservice/sysom_vul/conf/common.py index 111eb9b1..9a1d96a0 100644 --- a/microservice/sysom_vul/conf/common.py +++ b/microservice/sysom_vul/conf/common.py @@ -21,7 +21,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent}/conf/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) diff --git a/microservice/sysom_vul/scripts/server_init.sh b/microservice/sysom_vul/scripts/server_init.sh index 4ede0d19..d7947298 100644 --- a/microservice/sysom_vul/scripts/server_init.sh +++ b/microservice/sysom_vul/scripts/server_init.sh @@ -1,6 +1,6 @@ #!/bin/bash -SERVICE_HOME=${MICROSERVICE_HOME}/sysom_vul -VIRTUALENV_HOME=${INFRASTRUCTURE_HOME}/virtualenv +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-vul if [ "$UID" -ne 0 ]; then @@ -28,7 +28,7 @@ init_conf() { ###change the install dir base on param $1### sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` - sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVER_HOME}/conf/vul_gunicorn.py + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/gunicorn.py } start_app() { diff --git a/microservice/sysom_vul/sysom-vul.ini b/microservice/sysom_vul/sysom-vul.ini index 4efe27c5..9e794178 100644 --- a/microservice/sysom_vul/sysom-vul.ini +++ b/microservice/sysom_vul/sysom-vul.ini @@ -1,9 +1,9 @@ [program:sysom-vul] directory = /usr/local/sysom/server/microservice/sysom_vul -command=/usr/local/sysom/server/infrastructure/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vul.wsgi:application +command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vul.wsgi:application startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/infrastructure/virtualenv/bin/" +environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-vul-error.log stdout_logfile=/var/log/sysom/sysom-vul.log diff --git a/package.sh b/package.sh old mode 100644 new mode 100755 index 5f8c10c1..61f8cd4d --- a/package.sh +++ b/package.sh @@ -16,7 +16,8 @@ check_cmd yarn check_cmd tar RELEASE=sysomRelease-$(date +"%Y%m%d%H%M%S") -SERVERDIR=sysom_server +MICROSERVICE_DIR=microservice +INFRASTRUCTURE_DIR=infrastructure WEBDIR=sysom_web CONFDIR=conf SCRIPTDIR=script @@ -28,11 +29,11 @@ yarn build popd || exit mkdir -p "${RELEASE}" -cp -r ${SERVERDIR}/ ${TOOLSDIR}/ ${CONFDIR}/ "${RELEASE}"/ +cp -r ${MICROSERVICE_DIR}/ ${INFRASTRUCTURE_DIR}/ ${TOOLSDIR}/ ${CONFDIR}/ "${RELEASE}"/ cp -r ${WEBDIR}/dist/ "${RELEASE}"/${WEBDIR}/ mkdir -p "${RELEASE}"/${WEBDIR}/download/ -cp ${TOOLSDIR}/deploy/deploy.sh "${RELEASE}"/ -cp ${TOOLSDIR}/deploy/clear.sh "${RELEASE}"/ +cp ${SCRIPTDIR}/deploy/deploy.sh "${RELEASE}"/ +cp ${SCRIPTDIR}/deploy/clear.sh "${RELEASE}"/ cp -r ${SCRIPTDIR} "${RELEASE}"/ tar czf "${RELEASE}".tar.gz "${RELEASE}"/ rm -rf "${RELEASE}" diff --git a/package_rpm_offline.sh b/package_rpm_offline.sh old mode 100644 new mode 100755 diff --git a/package_rpm_online.sh b/package_rpm_online.sh old mode 100644 new mode 100755 diff --git a/script/clear.sh b/script/clear.sh old mode 100644 new mode 100755 index 26e5994f..c46a3af1 --- a/script/clear.sh +++ b/script/clear.sh @@ -18,20 +18,23 @@ do_clear_infrastructure() { targets=(${target//,/ }) for target in ${targets[*]} do - target_dir=${INFRASTRUCTURE_HOME}/${target} - pushd ${target_dir} - - green "$target_dir clear..................................." - bash -x ${target_dir}/clear.sh - if [ $? -ne 0 ];then - red "$target_dir clear fail, please check..................................." - red "sysom init failed, exit 1" - exit 1 - fi - popd - - # rm dir - rm -rf ${target_dir} + for service_dir in $(ls ${INFRASTRUCTURE_HOME} | grep ${target}) + do + target_dir=${INFRASTRUCTURE_HOME}/${service_dir} + pushd ${target_dir} + + green "$target_dir clear..................................." + bash -x ${target_dir}/clear.sh + if [ $? -ne 0 ];then + red "$target_dir clear fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + popd + + # rm dir + rm -rf ${target_dir} + done done # rm infrastructure if the folder is empty @@ -46,24 +49,32 @@ do_clear_microservice() { targets=(${target//,/ }) for target in ${targets[*]} do - target_dir=${MICROSERVICE_HOME}/${target} - if [ ! -d "$target_dir" ] - then - continue - fi - pushd ${target_dir} - - green "$target_dir clear..................................." - bash -x ${target_dir}/scripts/server_clear.sh - if [ $? -ne 0 ];then - red "$target_dir clear fail, please check..................................." - red "sysom init failed, exit 1" - exit 1 - fi - popd - - # rm dir - rm -rf ${target_dir} + for service_dir in $(ls ${MICROSERVICE_HOME} | grep ${target}) + do + target_dir=${MICROSERVICE_HOME}/${service_dir} + if [ ! -d "$target_dir" ] + then + continue + fi + pushd ${target_dir} + + if [ ! -f ${target_dir}/scripts/server_clear.sh ] + then + continue + fi + + green "$target_dir clear..................................." + bash -x ${target_dir}/scripts/server_clear.sh + if [ $? -ne 0 ];then + red "$target_dir clear fail, please check..................................." + # red "sysom init failed, exit 1" + # exit 1 + fi + popd + + # rm dir + rm -rf ${target_dir} + done done dircount=`ls -l ${MICROSERVICE_HOME}| grep "^d" | wc -l` @@ -112,13 +123,21 @@ sub_infrastructure() { target=$1 if [ "$target" == "ALL" ] then - # Deploy all microservices - for infrastructure in $INFRASTRUCTURE_HOME + # Load clear_excludes + clear_excludes="" + if [ -f "${INFRASTRUCTURE_HOME}/clear_exclude" ];then + clear_excludes=`cat ${INFRASTRUCTURE_HOME}/clear_exclude` + fi + # Clear all microservices + for infrastructure in $(ls -r $INFRASTRUCTURE_HOME) do + if [[ $clear_excludes =~ $infrastructure ]] || [ ! -d "${INFRASTRUCTURE_HOME}/${infrastructure}" ];then + continue + fi do_clear_infrastructure $infrastructure done else - # Deploy specific microservices + # Clear specific microservices do_clear_infrastructure $target fi } @@ -132,10 +151,17 @@ sub_microservice() { target=$1 if [ "$target" == "ALL" ] then + # Load clear_excludes + clear_excludes="" + if [ -f "${MICROSERVICE_HOME}/clear_exclude" ];then + clear_excludes=`cat ${MICROSERVICE_HOME}/clear_exclude` + fi # Deploy all microservices - for microservice in $(ls $MICROSERVICE_HOME) + for microservice in $(ls -r $MICROSERVICE_HOME) do - echo ${microservice} + if [[ $clear_excludes =~ $microservice ]] || [ ! -d "${MICROSERVICE_HOME}/${microservice}" ];then + continue + fi do_clear_microservice ${microservice} done else diff --git a/script/deploy.sh b/script/deploy.sh index 728d2786..6df9b3d8 100755 --- a/script/deploy.sh +++ b/script/deploy.sh @@ -2,9 +2,6 @@ set -x ProgName=$(basename $0) BaseDir=$(dirname $(readlink -f "$0")) - -SYSOM_CONF_DIR=${MICROSERVICE_HOME}/conf -SYSOM_CONF=${SYSOM_CONF_DIR}/config.yml # SYSOM_DATABASE_HOST=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a host | awk '{print $2}'` # SYSOM_DATABASE_PORT=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a port | awk '{print $2}'` # SYSOM_DATABASE_USER=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a user | awk '{print $2}'` @@ -46,6 +43,13 @@ SERVER_PORT=${SERVER_PORT} EOF } +ensure_sysom_conf_exists() { + if [ ! -f "$SYSOM_CONF" ] + then + cp `dirname $BaseDir`/conf/config.yml $SYSOM_CONF + fi +} + update_global_config() { sed "s/SERVER_LOCAL_IP: 127.0.0.1/SERVER_LOCAL_IP: $SERVER_LOCAL_IP/g" -i ${SYSOM_CONF} sed "s/SERVER_PUBLIC_IP: 127.0.0.1/SERVER_PUBLIC_IP: $SERVER_PUBLIC_IP/g" -i ${SYSOM_CONF} @@ -76,41 +80,57 @@ update_global_config() { fi } -ensure_sysom_conf_exists() { - mkdir -p $SYSOM_CONF_DIR - if [ ! -f "$SYSOM_CONF" ] - then - cp `dirname $BaseDir`/conf/config.yml $SYSOM_CONF - fi -} - do_deploy_microservices() { - ensure_sysom_conf_exists - mkdir -p ${MICROSERVICE_HOME} local_microservice_dir=`dirname $BaseDir`/microservice target=$1 targets=(${target//,/ }) for target in ${targets[*]} do - if [ "$local_microservice_dir" != "$MICROSERVICE_HOME" ] - then - # Copy microservice dir to deploy path if need - cp -r $local_microservice_dir/${target} $MICROSERVICE_HOME/${target} - fi + for service_dir in $(ls ${local_microservice_dir} | grep ${target}) + do + if [ "$local_microservice_dir" != "$MICROSERVICE_HOME" ] + then + # Copy microservice dir to deploy path if need + cp -r $local_microservice_dir/${service_dir} $MICROSERVICE_HOME/${service_dir} + fi - target_dir=${MICROSERVICE_HOME}/${target} - pushd ${target_dir} + target_dir=${MICROSERVICE_HOME}/${service_dir} - green "$target_dir initialing..................................." - bash -x ${target_dir}/scripts/server_init.sh - if [ $? -ne 0 ];then - red "$target_dir initial fail, please check..................................." - red "sysom init failed, exit 1" - exit 1 - fi + pushd ${target_dir} + + # update microservice common.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} + if [ -f "${target_dir}/conf/common.py" ]; then + sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${target_dir}/conf/common.py + fi + # update fastapi based microservice alembic/env.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} + if [ -f "${target_dir}/alembic/env.py" ]; then + sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${target_dir}/alembic/env.py + fi + # update microservice gunicorn.py replace /var/log/sysom to ${LOG_HOME} + if [ -f "${target_dir}/conf/gunicorn.py" ]; then + sed -i "s;/var/log/sysom;${LOG_HOME};g" ${target_dir}/conf/gunicorn.py + fi + # update *.ini replace /var/log/sysom to ${LOG_HOME} + for ini_conf in $(ls sysom-*.ini) + do + sed -i "s;/var/log/sysom;${LOG_HOME};g" ${target_dir}/${ini_conf} + done + + green "$target_dir initialing..................................." + if [ ! -f "${target_dir}/scripts/server_init.sh" ];then + continue + fi + bash -x ${target_dir}/scripts/server_init.sh + if [ $? -ne 0 ];then + red "$target_dir initial fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi # green "setup_web_menu_enable $dir" # setup_web_menu_enable $dir + done + popd done } @@ -122,24 +142,29 @@ do_deploy_infrastructure() { targets=(${target//,/ }) for target in ${targets[*]} do - if [ "$local_infrastructure_dir" != "$INFRASTRUCTURE_HOME" ] - then - # Copy microservice dir to deploy path if need - cp -r $local_infrastructure_dir/${target} $INFRASTRUCTURE_HOME/${target} - fi - target_dir=${INFRASTRUCTURE_HOME}/${target} - pushd ${target_dir} - - green "$target_dir initialing..................................." - bash -x ${target_dir}/init.sh - if [ $? -ne 0 ];then - red "$target_dir initial fail, please check..................................." - red "sysom init failed, exit 1" - exit 1 - fi - # green "setup_web_menu_enable $dir" - # setup_web_menu_enable $dir - popd + for service_dir in $(ls ${local_infrastructure_dir} | grep ${target}) + do + if [ "$local_infrastructure_dir" != "$INFRASTRUCTURE_HOME" ] + then + # Copy microservice dir to deploy path if need + cp -r $local_infrastructure_dir/${service_dir} $INFRASTRUCTURE_HOME/${service_dir} + fi + target_dir=${INFRASTRUCTURE_HOME}/${service_dir} + pushd ${target_dir} + + green "$target_dir initialing..................................." + bash -x ${target_dir}/init.sh + if [ $? -ne 0 ];then + red "$target_dir initial fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + # green "setup_web_menu_enable $dir" + # setup_web_menu_enable $dir + popd + echo ${service_dir} + done + done } @@ -177,9 +202,18 @@ sub_infrastructure() { target=$1 if [ "$target" == "ALL" ] then + local_infrastructure_dir=`dirname $BaseDir`/infrastructure + # Load deploy_excludes + deploy_excludes="" + if [ -f "${local_infrastructure_dir}/deploy_exclude" ];then + deploy_excludes=`cat ${local_infrastructure_dir}/deploy_exclude` + fi # Deploy all microservices - for infrastructure in $INFRASTRUCTURE_HOME + for infrastructure in $(ls $local_infrastructure_dir) do + if [[ $deploy_excludes =~ $infrastructure ]] || [ ! -d "${local_infrastructure_dir}/${infrastructure}" ];then + continue + fi do_deploy_infrastructure $infrastructure done else @@ -197,10 +231,19 @@ sub_microservice() { target=$1 if [ "$target" == "ALL" ] then + local_microservice_dir=`dirname $BaseDir`/microservice + # Load deploy_excludes + deploy_excludes="" + echo "hhhhh ${local_microservice_dir}" + if [ -f "${local_microservice_dir}/deploy_exclude" ];then + deploy_excludes=`cat ${local_microservice_dir}/deploy_exclude` + fi # Deploy all microservices - for microservice in $(ls $MICROSERVICE_HOME) + for microservice in $(ls $local_microservice_dir) do - echo ${microservice} + if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_dir}/${microservice}" ];then + continue + fi do_deploy_microservices ${microservice} done else @@ -216,6 +259,8 @@ case $subcommand in sub_help ;; *) + ensure_sysom_conf_exists + update_global_config shift sub_${subcommand} $@ if [ $? = 127 ]; then diff --git a/script/deploy/clear.sh b/script/deploy/clear.sh new file mode 100755 index 00000000..85ffdcd2 --- /dev/null +++ b/script/deploy/clear.sh @@ -0,0 +1,61 @@ +#!/bin/bash +#****************************************************************# +# ScriptName: clear.sh +# Function: clear sysom +#***************************************************************# +APP_HOME=/usr/local/sysom + +if [ $# -lt 1 ] ; then + echo "USAGE: $0 INSTALL_DIR" + echo "Or we use default install dir: /usr/local/sysom/" + echo "E.g.: $0 /usr/local/sysom" +else + APP_HOME=$1 +fi + +export APP_HOME=${APP_HOME} +export SERVER_HOME=${APP_HOME}/server + +del_app_home() { + # Whether delete APP_HOME + if [ "`ls -A ${APP_HOME}`" = "" ]; then + rm -r ${APP_HOME} + fi +} + +del_conf() { + if [ -f "/etc/nginx/nginx.conf.bak" ];then + mv /etc/nginx/nginx.conf.bak /etc/nginx/nginx.conf + fi + if [ -f "/etc/nginx/conf.d/sysom.conf" ];then + rm -f /etc/nginx/conf.d/sysom.conf + fi + if [ -f "/etc/supervisord.d/sysom.ini" ];then + rm -f /etc/supervisord.d/sysom.ini + fi + if [ -f "/usr/lib/systemd/system/sysom-server.service" ];then + rm -f /usr/lib/systemd/system/sysom-server.service + fi +} + +stop_server() { + systemctl stop sysom-server.service + systemctl disable sysom-server.service + systemctl daemon-reload +} + +clear_server() { + bash -x ${SERVER_HOME}/init_scripts/server/clear.sh +} + + +clear() { + if [ -d "${APP_HOME}" ];then + stop_server + clear_server + del_conf + del_app_home + fi +} + +clear diff --git a/script/deploy/deploy.sh b/script/deploy/deploy.sh new file mode 100755 index 00000000..483833ab --- /dev/null +++ b/script/deploy/deploy.sh @@ -0,0 +1,94 @@ +#!/bin/bash +#****************************************************************# +# ScriptName: deploy.sh +# Author: algalon +# Create Date: 2021-11-13 22:42 +# Modify Date: 2021-11-16 00:02 +# Function: deploy sysom +#***************************************************************# +APP_NAME="sysom" +WEB_DIR="sysom_web" +MICROSERVICE_DIR="microservice" +INFRASTRUCTURE_DIR="infrastructure" +CONF_DIR="conf" +SCRIPT_DIR="script" +APP_HOME=/usr/local/sysom +SERVER_LOCAL_IP="" +SERVER_PUBLIC_IP="" +SERVER_PORT="" +SERVER_PORT=80 + +if [ $# -lt 3 ] ; then + echo "USAGE: $0 INSTALL_DIR Internal_IP EXTERNAL_IP WEB_PORT" + echo "Or we use default install dir: /usr/local/sysom/" + echo "If WEB_PORT not set, use port 80 default" + echo "E.g.: $0 /usr/local/sysom 192.168.0.100 120.26.xx.xx 80" + exit 1 +else + APP_HOME=$1 + SERVER_LOCAL_IP=$2 + SERVER_PUBLIC_IP=$3 +fi + +if [ $# -eq 4 ] ; then + SERVER_PORT=$4 +fi + +export APP_NAME=${APP_NAME} +export APP_HOME=${APP_HOME} +export SERVER_HOME=${APP_HOME}/server +export WEB_HOME=${APP_HOME}/web +export SCRIPT_HOME=${SERVER_HOME}/init_scripts +export SERVER_LOCAL_IP=${SERVER_LOCAL_IP} +export SERVER_PUBLIC_IP=${SERVER_PUBLIC_IP} +export SERVER_PORT=${SERVER_PORT} + +mkdir -p ${APP_HOME} + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + + +update_target() { + echo "INFO: copy project file..." + # WEB + if [ -d "${WEB_HOME}" ]; then + rm -rf ${WEB_HOME} + fi + + cp -r ${WEB_DIR} ${WEB_HOME} + + # SERVER + # if [ -d "${SERVER_HOME}" ]; then + # rm -rf ${SERVER_HOME} + # fi + mkdir -p ${SERVER_HOME} + if [ -d "${SERVER_HOME}/${MICROSERVICE_DIR}" ];then + rm -rf ${SERVER_HOME}/${MICROSERVICE_DIR} + fi + if [ -d "${SERVER_HOME}/${INFRASTRUCTURE_DIR}" ];then + rm -rf ${SERVER_HOME}/${INFRASTRUCTURE_DIR} + fi + cp -r ${MICROSERVICE_DIR} ${INFRASTRUCTURE_DIR} ${SERVER_HOME} + if [ ! -d "${SERVER_HOME}/conf" ];then + cp -r ${CONF_DIR} ${SERVER_HOME}/conf + fi + + + # SCRIPT + cp -r ${SCRIPT_DIR} ${SCRIPT_HOME} + cp tools/deploy/sysom-server.service /usr/lib/systemd/system/ +} + +start_script_server() { + bash -x ${SERVER_HOME}/init_scripts/server/init.sh +} + +deploy() { + update_target + start_script_server +} + +deploy \ No newline at end of file diff --git a/script/node/clear.sh b/script/node/clear.sh deleted file mode 100644 index 4563e236..00000000 --- a/script/node/clear.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/bin/bash -x - -basedir=`dirname $0` - -cd $basedir - -for i in `cat conf` -do - export $i -done - -for dir in `ls` -do - if [ -d $dir ] - then - pushd $dir - bash -x clear.sh - popd - fi -done diff --git a/script/node/diagnosis/clear.sh b/script/node/diagnosis/clear.sh deleted file mode 100755 index fb2200d6..00000000 --- a/script/node/diagnosis/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -x -main() -{ - yum erase -y `rpm -qa | grep sysak` - exit 0 -} - -main diff --git a/script/node/diagnosis/init.sh b/script/node/diagnosis/init.sh deleted file mode 100755 index b61a73c6..00000000 --- a/script/node/diagnosis/init.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -x - -RESOURCE_DIR=${NODE_HOME}/diagnosis -SYSAK_VER=1.3.0-2 -SYSAK_ARCH=x86_64 -SYSAK_PKG=sysak-${SYSAK_VER}.${SYSAK_ARCH}.rpm - -main() -{ - ###we need to update the sysak rpm### - rpm -qa | grep sysak - if [ $? -eq 0 ] - then - yum erase -y `rpm -qa | grep sysak` - fi - - yum install -y ${SYSAK_PKG} - exit $? -} - -main diff --git a/script/node/init.sh b/script/node/init.sh deleted file mode 100644 index 5ae9fb6d..00000000 --- a/script/node/init.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -x - -basedir=`dirname $0` - -cd $basedir - -for i in `cat conf` -do - export $i -done - -cp -f conf ${APP_HOME}/ - -for dir in `ls` -do - if [ -d $dir ] - then - pushd $dir - bash -x init.sh - popd - fi -done diff --git a/script/node/monitor/clear.sh b/script/node/monitor/clear.sh deleted file mode 100755 index 78c144ec..00000000 --- a/script/node/monitor/clear.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/bin/bash -x - -RESOURCE_DIR=${NODE_HOME}/monitor - -main() -{ - systemctl stop node_exporter - rm -f /usr/lib/systemd/system/node_exporter.service - rm -rf ${RESOURCE_DIR}/node_exporter - bash -x raptor_profiling_clear.sh - exit 0 -} - -main diff --git a/script/node/monitor/init.sh b/script/node/monitor/init.sh deleted file mode 100755 index 306f4933..00000000 --- a/script/node/monitor/init.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -x - -RESOURCE_DIR=${NODE_HOME}/monitor - -##设置node_exporter开机自动启动 -cat << EOF > node_exporter.service -[Unit] -Description=SysOM Monitor Prometheus -Documentation=SysOM Monitor Prometheus -Wants=network-online.target -After=network-online.target - -[Service] -ExecStart=${RESOURCE_DIR}/node_exporter/node_exporter -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target -EOF - -main() -{ - NODE_EXPORTER_PKG=`ls node_exporter-*.tar.gz | awk -F".tar.gz" '{print $1}'` - NODE_EXPORTER_TAR=${NODE_EXPORTER_PKG}.tar.gz - tar -zxvf ${NODE_EXPORTER_TAR} - rm -rf ${RESOURCE_DIR}/node_exporter - mkdir -p ${RESOURCE_DIR} - mv ${NODE_EXPORTER_PKG} ${RESOURCE_DIR}/node_exporter - mv node_exporter.service /usr/lib/systemd/system - systemctl daemon-reload - systemctl enable node_exporter - systemctl start node_exporter - ps -elf | grep "${RESOURCE_DIR}/node_exporter/node_exporter" | grep -v grep 1>/dev/null - if [ $? -ne 0 ] - then - exit 1 - fi - bash -x raptor_profiling_deploy.sh - exit 0 -} - -main diff --git a/script/node/monitor/raptor_profiling_clear.sh b/script/node/monitor/raptor_profiling_clear.sh deleted file mode 100755 index f718689a..00000000 --- a/script/node/monitor/raptor_profiling_clear.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/bin/bash -x -RESOURCE_DIR=${NODE_HOME}/monitor - -main() -{ - systemctl stop raptor - rm -f /usr/lib/systemd/system/raptor.service - rm -rf ${RESOURCE_DIR}/raptor - exit 0 -} - -main diff --git a/script/node/monitor/raptor_profiling_deploy.sh b/script/node/monitor/raptor_profiling_deploy.sh deleted file mode 100755 index 6809f422..00000000 --- a/script/node/monitor/raptor_profiling_deploy.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/bash -x - -RESOURCE_DIR=${NODE_HOME}/monitor - -##设置raptor开机自动启动 -cat << EOF > raptor.service -[Unit] -Description=SysOM Monitor Prometheus -Documentation=SysOM Monitor Prometheus -Wants=network-online.target -After=network-online.target - -[Service] -ExecStart=/usr/bin/sysak raptor oncpu --server http://${SERVER_LOCAL_IP}:4040 --app-name ${NODE_IP} -Restart=always -RestartSec=5 - -[Install] -WantedBy=multi-user.target -EOF - -main() -{ - mv raptor.service /usr/lib/systemd/system - systemctl daemon-reload - systemctl enable raptor - systemctl start raptor - ps -elf | grep "${RESOURCE_DIR}/raptor/raptor" | grep -v grep 1>/dev/null - if [ $? -ne 0 ] - then - exit 1 - fi - exit 0 -} - -main diff --git a/script/node/vmcore/clear.sh b/script/node/vmcore/clear.sh deleted file mode 120000 index 188102ab..00000000 --- a/script/node/vmcore/clear.sh +++ /dev/null @@ -1 +0,0 @@ -clear_vmcore.sh \ No newline at end of file diff --git a/script/node/vmcore/clear_vmcore.sh b/script/node/vmcore/clear_vmcore.sh deleted file mode 100644 index 1c631fe6..00000000 --- a/script/node/vmcore/clear_vmcore.sh +++ /dev/null @@ -1,6 +0,0 @@ -#! /bin/sh - -systemctl disable vmcore-collect.service -rm -f /usr/lib/systemd/system/vmcore-collect.service -systemctl daemon-reload -rm -rf ${VMCORE_HOME} diff --git a/script/node/vmcore/init.sh b/script/node/vmcore/init.sh deleted file mode 120000 index 6223fe60..00000000 --- a/script/node/vmcore/init.sh +++ /dev/null @@ -1 +0,0 @@ -init_vmcore.sh \ No newline at end of file diff --git a/script/node/vmcore/init_vmcore.sh b/script/node/vmcore/init_vmcore.sh deleted file mode 100644 index 239a7b39..00000000 --- a/script/node/vmcore/init_vmcore.sh +++ /dev/null @@ -1,37 +0,0 @@ -#! /bin/sh -rpm -q --quiet nfs-utils || yum install -y nfs-utils -rpm -q --quiet rpcbind || yum install -y rpcbind -systemctl start rpcbind && systemctl enable rpcbind -systemctl start nfs && systemctl enable nfs - -VMCORE_HOME=${NODE_HOME}/vmcore -VMCORE_NFS_HOME=${SERVER_HOME}/vmcore/vmcore-nfs -NODE_CONF=${APP_HOME}/conf -mkdir -p ${VMCORE_HOME} - -service=" -[Unit] -Description=Collect vmcore file to oss -After=network.target network-online.target remote-fs.target basic.target -DefaultDependencies=no - -[Service] -Type=forking -ExecStart=/usr/bin/python3 ${VMCORE_HOME}/vmcore_collect.py ${SERVER_LOCAL_IP} ${VMCORE_NFS_HOME} ${NODE_CONF} -StartLimitInterval=0 -StandardOutput=syslog -StandardError=inherit - -[Install] -WantedBy=multi-user.target -" - -cat << EOF > vmcore-collect.service -$service -EOF - -cp vmcore_collect.py ${VMCORE_HOME} -mv vmcore-collect.service /usr/lib/systemd/system/vmcore-collect.service -chmod 644 /usr/lib/systemd/system/vmcore-collect.service -systemctl daemon-reload -systemctl enable vmcore-collect.service diff --git a/script/node/vmcore/vmcore_collect.py b/script/node/vmcore/vmcore_collect.py deleted file mode 100644 index c8682a5f..00000000 --- a/script/node/vmcore/vmcore_collect.py +++ /dev/null @@ -1,160 +0,0 @@ -# -*- coding: utf-8 -*- -import os -import sys -from datetime import datetime -import json -import traceback -import socket - -nfs_ip = '127.0.0.1' -nfs_dir = '/usr/vmcore-nfs' -node_conf = '/usr/local/sysom/conf' -if len(sys.argv) >= 3 : - nfs_ip = sys.argv[1] - nfs_dir = sys.argv[2] - if len(sys.argv) == 4 and sys.argv[3] != "": - node_conf = sys.argv[3] - -def get_crash_path(): - try: - if os.path.exists('/etc/kdump.conf'): - with open('/etc/kdump.conf', 'r') as f1: - lines = f1.readlines() - part = '' - var_path = '' - for line in lines: - if line.startswith('ext4'): - if len(line.split()) > 1: - part0 = line.split()[1] - else: - continue - if part0.startswith('/dev/'): - cmd = 'lsblk %s' % (part0) - output = os.popen(cmd) - ret = output.read().strip() - output.close() - part = ret.splitlines()[-1].split()[-1] - elif part0.startswith('LABEL='): - part = part0.split('=')[-1] - elif line.startswith('path'): - var_path = line.split()[-1] - if len(part) > 0 and len(var_path) > 0: - return "%s%s" % (part, var_path) - elif len(var_path) > 0: - return var_path - else: - return '/var/crash/' - except: - pass - return '/var/crash/' - -def unmount_nfs(): - cmd = 'umount /tmp/vmcore-nfs' - ret = os.system(cmd) - if ret != 0: - raise Exception('failed to unmount nfs at /tmp/vmcore-nfs') - -def upload_nfs(vmcore_dir): - try: - s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - s.settimeout(0) - s.connect(('10.255.255.255', 1)) - ip = s.getsockname()[0] - except Exception: - hostname = socket.gethostname() - ip = socket.gethostbyname(hostname) - finally: - s.close() - timelist = vmcore_dir.split('-')[1:] - core_time = ''.join(timelist) - core_time = core_time.replace(':','') - vmcore_name ='%s_%s'%(core_time,ip) - - if os.path.exists("/tmp/vmcore-nfs") == False: - cmd = 'mkdir -p /tmp/vmcore-nfs' - ret = os.system(cmd) - if ret != 0: - raise Exception('failed to make nfs mount point at /tmp/vmcore-nfs') - - cmd = 'mount -t nfs %s:%s /tmp/vmcore-nfs' % (nfs_ip,nfs_dir) - ret = os.system(cmd) - if ret != 0: - raise Exception('failed to mount to nfs %s' % vmcore_dir) - - if os.path.exists("/tmp/vmcore-nfs/%s" % vmcore_name) == False: - cmd = 'mkdir /tmp/vmcore-nfs/%s' % vmcore_name - ret = os.system(cmd) - if ret != 0: - unmount_nfs() - raise Exception('failed to make dir at mount point (/tmp/vmcore-nfs/%s)' % vmcore_name) - - cmd = 'cp %s/vmcore-dmesg.txt /tmp/vmcore-nfs/%s/vmcore-dmesg.txt' % (vmcore_dir,vmcore_name) - ret = os.system(cmd) - if ret != 0: - unmount_nfs() - raise Exception('failed to copy to nfs /tmp/vmcore-nfs/%s/vmcore-dmesg.txt' % vmcore_name) - - cmd = 'cp %s/vmcore /tmp/vmcore-nfs/%s/vmcore' % (vmcore_dir,vmcore_name) - ret = os.system(cmd) - if ret != 0: - unmount_nfs() - raise Exception('failed to copy to nfs /tmp/vmcore-nfs/%s/vmcore' % vmcore_name) - - unmount_nfs() - with open('%s/.upload' % vmcore_dir,'w') as f: - pass - -def nfs_config(): - global nfs_ip, nfs_dir,node_conf - server_local_ip = "" - server_port = "80" - try: - with open(node_conf,'r') as fin: - line = fin.readline() - while len(line): - if line.startswith("SERVER_LOCAL_IP"): - server_local_ip = line.split("SERVER_LOCAL_IP=")[1].strip() - if line.startswith("SERVER_PORT"): - server_port = line.split("SERVER_PORT=")[1].strip() - line = fin.readline() - if server_local_ip != "": - cmd = f'wget -T 3 -t 1 http://{server_local_ip}:{server_port}/download/vmcore_nfs_config -O vmcore_nfs_config' - ret = os.system(cmd) - if ret: - return False - with open("vmcore_nfs_config",'r') as fin: - line = fin.readline() - while len(line): - if line.startswith("server_host"): - nfs_ip = line.split("server_host=")[1].strip() - if line.startswith("mount_point"): - nfs_dir = line.split("mount_point=")[1].strip() - line = fin.readline() - except: - import traceback - traceback.print_exc() - return False - return True - -def main(): - nfs_config() - crash_path = get_crash_path() - dirs_list = [] - files = os.listdir(crash_path) - files_path = [f'{crash_path}/{file}' for file in files] - for file in files_path: - if os.path.isfile(file): - continue - if file.find('-') < 0: - continue - dirs_list.append(file) - dirs_list.sort(key=lambda fp: os.path.getmtime(fp),reverse=True) - for dir in dirs_list: - tmp = '%s/.upload' % dir - if os.path.exists(tmp): - break - upload_nfs(dir) - -if __name__=="__main__": - main() - diff --git a/script/server/0_env/init.sh b/script/server/0_env/init.sh deleted file mode 100755 index 9fa21289..00000000 --- a/script/server/0_env/init.sh +++ /dev/null @@ -1,119 +0,0 @@ -#!/bin/bash -#****************************************************************# -# ScriptName: init sysom env -# Author: huangtuquan -#***************************************************************# - -ALIYUN_MIRROR="https://mirrors.aliyun.com/pypi/simple/" -SERVER_DIR="sysom_server" -SDK_DIR=$SERVER_DIR/sdk - -VIRTUALENV_HOME="${SERVER_HOME}/virtualenv" -TARGET_PATH="${SERVER_HOME}/target" - -if [ "$UID" -ne 0 ]; then - echo "Please run as root" - exit 1 -fi - -mkdir -p ${SERVER_HOME} - -check_selinux_status() -{ - ###check selinux rpm### - rpm -qa | grep selinux-policy - if [ $? -eq 0 ] - then - cat /etc/selinux/config | grep "SELINUX=disabled" - if [ $? -eq 0 ] - then - echo "selinux disable..." - else - echo "selinux enable, please set selinux disable" - exit 1 - fi - else - echo "selinux rpm package not install" - fi -} - -touch_env_rpms() { - if [ -f /etc/alios-release ]; then - if [ ! -f /etc/yum.repos.d/epel.repo ]; then - wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo - fi - elif [ -f /etc/anolis-release ]; then - sed -i '/epel\/8\/Everything/{n;s/enabled=0/enabled=1/;}' /etc/yum.repos.d/AnolisOS-DDE.repo - fi - rpm -q --quiet python3 || yum install -y python3 - rpm -q --quiet mariadb-server || yum install -y mariadb-server - rpm -q --quiet supervisor || yum install -y supervisor - rpm -q --quiet nginx || yum install -y nginx - rpm -q --quiet gcc || yum install -y gcc - rpm -q --quiet make || yum install -y make - rpm -q --quiet redis || yum install -y redis - rpm -q --quiet wget || yum install -y wget - rpm -q --quiet rpcbind || yum install -y rpcbind - rpm -q --quiet nfs-utils || yum install -y nfs-utils -} - -check_requirements() { - echo "INFO: begin install requirements..." - if ! [ -d ${SERVER_HOME}/logs/ ]; then - mkdir -p ${SERVER_HOME}/logs/ || exit 1 - fi - - local requirements_log="${SERVER_HOME}/logs/${APP_NAME}_requirements.log" - local requirements="requirements.txt" - touch "$requirements_log" || exit - ### atomic-0.7.3 need cffi, we show install cffi first### - pip install --upgrade pip - pip install cffi - pip install -r ${requirements} -i "${ALIYUN_MIRROR}" |tee -a "${requirements_log}" || exit 1 - local pip_res=$? - if [ $pip_res -ne 0 ]; then - echo "ERROR: requirements not satisfied and auto install failed, please check ${requirements_log}" - exit 1 - fi -} - -touch_virtualenv() { - if [ -d ${VIRTUALENV_HOME} ]; then - echo "virtualenv exists, skip" - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 - else - mkdir -p ~/.pip - cp pip.conf ~/.pip/ - python3 -m venv ${VIRTUALENV_HOME} - if [ "$?" = 0 ]; then - echo "INFO: create virtualenv success" - else - echo "ERROR: create virtualenv failed" - exit 1 - fi - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 - check_requirements - fi -} - -install_sdk() { - pushd ${TARGET_PATH}/${SDK_DIR} - python setup_cec_base.py develop - python setup_cec_redis.py develop - python setup_channel_job.py develop - python setup_sysom_utils.py develop - python setup_clogger.py develop - sudo rm -r *.egg-info build dist - popd -} - -deploy() { - check_selinux_status - touch_env_rpms - touch_virtualenv - install_sdk -} - -deploy diff --git a/script/server/0_env/pip.conf b/script/server/0_env/pip.conf deleted file mode 100644 index ccb91ab5..00000000 --- a/script/server/0_env/pip.conf +++ /dev/null @@ -1,8 +0,0 @@ -## Note, this file is written by cloud-init on first boot of an instance -## modifications made here will not survive a re-bundle. -### -[global] -index-url=https://mirrors.aliyun.com/pypi/simple/ - -[install] -trusted-host=mirrors.aliyun.com diff --git a/script/server/0_env/requirements.txt b/script/server/0_env/requirements.txt deleted file mode 100644 index a3f487ab..00000000 --- a/script/server/0_env/requirements.txt +++ /dev/null @@ -1,38 +0,0 @@ -cffi==1.15.1 -aiofiles==0.8.0 -alembic==1.7.7 -anyio==3.6.2 -aiohttp==3.8.3 -asyncer==0.0.2 -asyncssh==2.12.0 -autopep8==2.0.0 -channels==3.0.4 -django==3.2.16 -django-apscheduler==0.6.2 -django-cors-headers==3.10.1 -django-filter==21.1 -django-redis==5.2.0 -djangorestframework==3.14.0 -drf-yasg==1.21.4 -daphne==3.0.2 -fastapi==0.83.0 -gevent==22.10.2 -gunicorn==20.1.0 -pandas>=1.1.5 -paramiko==2.12.0 -prompt-toolkit==3.0.32 -PyJWT==2.4.0 -PyMySQL==1.0.2 -pytest-runner==5.3.2 -python-multipart==0.0.5 -rich==12.6.0 -requests==2.27.1 -redis==4.3.4 -schedule==1.1.0 -uvicorn==0.16.0 -wheel==0.37.1 -xlwt==1.3.0 -xlrd==2.0.1 -prometheus-client==0.16.0 -pyyaml==6.0 -pyyaml-include==1.3 \ No newline at end of file diff --git a/script/server/0_local_services/clear.sh b/script/server/0_local_services/clear.sh deleted file mode 100755 index e81c8ddb..00000000 --- a/script/server/0_local_services/clear.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-redis -clear_local_redis() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} - -stop_redis() { - ###we need redis version >= 5.0.0, check redis version### - redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` - echo ${redis_version} - if [ $redis_version -lt 5 ] - then - clear_local_redis - else - systemctl stop redis.service - fi -} - -stop_app() { - systemctl stop nginx.service - systemctl stop supervisord - stop_redis -} - -clear_db() { - systemctl start mariadb.service - systemctl stop mariadb.service -} - -clear_app() { - stop_app - clear_db -} - -clear_app diff --git a/script/server/0_local_services/init.sh b/script/server/0_local_services/init.sh deleted file mode 100755 index e3acc951..00000000 --- a/script/server/0_local_services/init.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/bin/bash -#****************************************************************# -# ScriptName: start local service -# Author: huangtuquan -#***************************************************************# -SERVICE_NAME=sysom-redis -OSS_URL=https://sysom.oss-cn-beijing.aliyuncs.com/redis -REDIS_DL_URL=https://download.redis.io/releases -REDIS_DIR=redis-5.0.14 -REDIS_PKG=redis-5.0.14.tar.gz -usr_local_redis=0 - -setup_database() { - echo "INFO: begin create db..." - - systemctl restart mariadb.service - systemctl enable mariadb.service - mysql -uroot -e "create user if not exists 'sysom'@'%' identified by 'sysom_admin';" - mysql -uroot -e "grant usage on *.* to 'sysom'@'localhost' identified by 'sysom_admin'" - mysql -uroot -e "create database sysom character set utf8;" - mysql -uroot -e "create database grafana character set utf8;" - mysql -uroot -e "grant all privileges on sysom.* to 'sysom'@'%';" - mysql -uroot -e "grant all privileges on grafana.* to 'sysom'@'%';" - mysql -uroot -e "flush privileges;" -} -setup_nginx() { - mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak - cp nginx.conf /etc/nginx/ - cp sysom.conf /etc/nginx/conf.d/ - ###change the install dir base on param $1### - sed -i "s;SERVER_PORT;${SERVER_PORT};g" /etc/nginx/conf.d/sysom.conf - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/nginx/conf.d/sysom.conf -} - -init_conf() { - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini -} - -setup_redis() { - ###we need redis version >= 5.0.0, check redis version### - redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` - echo ${redis_version} - if [ $redis_version -lt 5 ] - then - echo "redis version in yum repo is less than 5.0.0, we will compile redis(5.0.14) and install it." - if [ ! -e ${REDIS_PKG} ] - then - wget ${OSS_URL}/${REDIS_PKG} || wget ${REDIS_DL_URL}/${REDIS_PKG} - if [ ! -e ${REDIS_PKG} ] - then - echo "download ${REDIS_PKG} fail" - exit 1 - fi - fi - echo "now uncompress ${REDIS_PKG}, then compile and install it." - tar -zxvf ${REDIS_PKG} - pushd ${REDIS_DIR} - make - mkdir -p ${SERVER_HOME}/redis - cp redis.conf ${SERVER_HOME}/redis/ - cp src/redis-server ${SERVER_HOME}/redis/ - if [ $? -ne 0 ] - then - echo "redis compile or install error, exit 1" - exit 1 - fi - usr_local_redis=1 - popd - fi -} - -start_local_redis() { - init_conf - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -start_app() { - systemctl enable nginx.service - systemctl restart nginx.service - systemctl enable supervisord - systemctl start supervisord - if [ $usr_local_redis == 1 ] - then - ###if redis systemd service has been start, we need stop it first### - systemctl status redis - if [ $? -eq 0 ] - then - systemctl stop redis - fi - start_local_redis - else - systemctl enable redis.service - systemctl restart redis.service - fi -} - -deploy() { - setup_database | tee -a ${SERVER_HOME}/logs/${APP_NAME}_setup_database.log 2>&1 - setup_nginx - setup_redis - start_app -} - -deploy diff --git a/script/server/0_local_services/nginx.conf b/script/server/0_local_services/nginx.conf deleted file mode 100644 index 959830cd..00000000 --- a/script/server/0_local_services/nginx.conf +++ /dev/null @@ -1,68 +0,0 @@ -# For more information on configuration, see: -# * Official English Documentation: http://nginx.org/en/docs/ -# * Official Russian Documentation: http://nginx.org/ru/docs/ - -user nginx; -worker_processes auto; -error_log /var/log/nginx/error.log; -pid /run/nginx.pid; - -# Load dynamic modules. See /usr/share/doc/nginx/README.dynamic. -include /usr/share/nginx/modules/*.conf; - -events { - worker_connections 1024; -} - -http { - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - access_log /var/log/nginx/access.log main; - - sendfile on; - tcp_nopush on; - tcp_nodelay on; - server_tokens off; - keepalive_timeout 0; - types_hash_max_size 4096; - - include /etc/nginx/mime.types; - default_type application/octet-stream; - - # Load modular configuration files from the /etc/nginx/conf.d directory. - # See http://nginx.org/en/docs/ngx_core_module.html#include - # for more information. - include /etc/nginx/conf.d/*.conf; - - -# Settings for a TLS enabled server. -# -# server { -# listen 443 ssl http2; -# listen [::]:443 ssl http2; -# server_name _; -# root /usr/share/nginx/html; -# -# ssl_certificate "/etc/pki/nginx/server.crt"; -# ssl_certificate_key "/etc/pki/nginx/private/server.key"; -# ssl_session_cache shared:SSL:1m; -# ssl_session_timeout 10m; -# ssl_ciphers HIGH:!aNULL:!MD5; -# ssl_prefer_server_ciphers on; -# -# # Load configuration files for the default server block. -# include /etc/nginx/default.d/*.conf; -# -# error_page 404 /404.html; -# location = /40x.html { -# } -# -# error_page 500 502 503 504 /50x.html; -# location = /50x.html { -# } -# } - -} - diff --git a/script/server/0_local_services/start.sh b/script/server/0_local_services/start.sh deleted file mode 100755 index 0f32583c..00000000 --- a/script/server/0_local_services/start.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-redis -start_local_redis() { - supervisorctl start $SERVICE_NAME -} - -start_redis() { - ###we need redis version >= 5.0.0, check redis version### - redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` - echo ${redis_version} - if [ $redis_version -lt 5 ] - then - start_local_redis - else - systemctl start redis.service - fi -} - -start_app() { - systemctl start mariadb.service - systemctl start nginx.service - systemctl start supervisord - start_redis -} - -start_app diff --git a/script/server/0_local_services/stop.sh b/script/server/0_local_services/stop.sh deleted file mode 100755 index 9dda6548..00000000 --- a/script/server/0_local_services/stop.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-redis -stop_local_redis() { - supervisorctl stop $SERVICE_NAME -} -stop_redis() { - ###we need redis version >= 5.0.0, check redis version### - redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` - echo ${redis_version} - if [ $redis_version -lt 5 ] - then - stop_local_redis - else - systemctl stop redis.service - fi -} - -stop_app() { - systemctl stop nginx.service - systemctl stop mariadb.service - systemctl stop supervisord - stop_redis -} - -stop_app diff --git a/script/server/0_local_services/sysom.conf b/script/server/0_local_services/sysom.conf deleted file mode 100644 index 1addd31f..00000000 --- a/script/server/0_local_services/sysom.conf +++ /dev/null @@ -1,141 +0,0 @@ -map $http_upgrade $connection_upgrade { - default upgrade; - '' close; -} - -server { - listen SERVER_PORT; - server_name _; - root /usr/local/sysom/server/target/sysom_web; - index index.html; - client_max_body_size 20m; - - gzip on; - gzip_min_length 1k; - gzip_buffers 4 16k; - gzip_http_version 1.1; - gzip_comp_level 7; - gzip_types text/plain text/css text/javascript application/javascript application/json; - gzip_vary on; - - location /grafana/ { - proxy_pass http://localhost:3000/; - proxy_set_header Host $http_host; - } - - location /ws/ { - proxy_pass http://127.0.0.1:7001; - proxy_read_timeout 180s; - proxy_redirect off; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $connection_upgrade; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/tasks/ { - proxy_pass http://127.0.0.1:7002; - proxy_read_timeout 180s; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v2/tasks/ { - proxy_pass http://127.0.0.1:7002; - proxy_read_timeout 180s; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/vmcore/ { - proxy_pass http://127.0.0.1:7004; - proxy_read_timeout 180s; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/channel/ { - proxy_pass http://127.0.0.1:7003; - proxy_read_timeout 180s; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - client_max_body_size 500m; - } - - location /api/v1/vul/ { - proxy_pass http://127.0.0.1:7005; - proxy_read_timeout 180; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/migration/ { - proxy_pass http://127.0.0.1:7006; - proxy_read_timeout 180; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/assessment/ { - proxy_pass http://127.0.0.1:7006; - proxy_read_timeout 180; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/implementation/ { - proxy_pass http://127.0.0.1:7006; - proxy_read_timeout 180; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/hotfix/ { - proxy_pass http://127.0.0.1:7007; - proxy_read_timeout 180; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/demo/ { - proxy_pass http://127.0.0.1:7008; - proxy_read_timeout 180; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/v1/monitor/ { - proxy_pass http://127.0.0.1:7009; - proxy_read_timeout 180; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location /api/ { - proxy_pass http://127.0.0.1:7001; - proxy_read_timeout 180s; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - - location /checkpreload.htm { - proxy_pass http://127.0.0.1:7001/checkpreload.htm; - proxy_read_timeout 180s; - proxy_redirect off; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - } - - location / { - try_files $uri /index.html; - } - - location /download/ { - alias /usr/local/sysom/server/target/sysom_web/download/; - autoindex on; - autoindex_exact_size off; - autoindex_localtime on; - index index.html index.htm; - limit_rate 20m; - } - -} diff --git a/script/server/0_sysom_api/clear.sh b/script/server/0_sysom_api/clear.sh deleted file mode 100755 index 07b92432..00000000 --- a/script/server/0_sysom_api/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-api -clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} -clear_app diff --git a/script/server/0_sysom_api/init.sh b/script/server/0_sysom_api/init.sh deleted file mode 100755 index 9713b601..00000000 --- a/script/server/0_sysom_api/init.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/bin/bash -#****************************************************************# -# ScriptName: deploy.sh -# Author: algalon -# Create Date: 2021-11-13 22:42 -# Modify Date: 2021-11-16 00:02 -# Function: deploy sysom -#***************************************************************# -SERVER_DIR="sysom_server" -API_DIR=${SERVER_DIR}/sysom_api -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -TARGET_PATH=${SERVER_HOME}/target -SERVICE_NAME=sysom-api - -if [ "$UID" -ne 0 ]; then - echo "Please run as root" - exit 1 -fi - -mkdir -p ${SERVER_HOME} - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - mkdir -p /run/daphne - pushd ${TARGET_PATH}/${API_DIR} - python manage.py migrate - popd - - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME}:0 - if [ $? -eq 0 ] - then - echo "${SERVICE_NAME} service start success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - start_app -} - -deploy diff --git a/script/server/0_sysom_api/start.sh b/script/server/0_sysom_api/start.sh deleted file mode 100755 index 953f2cb9..00000000 --- a/script/server/0_sysom_api/start.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-api -stop_app() { - for service in `supervisorctl status | grep ${SERVICE_NAME} | awk '{print $1}'` - do - supervisorctl start $service - done -} - -stop_app diff --git a/script/server/0_sysom_api/stop.sh b/script/server/0_sysom_api/stop.sh deleted file mode 100755 index eb4150ad..00000000 --- a/script/server/0_sysom_api/stop.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-api -stop_app() { - for service in `supervisorctl status | grep ${SERVICE_NAME} | awk '{print $1}'` - do - supervisorctl stop $service - done -} - -stop_app diff --git a/script/server/0_sysom_api/sysom-api.ini b/script/server/0_sysom_api/sysom-api.ini deleted file mode 100644 index c81b21c8..00000000 --- a/script/server/0_sysom_api/sysom-api.ini +++ /dev/null @@ -1,13 +0,0 @@ -[fcgi-program:sysom-api] -socket=tcp://localhost:7001 -directory=/usr/local/sysom/server/target/sysom_server/sysom_api -command=/usr/local/sysom/server/virtualenv/bin/daphne -u /run/daphne%(process_num)d.sock --fd 0 --access-log - --proxy-headers sysom.asgi:application -numprocs=4 -process_name=%(process_num)d -autostart=true -autorestart=true -redirect_stderr=true -stopasgroup=true -stdout_logfile=/usr/local/sysom/server/logs/sysom-api.log -stderr_logfile=/usr/local/sysom/server/logs/sysom-api-error.log -environment=PATH=/usr/local/sysom/server/virtualenv/bin:%(ENV_PATH)s diff --git a/script/server/0_sysom_channel/clear.sh b/script/server/0_sysom_channel/clear.sh deleted file mode 100755 index 63e6c7c8..00000000 --- a/script/server/0_sysom_channel/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-channel -clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} -clear_app diff --git a/script/server/0_sysom_channel/init.sh b/script/server/0_sysom_channel/init.sh deleted file mode 100755 index 60df72b8..00000000 --- a/script/server/0_sysom_channel/init.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -SERVER_DIR="sysom_server" -CHANNEL_DIR=${SERVER_DIR}/sysom_channel -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -TARGET_PATH=${SERVER_HOME}/target -SERVICE_NAME=sysom-channel - -if [ "$UID" -ne 0 ]; then - echo "Please run as root" - exit 1 -fi - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - pushd ${TARGET_PATH}/${CHANNEL_DIR} - alembic upgrade head - popd - - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - start_app -} - -deploy diff --git a/script/server/0_sysom_channel/start.sh b/script/server/0_sysom_channel/start.sh deleted file mode 100755 index 080f5f53..00000000 --- a/script/server/0_sysom_channel/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-channel -start_app() { - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/0_sysom_channel/stop.sh b/script/server/0_sysom_channel/stop.sh deleted file mode 100755 index f58c6440..00000000 --- a/script/server/0_sysom_channel/stop.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-channel -stop_app() { - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/0_sysom_channel/sysom-channel.ini b/script/server/0_sysom_channel/sysom-channel.ini deleted file mode 100644 index 717eaf94..00000000 --- a/script/server/0_sysom_channel/sysom-channel.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-channel] -directory = /usr/local/sysom/server/target/sysom_server/sysom_channel -command=/usr/local/sysom/server/virtualenv/bin/uvicorn main:app --port 7003 -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/sysom-channel-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-channel.log diff --git a/script/server/1_sysom_monitor/sysom-prometheus.ini b/script/server/1_sysom_monitor/sysom-prometheus.ini deleted file mode 100644 index 90c07d65..00000000 --- a/script/server/1_sysom_monitor/sysom-prometheus.ini +++ /dev/null @@ -1,8 +0,0 @@ -[program:sysom-prometheus] -directory = /usr/local/sysom/server/monitor/prometheus -command=/usr/local/sysom/server/monitor/prometheus/prometheus -startsecs=3 -autostart=true -autorestart=true -stderr_logfile=/usr/local/sysom/server/logs/sysom-prometheus-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-prometheus.log diff --git a/script/server/1_sysom_monitor_server/clear.sh b/script/server/1_sysom_monitor_server/clear.sh deleted file mode 100755 index ca965e5c..00000000 --- a/script/server/1_sysom_monitor_server/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-monitor-server -clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} -clear_app diff --git a/script/server/1_sysom_monitor_server/init.sh b/script/server/1_sysom_monitor_server/init.sh deleted file mode 100755 index 876e6910..00000000 --- a/script/server/1_sysom_monitor_server/init.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -SERVER_DIR="sysom_server" -CHANNEL_DIR=${SERVER_DIR}/sysom_monitor_server -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -TARGET_PATH=${SERVER_HOME}/target -SERVICE_NAME=sysom-monitor-server - -if [ "$UID" -ne 0 ]; then - echo "Please run as root" - exit 1 -fi - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - pushd ${TARGET_PATH}/${CHANNEL_DIR} - alembic upgrade head - popd - - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - start_app -} - -deploy diff --git a/script/server/1_sysom_monitor_server/start.sh b/script/server/1_sysom_monitor_server/start.sh deleted file mode 100755 index e18fa9c5..00000000 --- a/script/server/1_sysom_monitor_server/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-monitor-server -start_app() { - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/1_sysom_monitor_server/stop.sh b/script/server/1_sysom_monitor_server/stop.sh deleted file mode 100755 index 1391f149..00000000 --- a/script/server/1_sysom_monitor_server/stop.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-monitor-server -stop_app() { - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/1_sysom_monitor_server/sysom-monitor-server.ini b/script/server/1_sysom_monitor_server/sysom-monitor-server.ini deleted file mode 100644 index 547ecf90..00000000 --- a/script/server/1_sysom_monitor_server/sysom-monitor-server.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-monitor-server] -directory = /usr/local/sysom/server/target/sysom_server/sysom_monitor_server -command=/usr/local/sysom/server/virtualenv/bin/uvicorn main:app --port 7009 -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/sysom-monitor-server-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-monitor-server.log diff --git a/script/server/2_sysom_vmcore/clear.sh b/script/server/2_sysom_vmcore/clear.sh deleted file mode 100644 index 06921d22..00000000 --- a/script/server/2_sysom_vmcore/clear.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-vmcore -clear_app() { - sed -i '/vmcore/d' /var/spool/cron/root - sed -i '/vmcore/d' /etc/exports - exportfs -rv - systemctl stop nfs-server - systemctl stop rpcbind - systemctl stop nfs - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} - -clear_app diff --git a/script/server/2_sysom_vmcore/init.sh b/script/server/2_sysom_vmcore/init.sh deleted file mode 100644 index 54ac7927..00000000 --- a/script/server/2_sysom_vmcore/init.sh +++ /dev/null @@ -1,71 +0,0 @@ -#! /bin/bash -NODE_INIT_DIR=${SERVER_HOME}/target/sysom_web/download/sysom_node_init -SERVER_DIR="sysom_server" -VMCORE_DIR=${SERVER_DIR}/sysom_vmcore -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -TARGET_PATH=${SERVER_HOME}/target -SERVICE_NAME=sysom-vmcore -VIRTUALENV_PYTHON3=${SERVER_HOME}/virtualenv/bin/python3 - -BASE_DIR=`dirname $0` - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - pushd ${TARGET_PATH}/${VMCORE_DIR} - python manage.py migrate - popd - - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -start_nfs() -{ - systemctl start rpcbind && systemctl enable rpcbind - systemctl start nfs && systemctl enable nfs - if [ $? -ne 0 ];then - systemctl start nfs-server && systemctl enable nfs-server - fi - - nfs_mask=`ip -4 route | grep "link src" | grep $SERVER_LOCAL_IP | awk '{print $1}' | head -n 1` - file_path=${SERVER_HOME}/vmcore/vmcore-nfs - mkdir -p ${file_path} - echo "${file_path} ${nfs_mask}(rw,async)" >> /etc/exports - exportfs -rv - chmod -R 777 ${file_path} -} - -start_cron() -{ - cp parse_panic.py ${SERVER_HOME}/vmcore - cp vmcore_const.py ${SERVER_HOME}/vmcore - echo "* * * * * pushd ${SERVER_HOME}/vmcore;${VIRTUALENV_PYTHON3} parse_panic.py ${file_path} ${SERVER_PORT};popd" >> /var/spool/cron/root -} - -deploy() { - source_virtualenv - init_conf - start_app - start_nfs - start_cron -} - -deploy diff --git a/script/server/2_sysom_vmcore/parse_panic.py b/script/server/2_sysom_vmcore/parse_panic.py deleted file mode 100644 index 14ce4b68..00000000 --- a/script/server/2_sysom_vmcore/parse_panic.py +++ /dev/null @@ -1,565 +0,0 @@ -# -*- coding: utf-8 -*- -# @Author: lichen/zhilan - -import os -import sys -import time -import subprocess -import re -import sqlite3 -import json -import traceback -import importlib -import argparse -import requests -import vmcore_const -import time -from datetime import datetime -import threading -import queue -queue = queue.Queue() - -if sys.version[0] == '2': - reload(sys) - sys.setdefaultencoding('utf8') - -# crashkey_type={ -# 0:func_name -# 1:calltrace -# 2:crashkey -# 3:bugon_file -#} -nfs_root = '/usr/vmcore-nfs' -root_url = 'http://localhost' - -ltime_pattern = re.compile(r'^\[\s*([0-9]+)\..*\]') -rip_pattern = re.compile(r'\[\s*\S+\] RIP: 0010:.*\[<([0-9a-f]+)>\] (.+)') -rip_pattern_1 = re.compile(r'\[\s*\S+\] RIP: 0010:(\S+)') -rip_pattern_2 = re.compile(r'\[\s*\S+\] RIP .*\[<([0-9a-f]+)>\] (.+)') -ripmod_pattern = re.compile(r'\[\s*\S+\] RIP.* \[(\S+)\]$') -bugat_pattern = re.compile(r'.+\] kernel BUG at (\S+)!') -ver_pattern = re.compile(r'Comm: (\S*).*(Tainted:|Not tainted).* (\S+) #') -unload_pattern = re.compile(r'\[last unloaded: (\S+)\]') -title_pattern = re.compile(r'\[\s*\S+\] ((BUG:|Kernel panic|Bad pagetable:|divide error:|kernel BUG at|general protection fault:) .+)') -vertype_pattern = re.compile(r'(\d+)\.(\d+)\.') -last_strhost = '' - -ignore_funcs = ["schedule","schedule_timeout","ret_from_fork","kthread", - "do_syscall_64","entry_SYSCALL_64_after_swapgs","system_call_fastpath","fastpath", - "entry_SYSCALL_64_after_hwframe", - "start_secondary","cpu_startup_entry","arch_cpu_idle","default_idle", - "do_IRQ","common_interrupt","irq_exit","do_softirq", - "__schedule","io_schedule_timeout","io_schedule","dump_stack", - "exit_to_usermode_loop","stub_clone","schedule_preempt_disabled","oom_kill_process", - "unwind_backtrace","dump_header","show_stack","dump_backtrace","panic","watchdog_timer_fn", - "nmi_panic","watchdog_overflow_callback","__perf_event_overflow","perf_event_overflow","intel_pmu_handle_irq", - "perf_event_nmi_handler","nmi_handle","do_nmi","end_repeat_nmi","watchdog", - "__hrtimer_run_queues","hrtimer_interrupt","local_apic_timer_interrupt","smp_apic_timer_interrupt","apic_timer_interrupt", - "__pv_queued_spin_lock_slowpath","queued_spin_lock_slowpath" -] - -def get_column_value(column, line): - match = rip_pattern.match(line) - if match is None: - match = rip_pattern_2.match(line) - - if column['func_name']=='NA' and match: - column['rip']=match.group(1) - column['func_name']=match.group(2).split('+')[0] - column['func_name']=column['func_name'].split('.')[0] - ripmod_match = ripmod_pattern.match(line.strip()) - if ripmod_match: - column['ripmod']=ripmod_match.group(1) - match = rip_pattern_1.match(line) - if column['func_name']=='NA' and column.get('func_name_1','') =='NA' and match: - column['func_name_1']=match.group(1).split('+')[0] - column['func_name_1']=column['func_name_1'].split('.')[0] - - match = bugat_pattern.match(line) - if match: - column['bugat'] = match.group(1) - idx = line.find('Comm:') - if idx > 0: - match = ver_pattern.match(line, idx) - if match: - column['comm'] = match.group(1) - column['ver'] = match.group(3) - idx = line.find('[last unloaded:') - if idx > 0: - match = unload_pattern.match(line, idx) - if match: - column['unload'] = match.group(1) - match = title_pattern.match(line) - if match and column['title'] == 'NA': - column['title'] = match.group(1) - if column['func_name'] != 'NA': - column['tmp_func_name'] = column['func_name'] - column['tmp_rip'] = column['rip'] - column['tmp_ripmod'] = column['ripmod'] - column['func_name'] = '' - column['rip'] = '' - column['ripmod'] = '' - -def get_stamp(line): - match = ltime_pattern.match(line) - if match: - return int(match.group(1)) - return 0 - -def get_last_time(f): - ret = 10 - try: - f.seek(-512, os.SEEK_END) - except: - pass - for line in f.readlines(): - ret = get_stamp(line) - if ret > 0: - break - f.seek(0, os.SEEK_SET) - return ret-10 - -def fix_func_name(column): - if column['dmesg'].find('SysRq : Trigger a crash') > 0: - column['func_name'] = 'sysrq_handle_crash' - column['title'] = 'sysrq: SysRq : Trigger a crash' - column['status'] = vmcore_const.STATUS_SYSRQ - column['crashkey_type'] = 2 - column['crashkey'] = 'sysrq_handle_crash' - if column['dmesg'].find('Kernel panic - not syncing: Fatal machine check') > 0: - column['func_name'] = 'fatal_machine_check' - column['title'] = 'Kernel panic - not syncing: Fatal machine check' - column['status'] = vmcore_const.STATUS_HWERROR - column['crashkey_type'] = 2 - column['crashkey'] = 'fatal_machine_check' - column['panic_class'] = 'HardwareError' - if column['dmesg'].find('Kernel panic - not syncing: Fatal hardware error') > 0: - column['func_name'] = 'fatal_hardware_error' - column['title'] = 'Kernel panic - not syncing: Fatal machine check' - column['status'] = vmcore_const.STATUS_HWERROR - column['crashkey_type'] = 2 - column['crashkey'] = 'fatal_hardware_error' - column['panic_class'] = 'HardwareError' - if column['dmesg'].find('Fatal local machine check') > 0: - column['func_name'] = 'fatal_machine_check' - column['title'] = 'Kernel panic - not syncing: Fatal local machine check' - column['status'] = vmcore_const.STATUS_HWERROR - column['crashkey_type'] = 2 - column['crashkey'] = 'fatal_machine_check' - column['panic_class'] = 'HardwareError' - if 'bugat' in column: - column['bugon_file'] = column['bugat'].split(':')[0] - column['crashkey_type'] = 3 - - -def parse_rawdmesg(column): - dmesgs = column['rawdmesg'].splitlines() - column['rawdmesg'] = '' - result = '' - for line in dmesgs: - if line.find('Modules linked in') >= 0: - column['modules'] = line[line.find(':')+1:] - result += line+'\n' - get_column_value(column, line) - column['dmesg'] = result - fix_func_name(column) - -def parse_file(name, column): - f = open(name, 'r') - result = '' - for line in f.readlines(): - if line.find('Modules linked in') >= 0: - column['modules'] = line[line.find(':')+1:] - if len(column['modules']) >= 512: - column['modules'] = column['modules'][:-512] - result += line - get_column_value(column, line) - f.close() - column['dmesg'] = result - column['dmesg_file'] = name - if 'tmp_func_name' in column and column['func_name'] == 'NA' and column['tmp_func_name'] != 'NA': - column['func_name'] = column['tmp_func_name'] - column['rip'] = column['tmp_rip'] - column['ripmod'] = column['ripmod'] - fix_func_name(column) - if column['ripmod'] != 'NA': - if column['ripmod'] not in vmcore_const.BASEMODS: - column['panic_class'] = 'Module(%s)'%(column['ripmod']) - -line_pattern = re.compile(r'.+[0-9]+\].+\[.*\][? ]* (\S+)\+0x(\S+)/0x(\S+)') -line_pattern_1 = re.compile(r'.+[0-9]+\][? ]*(\S+)\+0x(\S+)/0x(\S+)') -def get_calltrace(column): - meettitle = 0 - list1 = [] - lines = column['dmesg'].split('\n') - modname = [] - tmplist = [] - workqueue = '' - nocalltrace = True - hung_flag = False - if column['title'].find('unrecovered softlockup') >= 0: - hung_flag = True - - invalidrip = False - if (column['rip'] == 'NA'and column['func_name'] == 'NA') or column['func_name'].startswith('0x'): - invalidrip = True - - badrip = False - if column['dmesg'].find('Code: Bad RIP value.') >= 0: - badrip = True - - question_continue = True - question_count = 0 - - for r in lines: - if r.find(column['title']) >= 0: - nocalltrace = True - meettitle = 1 - tmplist.extend(list1) - del list1[:] - question_count = 0 - question_continue = True - continue - - if r.find('Workqueue: events ') >= 0: - idx = r.find('Workqueue: events ') - workqueue = r[idx+18:] - - if r.find('EFLAGS: ') >= 0: - idx = r.find('EFLAGS: ') - eflags = r[idx+8:] - #print 'eflags',eflags - try: - eflags = int(eflags,16) - if (eflags >> 9) % 2 == 0: - badrip = True - except: - pass - if r.find("<>") >= 0: - if column['func_name'] == 'NA': - tmpline = lines[lines.index(r)-1] - m = line_pattern.match(tmpline) - if m: - column['func_name'] = m.group(1) - else: - m = line_pattern_1.match(tmpline) - if m: - column['func_name'] = m.group(1) - - if r.find('') >= 0: - badrip = True - - if hung_flag and r.find('') >= 0: - try: - if r.find('> ') >= 0 and r.find(' <') >= 0: - idx = r.find(' <') - idx2 = r.rfind('> ',0) - r = r[0:idx] + r[idx2+1:] - except: - import traceback - traceback.print_exc() - del list1[:] - question_count = 0 - question_continue = True - - if r.find("Call Trace:") > 0 or r.find("<>") > 0 or r.find("") > 0 or r.find("") >= 0: - try: - if r.find('> ') >= 0 and r.find(' <') >= 0: - idx = r.find(' <') - idx2 = r.rfind('> ',0) - r = r[0:idx] + r[idx2+1:] - except: - import traceback - traceback.print_exc() - del list1[:] - question_count = 0 - question_continue = True - modname = [] - - if r.find('?') >= 0: - if workqueue != '' and r.find(workqueue) >= 0: - list1.append(workqueue) - #print r - #print invalidrip,badrip,question_continue - if invalidrip and badrip and question_continue: - m2 = line_pattern.match(r) - if m2: - #print m2.group(1),m2.group(2),m2.group(3) - if m2.group(1).split('.')[0] == column['func_name'] or m2.group(1) in ignore_funcs: - continue - nocalltrace = False - if m2.group(2) != m2.group(3): - tmp = m2.group(1) - tmp = tmp.split('.')[0] - list1.append(tmp) - #print 'append: ',m2.group(1) - #print list1 - question_count += 1 - else: - m2 = line_pattern_1.match(r) - if m2: - #print m2.group(1),m2.group(2),m2.group(3) - if m2.group(1).split('.')[0] == column['func_name'] or m2.group(1) in ignore_funcs: - continue - nocalltrace = False - if m2.group(2) != m2.group(3): - tmp = m2.group(1) - tmp = tmp.split('.')[0] - list1.append(tmp) - #print 'append: ',m2.group(1) - #print list1 - question_count += 1 - continue - if question_count > 0: - question_continue = False - - m = line_pattern.match(r) - if m: - nocalltrace = False - if m.group(1).split('.')[0] == column['func_name'] or m.group(1) in ignore_funcs: - continue - if m.group(1) == 'panic': - del list1[:] - question_count = 0 - question_continue = True - modname = [] - continue - if len(list1) == 0 and m.group(1) in ignore_funcs: - continue - if len(modname) < 2: - modname.append(r.strip()) - tmp = m.group(1) - tmp = tmp.split('.')[0] - list1.append(tmp) - #print 'append: ',m.group(1) - #print list1 - else: - m = line_pattern_1.match(r) - if m: - nocalltrace = False - if m.group(1).split('.')[0] == column['func_name'] or m.group(1) in ignore_funcs: - continue - if m.group(1) == 'panic': - del list1[:] - question_count = 0 - question_continue = True - modname = [] - continue - if len(list1) == 0 and m.group(1) in ignore_funcs: - continue - if len(modname) < 2: - modname.append(r.strip()) - tmp = m.group(1) - tmp = tmp.split('.')[0] - list1.append(tmp) - #print 'append: ',m.group(1) - #print list1 - else: - if len(list1) > 0 and meettitle == 1: - break - if len(list1) == 0 and nocalltrace: - list1 = tmplist - - if column['func_name'] == 'NA' and len(list1) > 0: - column['func_name'] = list1[0] - del list1[0] - - column['calltrace_list'] = [] - column['calltrace_list'].extend(list1) - - calltrace = column['func_name'] - if calltrace != '': - calltrace = calltrace.split('+')[0] - if len(list1) > 2: - list1 = list1[0:2] - for i in list1: - calltrace = ''.join([calltrace,'$',i]) - column['calltrace'] = calltrace - - -def clarify_panic_type(column): - column['panic_type'] = 0 - if column['title'].find('divide error') >= 0: - column['panic_type'] = vmcore_const.PANIC_DIVIDEERROR - elif column['bugon_file'] != 'NA': - column['panic_type'] = vmcore_const.PANIC_BUGON - elif column['title'].find('NULL pointer dereference') >= 0: - column['panic_type'] = vmcore_const.PANIC_NULLPOINTER - elif column['title'].find('Kernel stack is corrupted') >= 0: - column['panic_type'] = vmcore_const.PANIC_STACKCORRUPTION - elif column['title'].find('hard LOCKUP') >= 0: - column['panic_type'] = vmcore_const.PANIC_HARDLOCKUP - elif column['title'].find('hung_task') >= 0: - column['panic_type'] = vmcore_const.PANIC_HUNGTASK - elif column['title'].find('RCU Stall') >= 0: - column['panic_type'] = vmcore_const.PANIC_RCUSTALL - elif (column['title'].find('soft lockup') >= 0 or column['title'].find('softlockup') >= 0): - column['panic_type'] = vmcore_const.PANIC_SOFTLOCKUP - -def check_panic(column): - if 'rawdmesg' not in column and os.path.isfile(column['dmesg_file']) == False: - return False - - matched = False - if 'rawdmesg' in column: - parse_rawdmesg(column) - else: - parse_file(column['dmesg_file'], column) - - m = vertype_pattern.match(column['ver']) - if m: - column['vertype'] = int(m.group(1)) * 100 + int(m.group(2)) - - get_calltrace(column) - if column['calltrace'] == 'NA': - column['crashkey_type'] = 0 - if column['crashkey_type'] == 0 and column['func_name'] != 'NA': - column['crashkey'] = '%d$%s'%(column['vertype'],column['func_name']) - elif column['crashkey_type'] == 1 and column['calltrace'] != 'NA': - column['crashkey'] = '%d$%s'%(column['vertype'],column['calltrace']) - elif column['crashkey_type'] == 2 and column['crashkey'] != 'NA': - column['crashkey'] = '%d$%s'%(column['vertype'],column['crashkey']) - elif column['crashkey_type'] == 3 and column['bugon_file'] != 'NA': - column['crashkey'] = '%d$%s$%s'%(column['vertype'],column['bugon_file'],column['calltrace']) - - clarify_panic_type(column) - #return False - - ip={'ip':column['ip']} - host_url = root_url+"/api/v1/host/" - res = requests.get(host_url,params=ip) - if res.status_code != 200 or res.text == '[]': - print("查询主机名失败") - return False - - column['hostname'] = res.json()['data'][0]['hostname'] - vmcore_url = root_url+"/api/v1/vmcore/" - data = json.dumps(column) - headers = {'content-type': 'application/json'} - res = requests.post(url=vmcore_url, data=data, headers=headers) - print(res.json()) - if res.status_code == 200: - print(f"add {column['name']} to db") - else: - print("插入失败") - return False - - idx = 0 - column['calltrace_list'].insert(0,column['func_name']) - for line in column['calltrace_list']: - calltrace_info = {'name':column['name'], 'line':line, 'idx':idx} - calltrace_url = root_url+"/api/v1/vmcore/" - data = json.dumps(calltrace_info) - headers = {'content-type': 'application/json'} - res = requests.post(url=calltrace_url, data=data, headers=headers) - if res.status_code != 200: - print(f"{column['name']} 插入calltrace line失败") - return False - idx += 1 - print(f"add {column['name']} calltrace line to db") - return True -def do_cmd(cmd): - output = os.popen(cmd) - ret = output.read().strip() - output.close() - return ret - -def init_column(column): - column['upload_time'] = int(time.time()) - column['vmcore_file'] = 'NA' - column['dmesg_file'] = 'NA' - column['rip'] = 'NA' - column['ripmod'] = 'NA' - column['comm'] = 'NA' - column['ver'] = 'NA' - column['vertype'] = 0 - column['func_name'] = 'NA' - column['title'] = 'NA' - column['status'] = 0 - column['calltrace'] = 'NA' - column['bugon_file'] = 'NA' - column['crashkey_type'] = 1 - column['crashkey'] = 'NA' - column['modules'] = 'NA' - column['panic_type'] = 0 - column['panic_class'] = 'BaseKernel' - column['issue_id'] = 0 - -def parse_new_crash(crash_dir): - try: - column = {} - column['name'] = crash_dir.split('/')[-1] - core_time = column['name'].split('_')[0] - core_time = datetime.strptime(core_time, "%Y%m%d%H%M%S") - column['core_time'] = core_time.strftime("%Y-%m-%d %H:%M:%S") - column['ip'] = column['name'].split('_')[1] - column['hostname'] = column['name'].split('_')[1] - init_column(column) - column['dmesg_file'] = '%s/vmcore-dmesg.txt' % crash_dir - column['vmcore_file'] = '%s/vmcore' % crash_dir - ret = check_panic(column) - if ret: - with open('%s/.upload'%crash_dir,'w'): - pass - except: - import traceback - traceback.print_exc() - -def mount_nfs(): - global nfs_root - get_config={'get_config':'1'} - host_url = root_url+"/api/v1/vmcore/" - res = requests.get(host_url,params=get_config) - if res.status_code != 200 or res.text == '[]': - print("无法查询nfs配置") - return False - server_host = res.json()['data']['server_host'] - mount_point = res.json()['data']['mount_point'] - if not server_host : - return False - - cmd = 'mount -t nfs %s:%s %s' % (server_host,mount_point,nfs_root) - ret = os.system(cmd) - if ret != 0: - print('failed to mount to nfs %s' % nfs_root) - return False - return True - -def unmount_nfs(): - global nfs_root - cmd = 'umount %s' % nfs_root - ret = os.system(cmd) - if ret != 0: - print(f'failed to unmount nfs at {nfs_root}') - -def main(): - global nfs_root - global root_url - server_port = 80 - if len(sys.argv) > 1: - nfs_root = sys.argv[1] - if len(sys.argv) > 2: - server_port = sys.argv[2] - root_url = root_url + ":" + server_port - dirs_list = [] - #while True: - hasnfs = mount_nfs() - files = os.listdir(nfs_root) - files_path = [f'{nfs_root}/{file}' for file in files] - for file in files_path: - if os.path.isfile(file): - continue - dirs_list.append(file) - dirs_list.sort(key=lambda fp: os.path.getmtime(fp),reverse=True) - for dir in dirs_list: - tmp = '%s/.upload' % dir - if os.path.exists(tmp): - break - parse_new_crash(dir) - #time.sleep(20) - if hasnfs: - unmount_nfs() - - -if __name__ == "__main__": - main() diff --git a/script/server/2_sysom_vmcore/start.sh b/script/server/2_sysom_vmcore/start.sh deleted file mode 100644 index 1bb89fb1..00000000 --- a/script/server/2_sysom_vmcore/start.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-vmcore -start_app() { - systemctl start nfs-server - systemctl start rpcbind - systemctl start nfs - sed -i '/vmcore/s;^#;;g' /var/spool/cron/root - sed -i '/vmcore/s;^#;;g' /etc/exports - exportfs -rv - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/2_sysom_vmcore/stop.sh b/script/server/2_sysom_vmcore/stop.sh deleted file mode 100644 index aadecadc..00000000 --- a/script/server/2_sysom_vmcore/stop.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-vmcore -stop_app() { - sed -i '/vmcore/s;^;#;g' /var/spool/cron/root - sed -i '/vmcore/s;^;#;g' /etc/exports - exportfs -rv - systemctl stop nfs-server - systemctl stop rpcbind - systemctl stop nfs - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/2_sysom_vmcore/sysom-vmcore.ini b/script/server/2_sysom_vmcore/sysom-vmcore.ini deleted file mode 100644 index b7421943..00000000 --- a/script/server/2_sysom_vmcore/sysom-vmcore.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-vmcore] -directory = /usr/local/sysom/server/target/sysom_server/sysom_vmcore -command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/vmcore_gunicorn.py sysom_vmcore.wsgi:application -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/sysom-vmcore-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-vmcore.log diff --git a/script/server/2_sysom_vmcore/vmcore_const.py b/script/server/2_sysom_vmcore/vmcore_const.py deleted file mode 100644 index 8f9ac84a..00000000 --- a/script/server/2_sysom_vmcore/vmcore_const.py +++ /dev/null @@ -1,25 +0,0 @@ -STATUS_HWERROR=5 -STATUS_SYSRQ=6 -# panic type -PANIC_UAF=1 -PANIC_DOUBLEFREE=2 -PANIC_OOBREAD=3 -PANIC_OOBWRITE=4 -PANIC_NULLPOINTER=5 -PANIC_UNINITVAR=6 -PANIC_STACKCORRUPTION=7 -PANIC_INVALIDIPPTR=8 -PANIC_INVALIDDATAPTR=9 -PANIC_BUGON=10 -PANIC_DIVIDEERROR=11 -PANIC_HARDLOCKUP=12 -PANIC_SOFTLOCKUP=13 -PANIC_HUNGTASK=14 -PANIC_RCUSTALL=15 - -BASEMODS = ['ext4','jbd2','overlay','libata','libiscsi','bridge','nf_conntrack','nf_conntrack_ipv4', - 'nf_nat','nf_nat_ipv4','iptable_nat','tun','binfmt_misc','xt_CHECKSUM','iptable_mangle', - 'nf_defrag_ipv4','xt_conntrack','ipt_REJECT','nf_reject_ipv4','stp','llc','ebtable_filter', - 'ebtables','ip6_tables','iptable_filter','iscsi_tcp','libiscsi_tcp','scsi_transport_iscsi', - 'bonding','dm_mod','sg','ip_tables','mbcache','sd_mod','mpt3sas','raid_class','scsi_transport_sas', - 'ahci','libahci','btrfs','zram','numa_balancer'] diff --git a/script/server/3_sysom_diagnosis/clear.sh b/script/server/3_sysom_diagnosis/clear.sh deleted file mode 100755 index 310be427..00000000 --- a/script/server/3_sysom_diagnosis/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-diagnosis -clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} -clear_app diff --git a/script/server/3_sysom_diagnosis/init.sh b/script/server/3_sysom_diagnosis/init.sh deleted file mode 100755 index 1d1c9e6c..00000000 --- a/script/server/3_sysom_diagnosis/init.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -NODE_INIT_DIR=${SERVER_HOME}/target/sysom_web/download/sysom_node_init -SERVER_DIR="sysom_server" -TARGET_PATH=${SERVER_HOME}/target -DIAGNOSIS_DIR=${SERVER_DIR}/sysom_diagnosis -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -SERVICE_NAME=sysom-diagnosis -SYSAK_DOWNLOAD_URL=https://mirrors.openanolis.cn/sysak/packages/ -SYSAK_PKG=sysak-1.3.0-2.x86_64.rpm - -BASE_DIR=`dirname $0` - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - pushd ${TARGET_PATH}/${DIAGNOSIS_DIR} - python manage.py migrate - popd - - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini - cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` - sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${DIAGNOSIS_DIR}/conf/diagnosis_gunicorn.py -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - start_app -} - -deploy diff --git a/script/server/3_sysom_diagnosis/start.sh b/script/server/3_sysom_diagnosis/start.sh deleted file mode 100755 index 025c5f29..00000000 --- a/script/server/3_sysom_diagnosis/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-diagnosis -start_app() { - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/3_sysom_diagnosis/stop.sh b/script/server/3_sysom_diagnosis/stop.sh deleted file mode 100755 index 587cefea..00000000 --- a/script/server/3_sysom_diagnosis/stop.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-diagnosis -stop_app() { - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/3_sysom_diagnosis/sysom-diagnosis.ini b/script/server/3_sysom_diagnosis/sysom-diagnosis.ini deleted file mode 100644 index 6065104f..00000000 --- a/script/server/3_sysom_diagnosis/sysom-diagnosis.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-diagnosis] -directory = /usr/local/sysom/server/target/sysom_server/sysom_diagnosis -command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/diagnosis_gunicorn.py sysom_diagnosis.wsgi:application -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/sysom-diagnosis-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-diagnosis.log diff --git a/script/server/4_sysom_vul/clear.sh b/script/server/4_sysom_vul/clear.sh deleted file mode 100644 index 9f3c3130..00000000 --- a/script/server/4_sysom_vul/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-vul -clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} -clear_app diff --git a/script/server/4_sysom_vul/init.sh b/script/server/4_sysom_vul/init.sh deleted file mode 100644 index dba6938c..00000000 --- a/script/server/4_sysom_vul/init.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/bin/bash -SERVER_DIR="sysom_server" -TARGET_PATH=${SERVER_HOME}/target -VUL_DIR=${SERVER_DIR}/sysom_vul -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -SERVICE_NAME=sysom-vul - -if [ "$UID" -ne 0 ]; then - echo "Please run as root" - exit 1 -fi - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - pushd ${TARGET_PATH}/${VUL_DIR} - python manage.py migrate - popd - - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini - cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` - sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${VUL_DIR}/conf/vul_gunicorn.py -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - start_app -} - -deploy diff --git a/script/server/4_sysom_vul/start.sh b/script/server/4_sysom_vul/start.sh deleted file mode 100644 index eeb0d1de..00000000 --- a/script/server/4_sysom_vul/start.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - - -SERVICE_NAME=sysom-vul - -start_app() { - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/4_sysom_vul/stop.sh b/script/server/4_sysom_vul/stop.sh deleted file mode 100644 index 594c064f..00000000 --- a/script/server/4_sysom_vul/stop.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -SERVICE_NAME=sysom-vul - -stop_app() { - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/4_sysom_vul/sysom-vul.ini b/script/server/4_sysom_vul/sysom-vul.ini deleted file mode 100644 index ae9a6b40..00000000 --- a/script/server/4_sysom_vul/sysom-vul.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-vul] -directory = /usr/local/sysom/server/target/sysom_server/sysom_vul -command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/vul_gunicorn.py sysom_vul.wsgi:application -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/sysom-vul-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-vul.log diff --git a/script/server/5_sysom_migration/clear.sh b/script/server/5_sysom_migration/clear.sh deleted file mode 100644 index a958999b..00000000 --- a/script/server/5_sysom_migration/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-migration -clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} -clear_app diff --git a/script/server/5_sysom_migration/init.sh b/script/server/5_sysom_migration/init.sh deleted file mode 100644 index c088c644..00000000 --- a/script/server/5_sysom_migration/init.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash -SERVER_DIR="sysom_server" -TARGET_PATH=${SERVER_HOME}/target -MIGRATION_DIR=${SERVER_DIR}/sysom_migration -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -SERVICE_NAME=sysom-migration -ANCE_X86_PKG=ance-0.1.1-1.x86_64.rpm -ANCE_ARM_PKG=ance-0.1.1-1.aarch64.rpm -ANOLIS_X86_SQLITE=Anolis_OS-8.6.x86_64.sqlite -ANOLIS_ARM_SQLITE=Anolis_OS-8.6.aarch64.sqlite -ANOLIS_MIGRATION_PKGS_X86=anolis_migration_pkgs_x86_64.tar.gz -ANOLIS_MIGRATION_PKGS_ARM=anolis_migration_pkgs_aarch64.tar.gz - -if [ "$UID" -ne 0 ]; then - echo "Please run as root" - exit 1 -fi - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - pushd ${TARGET_PATH}/${MIGRATION_DIR} - python manage.py migrate - popd - - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini - cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` - sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${MIGRATION_DIR}/conf/migration_gunicorn.py -} - -check_or_download_ance() { - mkdir -p ${TARGET_PATH}/${MIGRATION_DIR}/ance - pushd ${TARGET_PATH}/${MIGRATION_DIR}/ance - if [ ! -f "${ANCE_X86_PKG}" ]; then - wget "https://ance.oss-cn-hangzhou.aliyuncs.com/release/x86_64/${ANCE_X86_PKG}" - fi - if [ ! -f "${ANCE_ARM_PKG}" ]; then - wget "https://ance.oss-cn-hangzhou.aliyuncs.com/release/aarch64/${ANCE_ARM_PKG}" - fi - if [ ! -f "${ANOLIS_X86_SQLITE}" ]; then - wget "https://ance.oss-cn-hangzhou.aliyuncs.com/databases/${ANOLIS_X86_SQLITE}" - fi - if [ ! -f "${ANOLIS_ARM_SQLITE}" ]; then - wget "https://ance.oss-cn-hangzhou.aliyuncs.com/databases/${ANOLIS_ARM_SQLITE}" - fi - if [ ! -f "${ANOLIS_MIGRATION_PKGS_X86}" ]; then - wget "https://gitee.com/src-anolis-sig/leapp/releases/download/v1.0.2-all-in-one/${ANOLIS_MIGRATION_PKGS_X86}" - fi - if [ ! -f "${ANOLIS_MIGRATION_PKGS_ARM}" ]; then - wget "https://gitee.com/src-anolis-sig/leapp/releases/download/v1.0.2-all-in-one/${ANOLIS_MIGRATION_PKGS_ARM}" - fi - popd -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - check_or_download_ance - start_app -} - -deploy diff --git a/script/server/5_sysom_migration/start.sh b/script/server/5_sysom_migration/start.sh deleted file mode 100644 index d8b87908..00000000 --- a/script/server/5_sysom_migration/start.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash - - -SERVICE_NAME=sysom-migration - -start_app() { - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/5_sysom_migration/stop.sh b/script/server/5_sysom_migration/stop.sh deleted file mode 100644 index ff65275f..00000000 --- a/script/server/5_sysom_migration/stop.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -SERVICE_NAME=sysom-migration - -stop_app() { - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/5_sysom_migration/sysom-migration.ini b/script/server/5_sysom_migration/sysom-migration.ini deleted file mode 100644 index 036d45b3..00000000 --- a/script/server/5_sysom_migration/sysom-migration.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-migration] -directory = /usr/local/sysom/server/target/sysom_server/sysom_migration -command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/migration_gunicorn.py sysom_migration.wsgi:application -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/sysom-migration-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-migration.log diff --git a/script/server/6_sysom_hotfix/clear.sh b/script/server/6_sysom_hotfix/clear.sh deleted file mode 100644 index 5103464e..00000000 --- a/script/server/6_sysom_hotfix/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-hotfix -clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} -clear_app diff --git a/script/server/6_sysom_hotfix/init.sh b/script/server/6_sysom_hotfix/init.sh deleted file mode 100644 index b7ab3c6b..00000000 --- a/script/server/6_sysom_hotfix/init.sh +++ /dev/null @@ -1,61 +0,0 @@ -#! /bin/bash -SERVER_DIR="sysom_server" -HOTFIX_DIR=${SERVER_DIR}/sysom_hotfix -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -TARGET_PATH=${SERVER_HOME}/target -SERVICE_NAME=sysom-hotfix - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - pushd ${TARGET_PATH}/${HOTFIX_DIR} - python manage.py migrate - popd - - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini - cpu_num=`cat /proc/cpuinfo | grep processor | wc -l` - sed -i "s/threads = 3/threads = $cpu_num/g" ${TARGET_PATH}/${HOTFIX_DIR}/conf/hotfix_gunicorn.py -} - -start_nfs() -{ - systemctl start rpcbind && systemctl enable rpcbind - systemctl start nfs && systemctl enable nfs - if [ $? -ne 0 ];then - systemctl start nfs-server && systemctl enable nfs-server - fi - - nfs_mask=`ip -4 route | grep "link src" | grep $SERVER_LOCAL_IP | awk '{print $1}' | head -n 1` - file_path=${SERVER_HOME}/hotfix_builder/hotfix-nfs - mkdir -p ${file_path} - echo "${file_path} ${nfs_mask}(rw,async)" >> /etc/exports - exportfs -rv - chmod -R 777 ${file_path} -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - start_app - start_nfs -} - -deploy diff --git a/script/server/6_sysom_hotfix/start.sh b/script/server/6_sysom_hotfix/start.sh deleted file mode 100644 index f967321b..00000000 --- a/script/server/6_sysom_hotfix/start.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-hotfix - -start_app() { - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/6_sysom_hotfix/stop.sh b/script/server/6_sysom_hotfix/stop.sh deleted file mode 100644 index 88041451..00000000 --- a/script/server/6_sysom_hotfix/stop.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -SERVICE_NAME=sysom-hotfix - -stop_app() { - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/6_sysom_hotfix/sysom-hotfix.ini b/script/server/6_sysom_hotfix/sysom-hotfix.ini deleted file mode 100644 index 16689afd..00000000 --- a/script/server/6_sysom_hotfix/sysom-hotfix.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-hotfix] -directory = /usr/local/sysom/server/target/sysom_server/sysom_hotfix -command=/usr/local/sysom/server/virtualenv/bin/gunicorn -c ./conf/hotfix_gunicorn.py sysom_hotfix.wsgi:application -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/sysom-hotfix-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-hotfix.log diff --git a/script/server/6_sysom_hotfix_builder/clear.sh b/script/server/6_sysom_hotfix_builder/clear.sh deleted file mode 100644 index ea55d38b..00000000 --- a/script/server/6_sysom_hotfix_builder/clear.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-hotfix-builder - -clear_app() { - LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix - sed -i '/hotfix_builder/d' /etc/exports - - # kill all kpatch-build process - echo "find : `ps -eo comm,pid | grep "kpatch-build"`" - for each_line in `ps -eo comm,pid | grep "kpatch-build"`; do - echo $each_line - if [[ $each_line =~ "kpatch-build" ]]; then - echo "find kpatch-build" - else - kill -9 $each_line - fi - done - - umount $LOCAL_NFS_HOME - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} - -clear_app diff --git a/script/server/6_sysom_hotfix_builder/init.sh b/script/server/6_sysom_hotfix_builder/init.sh deleted file mode 100644 index 378d6ca8..00000000 --- a/script/server/6_sysom_hotfix_builder/init.sh +++ /dev/null @@ -1,56 +0,0 @@ -#! /bin/bash -SERVER_DIR="sysom_server" -HOTFIX_BUILDER_DIR=${SERVER_DIR}/sysom_hotfix_builder -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -SERVICE_NAME=sysom-hotfix-builder -NFS_SERVER_IP=${SERVER_LOCAL_IP} - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ -} - -install_package() { - - rpm -q --quiet make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || yum install -y make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || exit 1 - - rpm -q --quiet docker git || yum install -y docker git || echo "Warngin : Docker is not installed in this machine!" -} - -mount_nfs() -{ - HOTFIX_NFS_HOME=${SERVER_HOME}/hotfix_builder/hotfix-nfs - LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix - - mkdir -p ${LOCAL_NFS_HOME} - - sudo umount $LOCAL_NFS_HOME # Remove the mounted directory in case it mounted before - sudo mount -t nfs ${NFS_SERVER_IP}:${HOTFIX_NFS_HOME} ${LOCAL_NFS_HOME} || exit 1 -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - install_package - mount_nfs - start_app -} - -deploy diff --git a/script/server/6_sysom_hotfix_builder/start.sh b/script/server/6_sysom_hotfix_builder/start.sh deleted file mode 100644 index ead33613..00000000 --- a/script/server/6_sysom_hotfix_builder/start.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-hotfix-builder - -start_app() { - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/6_sysom_hotfix_builder/stop.sh b/script/server/6_sysom_hotfix_builder/stop.sh deleted file mode 100644 index 6817fbbc..00000000 --- a/script/server/6_sysom_hotfix_builder/stop.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -SERVICE_NAME=sysom-hotfix-builder - -stop_app() { - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/6_sysom_hotfix_builder/sysom-hotfix-builder.ini b/script/server/6_sysom_hotfix_builder/sysom-hotfix-builder.ini deleted file mode 100644 index b6aa5f8f..00000000 --- a/script/server/6_sysom_hotfix_builder/sysom-hotfix-builder.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-hotfix-builder] -directory = /usr/local/sysom/server/target/sysom_server/sysom_hotfix_builder -command=/usr/local/sysom/server/virtualenv/bin/python3 builder.py -startsecs=3 -autostart=true -autorestart=true -environment=PATH=/usr/local/sysom/server/virtualenv/bin:%(ENV_PATH)s -stderr_logfile=/usr/local/sysom/server/logs/sysom-hotfix-builder-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-hotfix-builder.log diff --git a/script/server/7_sysom_demo/clear.sh b/script/server/7_sysom_demo/clear.sh deleted file mode 100755 index e3b06f08..00000000 --- a/script/server/7_sysom_demo/clear.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-demo -clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini - ###use supervisorctl update to stop and clear services### - supervisorctl update -} -clear_app diff --git a/script/server/7_sysom_demo/init.sh b/script/server/7_sysom_demo/init.sh deleted file mode 100755 index eb4706d0..00000000 --- a/script/server/7_sysom_demo/init.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -SERVER_DIR="sysom_server" -DEMO_DIR=${SERVER_DIR}/sysom_demo -VIRTUALENV_HOME=${SERVER_HOME}/virtualenv -TARGET_PATH=${SERVER_HOME}/target -SERVICE_NAME=sysom-demo - -if [ "$UID" -ne 0 ]; then - echo "Please run as root" - exit 1 -fi - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -init_conf() { - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ - ###change the install dir base on param $1### - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini -} - -start_app() { - ###if supervisor service started, we need use "supervisorctl update" to start new conf#### - supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 -} - -deploy() { - source_virtualenv - init_conf - start_app -} - -deploy diff --git a/script/server/7_sysom_demo/start.sh b/script/server/7_sysom_demo/start.sh deleted file mode 100755 index f42c6ddc..00000000 --- a/script/server/7_sysom_demo/start.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-demo -start_app() { - supervisorctl start $SERVICE_NAME -} - -start_app diff --git a/script/server/7_sysom_demo/stop.sh b/script/server/7_sysom_demo/stop.sh deleted file mode 100755 index 61269da4..00000000 --- a/script/server/7_sysom_demo/stop.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash -SERVICE_NAME=sysom-demo -stop_app() { - supervisorctl stop $SERVICE_NAME -} - -stop_app diff --git a/script/server/7_sysom_demo/sysom-demo.ini b/script/server/7_sysom_demo/sysom-demo.ini deleted file mode 100644 index 80af4cee..00000000 --- a/script/server/7_sysom_demo/sysom-demo.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-demo] -directory = /usr/local/sysom/server/target/sysom_server/sysom_demo -command=/usr/local/sysom/server/virtualenv/bin/uvicorn main:app --port 7008 -startsecs=3 -autostart=true -autorestart=true -environment=PATH="/usr/local/sysom/server/virtualenv/bin/" -stderr_logfile=/usr/local/sysom/server/logs/sysom-demo-error.log -stdout_logfile=/usr/local/sysom/server/logs/sysom-demo.log diff --git a/script/server/clear.sh b/script/server/clear.sh index a911fa3a..c9f1bfba 100644 --- a/script/server/clear.sh +++ b/script/server/clear.sh @@ -1,21 +1,16 @@ #!/bin/bash -x -basedir=`dirname $0` -config=conf -env=${basedir}/../../env +BaseDir=$(dirname $(readlink -f "$0")) +pushd `dirname $BaseDir` +bash +x sysom.sh clear ms ALL +bash +x sysom.sh clear infra ALL +popd -for i in `cat $env` -do - export $i -done +# Whether delete SERVER_HOME +if [ "`ls -A ${SERVER_HOME}`" = "" ];then + rm -r ${SERVER_HOME} +fi -cd $basedir -for dir in `tac $config` -do - if [ -d $dir ] - then - pushd $dir - echo $dir - bash -x clear.sh - popd - fi -done +# Whether delete APP_HOME +if [ "`ls -A ${APP_HOME}`" = "" ]; then + rm -r ${APP_HOME} +fi \ No newline at end of file diff --git a/script/server/conf b/script/server/conf deleted file mode 100644 index 9408badd..00000000 --- a/script/server/conf +++ /dev/null @@ -1,32 +0,0 @@ -[base] -0_env -0_local_services -0_sysom_api -0_sysom_channel - -[monitor] -1_sysom_monitor - -[monitor-server] -1_sysom_monitor_server - -[vmcore] -2_sysom_vmcore - -[diagnosis] -3_sysom_diagnosis - -[vul] -4_sysom_vul - -[migration] -5_sysom_migration - -[hotfix] -6_sysom_hotfix - -[hotfix-builder] -6_sysom_hotfix_builder - -#[demo] -#7_sysom_demo diff --git a/script/server/init.sh b/script/server/init.sh index e332f779..e5848cf7 100644 --- a/script/server/init.sh +++ b/script/server/init.sh @@ -1,148 +1,6 @@ #!/bin/bash -x - -FIRST_INIT_DONE=0 - -if [ "$APP_NAME" == "" ] -then - export APP_NAME="sysom" -fi - -if [ "$APP_HOME" == "" ] -then - export APP_HOME=/usr/local/sysom - export SERVER_HOME=/usr/local/sysom/server - export NODE_HOME=/usr/local/sysom/node -fi -if [ "$SERVER_LOCAL_IP" == "" ] -then - local_ip=`ip -4 route | grep "link src" | awk -F"link src " '{print $2}' | awk '{print $1}' | head -n 1` - export SERVER_LOCAL_IP=$local_ip -fi - -if [ "$SERVER_PUBLIC_IP" == "" ] -then - export SERVER_PUBLIC_IP=$SERVER_LOCAL_IP -fi - -if [ "$SERVER_PORT" == "" ] -then - export SERVER_PORT=80 -fi - -config=conf -basedir=`dirname $0` -SYSOM_CONF=${SERVER_HOME}/target/conf/config.yml -SYSOM_DATABASE_HOST=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a host | awk '{print $2}'` -SYSOM_DATABASE_PORT=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a port | awk '{print $2}'` -SYSOM_DATABASE_USER=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a user | awk '{print $2}'` -SYSOM_DATABASE_PASSWORD=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a password | awk '{print $2}'` -UPLOAD_DIR=${SERVER_HOME}/target/sysom_web/download/ -NODE_INIT_DIR=sysom_node_init -NODE_INIT_PKG=sysom_node_init.tar.gz -NODE_DIR=${basedir}/../node -SYSOM_DEPLOY_LOG=/var/log/sysom/sysom.log - -mkdir -p /var/log/sysom/ - -red() { - printf '\33[1;31m%b\n\33[0m' "$1" -} - -green() { - printf '\33[1;32m%b\n\33[0m' "$1" -} - -###initial download sysom_node_init.tar.gz### - -###enable the service web menu### -setup_web_menu_enable() -{ - service=`echo $1 | grep sysom | awk -F"sysom_" '{print $NF}'` - if [ "$service" != "" ] - then - mysql -h ${SYSOM_DATABASE_HOST} -P ${SYSOM_DATABASE_PORT} -u ${SYSOM_DATABASE_USER} -p${SYSOM_DATABASE_PASSWORD} \ - -e "use sysom;insert into sys_service_info(service_name, created_at) values ('sysom_${service}', current_timestamp);" - fi -} - -generate_service_env() { - rm -f ${APP_HOME}/env - cat << EOF > ${APP_HOME}/env -APP_HOME=${APP_HOME} -SERVER_HOME=${APP_HOME}/server -NODE_HOME=${APP_HOME}/node -SERVER_LOCAL_IP=${SERVER_LOCAL_IP} -SERVER_PUBLIC_IP=${SERVER_PUBLIC_IP} -SERVER_PORT=${SERVER_PORT} -EOF -} - -update_global_config() { - sed "s/SERVER_LOCAL_IP: 127.0.0.1/SERVER_LOCAL_IP: $SERVER_LOCAL_IP/g" -i ${SYSOM_CONF} - sed "s/SERVER_PUBLIC_IP: 127.0.0.1/SERVER_PUBLIC_IP: $SERVER_PUBLIC_IP/g" -i ${SYSOM_CONF} - sed "s/SERVER_PORT: 80/SERVER_PORT: $SERVER_PORT/g" -i ${SYSOM_CONF} - sed "s/global_root_path \/usr\/local\/sysom/global_root_path $APP_HOME/g" -i ${SYSOM_CONF} - - ###update local timezone### - local_timezone=`timedatectl status | grep "Time zone" | awk '{print $3}'` - green "current timezone is : $local_timezone" - if [ "$local_timezone" == "" ] - then - echo "get time zone fail, use default time zone Asia/Shanghai" - else - sed "s;Asia/Shanghai;$local_timezone;g" -i ${SYSOM_CONF} - fi -} - -do_init_servers() -{ - dd=`date` - green "Now deploy new sysom, deploy time : $dd" - mkdir -p ${SERVER_HOME}/logs - pushd $basedir - if [ $FIRST_INIT_DONE == 0 ] - then - generate_service_env - update_global_config - for dir in `cat $config` - do - if [ -e ${dir}/init.sh ] - then - pushd $dir - green "$dir initialing..................................." - bash -x init.sh - if [ $? -ne 0 ];then - red "$dir initial fail, please check..................................." - red "sysom init failed, exit 1" - exit 1 - fi - green "setup_web_menu_enable $dir" - setup_web_menu_enable $dir - popd - fi - done - green "sysom init done...................................." - sed -i 's/^FIRST_INIT_DONE=0/FIRST_INIT_DONE=1/g' $0 - else - for dir in `cat $config` - do - if [ -e ${dir}/start.sh ] - then - pushd $dir - green "$dir starting..................................." - bash -x start.sh - if [ $? -ne 0 ];then - red "$dir start fail, please check..................................." - red "sysom start failed, exit 1" - exit 1 - fi - popd - fi - done - green "sysom start done...................................." - fi - popd -} - -###only save the last deploy log### -do_init_servers | tee ${SYSOM_DEPLOY_LOG} +BaseDir=$(dirname $(readlink -f "$0")) +pushd `dirname $BaseDir` +bash +x sysom.sh deploy infra env,sdk,local_services,monitor +bash +x sysom.sh deploy ms ALL +popd \ No newline at end of file diff --git a/script/server/stop.sh b/script/server/stop.sh deleted file mode 100644 index c6bd30d2..00000000 --- a/script/server/stop.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -x -basedir=`dirname $0` -config=conf -env=${basedir}/../../env - -for i in `cat $env` -do - export $i -done - -cd $basedir -for dir in `tac $config` -do - if [ -d $dir ] - then - pushd $dir - echo $dir - bash -x stop.sh - popd - fi -done diff --git a/script/sysom.sh b/script/sysom.sh old mode 100755 new mode 100644 index ed7b5c45..c45c0f9f --- a/script/sysom.sh +++ b/script/sysom.sh @@ -16,6 +16,8 @@ export SERVER_HOME=${APP_HOME}/server export NODE_HOME=${APP_HOME}/node export MICROSERVICE_HOME=${SERVER_HOME}/microservice export INFRASTRUCTURE_HOME=${SERVER_HOME}/infrastructure +export ENVIRONMENT_HOME=${SERVER_HOME}/environment +export GLOBAL_VIRTUALENV_HOME=${ENVIRONMENT_HOME}/virtualenv if [ "$SERVER_LOCAL_IP" == "" ] then @@ -39,6 +41,13 @@ then fi mkdir -p $LOG_HOME +if [ "$CONF_HOME" == "" ] +then + export CONF_HOME=/etc/sysom + export SYSOM_CONF=${CONF_HOME}/config.yml +fi +mkdir -p $CONF_HOME + if [ "$DB_MYSQL_HOST" == "" ] then export DB_MYSQL_HOST=localhost @@ -90,7 +99,7 @@ fi #################################################################################################################### ProgName=$(basename $0) -BaseDir=`dirname $0` +BaseDir=$(dirname $(readlink -f "$0")) sub_help(){ echo "Usage: $ProgName [options]" diff --git a/sysom_server/.gitignore b/sysom_server/.gitignore deleted file mode 100644 index 98c4b8d7..00000000 --- a/sysom_server/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/conf/ -__pycache__ -logs \ No newline at end of file diff --git a/sysom_server/sysom_demo/main.py b/sysom_server/sysom_demo/main.py deleted file mode 100644 index d983ab36..00000000 --- a/sysom_server/sysom_demo/main.py +++ /dev/null @@ -1,29 +0,0 @@ -from fastapi import FastAPI -from channel_job import default_channel_job_executor - -SYSOM_CEC_URL = "redis://localhost:6379?cec_default_max_len=1000&cec_auto_mk_topic=true" -CHANNEL_JOB_URL = f"{SYSOM_CEC_URL}&channel_job_target_topic=SYSOM_CEC_CHANNEL_TOPIC" \ - f"&channel_job_listen_topic=SYSOM_CEC_DEMO_TOPIC" \ - f"&channel_job_consumer_group=SYSOM_CEC_DEMO_CONSUMER_GROUP" - -default_channel_job_executor.init_config(CHANNEL_JOB_URL) -default_channel_job_executor.start() - -app = FastAPI() - -@app.get("/api/v1/demo/get_kernel_info/{instance}") -async def get_kernel_info(instance: str): - job = default_channel_job_executor.dispatch_job( - channel_type="ssh", - channel_opt="cmd", - params={ - "instance": instance, - "command": "uname -a" - } - ) - channel_result = await job.execute_async() - return { - "code": channel_result.code, - "err_msg": channel_result.err_msg, - "result": channel_result.result - } \ No newline at end of file -- Gitee From 65ef1d9f8939ed813edc61a7c442dec5edfecbe2 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 28 Apr 2023 14:40:27 +0800 Subject: [PATCH 032/196] fix(diagnosis): Add missing cache config --- microservice/.gitignore | 3 ++- microservice/sysom_diagnosis/conf/common.py | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/microservice/.gitignore b/microservice/.gitignore index 98c4b8d7..78e4319f 100644 --- a/microservice/.gitignore +++ b/microservice/.gitignore @@ -1,3 +1,4 @@ /conf/ __pycache__ -logs \ No newline at end of file +logs +.coverage \ No newline at end of file diff --git a/microservice/sysom_diagnosis/conf/common.py b/microservice/sysom_diagnosis/conf/common.py index 27d922b0..0430e581 100644 --- a/microservice/sysom_diagnosis/conf/common.py +++ b/microservice/sysom_diagnosis/conf/common.py @@ -76,6 +76,21 @@ DATABASES = { } } +CACHES = { + "default": { + "BACKEND": "django_redis.cache.RedisCache", + "LOCATION": ( + f"redis://{YAML_CONFIG.get_server_config().db.redis.host}" + f":{YAML_CONFIG.get_server_config().db.redis.port}/1" + ), + "OPTIONS": { + "CLIENT_CLASS": "django_redis.client.DefaultClient", + "CONNECTION_POOL_KWARGS": {"max_connections": 100}, + "DECODE_RESPONSES": True + } + } +} + ROOT_URLCONF = 'sysom_diagnosis.urls' WSGI_APPLICATION = 'sysom_diagnosis.wsgi.application' -- Gitee From 1c6d76d390790ba067fb741a557d213244e78600 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 5 May 2023 15:56:13 +0800 Subject: [PATCH 033/196] feat(sdk): Add redis_lua to extend redis --- infrastructure/1_sdk/redis_lua/__init__.py | 10 ++ infrastructure/1_sdk/redis_lua/x_hdel.lua | 15 +++ .../1_sdk/redis_lua/x_hdrop_table.lua | 9 ++ infrastructure/1_sdk/redis_lua/x_hexpire.lua | 22 +++++ infrastructure/1_sdk/redis_lua/x_hget.lua | 36 +++++++ infrastructure/1_sdk/redis_lua/x_hgetall.lua | 34 +++++++ infrastructure/1_sdk/redis_lua/x_hkeys.lua | 4 + infrastructure/1_sdk/redis_lua/x_hlen.lua | 4 + infrastructure/1_sdk/redis_lua/x_hset.lua | 26 +++++ .../1_sdk/redis_lua/xreadis_hash_table.py | 97 +++++++++++++++++++ infrastructure/1_sdk/setup_redis_lua.py | 20 ++++ 11 files changed, 277 insertions(+) create mode 100644 infrastructure/1_sdk/redis_lua/__init__.py create mode 100644 infrastructure/1_sdk/redis_lua/x_hdel.lua create mode 100644 infrastructure/1_sdk/redis_lua/x_hdrop_table.lua create mode 100644 infrastructure/1_sdk/redis_lua/x_hexpire.lua create mode 100644 infrastructure/1_sdk/redis_lua/x_hget.lua create mode 100644 infrastructure/1_sdk/redis_lua/x_hgetall.lua create mode 100644 infrastructure/1_sdk/redis_lua/x_hkeys.lua create mode 100644 infrastructure/1_sdk/redis_lua/x_hlen.lua create mode 100644 infrastructure/1_sdk/redis_lua/x_hset.lua create mode 100644 infrastructure/1_sdk/redis_lua/xreadis_hash_table.py create mode 100644 infrastructure/1_sdk/setup_redis_lua.py diff --git a/infrastructure/1_sdk/redis_lua/__init__.py b/infrastructure/1_sdk/redis_lua/__init__.py new file mode 100644 index 00000000..bf810da6 --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/27 16:46 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py.py +Description: +""" + +from .xreadis_hash_table import * \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hdel.lua b/infrastructure/1_sdk/redis_lua/x_hdel.lua new file mode 100644 index 00000000..b4eac4d3 --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/x_hdel.lua @@ -0,0 +1,15 @@ +-- 获取 KEY, FIELD +local hash_name = KEYS[1] + +-- 用来记录时间的hash表名字 +local hash_name_time = hash_name .. ":" .. "xhash_expire_time" + +for i = 1, #ARGV do + local field = ARGV[i] + -- 删除键值 + redis.call("HDEL", hash_name, field) + -- 删除过期时间 + redis.call("HDEL", hash_name_time, field) +end +return "OK" + diff --git a/infrastructure/1_sdk/redis_lua/x_hdrop_table.lua b/infrastructure/1_sdk/redis_lua/x_hdrop_table.lua new file mode 100644 index 00000000..8dafcebb --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/x_hdrop_table.lua @@ -0,0 +1,9 @@ +-- 获取 KEY +local hash_name = KEYS[1] +-- 用来记录时间的hash表名字 +local hash_name_time = hash_name .. ":" .. "xhash_expire_time" +-- 保存键值 +redis.call("DEL", hash_name) +redis.call("DEL", hash_name_time) +return "OK" + diff --git a/infrastructure/1_sdk/redis_lua/x_hexpire.lua b/infrastructure/1_sdk/redis_lua/x_hexpire.lua new file mode 100644 index 00000000..538aed9b --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/x_hexpire.lua @@ -0,0 +1,22 @@ +-- 获取 KEY, FIELD, EXPIRE +local hash_name = KEYS[1] +local field = ARGV[1] +local expire = ARGV[2] + +-- 用来记录时间的hash表名字 +local hash_name_time = hash_name .. ":" .. "xhash_expire_time" + +-- 获取当前时间（以秒为单位） +local time = redis.call('TIME') +local expire_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 + expire + +if expire ~= -1 then + -- 保存过期时间 + redis.call("HSET", hash_name_time, field, expire_time) +else + -- 删除过期限制 + redis.call("HDEL", hash_name_time, field) +end +return "OK" + + diff --git a/infrastructure/1_sdk/redis_lua/x_hget.lua b/infrastructure/1_sdk/redis_lua/x_hget.lua new file mode 100644 index 00000000..9e375e3e --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/x_hget.lua @@ -0,0 +1,36 @@ +-- 获取 KEY, FIELD +local hash_name = KEYS[1] +local field = ARGV[1] + +-- 用来记录时间的hash表名字 +local hash_name_time = hash_name .. ":" .. "xhash_expire_time" + +-- 获取当前时间（以秒为单位） +local time = redis.call('TIME') +local current_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 + +local expire_time_str = redis.call("HGET", hash_name_time, field) +local expire_time = -1 +if expire_time_str ~= false then + expire_time = tonumber(expire_time_str) +end + +-- 1. 过期则删除并返回nil +-- 2. 不存在则返回nil +-- 3. 否则返回值 +if (expire_time >= 0 and expire_time < current_time) then + -- 删除过期的成员 + redis.call("HDEL", hash_name, field) + redis.call("HDEL", hash_name_time, field) + return nil +else + local value = redis.call("HGET", hash_name, field) + if value == false then + return nil + else + return value + end +end + + + diff --git a/infrastructure/1_sdk/redis_lua/x_hgetall.lua b/infrastructure/1_sdk/redis_lua/x_hgetall.lua new file mode 100644 index 00000000..2f08effe --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/x_hgetall.lua @@ -0,0 +1,34 @@ +-- 获取 KEY +local hash_name = KEYS[1] + +-- 用来记录时间的hash表名字 +local hash_name_time = hash_name .. ":" .. "xhash_expire_time" + +-- 获取哈希的所有子键和值 +local hash_fields = redis.call('HGETALL', hash_name) + +-- 获取当前时间（以秒为单位） +local time = redis.call('TIME') +local current_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 + +local res = {} +-- 遍历所有子键和值 +for i = 1, #hash_fields, 2 do + -- 获取子键和值 + local field = hash_fields[i] + local value = hash_fields[i + 1] + local expire_time_str = redis.call("HGET", hash_name_time, field) + local expire_time = -1 + if expire_time_str ~= false then + expire_time = tonumber(expire_time_str) + end + if (expire_time >= 0 and expire_time < current_time) then + -- 删除过期的成员 + redis.call("HDEL", hash_name, field) + redis.call("HDEL", hash_name_time, field) + else + res[i] = field + res[i + 1] = value + end +end +return res \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hkeys.lua b/infrastructure/1_sdk/redis_lua/x_hkeys.lua new file mode 100644 index 00000000..826d1e3d --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/x_hkeys.lua @@ -0,0 +1,4 @@ +-- 获取 KEY, FIELD, VALUE, EXPIRE +local hash_name = KEYS[1] + +return redis.call("HKEYS", hash_name) \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hlen.lua b/infrastructure/1_sdk/redis_lua/x_hlen.lua new file mode 100644 index 00000000..c6377144 --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/x_hlen.lua @@ -0,0 +1,4 @@ +-- 获取 KEY, FIELD, VALUE, EXPIRE +local hash_name = KEYS[1] + +return redis.call("HLEN", hash_name) \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hset.lua b/infrastructure/1_sdk/redis_lua/x_hset.lua new file mode 100644 index 00000000..03b9e3a5 --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/x_hset.lua @@ -0,0 +1,26 @@ +-- 获取 KEY, FIELD, VALUE, EXPIRE +local hash_name = KEYS[1] +local expire = tonumber(ARGV[1]) +if #ARGV < 3 then + return "ERR: Missing filed,value" +end + +-- 获取当前时间（以秒为单位） +local time = redis.call('TIME') +local expire_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 + expire + +-- 用来记录时间的hash表名字 +local hash_name_time = hash_name .. ":" .. "xhash_expire_time" + +for i = 2, #ARGV, 2 do + local field = ARGV[i] + local value = ARGV[i + 1] + -- 保存键值 + redis.call("HSET", hash_name, field, value) + if expire ~= -1 then + -- 保存过期时间 + redis.call("HSET", hash_name_time, field, expire_time) + end +end +return "OK" + diff --git a/infrastructure/1_sdk/redis_lua/xreadis_hash_table.py b/infrastructure/1_sdk/redis_lua/xreadis_hash_table.py new file mode 100644 index 00000000..a932c9b7 --- /dev/null +++ b/infrastructure/1_sdk/redis_lua/xreadis_hash_table.py @@ -0,0 +1,97 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/27 16:47 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File xreadis_hash_table.py +Description: +""" +import os +from redis import Redis +from redis.client import Pipeline +from pathlib import Path +from typing import Optional, Union, Awaitable, List + + +class XRedisHashTable: + """ + An extended Redis Hash Table implement by lua script, features: + 1. Support set expire time for each filed + """ + + def __init__(self, redis_cli: Redis, lua_script_base_dir: str = None): + if lua_script_base_dir is None: + lua_script_base_dir = Path(__file__).resolve().parent + self._redis = redis_cli + self._lua_script_base_dir = lua_script_base_dir + self._script_hashes = { + + } + + def _load_lua_scripts_from_file(self, name: str): + lua_script_file = f"{os.path.join(self._lua_script_base_dir, name + '.lua')}" + with open(lua_script_file) as file: + script_value = file.read() + script_hash = self._redis.script_load(script_value) + return script_hash + + def _get_or_cache_script_sha(self, name: str): + target_hash = self._script_hashes.get(name, "") + if not target_hash or self._redis.script_exists(target_hash)[0]: + # Need load script + target_hash = self._load_lua_scripts_from_file(name) + self._script_hashes[name] = target_hash + return target_hash + + def _evalsha(self, name: str, *fields) -> Union[Awaitable[str], str]: + return self._redis.evalsha( + self._get_or_cache_script_sha(name), *fields + ) + + def _eval_by_pl(self, pl: Pipeline, name: str, *fields) -> Pipeline: + return pl.evalsha( + self._get_or_cache_script_sha(name), *fields + ) + + def hget(self, table_name: str, field: str) -> \ + Optional[str]: + res = self._evalsha("x_hget", 1, table_name, field) + return None if res is None else res + + def hget_pl(self, pl: Pipeline, table_name: str, field: str) -> Pipeline: + return self._eval_by_pl(pl, "x_hget", 1, table_name, field) + + def hgetall(self, table_name: str) -> dict: + kvs = self._evalsha("x_hgetall", 1, table_name) + res = {} + for i in range(0, len(kvs), 2): + res[kvs[i]] = kvs[i + 1] + return res + + def hset(self, table_name: str, *kvs: str, + expire: int = -1) -> bool: + try: + res = self._evalsha("x_hset", 1, table_name, expire, *kvs) + return res == "OK" + except Exception as _: + return False + + def hdrop_table(self, table_name: str) -> bool: + res = self._evalsha("x_hdrop_table", 1, table_name) + return res == "OK" + + def hdel(self, table_name: str, *fields: str) -> bool: + res = self._evalsha("x_hdel", 1, table_name, *fields) + return res == "OK" + + def hexpire(self, table_name: str, field: str, expire: int = -1) -> bool: + res = self._evalsha("x_hexpire", 1, table_name, field, expire) + return res == "OK" + + def hlen(self, table_name: str) -> int: + res = self._evalsha("x_hlen", 1, table_name) + return res + + def hkeys(self, table_name: str) -> List[str]: + res = self._evalsha("x_hkeys", 1, table_name) + return res diff --git a/infrastructure/1_sdk/setup_redis_lua.py b/infrastructure/1_sdk/setup_redis_lua.py new file mode 100644 index 00000000..4648da75 --- /dev/null +++ b/infrastructure/1_sdk/setup_redis_lua.py @@ -0,0 +1,20 @@ +import setuptools + +setuptools.setup( + name="redis_lua", + version="0.0.1", + author="mingfeng(SunnyQjm)", + author_email="mfeng@linux.alibaba.com", + description="redis_lua is a library of tools that encapsulates some common lua scripts", + url="", + include_package_data=True, + packages=setuptools.find_packages(), + install_requires=[ + "redis" + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ] +) -- Gitee From 34cb91746f9279833eea5bb22461cae3ae6f35e7 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 5 May 2023 15:57:02 +0800 Subject: [PATCH 034/196] refactor(sdk.cmg): Use x_redis_hash_table to implement registry center --- .../1_sdk/cmg_base/service_discovery.py | 5 ++ .../1_sdk/cmg_base/service_instance.py | 11 +++ .../1_sdk/cmg_base/service_registry.py | 2 +- infrastructure/1_sdk/cmg_redis/__init__.py | 4 + .../cmg_redis/redis_service_discovery.py | 24 +++--- .../1_sdk/cmg_redis/redis_service_registry.py | 76 +++++++++---------- infrastructure/1_sdk/init.sh | 5 +- infrastructure/1_sdk/redis_lua/x_hkeys.lua | 19 +++++ infrastructure/1_sdk/redis_lua/x_hlen.lua | 19 +++++ 9 files changed, 114 insertions(+), 51 deletions(-) diff --git a/infrastructure/1_sdk/cmg_base/service_discovery.py b/infrastructure/1_sdk/cmg_base/service_discovery.py index de0bd437..3b375091 100644 --- a/infrastructure/1_sdk/cmg_base/service_discovery.py +++ b/infrastructure/1_sdk/cmg_base/service_discovery.py @@ -33,6 +33,11 @@ class ServiceDiscovery(metaclass=ABCMeta): """Get services from register center""" pass + @abstractmethod + def get_instance_count(self, service_name: str) -> int: + """Get instances count of specific service""" + pass + @abstractmethod def get_instances(self, service_id: str, force: bool = False) \ -> List[ServiceInstance]: diff --git a/infrastructure/1_sdk/cmg_base/service_instance.py b/infrastructure/1_sdk/cmg_base/service_instance.py index 457ef998..8e74459f 100644 --- a/infrastructure/1_sdk/cmg_base/service_instance.py +++ b/infrastructure/1_sdk/cmg_base/service_instance.py @@ -72,3 +72,14 @@ class ServiceInstance: "metadata": json.dumps(self.metadata), "check": json.dumps(self.check) } + + def to_dict(self): + return { + "service_id": self.service_id, + "service_name": self.service_name, + "host": self.host, + "port": self.port, + "tags": self.tags, + "metadata": self.metadata, + "check": self.check + } diff --git a/infrastructure/1_sdk/cmg_base/service_registry.py b/infrastructure/1_sdk/cmg_base/service_registry.py index e41a9ac2..a1ed9065 100644 --- a/infrastructure/1_sdk/cmg_base/service_registry.py +++ b/infrastructure/1_sdk/cmg_base/service_registry.py @@ -29,7 +29,7 @@ class ServiceRegistry(metaclass=ABCMeta): pass @abstractmethod - def unregister(self, service_id: str): + def unregister(self, service_name: str, service_id: str): """Unregister specific service from register center""" pass diff --git a/infrastructure/1_sdk/cmg_redis/__init__.py b/infrastructure/1_sdk/cmg_redis/__init__.py index 400ddf83..40dc9ea9 100644 --- a/infrastructure/1_sdk/cmg_redis/__init__.py +++ b/infrastructure/1_sdk/cmg_redis/__init__.py @@ -6,4 +6,8 @@ Email mfeng@linux.alibaba.com File __init__.py.py Description: """ + +from .redis_service_discovery import * +from .redis_service_registry import * + name = "cmg_redis" diff --git a/infrastructure/1_sdk/cmg_redis/redis_service_discovery.py b/infrastructure/1_sdk/cmg_redis/redis_service_discovery.py index eaa5b089..dd903d70 100644 --- a/infrastructure/1_sdk/cmg_redis/redis_service_discovery.py +++ b/infrastructure/1_sdk/cmg_redis/redis_service_discovery.py @@ -6,11 +6,13 @@ Email mfeng@linux.alibaba.com File redis_service_discovery.py Description: """ +import json import time from typing import List, Dict from cmg_base import ServiceDiscovery, ServiceInstance, CmgUrl, \ LoadBalancingStrategy, create_load_balancing_strategy, \ LoadBalancingStrategyBase +from redis_lua import XRedisHashTable from .common import ClientBase, StaticConst @@ -22,6 +24,8 @@ class RedisServiceDiscovery(ServiceDiscovery, ClientBase): ServiceDiscovery.__init__(self, fetch_interval) ClientBase.__init__(self, url) + self._x_redis_hash_table = XRedisHashTable(self.redis_client) + # Handles Redis implementation of event-centric specialization self._auto_fetch_thread = None @@ -36,18 +40,20 @@ class RedisServiceDiscovery(ServiceDiscovery, ClientBase): } def get_services(self) -> List[str]: - return [StaticConst.get_origin_service_name(inner_service_name) for - inner_service_name in - self.redis_client.smembers(StaticConst.CMG_REDIS_SERVICES)] + return [StaticConst.get_origin_service_name(item) for item in + self._x_redis_hash_table.hkeys(StaticConst.CMG_REDIS_SERVICES)] + + def get_instance_count(self, service_name: str) -> int: + return self._x_redis_hash_table.hlen( + StaticConst.get_inner_service_name(service_name)) def _get_instances(self, service_name: str) -> List[ServiceInstance]: inner_service_name = StaticConst.get_inner_service_name(service_name) - service_ids = self.redis_client.smembers(inner_service_name) - pl = self.redis_client.pipeline() - for service_id in service_ids: - pl.hgetall(service_id) - return [ServiceInstance.from_redis_mapping(item) for item in - pl.execute()] + res = self._x_redis_hash_table.hgetall(inner_service_name) + return [ + ServiceInstance.from_redis_mapping(json.loads(res[k])) for k in + res + ] def get_instances(self, service_name: str, force: bool = False) \ -> List[ServiceInstance]: diff --git a/infrastructure/1_sdk/cmg_redis/redis_service_registry.py b/infrastructure/1_sdk/cmg_redis/redis_service_registry.py index 24842200..d2c96e06 100644 --- a/infrastructure/1_sdk/cmg_redis/redis_service_registry.py +++ b/infrastructure/1_sdk/cmg_redis/redis_service_registry.py @@ -7,9 +7,12 @@ File redis_service_registry.py Description: """ import functools +import json + from clogger import logger from cmg_base import ServiceRegistry, ServiceInstance, CmgUrl, CmgException, \ ServiceCheck, HealthState +from redis_lua import XRedisHashTable from .common import ClientBase, StaticConst from .utils import RedisLocker @@ -31,19 +34,22 @@ class RedisServiceRegistry(ServiceRegistry, ClientBase): # 1. Connect to the Redis server self._current_state = HealthState.OFFLINE + self._x_redis_hash_table = XRedisHashTable(self.redis_client) + def register(self, service_instance: ServiceInstance): """Register one service to redis-based register center""" - def _on_health_check(service_id: str, state: HealthState): + def _on_health_check(service_id: str, expire_time: int, state: HealthState): if state == HealthState.ONLINE: - pl.expire( - service_id, - service_instance.check.get("deregister", 20) * 1.2 - ) + self._x_redis_hash_table.hexpire(inner_service_name, + service_id, expire=expire_time) + self._x_redis_hash_table.hexpire( + StaticConst.CMG_REDIS_SERVICES, + inner_service_name, expire=expire_time) - def _do_unregister(service_id: str): + def _do_unregister(inner_service_name: str, inner_service_id: str): try: - self.unregister(service_id) + self.unregister(inner_service_name, inner_service_id) except Exception as exc: print(exc) @@ -57,60 +63,50 @@ class RedisServiceRegistry(ServiceRegistry, ClientBase): StaticConst.CMG_REDIS_SERVICES_LOCKER) as res: if not res: raise CmgException("Get locker failed") - # 1. Record service info to redis - pl = self.redis_client.pipeline() - pl.hset( - inner_service_id, mapping=service_instance.to_redis_mapping() + expire_time = int( + service_instance.check.get("deregister", 20) * 1.2) + self._x_redis_hash_table.hset( + inner_service_name, inner_service_id, + json.dumps(service_instance.to_redis_mapping()), + expire=expire_time ) - pl.sadd(inner_service_name, inner_service_id) - pl.sadd(StaticConst.CMG_REDIS_SERVICES, inner_service_name) - pl.expire( - inner_service_id, - int(service_instance.check.get("deregister", 20) * 1.2) + self._x_redis_hash_table.hset( + StaticConst.CMG_REDIS_SERVICES, inner_service_name, "", + expire=expire_time ) - try: - pl.execute() - except Exception as exc: - logger.exception(exc) # 2. Deal health check if self._self_health_check: ServiceCheck.create_health_check_instance( service_instance.check, on_check=functools.partial(_on_health_check, - inner_service_id), + inner_service_id, expire_time), do_unregister=functools.partial( - _do_unregister, service_instance.service_id + _do_unregister, inner_service_name, inner_service_id ) ).start() - def unregister(self, service_id: str): + def unregister(self, service_name: str, service_id: str): """Unregister one service from redis-based register center""" # 1. Get service instance inner_service_id = StaticConst.get_inner_service_id( service_id ) - values = self.redis_client.hgetall(inner_service_id) - if not values: - raise CmgException(f"Service(service_id={service_id}) not exists") - # 2. Get service_name from service instance - service_instance = ServiceInstance.from_redis_mapping(values) inner_service_name = StaticConst.get_inner_service_name( - service_instance.service_name + service_name ) + service_instance_str = self._x_redis_hash_table.hget( + inner_service_name, inner_service_id) + if service_instance_str is None: + raise CmgException( + f"Service(service_id={service_id}) not exists") with RedisLocker(self.redis_client, StaticConst.CMG_REDIS_SERVICES_LOCKER) as res: if not res: raise CmgException("Get locker failed") - # 3. Delete service instance - pl = self.redis_client.pipeline() - pl.delete(inner_service_id) - pl.srem(inner_service_name, inner_service_id) - pl.scard(inner_service_name) - results = pl.execute() - # 4. Delete the service if the service does not contain any - # instances - if results[2] <= 0: - self.redis_client.srem(StaticConst.CMG_REDIS_SERVICES, - inner_service_name) + # 3. Delete service instance + self._x_redis_hash_table.hdel(inner_service_name, inner_service_id) + if self._x_redis_hash_table.hlen(inner_service_name) <= 0: + self._x_redis_hash_table.hdel(StaticConst.CMG_REDIS_SERVICES, + inner_service_name) diff --git a/infrastructure/1_sdk/init.sh b/infrastructure/1_sdk/init.sh index e8465846..fd05380f 100755 --- a/infrastructure/1_sdk/init.sh +++ b/infrastructure/1_sdk/init.sh @@ -14,11 +14,14 @@ source_virtualenv() { install_sdk() { pushd ${SDK_DIR} + python setup_clogger.py develop + python setup_redis_lua.py develop python setup_cec_base.py develop python setup_cec_redis.py develop python setup_channel_job.py develop python setup_sysom_utils.py develop - python setup_clogger.py develop + python setup_cmg_base.py develop + python setup_cmg_redis.py develop popd } diff --git a/infrastructure/1_sdk/redis_lua/x_hkeys.lua b/infrastructure/1_sdk/redis_lua/x_hkeys.lua index 826d1e3d..0bb0c50a 100644 --- a/infrastructure/1_sdk/redis_lua/x_hkeys.lua +++ b/infrastructure/1_sdk/redis_lua/x_hkeys.lua @@ -1,4 +1,23 @@ -- 获取 KEY, FIELD, VALUE, EXPIRE local hash_name = KEYS[1] +-- 用来记录时间的hash表名字 +local hash_name_time = hash_name .. ":" .. "xhash_expire_time" + +-- 获取当前时间（以秒为单位） +local time = redis.call('TIME') +local current_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 + +-- 删除过期条目 +local expire_times = redis.call("HGETALL", hash_name_time) +for i = 1, #expire_times, 2 do + local field = expire_times[i] + local expire_time = tonumber(expire_times[i + 1]) + if (expire_time >= 0 and expire_time < current_time) then + -- 删除过期的成员 + redis.call("HDEL", hash_name, field) + redis.call("HDEL", hash_name_time, field) + end +end + return redis.call("HKEYS", hash_name) \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hlen.lua b/infrastructure/1_sdk/redis_lua/x_hlen.lua index c6377144..62c44740 100644 --- a/infrastructure/1_sdk/redis_lua/x_hlen.lua +++ b/infrastructure/1_sdk/redis_lua/x_hlen.lua @@ -1,4 +1,23 @@ -- 获取 KEY, FIELD, VALUE, EXPIRE local hash_name = KEYS[1] +-- 用来记录时间的hash表名字 +local hash_name_time = hash_name .. ":" .. "xhash_expire_time" + +-- 获取当前时间（以秒为单位） +local time = redis.call('TIME') +local current_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 + +-- 删除过期条目 +local expire_times = redis.call("HGETALL", hash_name_time) +for i = 1, #expire_times, 2 do + local field = expire_times[i] + local expire_time = tonumber(expire_times[i + 1]) + if (expire_time >= 0 and expire_time < current_time) then + -- 删除过期的成员 + redis.call("HDEL", hash_name, field) + redis.call("HDEL", hash_name_time, field) + end +end + return redis.call("HLEN", hash_name) \ No newline at end of file -- Gitee From 1f172f894616980f7d251357cc531df742ce0cab Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Sat, 6 May 2023 10:45:44 +0800 Subject: [PATCH 035/196] feat(monitor): Support monitor microservices status 1. You can get a list of services and how many online instances each service has by accessing the A interface; 2. The menu bar on the front end of the page will automatically show or hide the menu depending on whether the service is online or not. --- .../app/routeres/health.py | 21 ++++++++++++ .../app/routeres/services.py | 28 +++++++++++++++ microservice/sysom_monitor_server/config.yml | 32 ++++++++++++++++- microservice/sysom_monitor_server/main.py | 15 ++++---- sysom_web/src/app.jsx | 34 +++++++++++-------- 5 files changed, 109 insertions(+), 21 deletions(-) create mode 100644 microservice/sysom_monitor_server/app/routeres/health.py create mode 100644 microservice/sysom_monitor_server/app/routeres/services.py diff --git a/microservice/sysom_monitor_server/app/routeres/health.py b/microservice/sysom_monitor_server/app/routeres/health.py new file mode 100644 index 00000000..9b6f43fc --- /dev/null +++ b/microservice/sysom_monitor_server/app/routeres/health.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/04/17 19:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get("/check") +async def get_channel_config(): + return { + "code": 0, + "err_msg": "", + "data": "" + } diff --git a/microservice/sysom_monitor_server/app/routeres/services.py b/microservice/sysom_monitor_server/app/routeres/services.py new file mode 100644 index 00000000..edaff1b6 --- /dev/null +++ b/microservice/sysom_monitor_server/app/routeres/services.py @@ -0,0 +1,28 @@ +import json +from fastapi import APIRouter +from conf.settings import * +from cmg_base import dispatch_service_discovery +router = APIRouter() + + +discovery = dispatch_service_discovery(YAML_CONFIG.get_cmg_url()) + +@router.get("/list") +async def list_services(): + try: + res = {} + services = discovery.get_services() + for service in services: + count = discovery.get_instance_count(service) + res[service] = count + except Exception as exc: + return { + "code": 1, + "data": "", + "err_msg": str(exc) + } + return { + "code": 0, + "data": res, + "err_msg": "" + } \ No newline at end of file diff --git a/microservice/sysom_monitor_server/config.yml b/microservice/sysom_monitor_server/config.yml index 33927d6b..a57f52f2 100644 --- a/microservice/sysom_monitor_server/config.yml +++ b/microservice/sysom_monitor_server/config.yml @@ -1,7 +1,7 @@ vars: NODE_EXPORT_BASE_DOWNLOAD_URL: &NODE_EXPORT_BASE_DOWNLOAD_URL https://sysom.oss-cn-beijing.aliyuncs.com/monitor/ NODE_EXPORT_VERSION: &NODE_EXPORT_VERSION 1.5.0 - SERVICE_NAME: &SERVICE_NAME sysom_monitor + SERVICE_NAME: &SERVICE_NAME sysom_monitor_server SERVICE_CONSUMER_GROUP: !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] @@ -21,6 +21,36 @@ sysom_service: url: http://127.0.0.1:3000 username: admin password: sysom@openanolis.cn + protocol: &SERVICE_PROTO http + host: &SERVICE_HOST 127.0.0.1 + port: &SERVICE_PORT 7009 + plugins: + node_dispatch: + cmg: + service_name: *SERVICE_NAME + host: *SERVICE_HOST + port: *SERVICE_PORT + tags: + - MonitorServer + - FastApi + # Metadata of service + metadata: + check: + type: http + url: + !concat [ + *SERVICE_PROTO, + "://", + *SERVICE_HOST, + ":", + *SERVICE_PORT, + "/api/v1/monitor/health/check", + ] + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false # 节点测配置 sysom_node: diff --git a/microservice/sysom_monitor_server/main.py b/microservice/sysom_monitor_server/main.py index becfda89..333e9cf1 100644 --- a/microservice/sysom_monitor_server/main.py +++ b/microservice/sysom_monitor_server/main.py @@ -1,27 +1,30 @@ from channel_job import default_channel_job_executor from fastapi import FastAPI from conf.settings import * -from app.routeres import home, grafana -from sysom_utils import PluginEventExecutor +from app.routeres import home, services, health, grafana +from sysom_utils import FrameworkPlugMag, CmgPlugin app = FastAPI() app.include_router(home.router, prefix="/api/v1/monitor/home") app.include_router(grafana.router, prefix="/api/v1/monitor/grafana") - +app.include_router(services.router, prefix="/api/v1/monitor/services") +app.include_router(health.router, prefix="/api/v1/monitor/health") def init_monitor(): # 从 Channel 为服务拉取配置初始化 channel_job default_channel_job_executor.init_config(CHANNEL_JOB_URL) default_channel_job_executor.start() - PluginEventExecutor(YAML_CONFIG, default_channel_job_executor).start() - +def init_frame_plugin_mag(): + FrameworkPlugMag(YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() @app.on_event("startup") async def on_start(): init_monitor() - pass + init_frame_plugin_mag() @app.on_event("shutdown") diff --git a/sysom_web/src/app.jsx b/sysom_web/src/app.jsx index a41e9d0f..e2e88e5f 100644 --- a/sysom_web/src/app.jsx +++ b/sysom_web/src/app.jsx @@ -115,11 +115,12 @@ let menuNameMapServiceName = { welcome: "sysom_api", host: "sysom_api", journal: "sysom_api", - monitor: "sysom_monitor", + monitor: "sysom_monitor_server", vmcore: "sysom_vmcore", diagnose: "sysom_diagnosis", - migtaion: "sysom_migration", + migrate: "sysom_migration", security: "sysom_vul", + hotfix: "sysom_hotfix" } let enable_services = []; @@ -136,14 +137,17 @@ export function patchRoutes({ routes }) { if (enable_services.length > 0) { let rootPath = routes.find((item) => item.path == "/") - for (let i = 0; i < rootPath.routes.length; i++) { - if (!!rootPath.routes[i].name && !!menuNameMapServiceName[rootPath.routes[i].name]) { - if (!enable_services.find(v => v == menuNameMapServiceName[rootPath.routes[i].name])) { - rootPath.routes[i].disabled = true - rootPath.routes[i].hideChildrenInMenu = true - } - } - } + // for (let i = 0; i < rootPath.routes.length; i++) { + // if (!!rootPath.routes[i].name && !!menuNameMapServiceName[rootPath.routes[i].name]) { + // if (!enable_services.find(v => v == menuNameMapServiceName[rootPath.routes[i].name])) { + // rootPath.routes[i].disabled = true + // rootPath.routes[i].hideChildrenInMenu = true + // // delete rootPath.routes[i] + // } + // } + // // routes.routes = routes.routes.filter(item => !!item.name && enable_services.find(v => v == menuNameMapServiceName[item.name])) + // } + rootPath.routes = rootPath.routes.filter(v => !menuNameMapServiceName[v.name] || enable_services.find(v2 => v2 == menuNameMapServiceName[v.name])) } //Find the array of diagonse's children. ex: io, net, memory @@ -239,12 +243,14 @@ export function render(oldRender) { } }) // Request services list, used to disable not running services - return requestURL("/api/v1/services/") + return requestURL("/api/v1/monitor/services/list") }) .then(res => { - if (res.code == 200) { - for (let i = 0; i < res.data.length; i++) { - enable_services.push(res.data[i].service_name); + if (res.code == 0) { + for (let k in res.data) { + if (res.data[k] > 0) { + enable_services.push(k) + } } } oldRender(); -- Gitee From 7da278bbeb76ec5d83b442ba497e5181813afc92 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Sat, 6 May 2023 11:18:23 +0800 Subject: [PATCH 036/196] fix(sdk): clogger hit wrong version --- infrastructure/1_sdk/setup_cec_base.py | 2 +- infrastructure/1_sdk/setup_cec_redis.py | 2 +- infrastructure/1_sdk/setup_channel_job.py | 2 +- infrastructure/1_sdk/setup_cmg_base.py | 2 +- infrastructure/1_sdk/setup_cmg_redis.py | 2 +- infrastructure/1_sdk/setup_sysom_utils.py | 2 +- microservice/0_sysom_channel/requirements.txt | 2 +- microservice/sysom_diagnosis/requirements.txt | 2 +- microservice/sysom_hotfix/requirements.txt | 2 +- microservice/sysom_hotfix_builder/requirements.txt | 2 +- microservice/sysom_migration/requirements.txt | 2 +- microservice/sysom_monitor_server/requirements.txt | 2 +- microservice/sysom_vmcore/requirements.txt | 2 +- microservice/sysom_vul/requirements.txt | 2 +- 14 files changed, 14 insertions(+), 14 deletions(-) diff --git a/infrastructure/1_sdk/setup_cec_base.py b/infrastructure/1_sdk/setup_cec_base.py index b73735eb..ba387061 100644 --- a/infrastructure/1_sdk/setup_cec_base.py +++ b/infrastructure/1_sdk/setup_cec_base.py @@ -15,7 +15,7 @@ setuptools.setup( url="", packages=["cec_base"], install_requires=[ - "clogger>=0.0.1", + "clogger==0.0.1", "anyio>=3.6.2" ], classifiers=[ diff --git a/infrastructure/1_sdk/setup_cec_redis.py b/infrastructure/1_sdk/setup_cec_redis.py index 9954cc00..3705b7fe 100644 --- a/infrastructure/1_sdk/setup_cec_redis.py +++ b/infrastructure/1_sdk/setup_cec_redis.py @@ -12,7 +12,7 @@ setuptools.setup( "cec_base>=0.0.1", "redis>=4.3.4", "schedule>=1.1.0", - "clogger>=0.0.1" + "clogger==0.0.1" ], classifiers=[ "Programming Language :: Python :: 3", diff --git a/infrastructure/1_sdk/setup_channel_job.py b/infrastructure/1_sdk/setup_channel_job.py index 7a977e7e..c2d9b40f 100644 --- a/infrastructure/1_sdk/setup_channel_job.py +++ b/infrastructure/1_sdk/setup_channel_job.py @@ -11,7 +11,7 @@ setuptools.setup( install_requires=[ "cec_base>=0.0.1", "cec_redis>=0.0.1", - "clogger>=0.0.1", + "clogger==0.0.1", "anyio>=3.6.2", "requests" ], diff --git a/infrastructure/1_sdk/setup_cmg_base.py b/infrastructure/1_sdk/setup_cmg_base.py index 82fbedf0..f40e886b 100644 --- a/infrastructure/1_sdk/setup_cmg_base.py +++ b/infrastructure/1_sdk/setup_cmg_base.py @@ -17,7 +17,7 @@ setuptools.setup( url="", packages=["sysom_utils"], install_requires=[ - "clogger>=0.0.1", + "clogger==0.0.1", "requests>=2.27.1", ], classifiers=[ diff --git a/infrastructure/1_sdk/setup_cmg_redis.py b/infrastructure/1_sdk/setup_cmg_redis.py index 1c419d21..5d786a45 100644 --- a/infrastructure/1_sdk/setup_cmg_redis.py +++ b/infrastructure/1_sdk/setup_cmg_redis.py @@ -15,7 +15,7 @@ setuptools.setup( author_email="mfeng@linux.alibaba.com", description="A redis implement for common event center", install_requires=[ - "clogger>=0.0.1", + "clogger==0.0.1", "cmg_base>=0.0.1" ], classifiers=[ diff --git a/infrastructure/1_sdk/setup_sysom_utils.py b/infrastructure/1_sdk/setup_sysom_utils.py index f30006d1..65ab1206 100644 --- a/infrastructure/1_sdk/setup_sysom_utils.py +++ b/infrastructure/1_sdk/setup_sysom_utils.py @@ -19,7 +19,7 @@ setuptools.setup( install_requires=[ "cec_base>=0.0.1", "channel_job>=0.0.1", - "clogger>=0.0.1", + "clogger==0.0.1", "aiohttp>=3.8.3", "aiofiles==0.8.0", "anyio>=3.6.2", diff --git a/microservice/0_sysom_channel/requirements.txt b/microservice/0_sysom_channel/requirements.txt index b34b6ab5..d9ab7731 100644 --- a/microservice/0_sysom_channel/requirements.txt +++ b/microservice/0_sysom_channel/requirements.txt @@ -1,4 +1,4 @@ -clogger>=0.0.1 +clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 diff --git a/microservice/sysom_diagnosis/requirements.txt b/microservice/sysom_diagnosis/requirements.txt index f84443b8..dbf7b077 100644 --- a/microservice/sysom_diagnosis/requirements.txt +++ b/microservice/sysom_diagnosis/requirements.txt @@ -1,4 +1,4 @@ -clogger>=0.0.1 +clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 diff --git a/microservice/sysom_hotfix/requirements.txt b/microservice/sysom_hotfix/requirements.txt index 3952b08c..572c8d98 100644 --- a/microservice/sysom_hotfix/requirements.txt +++ b/microservice/sysom_hotfix/requirements.txt @@ -1,4 +1,4 @@ -clogger>=0.0.1 +clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 diff --git a/microservice/sysom_hotfix_builder/requirements.txt b/microservice/sysom_hotfix_builder/requirements.txt index f3e2139b..d3d398e7 100644 --- a/microservice/sysom_hotfix_builder/requirements.txt +++ b/microservice/sysom_hotfix_builder/requirements.txt @@ -1,4 +1,4 @@ -clogger>=0.0.1 +clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 diff --git a/microservice/sysom_migration/requirements.txt b/microservice/sysom_migration/requirements.txt index c114ec30..c9de9741 100644 --- a/microservice/sysom_migration/requirements.txt +++ b/microservice/sysom_migration/requirements.txt @@ -1,4 +1,4 @@ -clogger>=0.0.1 +clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 diff --git a/microservice/sysom_monitor_server/requirements.txt b/microservice/sysom_monitor_server/requirements.txt index ee35d62b..aafe520b 100644 --- a/microservice/sysom_monitor_server/requirements.txt +++ b/microservice/sysom_monitor_server/requirements.txt @@ -1,4 +1,4 @@ -clogger>=0.0.1 +clogger==0.0.1 sysom_utils>=0.0.1 alembic==1.7.7 anyio==3.6.2 diff --git a/microservice/sysom_vmcore/requirements.txt b/microservice/sysom_vmcore/requirements.txt index c114ec30..c9de9741 100644 --- a/microservice/sysom_vmcore/requirements.txt +++ b/microservice/sysom_vmcore/requirements.txt @@ -1,4 +1,4 @@ -clogger>=0.0.1 +clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 diff --git a/microservice/sysom_vul/requirements.txt b/microservice/sysom_vul/requirements.txt index ed045a76..de440558 100644 --- a/microservice/sysom_vul/requirements.txt +++ b/microservice/sysom_vul/requirements.txt @@ -1,5 +1,5 @@ aiohttp==3.8.4 -clogger>=0.0.1 +clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 -- Gitee From 72983af2c9a5908a2b59426a345191c23348efd5 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Sat, 6 May 2023 20:21:35 +0800 Subject: [PATCH 037/196] fix(sdk): CecUrl parse error --- infrastructure/1_sdk/cec_base/url.py | 5 +++-- infrastructure/1_sdk/channel_job/job.py | 2 +- infrastructure/1_sdk/cmg_base/url.py | 5 +++-- infrastructure/1_sdk/sysom_utils/config_parser.py | 9 +++++++-- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/infrastructure/1_sdk/cec_base/url.py b/infrastructure/1_sdk/cec_base/url.py index 037d95c6..99e54f9c 100644 --- a/infrastructure/1_sdk/cec_base/url.py +++ b/infrastructure/1_sdk/cec_base/url.py @@ -36,8 +36,9 @@ class CecUrl: self.params = params def __str__(self): + query = "&".join([f'{k}={v}' for k, v in self.params.items()]) return f"{self.proto}://{self.netloc}?" \ - f"{urllib.parse.urlencode(self.params)}" + f"{query}" @staticmethod def parse(url: str): @@ -49,7 +50,7 @@ class CecUrl: Returns: CecUrl """ - parse_result = urllib.parse.urlparse(url) + parse_result = urllib.parse.urlparse(url, allow_fragments=False) proto, netloc = parse_result.scheme, parse_result.netloc query_str, params = parse_result.query, {} if proto == '' or netloc == '': diff --git a/infrastructure/1_sdk/channel_job/job.py b/infrastructure/1_sdk/channel_job/job.py index 7460c4dc..d343ef20 100644 --- a/infrastructure/1_sdk/channel_job/job.py +++ b/infrastructure/1_sdk/channel_job/job.py @@ -310,7 +310,7 @@ class ChannelJobExecutor: "channel_job_auto_recover", True ) if None in [self._target_topic, self._listen_topic, listen_consumer_group]: - raise (ChannelJobException("CecUrl missing parameters")) + raise (ChannelJobException(f"CecUrl missing parameters: {cec_url}")) # 2. Create Consumer, Producer, Admin instance self._consumer: Consumer = dispatch_consumer( diff --git a/infrastructure/1_sdk/cmg_base/url.py b/infrastructure/1_sdk/cmg_base/url.py index 75c5a025..1006e397 100644 --- a/infrastructure/1_sdk/cmg_base/url.py +++ b/infrastructure/1_sdk/cmg_base/url.py @@ -36,8 +36,9 @@ class CmgUrl: self.params = params def __str__(self): + query = "&".join([f'{k}={v}' for k, v in self.params.items()]) return f"{self.proto}://{self.netloc}?" \ - f"{urllib.parse.urlencode(self.params)}" + f"{query}" @staticmethod def parse(url: str): @@ -49,7 +50,7 @@ class CmgUrl: Returns: CecUrl """ - parse_result = urllib.parse.urlparse(url) + parse_result = urllib.parse.urlparse(url, allow_fragments=False) proto, netloc = parse_result.scheme, parse_result.netloc query_str, params = parse_result.query, {} if proto == '' or netloc == '': diff --git a/infrastructure/1_sdk/sysom_utils/config_parser.py b/infrastructure/1_sdk/sysom_utils/config_parser.py index 3e7b2dcf..461d299a 100644 --- a/infrastructure/1_sdk/sysom_utils/config_parser.py +++ b/infrastructure/1_sdk/sysom_utils/config_parser.py @@ -134,8 +134,8 @@ class ConfigParser: else: raise ConfigParserException( f"Not support cec protocol: {cec_config.protocol}") - for k in params: - params.append(f"{k}={params[k]}") + for k in special_param: + params.append(f"{k}={special_param[k]}") cec_url += "&".join(params) return cec_url @@ -156,6 +156,9 @@ class ConfigParser: params.append(f"username={redis_config.username}") if redis_config.password: params.append(f"password={redis_config.password}") + for k in special_param: + params.append(f"{k}={special_param[k]}") + cmg_url += "&".join(params) return cmg_url def get_local_channel_job_url(self) -> str: @@ -167,4 +170,6 @@ class ConfigParser: f"channel_job_listen_topic={channel_job_config.listen_topic}", f"channel_job_consumer_group={channel_job_config.consumer_group}" ] + if cec_url != "" and cec_url[-1] != "?": + cec_url += "&" return cec_url + "&".join(params) -- Gitee From 9d4251a459da57d0408911748bda77c34f709a39 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 8 May 2023 11:26:25 +0800 Subject: [PATCH 038/196] fix(lua): Modifying lua scripts to be compatible with Redis clusters error: bad lua script for redis cluster, all the keys that the script uses should be passed using the KEYS array, and KEYS should not be in expression --- infrastructure/1_sdk/redis_lua/x_hdel.lua | 13 ++----- .../1_sdk/redis_lua/x_hdrop_table.lua | 11 ++---- infrastructure/1_sdk/redis_lua/x_hexpire.lua | 12 ++----- infrastructure/1_sdk/redis_lua/x_hget.lua | 17 +++------ infrastructure/1_sdk/redis_lua/x_hgetall.lua | 14 +++----- infrastructure/1_sdk/redis_lua/x_hkeys.lua | 14 +++----- infrastructure/1_sdk/redis_lua/x_hlen.lua | 14 +++----- infrastructure/1_sdk/redis_lua/x_hset.lua | 11 ++---- .../1_sdk/redis_lua/xreadis_hash_table.py | 35 +++++++++++++------ 9 files changed, 54 insertions(+), 87 deletions(-) diff --git a/infrastructure/1_sdk/redis_lua/x_hdel.lua b/infrastructure/1_sdk/redis_lua/x_hdel.lua index b4eac4d3..456c4218 100644 --- a/infrastructure/1_sdk/redis_lua/x_hdel.lua +++ b/infrastructure/1_sdk/redis_lua/x_hdel.lua @@ -1,15 +1,8 @@ --- 获取 KEY, FIELD -local hash_name = KEYS[1] - --- 用来记录时间的hash表名字 -local hash_name_time = hash_name .. ":" .. "xhash_expire_time" - for i = 1, #ARGV do local field = ARGV[i] -- 删除键值 - redis.call("HDEL", hash_name, field) + redis.call("HDEL", KEYS[1], field) -- 删除过期时间 - redis.call("HDEL", hash_name_time, field) + redis.call("HDEL", KEYS[2], field) end -return "OK" - +return "OK" \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hdrop_table.lua b/infrastructure/1_sdk/redis_lua/x_hdrop_table.lua index 8dafcebb..29e53df1 100644 --- a/infrastructure/1_sdk/redis_lua/x_hdrop_table.lua +++ b/infrastructure/1_sdk/redis_lua/x_hdrop_table.lua @@ -1,9 +1,4 @@ --- 获取 KEY -local hash_name = KEYS[1] --- 用来记录时间的hash表名字 -local hash_name_time = hash_name .. ":" .. "xhash_expire_time" -- 保存键值 -redis.call("DEL", hash_name) -redis.call("DEL", hash_name_time) -return "OK" - +redis.call("DEL", KEYS[1]) +redis.call("DEL", KEYS[2]) +return "OK" \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hexpire.lua b/infrastructure/1_sdk/redis_lua/x_hexpire.lua index 538aed9b..bf1d9963 100644 --- a/infrastructure/1_sdk/redis_lua/x_hexpire.lua +++ b/infrastructure/1_sdk/redis_lua/x_hexpire.lua @@ -1,22 +1,16 @@ -- 获取 KEY, FIELD, EXPIRE -local hash_name = KEYS[1] local field = ARGV[1] local expire = ARGV[2] --- 用来记录时间的hash表名字 -local hash_name_time = hash_name .. ":" .. "xhash_expire_time" - -- 获取当前时间（以秒为单位） local time = redis.call('TIME') local expire_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 + expire if expire ~= -1 then -- 保存过期时间 - redis.call("HSET", hash_name_time, field, expire_time) + redis.call("HSET", KEYS[2], field, expire_time) else -- 删除过期限制 - redis.call("HDEL", hash_name_time, field) + redis.call("HDEL", KEYS[2], field) end -return "OK" - - +return "OK" \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hget.lua b/infrastructure/1_sdk/redis_lua/x_hget.lua index 9e375e3e..35f37f43 100644 --- a/infrastructure/1_sdk/redis_lua/x_hget.lua +++ b/infrastructure/1_sdk/redis_lua/x_hget.lua @@ -1,15 +1,11 @@ -- 获取 KEY, FIELD -local hash_name = KEYS[1] local field = ARGV[1] --- 用来记录时间的hash表名字 -local hash_name_time = hash_name .. ":" .. "xhash_expire_time" - -- 获取当前时间（以秒为单位） local time = redis.call('TIME') local current_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 -local expire_time_str = redis.call("HGET", hash_name_time, field) +local expire_time_str = redis.call("HGET", KEYS[2], field) local expire_time = -1 if expire_time_str ~= false then expire_time = tonumber(expire_time_str) @@ -20,17 +16,14 @@ end -- 3. 否则返回值 if (expire_time >= 0 and expire_time < current_time) then -- 删除过期的成员 - redis.call("HDEL", hash_name, field) - redis.call("HDEL", hash_name_time, field) + redis.call("HDEL", KEYS[1], field) + redis.call("HDEL", KEYS[2], field) return nil else - local value = redis.call("HGET", hash_name, field) + local value = redis.call("HGET", KEYS[1], field) if value == false then return nil else return value end -end - - - +end \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hgetall.lua b/infrastructure/1_sdk/redis_lua/x_hgetall.lua index 2f08effe..115854f9 100644 --- a/infrastructure/1_sdk/redis_lua/x_hgetall.lua +++ b/infrastructure/1_sdk/redis_lua/x_hgetall.lua @@ -1,11 +1,5 @@ --- 获取 KEY -local hash_name = KEYS[1] - --- 用来记录时间的hash表名字 -local hash_name_time = hash_name .. ":" .. "xhash_expire_time" - -- 获取哈希的所有子键和值 -local hash_fields = redis.call('HGETALL', hash_name) +local hash_fields = redis.call('HGETALL', KEYS[1]) -- 获取当前时间（以秒为单位） local time = redis.call('TIME') @@ -17,15 +11,15 @@ for i = 1, #hash_fields, 2 do -- 获取子键和值 local field = hash_fields[i] local value = hash_fields[i + 1] - local expire_time_str = redis.call("HGET", hash_name_time, field) + local expire_time_str = redis.call("HGET", KEYS[2], field) local expire_time = -1 if expire_time_str ~= false then expire_time = tonumber(expire_time_str) end if (expire_time >= 0 and expire_time < current_time) then -- 删除过期的成员 - redis.call("HDEL", hash_name, field) - redis.call("HDEL", hash_name_time, field) + redis.call("HDEL", KEYS[1], field) + redis.call("HDEL", KEYS[2], field) else res[i] = field res[i + 1] = value diff --git a/infrastructure/1_sdk/redis_lua/x_hkeys.lua b/infrastructure/1_sdk/redis_lua/x_hkeys.lua index 0bb0c50a..53aa74f2 100644 --- a/infrastructure/1_sdk/redis_lua/x_hkeys.lua +++ b/infrastructure/1_sdk/redis_lua/x_hkeys.lua @@ -1,23 +1,17 @@ --- 获取 KEY, FIELD, VALUE, EXPIRE -local hash_name = KEYS[1] - --- 用来记录时间的hash表名字 -local hash_name_time = hash_name .. ":" .. "xhash_expire_time" - -- 获取当前时间（以秒为单位） local time = redis.call('TIME') local current_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 -- 删除过期条目 -local expire_times = redis.call("HGETALL", hash_name_time) +local expire_times = redis.call("HGETALL", KEYS[2]) for i = 1, #expire_times, 2 do local field = expire_times[i] local expire_time = tonumber(expire_times[i + 1]) if (expire_time >= 0 and expire_time < current_time) then -- 删除过期的成员 - redis.call("HDEL", hash_name, field) - redis.call("HDEL", hash_name_time, field) + redis.call("HDEL", KEYS[1], field) + redis.call("HDEL", KEYS[2], field) end end -return redis.call("HKEYS", hash_name) \ No newline at end of file +return redis.call("HKEYS", KEYS[1]) \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hlen.lua b/infrastructure/1_sdk/redis_lua/x_hlen.lua index 62c44740..77b0f20e 100644 --- a/infrastructure/1_sdk/redis_lua/x_hlen.lua +++ b/infrastructure/1_sdk/redis_lua/x_hlen.lua @@ -1,23 +1,17 @@ --- 获取 KEY, FIELD, VALUE, EXPIRE -local hash_name = KEYS[1] - --- 用来记录时间的hash表名字 -local hash_name_time = hash_name .. ":" .. "xhash_expire_time" - -- 获取当前时间（以秒为单位） local time = redis.call('TIME') local current_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 -- 删除过期条目 -local expire_times = redis.call("HGETALL", hash_name_time) +local expire_times = redis.call("HGETALL", KEYS[2]) for i = 1, #expire_times, 2 do local field = expire_times[i] local expire_time = tonumber(expire_times[i + 1]) if (expire_time >= 0 and expire_time < current_time) then -- 删除过期的成员 - redis.call("HDEL", hash_name, field) - redis.call("HDEL", hash_name_time, field) + redis.call("HDEL", KEYS[1], field) + redis.call("HDEL", KEYS[2], field) end end -return redis.call("HLEN", hash_name) \ No newline at end of file +return redis.call("HLEN", KEYS[1]) \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/x_hset.lua b/infrastructure/1_sdk/redis_lua/x_hset.lua index 03b9e3a5..94f670dc 100644 --- a/infrastructure/1_sdk/redis_lua/x_hset.lua +++ b/infrastructure/1_sdk/redis_lua/x_hset.lua @@ -1,5 +1,4 @@ -- 获取 KEY, FIELD, VALUE, EXPIRE -local hash_name = KEYS[1] local expire = tonumber(ARGV[1]) if #ARGV < 3 then return "ERR: Missing filed,value" @@ -9,18 +8,14 @@ end local time = redis.call('TIME') local expire_time = tonumber(time[1]) + tonumber(time[2]) / 1000000 + expire --- 用来记录时间的hash表名字 -local hash_name_time = hash_name .. ":" .. "xhash_expire_time" - for i = 2, #ARGV, 2 do local field = ARGV[i] local value = ARGV[i + 1] -- 保存键值 - redis.call("HSET", hash_name, field, value) + redis.call("HSET", KEYS[1], field, value) if expire ~= -1 then -- 保存过期时间 - redis.call("HSET", hash_name_time, field, expire_time) + redis.call("HSET", KEYS[2], field, expire_time) end end -return "OK" - +return "OK" \ No newline at end of file diff --git a/infrastructure/1_sdk/redis_lua/xreadis_hash_table.py b/infrastructure/1_sdk/redis_lua/xreadis_hash_table.py index a932c9b7..8eeb7adf 100644 --- a/infrastructure/1_sdk/redis_lua/xreadis_hash_table.py +++ b/infrastructure/1_sdk/redis_lua/xreadis_hash_table.py @@ -53,16 +53,23 @@ class XRedisHashTable: self._get_or_cache_script_sha(name), *fields ) + @staticmethod + def _get_expire_table(table_name): + return "{" + table_name + "}:xhash_expire_time" + def hget(self, table_name: str, field: str) -> \ Optional[str]: - res = self._evalsha("x_hget", 1, table_name, field) + res = self._evalsha("x_hget", 2, table_name, + self._get_expire_table(table_name), field) return None if res is None else res def hget_pl(self, pl: Pipeline, table_name: str, field: str) -> Pipeline: - return self._eval_by_pl(pl, "x_hget", 1, table_name, field) + return self._eval_by_pl(pl, "x_hget", 2, table_name, + self._get_expire_table(table_name), field) def hgetall(self, table_name: str) -> dict: - kvs = self._evalsha("x_hgetall", 1, table_name) + kvs = self._evalsha("x_hgetall", 2, table_name, + self._get_expire_table(table_name)) res = {} for i in range(0, len(kvs), 2): res[kvs[i]] = kvs[i + 1] @@ -71,27 +78,35 @@ class XRedisHashTable: def hset(self, table_name: str, *kvs: str, expire: int = -1) -> bool: try: - res = self._evalsha("x_hset", 1, table_name, expire, *kvs) + res = self._evalsha("x_hset", 2, table_name, + self._get_expire_table(table_name), expire, + *kvs) return res == "OK" - except Exception as _: + except Exception as err: + print(err) return False def hdrop_table(self, table_name: str) -> bool: - res = self._evalsha("x_hdrop_table", 1, table_name) + res = self._evalsha("x_hdrop_table", 2, table_name, + self._get_expire_table(table_name)) return res == "OK" def hdel(self, table_name: str, *fields: str) -> bool: - res = self._evalsha("x_hdel", 1, table_name, *fields) + res = self._evalsha("x_hdel", 2, table_name, + self._get_expire_table(table_name), *fields) return res == "OK" def hexpire(self, table_name: str, field: str, expire: int = -1) -> bool: - res = self._evalsha("x_hexpire", 1, table_name, field, expire) + res = self._evalsha("x_hexpire", 2, table_name, + self._get_expire_table(table_name), field, expire) return res == "OK" def hlen(self, table_name: str) -> int: - res = self._evalsha("x_hlen", 1, table_name) + res = self._evalsha("x_hlen", 2, table_name, + self._get_expire_table(table_name)) return res def hkeys(self, table_name: str) -> List[str]: - res = self._evalsha("x_hkeys", 1, table_name) + res = self._evalsha("x_hkeys", 2, table_name, + self._get_expire_table(table_name)) return res -- Gitee From 6c974f3af7c5e5a3ad9e6f545df87e0b470dbc98 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Sat, 6 May 2023 20:22:20 +0800 Subject: [PATCH 039/196] feat(deploy): Support dbmigrate opt --- .../0_sysom_api/scripts/server_db_migrate.sh | 17 ++ .../scripts/server_db_migrate.sh | 17 ++ .../scripts/server_db_migrate.sh | 17 ++ .../sysom_hotfix/scripts/server_db_migrate.sh | 17 ++ .../scripts/server_db_migrate.sh | 17 ++ .../scripts/server_db_migrate.sh | 17 ++ .../sysom_vmcore/scripts/server_db_migrate.sh | 17 ++ .../sysom_vul/scripts/server_db_migrate.sh | 17 ++ script/dbmigrate.sh | 158 ++++++++++++++++++ script/deploy.sh | 7 +- script/sysom.sh | 4 + 11 files changed, 300 insertions(+), 5 deletions(-) create mode 100644 microservice/0_sysom_api/scripts/server_db_migrate.sh create mode 100644 microservice/0_sysom_channel/scripts/server_db_migrate.sh create mode 100644 microservice/sysom_diagnosis/scripts/server_db_migrate.sh create mode 100644 microservice/sysom_hotfix/scripts/server_db_migrate.sh create mode 100644 microservice/sysom_migration/scripts/server_db_migrate.sh create mode 100644 microservice/sysom_monitor_server/scripts/server_db_migrate.sh create mode 100644 microservice/sysom_vmcore/scripts/server_db_migrate.sh create mode 100644 microservice/sysom_vul/scripts/server_db_migrate.sh create mode 100755 script/dbmigrate.sh mode change 100644 => 100755 script/sysom.sh diff --git a/microservice/0_sysom_api/scripts/server_db_migrate.sh b/microservice/0_sysom_api/scripts/server_db_migrate.sh new file mode 100644 index 00000000..89ba9eb6 --- /dev/null +++ b/microservice/0_sysom_api/scripts/server_db_migrate.sh @@ -0,0 +1,17 @@ +#!/bin/bash +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/0_sysom_channel/scripts/server_db_migrate.sh b/microservice/0_sysom_channel/scripts/server_db_migrate.sh new file mode 100644 index 00000000..df481d7b --- /dev/null +++ b/microservice/0_sysom_channel/scripts/server_db_migrate.sh @@ -0,0 +1,17 @@ +#!/bin/bash +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + alembic upgrade head + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/sysom_diagnosis/scripts/server_db_migrate.sh b/microservice/sysom_diagnosis/scripts/server_db_migrate.sh new file mode 100644 index 00000000..89ba9eb6 --- /dev/null +++ b/microservice/sysom_diagnosis/scripts/server_db_migrate.sh @@ -0,0 +1,17 @@ +#!/bin/bash +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/sysom_hotfix/scripts/server_db_migrate.sh b/microservice/sysom_hotfix/scripts/server_db_migrate.sh new file mode 100644 index 00000000..89ba9eb6 --- /dev/null +++ b/microservice/sysom_hotfix/scripts/server_db_migrate.sh @@ -0,0 +1,17 @@ +#!/bin/bash +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/sysom_migration/scripts/server_db_migrate.sh b/microservice/sysom_migration/scripts/server_db_migrate.sh new file mode 100644 index 00000000..89ba9eb6 --- /dev/null +++ b/microservice/sysom_migration/scripts/server_db_migrate.sh @@ -0,0 +1,17 @@ +#!/bin/bash +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/sysom_monitor_server/scripts/server_db_migrate.sh b/microservice/sysom_monitor_server/scripts/server_db_migrate.sh new file mode 100644 index 00000000..df481d7b --- /dev/null +++ b/microservice/sysom_monitor_server/scripts/server_db_migrate.sh @@ -0,0 +1,17 @@ +#!/bin/bash +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + alembic upgrade head + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/sysom_vmcore/scripts/server_db_migrate.sh b/microservice/sysom_vmcore/scripts/server_db_migrate.sh new file mode 100644 index 00000000..89ba9eb6 --- /dev/null +++ b/microservice/sysom_vmcore/scripts/server_db_migrate.sh @@ -0,0 +1,17 @@ +#!/bin/bash +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/sysom_vul/scripts/server_db_migrate.sh b/microservice/sysom_vul/scripts/server_db_migrate.sh new file mode 100644 index 00000000..89ba9eb6 --- /dev/null +++ b/microservice/sysom_vul/scripts/server_db_migrate.sh @@ -0,0 +1,17 @@ +#!/bin/bash +SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/script/dbmigrate.sh b/script/dbmigrate.sh new file mode 100755 index 00000000..0fb2f2a9 --- /dev/null +++ b/script/dbmigrate.sh @@ -0,0 +1,158 @@ +#!/bin/bash -x +set -x +ProgName=$(basename $0) +BaseDir=$(dirname $(readlink -f "$0")) + +#################################################################################################################### +# Helper functions +#################################################################################################################### + +red() { + printf '\33[1;31m%b\n\33[0m' "$1" +} + +green() { + printf '\33[1;32m%b\n\33[0m' "$1" +} + +ensure_sysom_conf_exists() { + if [ ! -f "$SYSOM_CONF" ] + then + cp `dirname $BaseDir`/conf/config.yml $SYSOM_CONF + fi +} + +update_global_config() { + sed "s/SERVER_LOCAL_IP: 127.0.0.1/SERVER_LOCAL_IP: $SERVER_LOCAL_IP/g" -i ${SYSOM_CONF} + sed "s/SERVER_PUBLIC_IP: 127.0.0.1/SERVER_PUBLIC_IP: $SERVER_PUBLIC_IP/g" -i ${SYSOM_CONF} + sed "s/SERVER_PORT: 80/SERVER_PORT: $SERVER_PORT/g" -i ${SYSOM_CONF} + sed "s/global_root_path \/usr\/local\/sysom/global_root_path $APP_HOME/g" -i ${SYSOM_CONF} + + # MySQL + sed "/mysql/,/host/s/host: localhost/host: $DB_MYSQL_HOST/g" -i ${SYSOM_CONF} + sed "/mysql/,/port/s/port: 3306/port: $DB_MYSQL_PORT/g" -i ${SYSOM_CONF} + sed "/mysql/,/user/s/user: sysom/user: $DB_MYSQL_USERNAME/g" -i ${SYSOM_CONF} + sed "/mysql/,/password/s/password: sysom_admin/password: $DB_MYSQL_PASSWORD/g" -i ${SYSOM_CONF} + sed "/mysql/,/database/s/database: sysom/database: $DB_MYSQL_DATABASE/g" -i ${SYSOM_CONF} + + # Redis + sed "/redis:/,/host/s/host: localhost/host: $REDIS_HOST/g" -i ${SYSOM_CONF} + sed "/redis:/,/port/s/port: 6379/port: $REDIS_PORT/g" -i ${SYSOM_CONF} + sed "/redis:/,/username/s/username:/username: $REDIS_USERNAME/g" -i ${SYSOM_CONF} + sed "/redis:/,/password/s/password:/password: $REDIS_PASSWORD/g" -i ${SYSOM_CONF} + + ###update local timezone### + local_timezone=`timedatectl status | grep "Time zone" | awk '{print $3}'` + green "current timezone is : $local_timezone" + if [ "$local_timezone" == "" ] + then + echo "get time zone fail, use default time zone Asia/Shanghai" + else + sed "s;Asia/Shanghai;$local_timezone;g" -i ${SYSOM_CONF} + fi +} + +#################################################################################################################### +# Subcommands +#################################################################################################################### + +sub_help(){ + echo "Usage: $ProgName [options]" + echo "Subcommands:" + echo " all Deploy all modules" + echo " infrastructure [ALL | ] Deploy infrastructure components" + echo " Example: $ProgName infrastructure env" + echo " Example: $ProgName infrastructure local_services" + echo " microservice [ALL | ] Deploy all microservices or specific microservice" + echo " Example: $ProgName microservice sysom_diagnosis" + echo "" + echo "For help with each subcommand run:" + echo "$ProgName -h|--help" + echo "" +} + +sub_all() { + local_microservice_dir=`dirname $BaseDir`/microservice + # Load deploy_excludes + deploy_excludes="" + if [ -f "${local_microservice_dir}/deploy_exclude" ];then + deploy_excludes=`cat ${local_microservice_dir}/deploy_exclude` + fi + # Deploy all microservices + for microservice in $(ls $local_microservice_dir) + do + if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_dir}/${microservice}" ];then + continue + fi + do_dbmigrate ${microservice} + done +} + +do_dbmigrate() { + mkdir -p ${MICROSERVICE_HOME} + local_microservice_dir=`dirname $BaseDir`/microservice + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]} + do + for service_dir in $(ls ${local_microservice_dir} | grep ${target}) + do + target_dir=${local_microservice_dir}/${service_dir} + + pushd ${target_dir} + + # update microservice common.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} + if [ -f "${target_dir}/conf/common.py" ]; then + sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${target_dir}/conf/common.py + fi + # update fastapi based microservice alembic/env.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} + if [ -f "${target_dir}/alembic/env.py" ]; then + sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${target_dir}/alembic/env.py + fi + # update microservice gunicorn.py replace /var/log/sysom to ${LOG_HOME} + if [ -f "${target_dir}/conf/gunicorn.py" ]; then + sed -i "s;/var/log/sysom;${LOG_HOME};g" ${target_dir}/conf/gunicorn.py + fi + # update *.ini replace /var/log/sysom to ${LOG_HOME} + for ini_conf in $(ls sysom-*.ini) + do + sed -i "s;/var/log/sysom;${LOG_HOME};g" ${target_dir}/${ini_conf} + done + + green "$target_dir db migrate start..................................." + if [ ! -f "${target_dir}/scripts/server_db_migrate.sh" ];then + popd + continue + fi + bash -x ${target_dir}/scripts/server_db_migrate.sh + if [ $? -ne 0 ];then + red "$target_dir db migrate fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + popd + done + done +} + +subcommand=$1 +case $subcommand in + "" | "-h" | "--help") + sub_help + ;; + "all" | "ALL") + ensure_sysom_conf_exists + update_global_config + sub_all + ;; + *) + ensure_sysom_conf_exists + update_global_config + do_dbmigrate $@ + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac \ No newline at end of file diff --git a/script/deploy.sh b/script/deploy.sh index 6df9b3d8..7a54c851 100755 --- a/script/deploy.sh +++ b/script/deploy.sh @@ -119,6 +119,7 @@ do_deploy_microservices() { green "$target_dir initialing..................................." if [ ! -f "${target_dir}/scripts/server_init.sh" ];then + popd continue fi bash -x ${target_dir}/scripts/server_init.sh @@ -127,11 +128,8 @@ do_deploy_microservices() { red "sysom init failed, exit 1" exit 1 fi - # green "setup_web_menu_enable $dir" - # setup_web_menu_enable $dir + popd done - - popd done } @@ -162,7 +160,6 @@ do_deploy_infrastructure() { # green "setup_web_menu_enable $dir" # setup_web_menu_enable $dir popd - echo ${service_dir} done done diff --git a/script/sysom.sh b/script/sysom.sh old mode 100644 new mode 100755 index c45c0f9f..120c58ba --- a/script/sysom.sh +++ b/script/sysom.sh @@ -136,6 +136,10 @@ sub_clear() { bash +x ${BaseDir}/clear.sh $@ } +sub_dbmigrate() { + bash +x ${BaseDir}/dbmigrate.sh $@ +} + sub_list() { echo "list" } -- Gitee From 944c7409d6d9abec9c77b17fea23fcd33d9166d7 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 10 May 2023 11:12:19 +0800 Subject: [PATCH 040/196] feat(sdk.gclient): Add gclient to support Inter-Microservice Communication --- conf/config.yml | 15 +- infrastructure/1_sdk/channel_job/job.py | 27 +-- infrastructure/1_sdk/gcache_base/__init__.py | 12 ++ .../1_sdk/gcache_base/exceptions.py | 36 ++++ infrastructure/1_sdk/gcache_base/gcache.py | 168 ++++++++++++++++++ infrastructure/1_sdk/gcache_base/url.py | 66 +++++++ infrastructure/1_sdk/gcache_redis/__init__.py | 12 ++ infrastructure/1_sdk/gcache_redis/common.py | 93 ++++++++++ .../1_sdk/gcache_redis/redis_gcache.py | 86 +++++++++ infrastructure/1_sdk/gcache_redis/utils.py | 24 +++ infrastructure/1_sdk/gclient_base/__init__.py | 12 ++ .../1_sdk/gclient_base/exceptions.py | 36 ++++ infrastructure/1_sdk/gclient_base/gclient.py | 127 +++++++++++++ infrastructure/1_sdk/gclient_base/url.py | 67 +++++++ infrastructure/1_sdk/gclient_cmg/__init__.py | 10 ++ .../1_sdk/gclient_cmg/cmg_gclient.py | 89 ++++++++++ infrastructure/1_sdk/gclient_cmg/common.py | 58 ++++++ infrastructure/1_sdk/gclient_http/__init__.py | 10 ++ infrastructure/1_sdk/gclient_http/common.py | 46 +++++ .../1_sdk/gclient_http/http_gclient.py | 56 ++++++ infrastructure/1_sdk/init.sh | 5 + infrastructure/1_sdk/setup_gcache_base.py | 18 ++ infrastructure/1_sdk/setup_gcache_redis.py | 19 ++ infrastructure/1_sdk/setup_gclient_base.py | 18 ++ infrastructure/1_sdk/setup_gclient_cmg.py | 19 ++ infrastructure/1_sdk/setup_gclient_http.py | 18 ++ infrastructure/1_sdk/sysom_utils/__init__.py | 3 +- .../1_sdk/sysom_utils/config_parser.py | 46 +++++ infrastructure/1_sdk/sysom_utils/framework.py | 83 +++++++++ .../1_sdk/sysom_utils/framework_plugins.py | 21 ++- .../1_sdk/sysom_utils/node_manager.py | 3 +- .../0_sysom_api/apps/accounts/apps.py | 2 +- .../apps/accounts/authentication.py | 5 +- .../0_sysom_api/apps/accounts/views.py | 13 +- microservice/0_sysom_api/apps/host/apps.py | 4 +- microservice/0_sysom_api/conf/common.py | 20 +-- microservice/0_sysom_api/config.yml | 25 +-- microservice/0_sysom_channel/conf/common.py | 4 +- microservice/0_sysom_channel/config.yml | 24 +-- microservice/0_sysom_channel/main.py | 8 +- .../sysom_diagnosis/apps/task/apps.py | 10 +- microservice/sysom_diagnosis/conf/common.py | 20 +-- microservice/sysom_diagnosis/config.yml | 23 +-- .../sysom_diagnosis/lib/authentications.py | 4 +- microservice/sysom_hotfix/apps/hotfix/apps.py | 9 +- microservice/sysom_hotfix/config.yml | 23 +-- .../sysom_migration/apps/migration/apps.py | 11 +- microservice/sysom_migration/config.yml | 27 +-- .../app/routeres/services.py | 20 +++ .../sysom_monitor_server/conf/common.py | 4 +- microservice/sysom_monitor_server/config.yml | 33 ++-- microservice/sysom_monitor_server/main.py | 10 +- microservice/sysom_vmcore/apps/vmcore/apps.py | 13 +- microservice/sysom_vmcore/config.yml | 23 +-- .../sysom_vmcore/scripts/server_init.sh | 4 + microservice/sysom_vul/config.yml | 23 +-- tools/deploy/deploy.sh | 1 - 57 files changed, 1415 insertions(+), 251 deletions(-) create mode 100644 infrastructure/1_sdk/gcache_base/__init__.py create mode 100644 infrastructure/1_sdk/gcache_base/exceptions.py create mode 100644 infrastructure/1_sdk/gcache_base/gcache.py create mode 100644 infrastructure/1_sdk/gcache_base/url.py create mode 100644 infrastructure/1_sdk/gcache_redis/__init__.py create mode 100644 infrastructure/1_sdk/gcache_redis/common.py create mode 100644 infrastructure/1_sdk/gcache_redis/redis_gcache.py create mode 100644 infrastructure/1_sdk/gcache_redis/utils.py create mode 100644 infrastructure/1_sdk/gclient_base/__init__.py create mode 100644 infrastructure/1_sdk/gclient_base/exceptions.py create mode 100644 infrastructure/1_sdk/gclient_base/gclient.py create mode 100644 infrastructure/1_sdk/gclient_base/url.py create mode 100644 infrastructure/1_sdk/gclient_cmg/__init__.py create mode 100644 infrastructure/1_sdk/gclient_cmg/cmg_gclient.py create mode 100644 infrastructure/1_sdk/gclient_cmg/common.py create mode 100644 infrastructure/1_sdk/gclient_http/__init__.py create mode 100644 infrastructure/1_sdk/gclient_http/common.py create mode 100644 infrastructure/1_sdk/gclient_http/http_gclient.py create mode 100644 infrastructure/1_sdk/setup_gcache_base.py create mode 100644 infrastructure/1_sdk/setup_gcache_redis.py create mode 100644 infrastructure/1_sdk/setup_gclient_base.py create mode 100644 infrastructure/1_sdk/setup_gclient_cmg.py create mode 100644 infrastructure/1_sdk/setup_gclient_http.py create mode 100644 infrastructure/1_sdk/sysom_utils/framework.py diff --git a/conf/config.yml b/conf/config.yml index de437335..9cb24e9d 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -10,7 +10,7 @@ sysom_server: root_path: !concat &sysom_server_root_path [ *global_root_path, - "/server/target/sysom_server", + "/server", ] db: redis: @@ -57,10 +57,21 @@ sysom_server: format: "%(asctime)s | %(levelname)s | %(message)s" level: "INFO" +sysom_service: + path: + root_path: !concat [*global_root_path, "/server/microservice"] + framework: + gclient: + protocol: cmg + special_param: + cmg_protocol: redis + cmg_load_balance_strategy: RANDOM + cmg_fetch_interval: 5 + # Unified configuration of Web sysom_web: path: - root_path: !concat [*global_root_path, "/server/target/sysom_web"] + root_path: !concat [*global_root_path, "/web"] # Unified configuration of Node sysom_node: diff --git a/infrastructure/1_sdk/channel_job/job.py b/infrastructure/1_sdk/channel_job/job.py index d343ef20..c8c1ed2d 100644 --- a/infrastructure/1_sdk/channel_job/job.py +++ b/infrastructure/1_sdk/channel_job/job.py @@ -19,6 +19,7 @@ from cec_base.event import Event from cec_base.url import CecUrl from cec_base.utils import StoppableThread from cec_base.exceptions import TopicAlreadyExistsException +from gclient_base import GClient, dispatch_g_client from .model import JobEntry, JobResult from .exception import ChannelJobException from .exception import ChannelJobException @@ -168,14 +169,14 @@ class ChannelFileJob: } def __init__( - self, base_url: str, + self, g_client: GClient, opt: str = "send-file", local_path: str = "", remote_path: str = "", instances: List[str] = [], instance: str = "", ) -> None: - self.base_url: str = base_url + self.g_client = g_client self.opt: str = opt self.local_path: str = local_path self.remote_path: str = remote_path @@ -183,7 +184,6 @@ class ChannelFileJob: self.instance = instance def _send_file(self) -> JobResult: - url = f"{self.base_url}{self._OPT_TABLE[self.opt]}" payload = { "target_path": self.remote_path, "target_instances": ";".join(self.instances) @@ -195,8 +195,8 @@ class ChannelFileJob: headers = { 'User-Agent': 'sysom_channel_job/1.0.0' } - response = requests.request( - "POST", url, headers=headers, data=payload, files=files) + response = self.g_client.post( + self._OPT_TABLE[self.opt], headers=headers, data=payload, files=files) if response.status_code != 200: return JobResult( code=1, @@ -220,7 +220,7 @@ class ChannelFileJob: 'User-Agent': 'sysom_channel_job/1.0.0', 'Content-Type': 'application/json' } - with requests.get(url, headers=headers, data=json.dumps(payload)) as r: + with self.g_client.get(self._OPT_TABLE[self.opt], headers=headers, data=json.dumps(payload)) as r: with open(self.local_path, "wb") as f: for chunk in r.iter_content(chunk_size=1024*1024): if chunk: @@ -260,7 +260,7 @@ class ChannelJobExecutor: self._admin: Optional[Admin] = None self._target_topic = "" self._listen_topic = "" - self._channel_base_url = "" + self._g_client: Optional[GClient] = None self._auto_recover = True self._job_mapper: Dict[str, ChannelJob] = { @@ -291,10 +291,10 @@ class ChannelJobExecutor: raise ChannelJobException(e) def init_config( - self, cec_url: str, channel_base_url: str = "http://127.0.0.1:7003" + self, url: str, g_client: GClient = dispatch_g_client("http://127.0.0.1:7003"), ): - cec_url = CecUrl.parse(cec_url) - self._channel_base_url = channel_base_url + cec_url = CecUrl.parse(url) + self._g_client = g_client # 1. Check require params self._target_topic = cec_url.params.pop( @@ -310,7 +310,8 @@ class ChannelJobExecutor: "channel_job_auto_recover", True ) if None in [self._target_topic, self._listen_topic, listen_consumer_group]: - raise (ChannelJobException(f"CecUrl missing parameters: {cec_url}")) + raise (ChannelJobException( + f"CecUrl missing parameters: {cec_url}")) # 2. Create Consumer, Producer, Admin instance self._consumer: Consumer = dispatch_consumer( @@ -375,7 +376,9 @@ class ChannelJobExecutor: return channel_job def dispatch_file_job(self, opt: str = "send-file", params: dict = {}) -> ChannelFileJob: - return ChannelFileJob(base_url=self._channel_base_url, opt=opt, **params) + if self._g_client is None: + raise ChannelJobException("GClient is None") + return ChannelFileJob(g_client=self._g_client, opt=opt, **params) def _deal_received_event(self, event: Event): job_result = JobResult.parse_by_cec_event_value(event.value) diff --git a/infrastructure/1_sdk/gcache_base/__init__.py b/infrastructure/1_sdk/gcache_base/__init__.py new file mode 100644 index 00000000..42068fcc --- /dev/null +++ b/infrastructure/1_sdk/gcache_base/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 14:45 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py.py +Description: +""" + +from .exceptions import * +from .gcache import * +from .url import * diff --git a/infrastructure/1_sdk/gcache_base/exceptions.py b/infrastructure/1_sdk/gcache_base/exceptions.py new file mode 100644 index 00000000..d1aeac39 --- /dev/null +++ b/infrastructure/1_sdk/gcache_base/exceptions.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 14:55 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File exceptions.py +Description: +""" + + +class GCacheException(Exception): + """GCache base exception + + This class defines the base exception for GCache, and all exceptions thrown + by GCache should inherit from this class. + """ + + +class GCacheProtoAlreadyExistsException(GCacheException): + """Exceptions thrown for duplicate proto(submodule) registration + + This exception should be thrown if the proto(submodule) already exists while + registering a submodule. + """ + + +class GCacheProtoNotExistsException(GCacheException): + """Exceptions thrown for trying to use a non-existent proto(submodule) + + Exceptions that will be thrown when trying to use a non-existent proto + (submodule). + """ + + +class GCacheNotValidGCacheUrlException(GCacheException): + """Exception thrown when an invalid GCacheUrl format is parsed.""" diff --git a/infrastructure/1_sdk/gcache_base/gcache.py b/infrastructure/1_sdk/gcache_base/gcache.py new file mode 100644 index 00000000..c6b0a947 --- /dev/null +++ b/infrastructure/1_sdk/gcache_base/gcache.py @@ -0,0 +1,168 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 14:47 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File gcache.py +Description: +""" +import importlib +from abc import ABCMeta, abstractmethod +from typing import Union, Optional +from clogger import logger +from .exceptions import GCacheProtoAlreadyExistsException, \ + GCacheProtoNotExistsException, GCacheException +from .url import GCacheUrl + + +class GCache(metaclass=ABCMeta): + """ + A generic cache used to caching data + """ + + proto_dict = {} + + @abstractmethod + def store(self, key: str, value: Union[int, float, dict, str], + expire: int = -1) -> bool: + """Store something to gcache + + Args: + key: key + value: Value, can be one of [int, float, dict, str] + expire: Expire time, the stored data will expire after A seconds + - -1 means never expires + Returns: + + """ + pass + + @abstractmethod + def load(self, key: str) -> Union[None, int, float, dict, str]: + pass + + @abstractmethod + def delete(self, key: str) -> bool: + """ + Delete specific key + Args: + key: + + Returns: + + """ + pass + + @abstractmethod + def clean(self): + """ + Clean all cache + Returns: + + """ + pass + + def load_int(self, key: str) -> Optional[int]: + res = self.load(key) + if res is None or isinstance(res, int): + return res + else: + raise GCacheException(f"GCache: expect int, get {type(res)}") + + def load_float(self, key: str) -> Optional[float]: + res = self.load(key) + if res is None or isinstance(res, float): + return res + else: + raise GCacheException(f"GCache: expect float, get {type(res)}") + + def load_dict(self, key: str) -> Optional[dict]: + res = self.load(key) + if res is None or isinstance(res, dict): + return res + else: + raise GCacheException(f"GCache: expect dict, get {type(res)}") + + def load_str(self, key: str) -> Optional[str]: + res = self.load(key) + if res is None or isinstance(res, str): + return res + else: + raise GCacheException(f"GCache: expect str, get {type(res)}") + + @staticmethod + def protocol_register(proto, sub_class): + """Register one new protocol => indicate one execution module + + Register a new protocol => This function is called by the executing + module to register its own implementation of Producer for the executing + module to take effect. + (Usually when the execution module is implemented according to the + specification, there is no need for the developer to call this method + manually, the abstraction layer will dynamically import) + + Args: + proto(str): Protocol identification + sub_class: Implementation class of Producer + + Returns: + + Examples: + >>> GCache.protocol_register('redis', RedisGCache) + + """ + if proto in GCache.proto_dict: + err = GCacheProtoAlreadyExistsException( + f"Proto '{proto}' already exists in Cmg-base-GCache." + ) + logger.error(err) + raise err + GCache.proto_dict[proto] = sub_class + logger.info( + f"Gcache-base-GCache register proto '{proto}' success" + ) + + +def dispatch_g_cache(cache_name: str, url: str, **kwargs) -> GCache: + """Construct one GCache instance according the url + + Construct a GCache instance of the corresponding type based on + the URL passed in. + + Args: + cache_name: + url(str): GCacheUrl + + Returns: + GCache: one GCache instance + + Examples: + >>> g_cache = dispatch_g_cache( + "cache_name", + ..."redis://localhost:6379?password=123456") + """ + cmg_url = GCacheUrl.parse(url) + if cmg_url.proto not in GCache.proto_dict: + # Check if dynamic import is possible + target_module = f"gcache_{cmg_url.proto}.{cmg_url.proto}_gcache" + try: + module = importlib.import_module(target_module) + GCache.protocol_register( + cmg_url.proto, + getattr(module, f'{cmg_url.proto.capitalize()}GCache') + ) + except ModuleNotFoundError as exc: + logger.error( + f"Try to auto import module {target_module} failed." + ) + raise GCacheProtoNotExistsException( + f"Proto '{cmg_url.proto}' not exists in GCache-base" + "-GCache." + ) from exc + g_cache_instance = GCache.proto_dict[cmg_url.proto](cache_name, + cmg_url, **kwargs) + logger.info( + f"GCache-base-GCache dispatch one GCache instance " + f"success. proto={cmg_url.proto}, url={url}" + ) + return g_cache_instance diff --git a/infrastructure/1_sdk/gcache_base/url.py b/infrastructure/1_sdk/gcache_base/url.py new file mode 100644 index 00000000..d8945189 --- /dev/null +++ b/infrastructure/1_sdk/gcache_base/url.py @@ -0,0 +1,66 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/3/17 19:43 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File url.py +Description: +""" + +import urllib.parse +from .exceptions import GCacheNotValidGCacheUrlException + + +class GCacheUrl: + """GCacheUrl definition + + GCacheUrl URL format definition, which consists of three parts + (proto, netloc, params) + + Args: + proto(str): Protocol identifier (e.g., redis) + netloc(str): Connection address, mainly used to connect to low-level + messaging middleware (e.g., localhost:6379) + params(dict): Connection parameters (e.g., {"password": "123456"}) + + Attributes: + proto(str): Protocol identifier (e.g., redis) + netloc(str): Connection address, mainly used to connect to low-level + messaging middleware (e.g., localhost:6379) + params(dict): Connection parameters (e.g., {"password": "123456"}) + """ + + def __init__(self, proto: str, netloc: str, params: dict): + self.proto = proto + self.netloc = netloc + self.params = params + + def __str__(self): + query = "&".join([f'{k}={v}' for k, v in self.params.items()]) + return f"{self.proto}://{self.netloc}?" \ + f"{query}" + + @staticmethod + def parse(url: str): + """Parses a string into a GCacheUrl object + + Args: + url(str) + + Returns: + GCacheUrl + """ + parse_result = urllib.parse.urlparse(url, allow_fragments=False) + proto, netloc = parse_result.scheme, parse_result.netloc + query_str, params = parse_result.query, {} + if proto == '' or netloc == '': + raise GCacheNotValidGCacheUrlException(url) + for param in query_str.split('&'): + if param.strip() == '': + continue + param_split = param.split('=') + if len(param_split) != 2: + raise GCacheNotValidGCacheUrlException( + f"params error: {param}, url: {url}") + params[param_split[0]] = param_split[1] + return GCacheUrl(proto, netloc, params) diff --git a/infrastructure/1_sdk/gcache_redis/__init__.py b/infrastructure/1_sdk/gcache_redis/__init__.py new file mode 100644 index 00000000..0c4dc4fe --- /dev/null +++ b/infrastructure/1_sdk/gcache_redis/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 15:07 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py.py +Description: +""" + +name = "gcache_redis" + +from .redis_gcache import * diff --git a/infrastructure/1_sdk/gcache_redis/common.py b/infrastructure/1_sdk/gcache_redis/common.py new file mode 100644 index 00000000..a40f7139 --- /dev/null +++ b/infrastructure/1_sdk/gcache_redis/common.py @@ -0,0 +1,93 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/7/29 13:33 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" +from typing import Optional +from clogger import logger +from redis import Redis +from gcache_base import GCacheUrl +from .utils import do_connect_by_gcache_url + + +class StaticConst: + """Static consts + + This class defines all the static constant values in the cmg-redis module + """ + # List of specialization parameters + REDIS_SPECIAL_PARM_G_CACHE_TABLE_NAME = \ + "table_name" + + _redis_special_parameter_list = [ + REDIS_SPECIAL_PARM_G_CACHE_TABLE_NAME + ] + + _redis_special_parameters_default_value = { + REDIS_SPECIAL_PARM_G_CACHE_TABLE_NAME: (str, True) + } + + @staticmethod + def parse_special_parameter(params: dict) -> dict: + """Parse specialization parameters + + Parse the specialization parameters and remove the specialization + parameters from the parameter list + + Args: + params(dict): CecUrl.params + + Returns: + + """ + res = {} + for key in StaticConst._redis_special_parameter_list: + _type, default = \ + StaticConst._redis_special_parameters_default_value[key] + res[key] = _type(params.pop(key, default)) + return res + + +class ClientBase: + """ + cec-redis client base class, Redis* requires inherit from this class, + which provides some generic implementation + """ + + def __init__(self, url: GCacheUrl): + self._redis_version = None + self._special_params = StaticConst.parse_special_parameter(url.params) + self.redis_client: Optional[Redis] = None + self.current_url = "" + self.connect_by_gcache_url(url) + + def get_special_param(self, key: str, default=''): + """Get specialization parameter by key + + Args: + key(str): specialization parameter key + default(Any): default value if key not exists + + Returns: + + """ + return self._special_params.get(key, default) + + def connect_by_gcache_url(self, url: GCacheUrl): + """Connect to redis server by CmgUrl + + Connecting to the Redis server via CmgUrl + + Args: + url(str): CmgUrl + """ + logger.debug( + f"{self} try to connect to '{url}'.") + self.redis_client = do_connect_by_gcache_url(url) + self.current_url = str(url) + logger.info( + f"{self} connect to '{url}' successfully.") + return self diff --git a/infrastructure/1_sdk/gcache_redis/redis_gcache.py b/infrastructure/1_sdk/gcache_redis/redis_gcache.py new file mode 100644 index 00000000..11058420 --- /dev/null +++ b/infrastructure/1_sdk/gcache_redis/redis_gcache.py @@ -0,0 +1,86 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 15:07 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File redis_gcache.py +Description: +""" +import json +from typing import Union +from gcache_base import GCache, GCacheUrl, GCacheException +from redis_lua import XRedisHashTable +from .common import ClientBase, StaticConst + +SEPARATOR = "_GCache_" + + +class RedisGCache(GCache, ClientBase): + """ + A redis-based gcache implement + """ + + def __init__(self, cache_name, url: GCacheUrl): + GCache.__init__(self) + ClientBase.__init__(self, url) + self._x_redis_hash_table = XRedisHashTable(self.redis_client) + self._table_name = cache_name + + @staticmethod + def _to_str(value: Union[int, float, dict, str]): + if isinstance(value, int): + return str(value) + elif isinstance(value, float): + return str(value) + elif isinstance(value, str): + return value + elif isinstance(value, dict): + return json.dumps(value) + + @staticmethod + def _get_store_value(value: Union[int, float, dict, str]): + return f"{type(value).__name__}{SEPARATOR}{RedisGCache._to_str(value)}" + + @staticmethod + def _from_str(type_v: str, value_str: str) -> Union[int, float, dict, str]: + if type_v == "int": + return int(value_str) + elif type_v == "float": + return float(value_str) + elif type_v == "str": + return value_str + elif type_v == "dict": + return json.loads(value_str) + + def store(self, key: str, value: Union[int, float, dict, str], + expire: int = -1) -> bool: + return self._x_redis_hash_table.hset( + self._table_name, key, + f"{self._get_store_value(value)}", + expire=expire + ) + + def load(self, key: str) -> Union[None, int, float, dict, str]: + res = self._x_redis_hash_table.hget(self._table_name, key) + if res is None: + return None + type_value = res.split(SEPARATOR) + if len(type_value) < 2: + raise GCacheException( + f"Load failed, expect value which is {res} start " + f"with {SEPARATOR}, " + ) + type_v = type_value[0] + value = "".join(type_value[1:]) + if type_v not in ["int", "float", "dict", "str"]: + raise GCacheException( + f"Got not supported type = {type_v}, expect one of " + f"[int, float, dict, str]" + ) + return self._from_str(type_v, value) + + def clean(self): + self._x_redis_hash_table.hdrop_table(self._table_name) + + def delete(self, key: str) -> bool: + return self._x_redis_hash_table.hdel(self._table_name, key) diff --git a/infrastructure/1_sdk/gcache_redis/utils.py b/infrastructure/1_sdk/gcache_redis/utils.py new file mode 100644 index 00000000..688b7fbe --- /dev/null +++ b/infrastructure/1_sdk/gcache_redis/utils.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 15:17 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File utils.py +Description: +""" +from gcache_base import GCacheUrl, GCacheException +from redis import Redis + + +def do_connect_by_gcache_url(cec_url: GCacheUrl) -> Redis: + host_port = cec_url.netloc.split(":") + if len(host_port) != 2: + raise GCacheException( + f"Not valid host:port => {host_port[0]}:{host_port[1]}") + host, port = host_port[0], int(host_port[1]) + try: + redis_client = Redis(host=host, port=port, db=0, decode_responses=True, + **cec_url.params) + except ConnectionError as e: + raise GCacheException(e) + return redis_client diff --git a/infrastructure/1_sdk/gclient_base/__init__.py b/infrastructure/1_sdk/gclient_base/__init__.py new file mode 100644 index 00000000..01eba34f --- /dev/null +++ b/infrastructure/1_sdk/gclient_base/__init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 14:45 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py.py +Description: +""" + +from .exceptions import * +from .gclient import * +from .url import * diff --git a/infrastructure/1_sdk/gclient_base/exceptions.py b/infrastructure/1_sdk/gclient_base/exceptions.py new file mode 100644 index 00000000..6a2b36b8 --- /dev/null +++ b/infrastructure/1_sdk/gclient_base/exceptions.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 14:55 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File exceptions.py +Description: +""" + + +class GClientException(Exception): + """GClient base exception + + This class defines the base exception for GClient, and all exceptions thrown + by GClient should inherit from this class. + """ + + +class GClientProtoAlreadyExistsException(GClientException): + """Exceptions thrown for duplicate proto(submodule) registration + + This exception should be thrown if the proto(submodule) already exists while + registering a submodule. + """ + + +class GClientProtoNotExistsException(GClientException): + """Exceptions thrown for trying to use a non-existent proto(submodule) + + Exceptions that will be thrown when trying to use a non-existent proto + (submodule). + """ + + +class GClientNotValidGClientUrlException(GClientException): + """Exception thrown when an invalid GClientUrl format is parsed.""" diff --git a/infrastructure/1_sdk/gclient_base/gclient.py b/infrastructure/1_sdk/gclient_base/gclient.py new file mode 100644 index 00000000..90ac7014 --- /dev/null +++ b/infrastructure/1_sdk/gclient_base/gclient.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/5/8 19:51 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File gclient.py +Description: +""" +import requests +import importlib +from clogger import logger +from abc import ABCMeta, abstractmethod +from .exceptions import GClientProtoAlreadyExistsException, \ + GClientProtoNotExistsException +from .url import GClientUrl + + +class GClient(metaclass=ABCMeta): + proto_dict = {} + + @abstractmethod + def get(self, path: str, params=None, + **kwargs) -> requests.Response: + pass + + @abstractmethod + def options(self, path: str, **kwargs) -> requests.Response: + pass + + @abstractmethod + def head(self, path: str, **kwargs) -> requests.Response: + pass + + @abstractmethod + def post(self, path: str, data=None, json=None, + **kwargs) -> requests.Response: + pass + + @abstractmethod + def put(self, path: str, data=None, + **kwargs) -> requests.Response: + pass + + @abstractmethod + def patch(self, path: str, data=None, + **kwargs) -> requests.Response: + pass + + @abstractmethod + def delete(self, path: str, **kwargs) -> requests.Response: + pass + + @staticmethod + def protocol_register(proto, sub_class): + """Register one new protocol => indicate one execution module + + Register a new protocol => This function is called by the executing + module to register its own implementation of Producer for the executing + module to take effect. + (Usually when the execution module is implemented according to the + specification, there is no need for the developer to call this method + manually, the abstraction layer will dynamically import) + + Args: + proto(str): Protocol identification + sub_class: Implementation class of Producer + + Returns: + + Examples: + >>> GClient.protocol_register('redis', HttpGClient) + + """ + if proto in GClient.proto_dict: + err = GClientProtoAlreadyExistsException( + f"Proto '{proto}' already exists in Cmg-base-GClient." + ) + logger.error(err) + raise err + GClient.proto_dict[proto] = sub_class + logger.info( + f"GClient-base-GClient register proto '{proto}' success" + ) + + +def dispatch_g_client(url: str, **kwargs) -> GClient: + """Construct one GClient instance according the url + + Construct a GClient instance of the corresponding type based on + the URL passed in. + + Args: + url(str): GClientUrl + + Returns: + GClient: one GClient instance + + Examples: + >>> g_client = dispatch_g_client( + "cache_name", + ..."http://localhost:6379") + """ + g_client_url = GClientUrl.parse(url) + if g_client_url.proto not in GClient.proto_dict: + # Check if dynamic import is possible + target_module = f"gclient_{g_client_url.proto}.{g_client_url.proto}_gclient" + try: + module = importlib.import_module(target_module) + GClient.protocol_register( + g_client_url.proto, + getattr(module, f'{g_client_url.proto.capitalize()}GClient') + ) + except ModuleNotFoundError as exc: + logger.error( + f"Try to auto import module {target_module} failed." + ) + raise GClientProtoNotExistsException( + f"Proto '{g_client_url.proto}' not exists in GClient-base" + "-GClient." + ) from exc + g_client_instance = GClient.proto_dict[g_client_url.proto](g_client_url, + **kwargs) + logger.info( + f"GClient-base-GClient dispatch one GClient instance " + f"success. proto={g_client_url.proto}, url={url}" + ) + return g_client_instance diff --git a/infrastructure/1_sdk/gclient_base/url.py b/infrastructure/1_sdk/gclient_base/url.py new file mode 100644 index 00000000..be7cf3b6 --- /dev/null +++ b/infrastructure/1_sdk/gclient_base/url.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/3/17 19:43 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File url.py +Description: +""" + +import urllib.parse +from .exceptions import GClientNotValidGClientUrlException + + +class GClientUrl: + """GClientUrl definition + + GClientUrl URL format definition, which consists of three parts + (proto, netloc, params) + + Args: + proto(str): Protocol identifier (e.g., redis) + netloc(str): Connection address, mainly used to connect to low-level + messaging middleware (e.g., localhost:6379) + params(dict): Connection parameters (e.g., {"password": "123456"}) + + Attributes: + proto(str): Protocol identifier (e.g., redis) + netloc(str): Connection address, mainly used to connect to low-level + messaging middleware (e.g., localhost:6379) + params(dict): Connection parameters (e.g., {"password": "123456"}) + """ + + def __init__(self, proto: str, netloc: str, path: str, params: dict): + self.proto = proto + self.netloc = netloc + self.params = params + self.path = path + + def __str__(self): + query = "&".join([f'{k}={v}' for k, v in self.params.items()]) + return f"{self.proto}://{self.netloc}{self.path}?" \ + f"{query}" + + @staticmethod + def parse(url: str): + """Parses a string into a GClientUrl object + + Args: + url(str) + + Returns: + GClientUrl + """ + parse_result = urllib.parse.urlparse(url, allow_fragments=False) + proto, netloc = parse_result.scheme, parse_result.netloc + query_str, params = parse_result.query, {} + if proto == '' or netloc == '': + raise GClientNotValidGClientUrlException(url) + for param in query_str.split('&'): + if param.strip() == '': + continue + param_split = param.split('=') + if len(param_split) != 2: + raise GClientNotValidGClientUrlException( + f"params error: {param}, url: {url}") + params[param_split[0]] = param_split[1] + return GClientUrl(proto, netloc, parse_result.path, params) diff --git a/infrastructure/1_sdk/gclient_cmg/__init__.py b/infrastructure/1_sdk/gclient_cmg/__init__.py new file mode 100644 index 00000000..4feff59b --- /dev/null +++ b/infrastructure/1_sdk/gclient_cmg/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 14:45 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py.py +Description: +""" + +from .cmg_gclient import * diff --git a/infrastructure/1_sdk/gclient_cmg/cmg_gclient.py b/infrastructure/1_sdk/gclient_cmg/cmg_gclient.py new file mode 100644 index 00000000..dbc4ea58 --- /dev/null +++ b/infrastructure/1_sdk/gclient_cmg/cmg_gclient.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/5/9 10:28 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File cmg_gclient.py +Description: +""" +import requests +from urllib.parse import urljoin +from gclient_base import GClient, GClientUrl, GClientException +from cmg_base import dispatch_service_discovery, LoadBalancingStrategy +from .common import StaticConst + + +class CmgGClient(GClient): + def __init__(self, url: GClientUrl): + special_params = StaticConst.parse_special_parameter(url.params) + cmg_protocol = special_params.get( + StaticConst.SPECIAL_PARM_CMG_PROTOCOL, "redis") + cmg_service_name = special_params.get( + StaticConst.SPECIAL_PARM_CMG_SERVICE_NAME, None + ) + cmg_load_balance_strategy = special_params.get( + StaticConst.SPECIAL_PARM_CMG_LOAD_BALANCE_STRATEGY, "RANDOM" + ) + cmg_fetch_interval = special_params.get( + StaticConst.SPECIAL_PARM_CMG_FETCH_INTERVAL, 5 + ) + params = [] + for k in url.params: + params.append(f"{k}={url.params[k]}") + cmg_url = f"{cmg_protocol}://{url.netloc}?{'&'.join(params)}" + if cmg_service_name is None: + raise GClientException( + f"CmgGClient: required " + f"{StaticConst.SPECIAL_PARM_CMG_SERVICE_NAME}") + + self.discover = dispatch_service_discovery( + cmg_url, fetch_interval=cmg_fetch_interval + ) + self.load_balance_strategy = self._get_load_balance_strategy( + cmg_load_balance_strategy + ) + self.service_name = cmg_service_name + + @staticmethod + def _get_load_balance_strategy(strategy_str: str) -> LoadBalancingStrategy: + strategies = LoadBalancingStrategy.__members__.keys() + if strategy_str not in strategies: + raise GClientException( + f"CmgGClient: Not support strategy => '{strategy_str}'" + ) + return LoadBalancingStrategy[strategy_str] + + def _get_http_base_url(self): + instance = self.discover.get_instance(self.service_name, + self.load_balance_strategy) + # TODO: Support automatic recognition of http and https + return f"http://{instance.host}:{instance.port}" + + def get(self, path: str, params=None, **kwargs) -> requests.Response: + url = urljoin(self._get_http_base_url(), path) + return requests.get(url, params, **kwargs) + + def options(self, path: str, **kwargs) -> requests.Response: + url = urljoin(self._get_http_base_url(), path) + return requests.options(url, **kwargs) + + def head(self, path: str, **kwargs) -> requests.Response: + url = urljoin(self._get_http_base_url(), path) + return requests.head(url, **kwargs) + + def post(self, path: str, data=None, json=None, + **kwargs) -> requests.Response: + url = urljoin(self._get_http_base_url(), path) + return requests.post(url, data, json, **kwargs) + + def put(self, path: str, data=None, **kwargs) -> requests.Response: + url = urljoin(self._get_http_base_url(), path) + return requests.put(url, data, **kwargs) + + def patch(self, path: str, data=None, **kwargs) -> requests.Response: + url = urljoin(self._get_http_base_url(), path) + return requests.patch(url, data, **kwargs) + + def delete(self, path: str, **kwargs) -> requests.Response: + url = urljoin(self._get_http_base_url(), path) + return requests.delete(url, **kwargs) diff --git a/infrastructure/1_sdk/gclient_cmg/common.py b/infrastructure/1_sdk/gclient_cmg/common.py new file mode 100644 index 00000000..88ac6a52 --- /dev/null +++ b/infrastructure/1_sdk/gclient_cmg/common.py @@ -0,0 +1,58 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/7/29 13:33 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" + + +class StaticConst: + """Static consts + + This class defines all the static constant values in the cmg-redis module + """ + # List of specialization parameters + SPECIAL_PARM_CMG_PROTOCOL = \ + "cmg_protocol" + SPECIAL_PARM_CMG_SERVICE_NAME = \ + "cmg_service_name" + SPECIAL_PARM_CMG_LOAD_BALANCE_STRATEGY = \ + "cmg_load_balance_strategy" + SPECIAL_PARM_CMG_FETCH_INTERVAL = \ + "cmg_fetch_interval" + + _special_parameter_list = [ + SPECIAL_PARM_CMG_PROTOCOL, + SPECIAL_PARM_CMG_SERVICE_NAME, + SPECIAL_PARM_CMG_LOAD_BALANCE_STRATEGY, + SPECIAL_PARM_CMG_FETCH_INTERVAL + ] + + _special_parameters_default_value = { + SPECIAL_PARM_CMG_PROTOCOL: (str, "redis"), + SPECIAL_PARM_CMG_SERVICE_NAME: (str, None), + SPECIAL_PARM_CMG_LOAD_BALANCE_STRATEGY: (str, "RANDOM"), + SPECIAL_PARM_CMG_FETCH_INTERVAL: (int, 5) + } + + @classmethod + def parse_special_parameter(cls, params: dict) -> dict: + """Parse specialization parameters + + Parse the specialization parameters and remove the specialization + parameters from the parameter list + + Args: + params(dict): CecUrl.params + + Returns: + + """ + res = {} + for key in cls._special_parameter_list: + _type, default = \ + cls._special_parameters_default_value[key] + res[key] = _type(params.pop(key, default)) + return res diff --git a/infrastructure/1_sdk/gclient_http/__init__.py b/infrastructure/1_sdk/gclient_http/__init__.py new file mode 100644 index 00000000..65cc45ec --- /dev/null +++ b/infrastructure/1_sdk/gclient_http/__init__.py @@ -0,0 +1,10 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/4/28 14:45 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py.py +Description: +""" + +from .http_gclient import * diff --git a/infrastructure/1_sdk/gclient_http/common.py b/infrastructure/1_sdk/gclient_http/common.py new file mode 100644 index 00000000..969b2dce --- /dev/null +++ b/infrastructure/1_sdk/gclient_http/common.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/7/29 13:33 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" + + +class StaticConst: + """Static consts + + This class defines all the static constant values in the cmg-redis module + """ + # List of specialization parameters + SPECIAL_PARM_TLS = \ + "tls" + + _special_parameter_list = [ + SPECIAL_PARM_TLS + ] + + _special_parameters_default_value = { + SPECIAL_PARM_TLS: (bool, False) + } + + @classmethod + def parse_special_parameter(cls, params: dict) -> dict: + """Parse specialization parameters + + Parse the specialization parameters and remove the specialization + parameters from the parameter list + + Args: + params(dict): CecUrl.params + + Returns: + + """ + res = {} + for key in cls._special_parameter_list: + _type, default = \ + cls._special_parameters_default_value[key] + res[key] = _type(params.pop(key, default)) + return res diff --git a/infrastructure/1_sdk/gclient_http/http_gclient.py b/infrastructure/1_sdk/gclient_http/http_gclient.py new file mode 100644 index 00000000..2509b3e0 --- /dev/null +++ b/infrastructure/1_sdk/gclient_http/http_gclient.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/5/9 10:28 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File http_gclient.py +Description: +""" +import requests +from urllib.parse import urljoin +from gclient_base import GClient, GClientUrl +from .common import StaticConst + + +class HttpGClient(GClient): + def __init__(self, url: GClientUrl): + special_params = StaticConst.parse_special_parameter(url.params) + tls = special_params.get("tls", False) + protocol = "https" if tls else "http" + self._base_url = f"{protocol}://{url.netloc}{url.path}" + + def get(self, path: str, params=None, **kwargs) -> requests.Response: + return requests.get( + urljoin(self._base_url, path), params, **kwargs + ) + + def options(self, path: str, **kwargs) -> requests.Response: + return requests.options( + urljoin(self._base_url, path), **kwargs + ) + + def head(self, path: str, **kwargs) -> requests.Response: + return requests.head( + urljoin(self._base_url, path), **kwargs + ) + + def post(self, path: str, data=None, json=None, + **kwargs) -> requests.Response: + return requests.post( + urljoin(self._base_url, path), data, json, **kwargs + ) + + def put(self, path: str, data=None, **kwargs) -> requests.Response: + return requests.put( + urljoin(self._base_url, path), data, **kwargs + ) + + def patch(self, path: str, data=None, **kwargs) -> requests.Response: + return requests.patch( + urljoin(self._base_url, path), data, **kwargs + ) + + def delete(self, path: str, **kwargs) -> requests.Response: + return requests.delete( + urljoin(self._base_url, path), **kwargs + ) diff --git a/infrastructure/1_sdk/init.sh b/infrastructure/1_sdk/init.sh index fd05380f..40e10df5 100755 --- a/infrastructure/1_sdk/init.sh +++ b/infrastructure/1_sdk/init.sh @@ -22,6 +22,11 @@ install_sdk() { python setup_sysom_utils.py develop python setup_cmg_base.py develop python setup_cmg_redis.py develop + python setup_gcache_base.py develop + python setup_gcache_redis.py develop + python setup_gclient_base.py develop + python setup_gclient_http.py develop + python setup_gclient_cmg.py develop popd } diff --git a/infrastructure/1_sdk/setup_gcache_base.py b/infrastructure/1_sdk/setup_gcache_base.py new file mode 100644 index 00000000..2840f1c3 --- /dev/null +++ b/infrastructure/1_sdk/setup_gcache_base.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="gcache_base", + version="0.0.1", + author="mingfeng(SunnyQjm)", + author_email="mfeng@linux.alibaba.com", + description="A generic cache base library that defines the basic functions that a generic cache has", + url="", + packages=setuptools.find_packages(), + install_requires=[ + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ] +) diff --git a/infrastructure/1_sdk/setup_gcache_redis.py b/infrastructure/1_sdk/setup_gcache_redis.py new file mode 100644 index 00000000..274cbc8a --- /dev/null +++ b/infrastructure/1_sdk/setup_gcache_redis.py @@ -0,0 +1,19 @@ +import setuptools + +setuptools.setup( + name="gcache_redis", + version="0.0.1", + author="mingfeng(SunnyQjm)", + author_email="mfeng@linux.alibaba.com", + description="A redis-based GCache implement", + url="", + packages=setuptools.find_packages(), + install_requires=[ + 'redis' + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ] +) diff --git a/infrastructure/1_sdk/setup_gclient_base.py b/infrastructure/1_sdk/setup_gclient_base.py new file mode 100644 index 00000000..03bdb8e4 --- /dev/null +++ b/infrastructure/1_sdk/setup_gclient_base.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="gclient_base", + version="0.0.1", + author="mingfeng(SunnyQjm)", + author_email="mfeng@linux.alibaba.com", + description="A generic http client lib", + url="", + packages=setuptools.find_packages(), + install_requires=[ + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ] +) diff --git a/infrastructure/1_sdk/setup_gclient_cmg.py b/infrastructure/1_sdk/setup_gclient_cmg.py new file mode 100644 index 00000000..1c48961a --- /dev/null +++ b/infrastructure/1_sdk/setup_gclient_cmg.py @@ -0,0 +1,19 @@ +import setuptools + +setuptools.setup( + name="gclient_cmg", + version="0.0.1", + author="mingfeng(SunnyQjm)", + author_email="mfeng@linux.alibaba.com", + description="A cmg-based http client lib", + url="", + packages=setuptools.find_packages(), + install_requires=[ + "cmg_base>=0.0.1" + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ] +) diff --git a/infrastructure/1_sdk/setup_gclient_http.py b/infrastructure/1_sdk/setup_gclient_http.py new file mode 100644 index 00000000..23543a3d --- /dev/null +++ b/infrastructure/1_sdk/setup_gclient_http.py @@ -0,0 +1,18 @@ +import setuptools + +setuptools.setup( + name="gclient_http", + version="0.0.1", + author="mingfeng(SunnyQjm)", + author_email="mfeng@linux.alibaba.com", + description="A requests-based http client lib", + url="", + packages=setuptools.find_packages(), + install_requires=[ + ], + classifiers=[ + "Programming Language :: Python :: 3", + "License :: OSI Approved :: MIT License", + "Operating System :: OS Independent", + ] +) diff --git a/infrastructure/1_sdk/sysom_utils/__init__.py b/infrastructure/1_sdk/sysom_utils/__init__.py index 107fedfd..147b161e 100644 --- a/infrastructure/1_sdk/sysom_utils/__init__.py +++ b/infrastructure/1_sdk/sysom_utils/__init__.py @@ -14,4 +14,5 @@ from .yaml_concat import * from .config_parser import * from .event_executor import * from .framework_plug_mag import * -from .framework_plugins import * \ No newline at end of file +from .framework_plugins import * +from .framework import * \ No newline at end of file diff --git a/infrastructure/1_sdk/sysom_utils/config_parser.py b/infrastructure/1_sdk/sysom_utils/config_parser.py index 461d299a..030dd984 100644 --- a/infrastructure/1_sdk/sysom_utils/config_parser.py +++ b/infrastructure/1_sdk/sysom_utils/config_parser.py @@ -68,6 +68,9 @@ class ConfigParser: self._overwrite_from_env() def _overwrite_from_env(self) -> None: + for env in os.environ: + if env.startswith("sysom"): + self._config.set_multi(env, os.environ[env]) for env, key_str in ENV_LIST.items(): if os.getenv(env): self._config.set_multi(key_str, os.getenv(env)) @@ -160,6 +163,49 @@ class ConfigParser: params.append(f"{k}={special_param[k]}") cmg_url += "&".join(params) return cmg_url + + def get_gcache_url(self) -> str: + service_config = self.get_service_config() + server_config = self.get_server_config() + gcache_config = service_config.framework.gcache + special_param = {} + params = [] + if gcache_config.special_param: + dict_merge(special_param, gcache_config.special_param) + gcache_url = "" + if gcache_config.protocol == "redis": + redis_config = server_config.db.redis + gcache_url = ( + f"{gcache_config.protocol}://{redis_config.host}:{redis_config.port}?" + ) + if redis_config.username: + params.append(f"username={redis_config.username}") + if redis_config.password: + params.append(f"password={redis_config.password}") + for k in special_param: + params.append(f"{k}={special_param[k]}") + gcache_url += "&".join(params) + return gcache_url + + def get_gclient_url(self, service_name: str) -> str: + service_config = self.get_service_config() + gclient_config = service_config.framework.gclient + special_param = {} + params = [] + if gclient_config.special_param: + dict_merge(special_param, gclient_config.special_param) + if service_name: + special_param["cmg_service_name"] = service_name + gclient_url = "" + if gclient_config.protocol == "cmg": + cmg_url = self.get_cmg_url() + gclient_url = f"cmg://{cmg_url.split('://')[1]}" + if not gclient_url.endswith("?"): + gclient_url += "&" + for k in special_param: + params.append(f"{k}={special_param[k]}") + gclient_url += "&".join(params) + return gclient_url def get_local_channel_job_url(self) -> str: server_config = self.get_server_config() diff --git a/infrastructure/1_sdk/sysom_utils/framework.py b/infrastructure/1_sdk/sysom_utils/framework.py new file mode 100644 index 00000000..b4ea13e8 --- /dev/null +++ b/infrastructure/1_sdk/sysom_utils/framework.py @@ -0,0 +1,83 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/05/08 14:13 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File framework.py +Description: +""" + +from typing import Optional, Dict, Type +from gcache_base import dispatch_g_cache, GCache +from channel_job.job import default_channel_job_executor +from gclient_base import dispatch_g_client, GClient +from .config_parser import ConfigParser +from .framework_plug_mag import FrameworkPlugMag, FrameworkPluginBase + +class SysomFrameworkException(Exception): + pass + +NOT_INIT_ERROR_TIP = "SysomFramework not init, please call SysomFramework.init first" +class SysomFramework: + _config: Optional[ConfigParser] = None + _gcache_map: Dict[str, GCache] = {} + _framework_plug_mag: Optional[FrameworkPlugMag] = None + + @classmethod + def init(cls, config: ConfigParser): + cls._config = config + cls._framework_plug_mag = FrameworkPlugMag(config) + return cls + + @classmethod + def gcache(cls, cache_name: str) -> GCache: + if cls._config is None: + raise SysomFrameworkException(NOT_INIT_ERROR_TIP) + if cache_name in cls._gcache_map: + return cls._gcache_map[cache_name] + else: + new_gcache = dispatch_g_cache(cache_name, cls._config.get_gcache_url()) + cls._gcache_map[cache_name] = new_gcache + return new_gcache + + @classmethod + def gclient(cls, service_name: str = "") -> GClient: + if cls._config is None: + raise SysomFrameworkException(NOT_INIT_ERROR_TIP) + return dispatch_g_client(cls._config.get_gclient_url(service_name)) + + @classmethod + def load_plugin(cls, plugin: FrameworkPluginBase): + if cls._framework_plug_mag is None: + raise SysomFrameworkException(NOT_INIT_ERROR_TIP) + cls._framework_plug_mag.load_plugin(plugin) + return cls + + @classmethod + def load_plugin_cls(cls, plugin_cls: Type[FrameworkPluginBase]): + if cls._framework_plug_mag is None: + raise SysomFrameworkException(NOT_INIT_ERROR_TIP) + cls._framework_plug_mag.load_plugin_cls(plugin_cls) + return cls + + + @classmethod + def enable_channel_job(cls): + if cls._config is None: + raise SysomFrameworkException(NOT_INIT_ERROR_TIP) + cls._config.get_cmg_url() + default_channel_job_executor.init_config( + cls._config.get_local_channel_job_url(), + cls.gclient("sysom_channel") + ) + return cls + + @classmethod + def start(cls): + cls._framework_plug_mag.start() + return cls + + def join(cls, timeout: Optional[float] = None): + cls._framework_plug_mag.join() + return cls + \ No newline at end of file diff --git a/infrastructure/1_sdk/sysom_utils/framework_plugins.py b/infrastructure/1_sdk/sysom_utils/framework_plugins.py index e71289dc..4802d6ac 100644 --- a/infrastructure/1_sdk/sysom_utils/framework_plugins.py +++ b/infrastructure/1_sdk/sysom_utils/framework_plugins.py @@ -7,9 +7,11 @@ File framework_plugins.py Description: """ from typing import Optional +from urllib.parse import urljoin from channel_job import ChannelJobExecutor from cmg_base import dispatch_service_registry, ServiceRegistry, \ ServiceInstance +from gclient_base import dispatch_g_client from .config_parser import ConfigParser from .event_executor import PluginEventExecutor from .framework_plug_mag import FrameworkPluginBase @@ -17,7 +19,7 @@ from .framework_plug_mag import FrameworkPluginBase class NodeDispatcherPlugin(FrameworkPluginBase): """Node Dispatcher Plugin - + NDP automatically implements the function of node file down and initialization by reading the configuration @@ -29,7 +31,8 @@ class NodeDispatcherPlugin(FrameworkPluginBase): self._config = config self._channel_job_executor = ChannelJobExecutor() self._channel_job_executor.init_config( - self._config.get_local_channel_job_url() + self._config.get_local_channel_job_url(), + dispatch_g_client(self._config.get_gclient_url("sysom_channel")) ) self._plugin_event_executor = PluginEventExecutor( config, self._channel_job_executor @@ -53,7 +56,19 @@ class CmgPlugin(FrameworkPluginBase): self._registry: ServiceRegistry = \ dispatch_service_registry(self._config.get_cmg_url()) self._service_instance: Optional[ServiceInstance] = None - cmg_plugin_config = self._config.get_service_config().plugins.cmg + service_config = self._config.get_service_config() + cmg_plugin_config = service_config.framework.cmg + if cmg_plugin_config.check.type == "http" and \ + (not cmg_plugin_config.check.url or + not cmg_plugin_config.check.url.startswith("http") + ): + cmg_plugin_config.check.url = urljoin( + f"{service_config.protocol}://{service_config.host}:{service_config.port}", + cmg_plugin_config.check.url + ) + cmg_plugin_config.service_name = service_config.service_name + cmg_plugin_config.host = service_config.host + cmg_plugin_config.port = service_config.port if cmg_plugin_config: self._service_instance = ServiceInstance(**dict(cmg_plugin_config)) diff --git a/infrastructure/1_sdk/sysom_utils/node_manager.py b/infrastructure/1_sdk/sysom_utils/node_manager.py index bad9db9f..59db1361 100644 --- a/infrastructure/1_sdk/sysom_utils/node_manager.py +++ b/infrastructure/1_sdk/sysom_utils/node_manager.py @@ -25,7 +25,6 @@ class NodeManagerException(Exception): class NodeManager: def __init__(self, config: ConfigParser, channel_job_executor: ChannelJobExecutor) -> None: self._config = config - self._server_config = config.get_server_config() self._service_config = config.get_service_config() self._node_config = config.get_node_config() self._channel_job_executor = channel_job_executor @@ -39,7 +38,7 @@ class NodeManager: def _get_service_root_path(self) -> str: return os.path.join( - self._server_config.path.root_path, + self._service_config.path.root_path, self._service_config.service_dir ) diff --git a/microservice/0_sysom_api/apps/accounts/apps.py b/microservice/0_sysom_api/apps/accounts/apps.py index 77784e56..be1b5aec 100644 --- a/microservice/0_sysom_api/apps/accounts/apps.py +++ b/microservice/0_sysom_api/apps/accounts/apps.py @@ -1,6 +1,6 @@ +import sys from clogger import logger from django.apps import AppConfig - from django.db.models.signals import post_migrate diff --git a/microservice/0_sysom_api/apps/accounts/authentication.py b/microservice/0_sysom_api/apps/accounts/authentication.py index 72adf8ab..5b6f5916 100644 --- a/microservice/0_sysom_api/apps/accounts/authentication.py +++ b/microservice/0_sysom_api/apps/accounts/authentication.py @@ -12,8 +12,7 @@ from rest_framework.authentication import BaseAuthentication from rest_framework.request import Request from apps.accounts.models import User from lib.authentications import decode_token -from django.core.cache import cache - +from sysom_utils import SysomFramework class Authentication(BaseAuthentication): def authenticate(self, request: Request): @@ -31,7 +30,7 @@ class Authentication(BaseAuthentication): payload = decode_token(token) user = self._check_user(payload=payload) # 判断用户是否已经手动注销登录 - if cache.get(token) is None: + if SysomFramework.gcache("JWT_TOKEN").load(token) is None: raise AuthenticationFailed('用户已退出登录!') logger.info(f"{user.username} 身份通过") diff --git a/microservice/0_sysom_api/apps/accounts/views.py b/microservice/0_sysom_api/apps/accounts/views.py index e078500c..898e7293 100644 --- a/microservice/0_sysom_api/apps/accounts/views.py +++ b/microservice/0_sysom_api/apps/accounts/views.py @@ -17,7 +17,7 @@ from . import models from . import serializer from lib.response import success, other_response from lib.exception import APIException -from django.core.cache import cache +from sysom_utils import SysomFramework class UserModelViewSet( GenericViewSet, @@ -160,18 +160,13 @@ class AccountAuthView(GenericViewSet): u_ser = serializer.UserListSerializer(instance=u, many=False) result = u_ser.data - cache_user_token = cache.get(t) - - if cache_user_token is not None: - result.update({"token": cache_user_token}) - else: - result.update({"token": t}) - cache.set(t, u.pk, timeout=settings.JWT_TOKEN_EXPIRE) + result.update({"token": t}) + SysomFramework.gcache("JWT_TOKEN").store(t, u.pk, expire=settings.JWT_TOKEN_EXPIRE) return other_response(message="登录成功", code=200, result=result) def logout(self, request: Request): """用户登出 清除缓存中的用户token信息""" - cache.delete(request.META.get('HTTP_AUTHORIZATION')) + SysomFramework.gcache("JWT_TOKEN").delete(request.META.get('HTTP_AUTHORIZATION')) return success(message='退出成功', result={}) def get_authenticators(self): diff --git a/microservice/0_sysom_api/apps/host/apps.py b/microservice/0_sysom_api/apps/host/apps.py index 8fccc152..c0ce1104 100644 --- a/microservice/0_sysom_api/apps/host/apps.py +++ b/microservice/0_sysom_api/apps/host/apps.py @@ -12,7 +12,7 @@ class HostConfig(AppConfig): def ready(self): if 'runserver' in sys.argv or 'manage.py' not in sys.argv: from django.conf import settings - from sysom_utils import FrameworkPlugMag, CmgPlugin + from sysom_utils import FrameworkPlugMag, CmgPlugin, SysomFramework # 这边微服务正式启动的时候执行一些处理代码 # 启动任务结果处理线程 default_channel_job_executor.init_config( @@ -27,7 +27,7 @@ class HostConfig(AppConfig): logger.warning(f'主机心跳未启动: {e}') # 启动框架插件系统 - FrameworkPlugMag(settings.YAML_CONFIG) \ + SysomFramework.init(settings.YAML_CONFIG) \ .load_plugin_cls(CmgPlugin) \ .start() diff --git a/microservice/0_sysom_api/conf/common.py b/microservice/0_sysom_api/conf/common.py index 29f75b96..ad4a65ab 100644 --- a/microservice/0_sysom_api/conf/common.py +++ b/microservice/0_sysom_api/conf/common.py @@ -1,9 +1,8 @@ from clogger import logger import os -import sys import datetime from pathlib import Path -from sysom_utils import ConfigParser, CecTarget +from sysom_utils import ConfigParser, CecTarget, SysomFramework BASE_DIR = Path(__file__).resolve().parent.parent @@ -16,6 +15,8 @@ YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) +SysomFramework.init(YAML_CONFIG) + ########################################################################################## # SysomAPI Service config ########################################################################################## @@ -76,21 +77,6 @@ DATABASES = { } } -CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": ( - f"redis://{YAML_CONFIG.get_server_config().db.redis.host}" - f":{YAML_CONFIG.get_server_config().db.redis.port}/1" - ), - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "CONNECTION_POOL_KWARGS": {"max_connections": 100}, - "DECODE_RESPONSES": True - } - } -} - ROOT_URLCONF = 'sysom.urls' AUTH_USER_MODEL = 'accounts.User' diff --git a/microservice/0_sysom_api/config.yml b/microservice/0_sysom_api/config.yml index 2a225cb0..03feb2c9 100644 --- a/microservice/0_sysom_api/config.yml +++ b/microservice/0_sysom_api/config.yml @@ -14,20 +14,19 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME - protocol: &SERVICE_PROTO http - host: &SERVICE_HOST 127.0.0.1 - port: &SERVICE_PORT 7001 + protocol: http + host: 127.0.0.1 + port: 7001 heartbeat: HEARTBEAT_INTERVAL: 20 app_host: # Host init timeout (seconds) HOST_INIT_TIMEOUT: 600 - plugins: + framework: + gcache: + protocol: redis node_dispatch: cmg: - service_name: *SERVICE_NAME - host: *SERVICE_HOST - port: *SERVICE_PORT tags: - Api - Django @@ -35,17 +34,9 @@ sysom_service: metadata: check: type: http - url: - !concat [ - *SERVICE_PROTO, - "://", - *SERVICE_HOST, - ":", - *SERVICE_PORT, - "/api/v1/host/health_check/", - ] + url: "/api/v1/host/health_check/" interval: 10 timeout: 10 deregister: 25 header: - tls_skip_verify: false + tls_skip_verify: false \ No newline at end of file diff --git a/microservice/0_sysom_channel/conf/common.py b/microservice/0_sysom_channel/conf/common.py index c495992a..47f42045 100644 --- a/microservice/0_sysom_channel/conf/common.py +++ b/microservice/0_sysom_channel/conf/common.py @@ -9,7 +9,7 @@ Description: from clogger import logger import os from pathlib import Path -from sysom_utils import ConfigParser, CecTarget +from sysom_utils import ConfigParser, CecTarget, SysomFramework BASE_DIR = Path(__file__).resolve().parent.parent @@ -25,6 +25,8 @@ cec_config = YAML_CONFIG.get_server_config().cec mysql_config = YAML_CONFIG.get_server_config().db.mysql service_config = YAML_CONFIG.get_service_config() +SysomFramework.init(YAML_CONFIG) + ################################################################## # fastapi config ################################################################## diff --git a/microservice/0_sysom_channel/config.yml b/microservice/0_sysom_channel/config.yml index 71f8bec2..b05b9b45 100644 --- a/microservice/0_sysom_channel/config.yml +++ b/microservice/0_sysom_channel/config.yml @@ -10,15 +10,14 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME - protocol: &SERVICE_PROTO http - host: &SERVICE_HOST 127.0.0.1 - port: &SERVICE_PORT 7003 - plugins: + protocol: http + host: 127.0.0.1 + port: 7003 + framework: + gcache: + protocol: redis node_dispatch: cmg: - service_name: *SERVICE_NAME - host: *SERVICE_HOST - port: *SERVICE_PORT tags: - Channel - FastApi @@ -26,17 +25,10 @@ sysom_service: metadata: check: type: http - url: - !concat [ - *SERVICE_PROTO, - "://", - *SERVICE_HOST, - ":", - *SERVICE_PORT, - "/api/v1/channel/health/check", - ] + url: "/api/v1/channel/health/check" interval: 10 timeout: 10 deregister: 25 header: tls_skip_verify: false + diff --git a/microservice/0_sysom_channel/main.py b/microservice/0_sysom_channel/main.py index 9231bd46..c635adb7 100644 --- a/microservice/0_sysom_channel/main.py +++ b/microservice/0_sysom_channel/main.py @@ -17,7 +17,7 @@ from app.schemas import ChannelSetting from lib.ssh import AsyncSSH from conf.settings import * from app.routers import file, config, cec_status, health -from sysom_utils import FrameworkPlugMag, CmgPlugin +from sysom_utils import CmgPlugin, SysomFramework app = FastAPI() @@ -99,8 +99,8 @@ def init_channel(): except Exception as e: logger.exception(e) -def init_frame_plugin_mag(): - FrameworkPlugMag(YAML_CONFIG) \ +def init_framwork(): + SysomFramework\ .load_plugin_cls(CmgPlugin) \ .start() @@ -108,7 +108,7 @@ def init_frame_plugin_mag(): @app.on_event("startup") async def on_start(): init_channel() - init_frame_plugin_mag() + init_framwork() @app.on_event("shutdown") diff --git a/microservice/sysom_diagnosis/apps/task/apps.py b/microservice/sysom_diagnosis/apps/task/apps.py index 515f3792..ee6be2a7 100644 --- a/microservice/sysom_diagnosis/apps/task/apps.py +++ b/microservice/sysom_diagnosis/apps/task/apps.py @@ -1,7 +1,7 @@ import sys from clogger import logger from django.apps import AppConfig -from sysom_utils import FrameworkPlugMag, NodeDispatcherPlugin, CmgPlugin +from sysom_utils import NodeDispatcherPlugin, CmgPlugin, SysomFramework class TaskConfig(AppConfig): @@ -12,15 +12,11 @@ class TaskConfig(AppConfig): from django.conf import settings if ('runserver' in sys.argv or 'manage.py' not in sys.argv): from apps.task.executor import DiagnosisTaskExecutor - from channel_job.job import default_channel_job_executor - # 初始化 channel_job sdk - default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) - default_channel_job_executor.start() - - FrameworkPlugMag(settings.YAML_CONFIG) \ + SysomFramework.init(settings.YAML_CONFIG) \ .load_plugin_cls(NodeDispatcherPlugin) \ .load_plugin_cls(CmgPlugin) \ + .enable_channel_job() \ .start() # 这边微服务正式启动的时候执行一些处理代码 diff --git a/microservice/sysom_diagnosis/conf/common.py b/microservice/sysom_diagnosis/conf/common.py index 0430e581..5fa3117f 100644 --- a/microservice/sysom_diagnosis/conf/common.py +++ b/microservice/sysom_diagnosis/conf/common.py @@ -1,8 +1,7 @@ from clogger import logger -import sys import os from pathlib import Path -from sysom_utils import ConfigParser, CecTarget +from sysom_utils import ConfigParser, CecTarget, SysomFramework BASE_DIR = Path(__file__).resolve().parent.parent @@ -15,6 +14,8 @@ YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) +SysomFramework.init(YAML_CONFIG) + ########################################################################################## # Diagnosis Service config ########################################################################################## @@ -76,21 +77,6 @@ DATABASES = { } } -CACHES = { - "default": { - "BACKEND": "django_redis.cache.RedisCache", - "LOCATION": ( - f"redis://{YAML_CONFIG.get_server_config().db.redis.host}" - f":{YAML_CONFIG.get_server_config().db.redis.port}/1" - ), - "OPTIONS": { - "CLIENT_CLASS": "django_redis.client.DefaultClient", - "CONNECTION_POOL_KWARGS": {"max_connections": 100}, - "DECODE_RESPONSES": True - } - } -} - ROOT_URLCONF = 'sysom_diagnosis.urls' WSGI_APPLICATION = 'sysom_diagnosis.wsgi.application' diff --git a/microservice/sysom_diagnosis/config.yml b/microservice/sysom_diagnosis/config.yml index a77c2639..799f69de 100644 --- a/microservice/sysom_diagnosis/config.yml +++ b/microservice/sysom_diagnosis/config.yml @@ -15,15 +15,14 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME - protocol: &SERVICE_PROTO http - host: &SERVICE_HOST 127.0.0.1 - port: &SERVICE_PORT 7002 - plugins: + protocol: http + host: 127.0.0.1 + port: 7002 + framework: + gcache: + protocol: redis node_dispatch: cmg: - service_name: *SERVICE_NAME - host: *SERVICE_HOST - port: *SERVICE_PORT tags: - Diagnosis - Django @@ -31,15 +30,7 @@ sysom_service: metadata: check: type: http - url: - !concat [ - *SERVICE_PROTO, - "://", - *SERVICE_HOST, - ":", - *SERVICE_PORT, - "/api/v1/tasks/health_check/", - ] + url: "/api/v1/tasks/health_check/" interval: 10 timeout: 10 deregister: 25 diff --git a/microservice/sysom_diagnosis/lib/authentications.py b/microservice/sysom_diagnosis/lib/authentications.py index bb26a66b..045392aa 100644 --- a/microservice/sysom_diagnosis/lib/authentications.py +++ b/microservice/sysom_diagnosis/lib/authentications.py @@ -7,7 +7,7 @@ from django.utils.translation import ugettext as _ from rest_framework.exceptions import AuthenticationFailed from rest_framework.request import Request from rest_framework.authentication import BaseAuthentication -from django.core.cache import cache +from sysom_utils import SysomFramework def get_jwt_decode_classes() -> List[BaseAuthentication]: @@ -45,7 +45,7 @@ class TokenAuthentication(BaseAuthentication): token = request.META.get('HTTP_AUTHORIZATION') payload = decode_token(token) # 判断用户是否已经手动注销登录 - if cache.get(token) is None: + if SysomFramework.gcache("JWT_TOKEN").load(token) is None: raise AuthenticationFailed('用户已退出登录!') payload['token'] = token diff --git a/microservice/sysom_hotfix/apps/hotfix/apps.py b/microservice/sysom_hotfix/apps/hotfix/apps.py index 27e78740..56f2514b 100644 --- a/microservice/sysom_hotfix/apps/hotfix/apps.py +++ b/microservice/sysom_hotfix/apps/hotfix/apps.py @@ -3,7 +3,6 @@ from clogger import logger from django.apps import AppConfig from django.db.models.signals import post_migrate from cec_base.admin import dispatch_admin -from channel_job.job import default_channel_job_executor class HotfixConfig(AppConfig): default_auto_field = 'django.db.models.BigAutoField' @@ -12,11 +11,9 @@ class HotfixConfig(AppConfig): def ready(self): from django.conf import settings if ('runserver' in sys.argv or 'manage.py' not in sys.argv): - from sysom_utils import FrameworkPlugMag, CmgPlugin + from sysom_utils import CmgPlugin, SysomFramework try: # 初始化 channel_job sdk - default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) - default_channel_job_executor.start() hotfix_cec_topic_name = "hotfix_job" post_migrate.connect(initialization_subscribe, sender=self) admin = dispatch_admin(settings.SYSOM_CEC_URL) @@ -25,9 +22,9 @@ class HotfixConfig(AppConfig): except Exception as e: logger.info(str(e)) logger.info(">> INIT_HOTFIX_VIEW : create hotfix_job cec topic failed") - # Framework Plug Mag init - FrameworkPlugMag(settings.YAML_CONFIG) \ + SysomFramework.init(settings.YAML_CONFIG) \ .load_plugin_cls(CmgPlugin) \ + .enable_channel_job() \ .start() else: # nothing to do when execute database migrations diff --git a/microservice/sysom_hotfix/config.yml b/microservice/sysom_hotfix/config.yml index 793b1685..4e312518 100644 --- a/microservice/sysom_hotfix/config.yml +++ b/microservice/sysom_hotfix/config.yml @@ -14,15 +14,14 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME - protocol: &SERVICE_PROTO http - host: &SERVICE_HOST 127.0.0.1 - port: &SERVICE_PORT 7007 - plugins: + protocol: http + host: 127.0.0.1 + port: 7007 + framework: + gcache: + protocol: redis node_dispatch: cmg: - service_name: *SERVICE_NAME - host: *SERVICE_HOST - port: *SERVICE_PORT tags: - Hotfix - Django @@ -30,15 +29,7 @@ sysom_service: metadata: check: type: http - url: - !concat [ - *SERVICE_PROTO, - "://", - *SERVICE_HOST, - ":", - *SERVICE_PORT, - "/api/v1/hotfix/health_check/", - ] + url: "/api/v1/hotfix/health_check/" interval: 10 timeout: 10 deregister: 25 diff --git a/microservice/sysom_migration/apps/migration/apps.py b/microservice/sysom_migration/apps/migration/apps.py index 14aa18ce..5ce2242b 100644 --- a/microservice/sysom_migration/apps/migration/apps.py +++ b/microservice/sysom_migration/apps/migration/apps.py @@ -8,18 +8,13 @@ class MigrationConfig(AppConfig): name = 'apps.migration' def ready(self): - from django.conf import settings if ('runserver' in sys.argv or 'manage.py' not in sys.argv): - from channel_job.job import default_channel_job_executor from django.conf import settings - from sysom_utils import FrameworkPlugMag, CmgPlugin + from sysom_utils import CmgPlugin, SysomFramework - # 初始化 channel_job sdk - default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) - default_channel_job_executor.start() - # Framework Plug Mag init - FrameworkPlugMag(settings.YAML_CONFIG) \ + SysomFramework.init(settings.YAML_CONFIG) \ .load_plugin_cls(CmgPlugin) \ + .enable_channel_job() \ .start() logger.info(">>> Migration module loading success") diff --git a/microservice/sysom_migration/config.yml b/microservice/sysom_migration/config.yml index 90924876..863b560a 100644 --- a/microservice/sysom_migration/config.yml +++ b/microservice/sysom_migration/config.yml @@ -1,7 +1,7 @@ vars: SERVICE_NAME: &SERVICE_NAME sysom_migration SERVICE_CONSUMER_GROUP: - &SERVICE_CONSUMER_GROUP !concat [*SERVICE_NAME, "_consumer_group"] + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] sysom_server: cec: @@ -14,15 +14,14 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME - protocol: &SERVICE_PROTO http - host: &SERVICE_HOST 127.0.0.1 - port: &SERVICE_PORT 7006 - plugins: + protocol: http + host: 127.0.0.1 + port: 7006 + framework: + gcache: + protocol: redis node_dispatch: cmg: - service_name: *SERVICE_NAME - host: *SERVICE_HOST - port: *SERVICE_PORT tags: - Migration - Django @@ -30,17 +29,9 @@ sysom_service: metadata: check: type: http - url: - !concat [ - *SERVICE_PROTO, - "://", - *SERVICE_HOST, - ":", - *SERVICE_PORT, - "/api/v1/migration/health_check/", - ] + url: "/api/v1/migration/health_check/" interval: 10 timeout: 10 deregister: 25 header: - tls_skip_verify: false \ No newline at end of file + tls_skip_verify: false diff --git a/microservice/sysom_monitor_server/app/routeres/services.py b/microservice/sysom_monitor_server/app/routeres/services.py index edaff1b6..4c479ef0 100644 --- a/microservice/sysom_monitor_server/app/routeres/services.py +++ b/microservice/sysom_monitor_server/app/routeres/services.py @@ -25,4 +25,24 @@ async def list_services(): "code": 0, "data": res, "err_msg": "" + } + +@router.get("/list_detail") +async def list_services(): + try: + res = {} + services = discovery.get_services() + for service in services: + ss = discovery.get_instances(service) + res[service] = [item.to_dict() for item in ss] + except Exception as exc: + return { + "code": 1, + "data": "", + "err_msg": str(exc) + } + return { + "code": 0, + "data": res, + "err_msg": "" } \ No newline at end of file diff --git a/microservice/sysom_monitor_server/conf/common.py b/microservice/sysom_monitor_server/conf/common.py index 19e9b46c..f3800c6c 100644 --- a/microservice/sysom_monitor_server/conf/common.py +++ b/microservice/sysom_monitor_server/conf/common.py @@ -9,7 +9,7 @@ Description: from clogger import logger import sys from pathlib import Path -from sysom_utils import ConfigParser +from sysom_utils import ConfigParser, SysomFramework # sysom_monitor_server root path BASE_DIR = Path(__file__).resolve().parent.parent @@ -22,6 +22,8 @@ YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) +SysomFramework.init(YAML_CONFIG) + ################################################################## # Cec settings ################################################################## diff --git a/microservice/sysom_monitor_server/config.yml b/microservice/sysom_monitor_server/config.yml index a57f52f2..952c7865 100644 --- a/microservice/sysom_monitor_server/config.yml +++ b/microservice/sysom_monitor_server/config.yml @@ -16,20 +16,14 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: !concat [*SERVICE_NAME, "_server"] - custom: - grafana: - url: http://127.0.0.1:3000 - username: admin - password: sysom@openanolis.cn - protocol: &SERVICE_PROTO http - host: &SERVICE_HOST 127.0.0.1 - port: &SERVICE_PORT 7009 - plugins: + protocol: http + host: 127.0.0.1 + port: 7009 + framework: + gcache: + protocol: redis node_dispatch: cmg: - service_name: *SERVICE_NAME - host: *SERVICE_HOST - port: *SERVICE_PORT tags: - MonitorServer - FastApi @@ -37,20 +31,17 @@ sysom_service: metadata: check: type: http - url: - !concat [ - *SERVICE_PROTO, - "://", - *SERVICE_HOST, - ":", - *SERVICE_PORT, - "/api/v1/monitor/health/check", - ] + url: "/api/v1/monitor/health/check" interval: 10 timeout: 10 deregister: 25 header: tls_skip_verify: false + custom: + grafana: + url: http://127.0.0.1:3000 + username: admin + password: sysom@openanolis.cn # 节点测配置 sysom_node: diff --git a/microservice/sysom_monitor_server/main.py b/microservice/sysom_monitor_server/main.py index 333e9cf1..c1991c8e 100644 --- a/microservice/sysom_monitor_server/main.py +++ b/microservice/sysom_monitor_server/main.py @@ -2,7 +2,7 @@ from channel_job import default_channel_job_executor from fastapi import FastAPI from conf.settings import * from app.routeres import home, services, health, grafana -from sysom_utils import FrameworkPlugMag, CmgPlugin +from sysom_utils import CmgPlugin, SysomFramework app = FastAPI() @@ -16,15 +16,11 @@ def init_monitor(): default_channel_job_executor.init_config(CHANNEL_JOB_URL) default_channel_job_executor.start() -def init_frame_plugin_mag(): - FrameworkPlugMag(YAML_CONFIG) \ - .load_plugin_cls(CmgPlugin) \ - .start() - @app.on_event("startup") async def on_start(): init_monitor() - init_frame_plugin_mag() + SysomFramework.load_plugin_cls(CmgPlugin) \ + .start() @app.on_event("shutdown") diff --git a/microservice/sysom_vmcore/apps/vmcore/apps.py b/microservice/sysom_vmcore/apps/vmcore/apps.py index 67cb7819..5d32ec4b 100644 --- a/microservice/sysom_vmcore/apps/vmcore/apps.py +++ b/microservice/sysom_vmcore/apps/vmcore/apps.py @@ -16,17 +16,12 @@ class VmcoreConfig(AppConfig): def ready(self) -> None: from django.conf import settings if ('runserver' in sys.argv or 'manage.py' not in sys.argv): - from channel_job.job import default_channel_job_executor - from sysom_utils import FrameworkPlugMag, CmgPlugin, NodeDispatcherPlugin + from sysom_utils import SysomFramework, CmgPlugin, NodeDispatcherPlugin - # 初始化 channel_job sdk - default_channel_job_executor.init_config(settings.CHANNEL_JOB_URL) - default_channel_job_executor.start() - - # Framework Plug Mag init - FrameworkPlugMag(settings.YAML_CONFIG) \ - .load_plugin_cls(CmgPlugin) \ + SysomFramework.init(settings.YAML_CONFIG) \ .load_plugin_cls(NodeDispatcherPlugin) \ + .load_plugin_cls(CmgPlugin) \ + .enable_channel_job() \ .start() else: # 这边执行数据库迁移等操作的时候执行一些处理代码 diff --git a/microservice/sysom_vmcore/config.yml b/microservice/sysom_vmcore/config.yml index 423c6da3..9871ebdc 100644 --- a/microservice/sysom_vmcore/config.yml +++ b/microservice/sysom_vmcore/config.yml @@ -14,15 +14,14 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME - protocol: &SERVICE_PROTO http - host: &SERVICE_HOST 127.0.0.1 - port: &SERVICE_PORT 7004 - plugins: + protocol: http + host: 127.0.0.1 + port: 7004 + framework: + gcache: + protocol: redis node_dispatch: cmg: - service_name: *SERVICE_NAME - host: *SERVICE_HOST - port: *SERVICE_PORT tags: - Vmcore - Django @@ -30,15 +29,7 @@ sysom_service: metadata: check: type: http - url: - !concat [ - *SERVICE_PROTO, - "://", - *SERVICE_HOST, - ":", - *SERVICE_PORT, - "/api/v1/vmcore/health_check/", - ] + url: "/api/v1/vmcore/health_check/" interval: 10 timeout: 10 deregister: 25 diff --git a/microservice/sysom_vmcore/scripts/server_init.sh b/microservice/sysom_vmcore/scripts/server_init.sh index f8e2ab70..7d72113c 100644 --- a/microservice/sysom_vmcore/scripts/server_init.sh +++ b/microservice/sysom_vmcore/scripts/server_init.sh @@ -66,6 +66,10 @@ start_cron() } deploy() { + rpm -q --quiet nfs-utils || yum install -y nfs-utils + rpm -q --quiet rpcbind || yum install -y rpcbind + systemctl start rpcbind && systemctl enable rpcbind + systemctl start nfs && systemctl enable nfs source_virtualenv install_requirement init_conf diff --git a/microservice/sysom_vul/config.yml b/microservice/sysom_vul/config.yml index b836255d..1f34f5fe 100644 --- a/microservice/sysom_vul/config.yml +++ b/microservice/sysom_vul/config.yml @@ -14,15 +14,14 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME service_dir: *SERVICE_NAME - protocol: &SERVICE_PROTO http - host: &SERVICE_HOST 127.0.0.1 - port: &SERVICE_PORT 7005 - plugins: + protocol: http + host: 127.0.0.1 + port: 7005 + framework: + gcache: + protocol: redis node_dispatch: cmg: - service_name: *SERVICE_NAME - host: *SERVICE_HOST - port: *SERVICE_PORT tags: - Vul - Django @@ -30,15 +29,7 @@ sysom_service: metadata: check: type: http - url: - !concat [ - *SERVICE_PROTO, - "://", - *SERVICE_HOST, - ":", - *SERVICE_PORT, - "/api/v1/vul/health_check/", - ] + url: "/api/v1/vul/health_check/" interval: 10 timeout: 10 deregister: 25 diff --git a/tools/deploy/deploy.sh b/tools/deploy/deploy.sh index 8c4caf8a..b13873a7 100755 --- a/tools/deploy/deploy.sh +++ b/tools/deploy/deploy.sh @@ -7,7 +7,6 @@ # Function: deploy sysom #***************************************************************# APP_NAME="sysom" -SERVER_DIR="sysom_server" WEB_DIR="sysom_web" CONF_DIR="conf" SCRIPT_DIR="script" -- Gitee From 0ad47f451e0b8503ae3aea563842dfb23e2f74ea Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 11 May 2023 14:59:40 +0800 Subject: [PATCH 041/196] refactor(sysom_monitor_server): Use SysomFramework to init default_channel_job executor --- microservice/sysom_monitor_server/config.yml | 2 +- microservice/sysom_monitor_server/main.py | 8 +------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/microservice/sysom_monitor_server/config.yml b/microservice/sysom_monitor_server/config.yml index 952c7865..4d7da764 100644 --- a/microservice/sysom_monitor_server/config.yml +++ b/microservice/sysom_monitor_server/config.yml @@ -15,7 +15,7 @@ sysom_server: sysom_service: service_name: *SERVICE_NAME - service_dir: !concat [*SERVICE_NAME, "_server"] + service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 port: 7009 diff --git a/microservice/sysom_monitor_server/main.py b/microservice/sysom_monitor_server/main.py index c1991c8e..4618f7cf 100644 --- a/microservice/sysom_monitor_server/main.py +++ b/microservice/sysom_monitor_server/main.py @@ -1,4 +1,3 @@ -from channel_job import default_channel_job_executor from fastapi import FastAPI from conf.settings import * from app.routeres import home, services, health, grafana @@ -11,15 +10,10 @@ app.include_router(grafana.router, prefix="/api/v1/monitor/grafana") app.include_router(services.router, prefix="/api/v1/monitor/services") app.include_router(health.router, prefix="/api/v1/monitor/health") -def init_monitor(): - # 从 Channel 为服务拉取配置初始化 channel_job - default_channel_job_executor.init_config(CHANNEL_JOB_URL) - default_channel_job_executor.start() - @app.on_event("startup") async def on_start(): - init_monitor() SysomFramework.load_plugin_cls(CmgPlugin) \ + .enable_channel_job() \ .start() -- Gitee From e0435aca2a7d2accce5a744e41ab38c0b2778dc9 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 10 May 2023 20:51:06 +0800 Subject: [PATCH 042/196] refactor(sysom): Adjust the directory layout to be compatible with sysom2.1 --- .../0_env/.pydistutils.cfg | 0 .../0_env/clear.sh | 0 {infrastructure => environment}/0_env/init.sh | 0 .../0_env/pip.conf | 0 .../0_env/requirements.txt | 0 .../0_env/start.sh | 0 {infrastructure => environment}/0_env/stop.sh | 0 .../1_sdk/.gitignore | 0 .../1_sdk/cec_base/__init__.py | 0 .../1_sdk/cec_base/admin.py | 0 .../1_sdk/cec_base/base.py | 0 .../1_sdk/cec_base/cec_client.py | 0 .../1_sdk/cec_base/cli.py | 0 .../1_sdk/cec_base/consumer.py | 0 .../1_sdk/cec_base/event.py | 0 .../1_sdk/cec_base/exceptions.py | 0 .../1_sdk/cec_base/meta.py | 0 .../1_sdk/cec_base/producer.py | 0 .../1_sdk/cec_base/url.py | 0 .../1_sdk/cec_base/utils.py | 0 .../1_sdk/cec_redis/__init__.py | 0 .../1_sdk/cec_redis/admin_static.py | 0 .../1_sdk/cec_redis/common.py | 0 .../1_sdk/cec_redis/consume_status_storage.py | 0 .../1_sdk/cec_redis/heartbeat.py | 0 .../1_sdk/cec_redis/redis_admin.py | 0 .../1_sdk/cec_redis/redis_consumer.py | 0 .../1_sdk/cec_redis/redis_producer.py | 0 .../1_sdk/cec_redis/utils.py | 0 .../1_sdk/channel_job/__init__.py | 0 .../1_sdk/channel_job/exception.py | 0 .../1_sdk/channel_job/job.py | 0 .../1_sdk/channel_job/model.py | 0 .../1_sdk/clear.sh | 0 .../1_sdk/clogger/__init__.py | 0 .../1_sdk/clogger/clogger.py | 0 .../1_sdk/cmg_base/__init__.py | 0 .../1_sdk/cmg_base/exceptions.py | 0 .../1_sdk/cmg_base/load_balancing_strategy.py | 0 .../1_sdk/cmg_base/service_check.py | 0 .../1_sdk/cmg_base/service_discovery.py | 0 .../1_sdk/cmg_base/service_instance.py | 0 .../1_sdk/cmg_base/service_invoker.py | 0 .../1_sdk/cmg_base/service_registry.py | 0 .../1_sdk/cmg_base/url.py | 0 .../1_sdk/cmg_base/utils.py | 0 .../1_sdk/cmg_redis/__init__.py | 0 .../1_sdk/cmg_redis/common.py | 0 .../cmg_redis/redis_service_discovery.py | 0 .../1_sdk/cmg_redis/redis_service_registry.py | 0 .../1_sdk/cmg_redis/utils.py | 0 .../1_sdk/gcache_base/__init__.py | 0 .../1_sdk/gcache_base/exceptions.py | 0 .../1_sdk/gcache_base/gcache.py | 0 .../1_sdk/gcache_base/url.py | 0 .../1_sdk/gcache_redis/__init__.py | 0 .../1_sdk/gcache_redis/common.py | 0 .../1_sdk/gcache_redis/redis_gcache.py | 0 .../1_sdk/gcache_redis/utils.py | 0 .../1_sdk/gclient_base/__init__.py | 0 .../1_sdk/gclient_base/exceptions.py | 0 .../1_sdk/gclient_base/gclient.py | 0 .../1_sdk/gclient_base/url.py | 0 .../1_sdk/gclient_cmg/__init__.py | 0 .../1_sdk/gclient_cmg/cmg_gclient.py | 0 .../1_sdk/gclient_cmg/common.py | 0 .../1_sdk/gclient_http/__init__.py | 0 .../1_sdk/gclient_http/common.py | 0 .../1_sdk/gclient_http/http_gclient.py | 0 {infrastructure => environment}/1_sdk/init.sh | 0 .../1_sdk/redis_lua/__init__.py | 0 .../1_sdk/redis_lua/x_hdel.lua | 0 .../1_sdk/redis_lua/x_hdrop_table.lua | 0 .../1_sdk/redis_lua/x_hexpire.lua | 0 .../1_sdk/redis_lua/x_hget.lua | 0 .../1_sdk/redis_lua/x_hgetall.lua | 0 .../1_sdk/redis_lua/x_hkeys.lua | 0 .../1_sdk/redis_lua/x_hlen.lua | 0 .../1_sdk/redis_lua/x_hset.lua | 0 .../1_sdk/redis_lua/xreadis_hash_table.py | 0 .../1_sdk/setup_cec_base.py | 0 .../1_sdk/setup_cec_redis.py | 0 .../1_sdk/setup_channel_job.py | 0 .../1_sdk/setup_clogger.py | 0 .../1_sdk/setup_cmg_base.py | 0 .../1_sdk/setup_cmg_redis.py | 0 .../1_sdk/setup_gcache_base.py | 0 .../1_sdk/setup_gcache_redis.py | 0 .../1_sdk/setup_gclient_base.py | 0 .../1_sdk/setup_gclient_cmg.py | 0 .../1_sdk/setup_gclient_http.py | 0 .../1_sdk/setup_redis_lua.py | 0 .../1_sdk/setup_sysom_utils.py | 0 .../1_sdk/start.sh | 0 {infrastructure => environment}/1_sdk/stop.sh | 0 .../1_sdk/sysom_utils/__init__.py | 0 .../1_sdk/sysom_utils/adddict.py | 0 .../1_sdk/sysom_utils/config_parser.py | 0 .../1_sdk/sysom_utils/event_executor.py | 0 .../1_sdk/sysom_utils/framework.py | 0 .../1_sdk/sysom_utils/framework_plug_mag.py | 0 .../1_sdk/sysom_utils/framework_plugins.py | 0 .../1_sdk/sysom_utils/node_manager.py | 0 .../1_sdk/sysom_utils/yaml_concat.py | 0 .../1_sdk/test_cec_redis/__init__.py | 0 .../1_sdk/test_cec_redis/test_heartbeat.py | 0 .../1_sdk/test_cec_redis/test_redis_admin.py | 0 .../test_cec_redis/test_redis_admin_async.py | 0 .../test_cec_redis/test_redis_consumer.py | 0 .../test_redis_consumer_async.py | 0 .../test_redis_multi_consumer.py | 0 .../test_cec_redis/test_redis_producer.py | 0 .../1_sdk/test_cec_redis/test_utils.py | 0 .../1_sdk/test_cmg_base/__init__.py | 0 .../1_sdk/test_cmg_base/test_health_check.py | 0 .../1_sdk/test_cmg_redis/__init__.py | 0 .../test_cmg_redis/test_service_discovery.py | 0 .../test_cmg_redis/test_service_registry.py | 0 .../2_local_services/clear.sh | 0 .../2_local_services/init.sh | 0 .../2_local_services/nginx.conf | 0 .../2_local_services/start.sh | 0 .../2_local_services/stop.sh | 0 .../2_local_services/sysom-redis.ini | 0 .../2_local_services/sysom.conf | 0 .../3_monitor/Readme.txt | 0 .../3_monitor/clear.sh | 3 + .../3_monitor/grafana_api_set.sh | 0 .../3_monitor/grafana_recover.sh | 0 .../3_monitor/init.sh | 0 .../3_monitor/local_copy_pkg.sh | 0 .../3_monitor/prometheus_get_node.py | 0 .../3_monitor/start.sh | 0 .../3_monitor/stop.sh | 0 .../3_monitor/sysom-cec-status-dashboard.json | 0 .../3_monitor/sysom-dashboard.json | 0 .../3_monitor/sysom-migration-dashboard.json | 0 .../3_monitor/sysom-netinfo-dashboard.json | 0 .../3_monitor/sysom-prometheus.ini | 4 +- {infrastructure => environment}/clear_exclude | 0 .../deploy_exclude | 0 .../sysom_vmcore/scripts/server_db_migrate.sh | 17 -- .../sysom_vul/scripts/server_db_migrate.sh | 17 -- package.sh | 6 +- script/deploy/clear.sh | 27 ++- script/deploy/deploy.sh | 73 ++------ {microservice => script/server}/clear_exclude | 0 .../server}/deploy_exclude | 0 .../server/sysom_api/clear.sh | 2 +- .../server/sysom_api/db_migrate.sh | 3 +- .../server/sysom_api/init.sh | 6 +- .../server/sysom_api}/requirements.txt | 0 .../server/sysom_api/start.sh | 0 .../server/sysom_api/stop.sh | 0 .../server/sysom_api}/sysom-api.ini | 6 +- .../server/sysom_channel/clear.sh | 0 .../server/sysom_channel/db_migrate.sh | 3 +- .../server/sysom_channel/init.sh | 6 +- .../server/sysom_channel}/requirements.txt | 0 .../server/sysom_channel/start.sh | 0 .../server/sysom_channel/stop.sh | 0 .../server/sysom_channel}/sysom-channel.ini | 6 +- .../server/sysom_diagnosis/clear.sh | 0 .../server/sysom_diagnosis/db_migrate.sh | 3 +- .../server/sysom_diagnosis/deploy.sh | 0 .../server/sysom_diagnosis/init.sh | 6 +- .../server}/sysom_diagnosis/requirements.txt | 0 .../server/sysom_diagnosis/start.sh | 0 .../server/sysom_diagnosis/stop.sh | 0 .../sysom_diagnosis/sysom-diagnosis.ini | 6 +- .../server/sysom_hotfix/clear.sh | 0 .../server/sysom_hotfix/db_migrate.sh | 3 +- .../server/sysom_hotfix/init.sh | 6 +- .../server}/sysom_hotfix/requirements.txt | 0 .../server/sysom_hotfix/start.sh | 0 .../server/sysom_hotfix/stop.sh | 0 .../server}/sysom_hotfix/sysom-hotfix.ini | 6 +- .../server/sysom_hotfix_builder/clear.sh | 0 .../server/sysom_hotfix_builder/init.sh | 6 +- .../sysom_hotfix_builder/requirements.txt | 0 .../server/sysom_hotfix_builder/start.sh | 0 .../server/sysom_hotfix_builder/stop.sh | 0 .../sysom-hotfix-builder.ini | 6 +- .../server/sysom_migration/clear.sh | 0 .../server/sysom_migration/db_migrate.sh | 3 +- .../server/sysom_migration/init.sh | 6 +- .../server}/sysom_migration/requirements.txt | 0 .../server/sysom_migration/start.sh | 0 .../server/sysom_migration/stop.sh | 0 .../sysom_migration/sysom-migration.ini | 6 +- .../server/sysom_monitor_server/clear.sh | 0 .../server/sysom_monitor_server/db_migrate.sh | 3 +- .../server/sysom_monitor_server/init.sh | 6 +- .../sysom_monitor_server/requirements.txt | 0 .../server/sysom_monitor_server/start.sh | 0 .../server/sysom_monitor_server/stop.sh | 0 .../sysom-monitor-server.ini | 6 +- .../server/sysom_vmcore/clear.sh | 3 + script/server/sysom_vmcore/db_migrate.sh | 18 ++ .../server/sysom_vmcore/init.sh | 6 +- .../server/sysom_vmcore}/parse_panic.py | 0 .../server}/sysom_vmcore/requirements.txt | 0 .../server/sysom_vmcore/start.sh | 0 .../server/sysom_vmcore/stop.sh | 0 .../server}/sysom_vmcore/sysom-vmcore.ini | 6 +- .../server/sysom_vmcore}/vmcore_const.py | 0 .../server/sysom_vul/clear.sh | 0 script/server/sysom_vul/db_migrate.sh | 18 ++ .../server/sysom_vul/init.sh | 6 +- .../server}/sysom_vul/requirements.txt | 0 .../server/sysom_vul/start.sh | 0 .../server/sysom_vul/stop.sh | 0 .../server}/sysom_vul/sysom-vul.ini | 6 +- script/sysom.sh | 20 ++- script/{clear.sh => sysom_clear.sh} | 62 ++++--- script/{dbmigrate.sh => sysom_dbmigrate.sh} | 0 script/{deploy.sh => sysom_deploy.sh} | 162 ++++++++++-------- {microservice => sysom_server}/.gitignore | 0 .../__init__.py => sysom_server/clear_exclude | 0 sysom_server/deploy_exclude | 1 + .../sysom_api}/.gitignore | 0 .../sysom_api}/apps/__init__.py | 0 .../sysom_api/apps/accounts}/__init__.py | 0 .../sysom_api}/apps/accounts/admin.py | 0 .../sysom_api}/apps/accounts/apps.py | 0 .../apps/accounts/authentication.py | 0 .../apps/accounts/migrations/0001_initial.py | 0 .../migrations/0002_user_allow_login.py | 0 .../apps/accounts}/migrations/__init__.py | 0 .../sysom_api}/apps/accounts/models.py | 0 .../sysom_api}/apps/accounts/permissions.py | 0 .../sysom_api}/apps/accounts/serializer.py | 0 .../sysom_api}/apps/accounts/tests.py | 0 .../sysom_api}/apps/accounts/urls.py | 0 .../sysom_api}/apps/accounts/views.py | 0 .../sysom_api}/apps/alarm/__init__.py | 0 .../sysom_api}/apps/alarm/admin.py | 0 .../sysom_api}/apps/alarm/apps.py | 0 .../apps/alarm/migrations/0001_initial.py | 0 .../apps/alarm}/migrations/__init__.py | 0 .../sysom_api}/apps/alarm/models.py | 0 .../sysom_api}/apps/alarm/serializer.py | 0 .../sysom_api}/apps/alarm/subscribe.json | 0 .../sysom_api}/apps/alarm/urls.py | 0 .../sysom_api}/apps/alarm/views.py | 0 .../sysom_api/apps/common}/__init__.py | 0 .../apps/common/common_model_viewset.py | 0 .../sysom_api/apps/host}/__init__.py | 0 .../sysom_api}/apps/host/admin.py | 0 .../sysom_api}/apps/host/apps.py | 0 .../sysom_api}/apps/host/cec_api.py | 0 .../sysom_api}/apps/host/heartbeat.py | 0 .../apps/host/migrations/0001_initial.py | 0 .../migrations/0002_hostmodel_host_info.py | 0 .../migrations/0003_alter_hostmodel_status.py | 0 .../migrations/0004_auto_20221227_1705.py | 0 .../apps/host}/migrations/__init__.py | 0 .../sysom_api}/apps/host/models.py | 0 .../sysom_api}/apps/host/serializer.py | 0 .../sysom_api}/apps/host/tests.py | 0 .../sysom_api}/apps/host/urls.py | 0 .../sysom_api}/apps/host/views.py | 0 .../sysom_api}/apps/services/__init__.py | 0 .../sysom_api}/apps/services/admin.py | 0 .../sysom_api}/apps/services/apps.py | 0 .../apps/services/migrations/0001_initial.py | 0 .../apps/services/migrations}/__init__.py | 0 .../sysom_api}/apps/services/models.py | 0 .../sysom_api}/apps/services/serializer.py | 0 .../sysom_api}/apps/services/tests.py | 0 .../sysom_api}/apps/services/urls.py | 0 .../sysom_api}/apps/services/views.py | 0 .../sysom_api/conf}/__init__.py | 0 .../sysom_api}/conf/common.py | 0 .../sysom_api}/conf/develop.py | 0 .../sysom_api}/conf/product.py | 0 .../sysom_api}/conf/testing.py | 0 .../sysom_api}/config.yml | 0 .../sysom_api}/consumer/__init__.py | 0 .../sysom_api}/consumer/consumers.py | 0 .../sysom_api}/consumer/middleware.py | 0 .../sysom_api}/consumer/routing.py | 0 .../sysom_api}/lib/__init__.py | 0 .../sysom_api}/lib/authentications.py | 0 .../sysom_api}/lib/base_model.py | 0 .../sysom_api}/lib/decode/sysom_decode.py | 0 .../sysom_api}/lib/excel.py | 0 .../sysom_api}/lib/exception.py | 0 .../sysom_api}/lib/paginations.py | 0 .../sysom_api}/lib/renderers.py | 0 .../sysom_api}/lib/response.py | 0 .../sysom_api}/lib/ssh.py | 0 .../sysom_api}/lib/utils.py | 0 .../sysom_api}/manage.py | 0 .../sysom_api}/sysom/__init__.py | 0 .../sysom_api}/sysom/asgi.py | 0 .../sysom_api}/sysom/settings.py | 0 .../sysom_api}/sysom/urls.py | 0 .../sysom_api}/sysom/wsgi.py | 0 .../sysom_channel}/alembic.ini | 0 .../sysom_channel}/alembic/README | 0 .../sysom_channel}/alembic/env.py | 0 .../sysom_channel}/alembic/script.py.mako | 0 .../alembic/versions/1663292d0bd8_init_db.py | 0 .../3f54250e3fca_add_channel_params_table.py | 0 .../sysom_channel}/app/__init__.py | 0 .../sysom_channel}/app/crud.py | 0 .../sysom_channel}/app/database.py | 0 .../sysom_channel}/app/executor.py | 0 .../sysom_channel}/app/models.py | 0 .../sysom_channel}/app/routers/cec_status.py | 0 .../sysom_channel}/app/routers/config.py | 0 .../sysom_channel}/app/routers/file.py | 0 .../sysom_channel}/app/routers/health.py | 0 .../sysom_channel}/app/schemas.py | 0 .../sysom_channel}/conf/common.py | 0 .../sysom_channel}/conf/develop.py | 0 .../sysom_channel}/conf/gunicorn.py | 0 .../sysom_channel}/conf/product.py | 0 .../sysom_channel}/conf/settings.py | 0 .../sysom_channel}/conf/testing.py | 0 .../sysom_channel}/config.yml | 0 .../sysom_channel}/lib/channels/__init__.py | 0 .../sysom_channel}/lib/channels/base.py | 0 .../sysom_channel}/lib/channels/ssh.py | 0 .../sysom_channel}/lib/ssh.py | 0 .../sysom_channel}/lib/utils.py | 0 .../sysom_channel}/main.py | 0 .../sysom_diagnosis/apps/task}/__init__.py | 0 .../sysom_diagnosis/apps/task/admin.py | 0 .../sysom_diagnosis/apps/task/apps.py | 0 .../sysom_diagnosis/apps/task/executor.py | 0 .../sysom_diagnosis/apps/task/filter.py | 0 .../sysom_diagnosis/apps/task/helper.py | 0 .../apps/task/migrations/0001_initial.py | 0 .../migrations/0002_auto_20230110_1738.py | 0 .../apps/task/migrations}/__init__.py | 0 .../sysom_diagnosis/apps/task/models.py | 0 .../sysom_diagnosis/apps/task/seriaizer.py | 0 .../sysom_diagnosis/apps/task/tests.py | 0 .../sysom_diagnosis/apps/task/urls.py | 0 .../sysom_diagnosis/apps/task/views.py | 0 .../sysom_diagnosis/conf/common.py | 0 .../sysom_diagnosis/conf/develop.py | 0 .../sysom_diagnosis/conf/gunicorn.py | 0 .../sysom_diagnosis/conf/product.py | 0 .../sysom_diagnosis/conf/testing.py | 0 .../sysom_diagnosis/config.yml | 0 .../sysom_diagnosis/lib/authentications.py | 0 .../sysom_diagnosis/lib/base_model.py | 0 .../sysom_diagnosis/lib/base_view.py | 0 .../lib/decode/sysom_decode.py | 0 .../sysom_diagnosis/lib/exception.py | 0 .../sysom_diagnosis/lib/paginations.py | 0 .../sysom_diagnosis/lib/renderers.py | 0 .../sysom_diagnosis/lib/response.py | 0 .../sysom_diagnosis/lib/utils.py | 0 .../sysom_diagnosis/manage.py | 0 .../sysom_diagnosis/scripts/node_clear.sh | 0 .../sysom_diagnosis/scripts/node_init.sh | 0 .../sysom_diagnosis/scripts/node_update.sh | 0 .../sysom_diagnosis/service_scripts/command | 0 .../service_scripts/command_post | 0 .../sysom_diagnosis/service_scripts/filecache | 0 .../service_scripts/filecache_post | 0 .../sysom_diagnosis/service_scripts/iofsstat | 0 .../service_scripts/iofsstat_post | 0 .../sysom_diagnosis/service_scripts/iohang | 0 .../service_scripts/iohang_post | 0 .../sysom_diagnosis/service_scripts/iolatency | 0 .../service_scripts/iolatency_post | 0 .../service_scripts/iosdiag_latency | 0 .../service_scripts/iosdiag_latency_post | 0 .../sysom_diagnosis/service_scripts/jitter | 0 .../service_scripts/jitter_post | 0 .../sysom_diagnosis/service_scripts/loadtask | 0 .../service_scripts/loadtask_post | 0 .../sysom_diagnosis/service_scripts/memgraph | 0 .../service_scripts/memgraph_post | 0 .../sysom_diagnosis/service_scripts/oomcheck | 0 .../service_scripts/oomcheck_post | 0 .../sysom_diagnosis/service_scripts/ossre | 0 .../service_scripts/ossre_post | 0 .../service_scripts/packetdrop | 0 .../service_scripts/packetdrop_post | 0 .../sysom_diagnosis/service_scripts/pingtrace | 0 .../service_scripts/pingtrace_post | 0 .../sysom_diagnosis/service_scripts/retran | 0 .../service_scripts/retran_post | 0 .../sysom_diagnosis/service_scripts/schedmoni | 0 .../service_scripts/schedmoni_post | 0 .../service_scripts/taskprofile | 0 .../service_scripts/taskprofile_post | 0 .../sysom_diagnosis/service_scripts/test | 0 .../sysom_diagnosis/service_scripts/test.sh | 0 .../sysom_diagnosis/service_scripts/tpython | 0 .../service_scripts/vmcore_analyse | 0 .../sysom_diagnosis/__init__.py | 0 .../sysom_diagnosis/sysom_diagnosis/asgi.py | 0 .../sysom_diagnosis/settings.py | 0 .../sysom_diagnosis/sysom_diagnosis/urls.py | 0 .../sysom_diagnosis/sysom_diagnosis/wsgi.py | 0 .../sysom_hotfix/apps/hotfix}/__init__.py | 0 .../sysom_hotfix/apps/hotfix/admin.py | 0 .../sysom_hotfix/apps/hotfix/apps.py | 0 .../sysom_hotfix/apps/hotfix/function.py | 0 .../apps/hotfix/migrations/0001_initial.py | 0 .../migrations/0002_auto_20230303_1601.py | 0 .../apps/hotfix/migrations}/__init__.py | 0 .../sysom_hotfix/apps/hotfix/models.py | 0 .../sysom_hotfix/apps/hotfix/serializer.py | 0 .../sysom_hotfix/apps/hotfix/urls.py | 0 .../sysom_hotfix/apps/hotfix/views.py | 0 .../sysom_hotfix/conf}/__init__.py | 0 .../sysom_hotfix/conf/common.py | 0 .../sysom_hotfix/conf/develop.py | 0 .../sysom_hotfix/conf/gunicorn.py | 0 .../sysom_hotfix/conf/product.py | 0 .../sysom_hotfix/conf/testing.py | 0 .../sysom_hotfix/config.yml | 0 .../sysom_hotfix/lib/__init__.py | 0 .../sysom_hotfix/lib/authentications.py | 0 .../sysom_hotfix/lib/base_model.py | 0 .../sysom_hotfix/lib/base_view.py | 0 .../sysom_hotfix/lib/decode/sysom_decode.py | 0 .../sysom_hotfix/lib/excel.py | 0 .../sysom_hotfix/lib/exception.py | 0 .../sysom_hotfix/lib/paginations.py | 0 .../sysom_hotfix/lib/renderers.py | 0 .../sysom_hotfix/lib/response.py | 0 .../sysom_hotfix/lib/utils.py | 0 .../sysom_hotfix/manage.py | 0 .../sysom_hotfix/sysom_hotfix/__init__.py | 0 .../sysom_hotfix/sysom_hotfix/asgi.py | 0 .../sysom_hotfix/sysom_hotfix/settings.py | 0 .../sysom_hotfix/sysom_hotfix/urls.py | 0 .../sysom_hotfix/sysom_hotfix/wsgi.py | 0 .../sysom_hotfix_builder/build_hotfix.sh | 0 .../sysom_hotfix_builder/build_rpm.sh | 0 .../sysom_hotfix_builder/builder.ini | 0 .../sysom_hotfix_builder/builder.py | 0 .../sysom_hotfix_builder/check_env.sh | 0 .../sysom_hotfix_builder/img_list.json | 0 .../apps/migration}/__init__.py | 0 .../sysom_migration/apps/migration/admin.py | 0 .../sysom_migration/apps/migration/apps.py | 0 .../apps/migration/migrations/0001_initial.py | 0 .../apps/migration/migrations}/__init__.py | 0 .../sysom_migration/apps/migration/models.py | 0 .../sysom_migration/apps/migration/tests.py | 0 .../sysom_migration/apps/migration/urls.py | 0 .../sysom_migration/apps/migration/views.py | 0 .../sysom_migration/conf/common.py | 0 .../sysom_migration/conf/develop.py | 0 .../sysom_migration/conf/gunicorn.py | 0 .../sysom_migration/conf/product.py | 0 .../sysom_migration/conf/testing.py | 0 .../sysom_migration/config.yml | 0 .../sysom_migration/lib/authentications.py | 0 .../sysom_migration/lib/base_model.py | 0 .../sysom_migration/lib/base_view.py | 0 .../sysom_migration/lib/channel.py | 0 .../lib/decode/sysom_decode.py | 0 .../sysom_migration/lib/exception.py | 0 .../sysom_migration/lib/paginations.py | 0 .../sysom_migration/lib/renderers.py | 0 .../sysom_migration/lib/response.py | 0 .../sysom_migration/lib/script.py | 0 .../sysom_migration/lib/utils.py | 0 .../sysom_migration/manage.py | 0 .../sysom_migration/__init__.py | 0 .../sysom_migration/sysom_migration/asgi.py | 0 .../sysom_migration/settings.py | 0 .../sysom_migration/sysom_migration/urls.py | 0 .../sysom_migration/sysom_migration/wsgi.py | 0 .../sysom_monitor_server/.gitignore | 0 .../app/routeres/grafana.py | 0 .../app/routeres/health.py | 0 .../sysom_monitor_server/app/routeres/home.py | 0 .../app/routeres/services.py | 0 .../sysom_monitor_server/conf/common.py | 0 .../sysom_monitor_server/conf/develop.py | 0 .../sysom_monitor_server/conf/gunicorn.py | 0 .../sysom_monitor_server/conf/product.py | 0 .../sysom_monitor_server/conf/settings.py | 0 .../sysom_monitor_server/conf/testing.py | 0 .../sysom_monitor_server/config.yml | 0 .../sysom_monitor_server/main.py | 0 .../scripts/node_clear.sh | 0 .../sysom_monitor_server/scripts/node_init.sh | 0 .../scripts/node_update.sh | 0 .../sysom_vmcore/apps/vmcore}/__init__.py | 0 .../sysom_vmcore/apps/vmcore/admin.py | 0 .../sysom_vmcore/apps/vmcore/apps.py | 0 .../apps/vmcore/migrations/0001_initial.py | 0 .../apps/vmcore/migrations}/__init__.py | 0 .../sysom_vmcore/apps/vmcore/models.py | 0 .../sysom_vmcore/apps/vmcore/serializer.py | 0 .../sysom_vmcore/apps/vmcore/tests.py | 0 .../sysom_vmcore/apps/vmcore/urls.py | 0 .../sysom_vmcore/apps/vmcore/views.py | 0 .../sysom_vmcore/apps/vmcore/vmcore.json | 0 .../sysom_vmcore/conf/common.py | 0 .../sysom_vmcore/conf/develop.py | 0 .../sysom_vmcore/conf/gunicorn.py | 0 .../sysom_vmcore/conf/product.py | 0 .../sysom_vmcore/conf/testing.py | 0 .../sysom_vmcore/config.yml | 0 .../sysom_vmcore/lib/authentications.py | 0 .../sysom_vmcore/lib/base_model.py | 0 .../sysom_vmcore/lib/decode/sysom_decode.py | 0 .../sysom_vmcore/lib/exception.py | 0 .../sysom_vmcore/lib/paginations.py | 0 .../sysom_vmcore/lib/renderers.py | 0 .../sysom_vmcore/lib/response.py | 0 .../sysom_vmcore/lib/utils.py | 0 .../sysom_vmcore/manage.py | 0 .../sysom_vmcore/scripts/node_clear.sh | 0 .../sysom_vmcore/scripts/node_init.sh | 0 .../sysom_vmcore/scripts/node_update.sh | 0 .../sysom_vmcore/scripts/vmcore_collect.py | 0 .../sysom_vmcore/sysom_vmcore/__init__.py | 0 .../sysom_vmcore/sysom_vmcore/asgi.py | 0 .../sysom_vmcore/sysom_vmcore/settings.py | 0 .../sysom_vmcore/sysom_vmcore/urls.py | 0 .../sysom_vmcore/sysom_vmcore/wsgi.py | 0 .../sysom_vul/apps/host/__init__.py | 0 .../sysom_vul/apps/host/app.py | 0 .../sysom_vul/apps/host/models.py | 0 .../sysom_vul/apps/host/urls.py | 0 .../sysom_vul/apps/vul/__init__.py | 0 .../sysom_vul/apps/vul/admin.py | 0 .../sysom_vul/apps/vul/apps.py | 0 .../sysom_vul/apps/vul/async_fetch.py | 0 .../sysom_vul/apps/vul/executor.py | 0 .../apps/vul/migrations/0001_initial.py | 0 .../vul/migrations/0002_auto_20230324_1840.py | 0 .../sysom_vul/apps/vul/migrations/__init__.py | 0 .../sysom_vul/apps/vul/models.py | 0 .../sysom_vul/apps/vul/serializer.py | 0 .../sysom_vul/apps/vul/ssh_pool.py | 0 .../sysom_vul/apps/vul/tests.py | 0 .../sysom_vul/apps/vul/urls.py | 0 .../sysom_vul/apps/vul/views.py | 0 .../sysom_vul/apps/vul/vul.py | 0 sysom_server/sysom_vul/conf/__init__.py | 0 .../sysom_vul/conf/common.py | 0 .../sysom_vul/conf/develop.py | 0 .../sysom_vul/conf/gunicorn.py | 0 .../sysom_vul/conf/product.py | 0 .../sysom_vul/conf/testing.py | 0 .../sysom_vul/config.yml | 0 .../sysom_vul/lib/authentications.py | 0 .../sysom_vul/lib/base_model.py | 0 .../sysom_vul/lib/decode/sysom_decode.py | 0 .../sysom_vul/lib/exception.py | 0 .../sysom_vul/lib/paginations.py | 0 .../sysom_vul/lib/response.py | 0 .../sysom_vul/lib/utils.py | 0 .../sysom_vul/manage.py | 0 .../sysom_vul/sysom_vul/__init__.py | 0 .../sysom_vul/sysom_vul/asgi.py | 0 .../sysom_vul/sysom_vul/settings.py | 0 .../sysom_vul/sysom_vul/urls.py | 0 .../sysom_vul/sysom_vul/wsgi.py | 0 565 files changed, 296 insertions(+), 263 deletions(-) rename {infrastructure => environment}/0_env/.pydistutils.cfg (100%) rename {infrastructure => environment}/0_env/clear.sh (100%) rename {infrastructure => environment}/0_env/init.sh (100%) rename {infrastructure => environment}/0_env/pip.conf (100%) rename {infrastructure => environment}/0_env/requirements.txt (100%) rename {infrastructure => environment}/0_env/start.sh (100%) rename {infrastructure => environment}/0_env/stop.sh (100%) rename {infrastructure => environment}/1_sdk/.gitignore (100%) rename {infrastructure => environment}/1_sdk/cec_base/__init__.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/admin.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/base.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/cec_client.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/cli.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/consumer.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/event.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/exceptions.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/meta.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/producer.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/url.py (100%) rename {infrastructure => environment}/1_sdk/cec_base/utils.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/__init__.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/admin_static.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/common.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/consume_status_storage.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/heartbeat.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/redis_admin.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/redis_consumer.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/redis_producer.py (100%) rename {infrastructure => environment}/1_sdk/cec_redis/utils.py (100%) rename {infrastructure => environment}/1_sdk/channel_job/__init__.py (100%) rename {infrastructure => environment}/1_sdk/channel_job/exception.py (100%) rename {infrastructure => environment}/1_sdk/channel_job/job.py (100%) rename {infrastructure => environment}/1_sdk/channel_job/model.py (100%) rename {infrastructure => environment}/1_sdk/clear.sh (100%) rename {infrastructure => environment}/1_sdk/clogger/__init__.py (100%) rename {infrastructure => environment}/1_sdk/clogger/clogger.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/__init__.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/exceptions.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/load_balancing_strategy.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/service_check.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/service_discovery.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/service_instance.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/service_invoker.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/service_registry.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/url.py (100%) rename {infrastructure => environment}/1_sdk/cmg_base/utils.py (100%) rename {infrastructure => environment}/1_sdk/cmg_redis/__init__.py (100%) rename {infrastructure => environment}/1_sdk/cmg_redis/common.py (100%) rename {infrastructure => environment}/1_sdk/cmg_redis/redis_service_discovery.py (100%) rename {infrastructure => environment}/1_sdk/cmg_redis/redis_service_registry.py (100%) rename {infrastructure => environment}/1_sdk/cmg_redis/utils.py (100%) rename {infrastructure => environment}/1_sdk/gcache_base/__init__.py (100%) rename {infrastructure => environment}/1_sdk/gcache_base/exceptions.py (100%) rename {infrastructure => environment}/1_sdk/gcache_base/gcache.py (100%) rename {infrastructure => environment}/1_sdk/gcache_base/url.py (100%) rename {infrastructure => environment}/1_sdk/gcache_redis/__init__.py (100%) rename {infrastructure => environment}/1_sdk/gcache_redis/common.py (100%) rename {infrastructure => environment}/1_sdk/gcache_redis/redis_gcache.py (100%) rename {infrastructure => environment}/1_sdk/gcache_redis/utils.py (100%) rename {infrastructure => environment}/1_sdk/gclient_base/__init__.py (100%) rename {infrastructure => environment}/1_sdk/gclient_base/exceptions.py (100%) rename {infrastructure => environment}/1_sdk/gclient_base/gclient.py (100%) rename {infrastructure => environment}/1_sdk/gclient_base/url.py (100%) rename {infrastructure => environment}/1_sdk/gclient_cmg/__init__.py (100%) rename {infrastructure => environment}/1_sdk/gclient_cmg/cmg_gclient.py (100%) rename {infrastructure => environment}/1_sdk/gclient_cmg/common.py (100%) rename {infrastructure => environment}/1_sdk/gclient_http/__init__.py (100%) rename {infrastructure => environment}/1_sdk/gclient_http/common.py (100%) rename {infrastructure => environment}/1_sdk/gclient_http/http_gclient.py (100%) rename {infrastructure => environment}/1_sdk/init.sh (100%) rename {infrastructure => environment}/1_sdk/redis_lua/__init__.py (100%) rename {infrastructure => environment}/1_sdk/redis_lua/x_hdel.lua (100%) rename {infrastructure => environment}/1_sdk/redis_lua/x_hdrop_table.lua (100%) rename {infrastructure => environment}/1_sdk/redis_lua/x_hexpire.lua (100%) rename {infrastructure => environment}/1_sdk/redis_lua/x_hget.lua (100%) rename {infrastructure => environment}/1_sdk/redis_lua/x_hgetall.lua (100%) rename {infrastructure => environment}/1_sdk/redis_lua/x_hkeys.lua (100%) rename {infrastructure => environment}/1_sdk/redis_lua/x_hlen.lua (100%) rename {infrastructure => environment}/1_sdk/redis_lua/x_hset.lua (100%) rename {infrastructure => environment}/1_sdk/redis_lua/xreadis_hash_table.py (100%) rename {infrastructure => environment}/1_sdk/setup_cec_base.py (100%) rename {infrastructure => environment}/1_sdk/setup_cec_redis.py (100%) rename {infrastructure => environment}/1_sdk/setup_channel_job.py (100%) rename {infrastructure => environment}/1_sdk/setup_clogger.py (100%) rename {infrastructure => environment}/1_sdk/setup_cmg_base.py (100%) rename {infrastructure => environment}/1_sdk/setup_cmg_redis.py (100%) rename {infrastructure => environment}/1_sdk/setup_gcache_base.py (100%) rename {infrastructure => environment}/1_sdk/setup_gcache_redis.py (100%) rename {infrastructure => environment}/1_sdk/setup_gclient_base.py (100%) rename {infrastructure => environment}/1_sdk/setup_gclient_cmg.py (100%) rename {infrastructure => environment}/1_sdk/setup_gclient_http.py (100%) rename {infrastructure => environment}/1_sdk/setup_redis_lua.py (100%) rename {infrastructure => environment}/1_sdk/setup_sysom_utils.py (100%) rename {infrastructure => environment}/1_sdk/start.sh (100%) rename {infrastructure => environment}/1_sdk/stop.sh (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/__init__.py (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/adddict.py (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/config_parser.py (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/event_executor.py (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/framework.py (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/framework_plug_mag.py (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/framework_plugins.py (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/node_manager.py (100%) rename {infrastructure => environment}/1_sdk/sysom_utils/yaml_concat.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/__init__.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/test_heartbeat.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/test_redis_admin.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/test_redis_admin_async.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/test_redis_consumer.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/test_redis_consumer_async.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/test_redis_multi_consumer.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/test_redis_producer.py (100%) rename {infrastructure => environment}/1_sdk/test_cec_redis/test_utils.py (100%) rename {infrastructure => environment}/1_sdk/test_cmg_base/__init__.py (100%) rename {infrastructure => environment}/1_sdk/test_cmg_base/test_health_check.py (100%) rename {infrastructure => environment}/1_sdk/test_cmg_redis/__init__.py (100%) rename {infrastructure => environment}/1_sdk/test_cmg_redis/test_service_discovery.py (100%) rename {infrastructure => environment}/1_sdk/test_cmg_redis/test_service_registry.py (100%) rename {infrastructure => environment}/2_local_services/clear.sh (100%) rename {infrastructure => environment}/2_local_services/init.sh (100%) rename {infrastructure => environment}/2_local_services/nginx.conf (100%) rename {infrastructure => environment}/2_local_services/start.sh (100%) rename {infrastructure => environment}/2_local_services/stop.sh (100%) rename {infrastructure => environment}/2_local_services/sysom-redis.ini (100%) rename {infrastructure => environment}/2_local_services/sysom.conf (100%) rename {infrastructure => environment}/3_monitor/Readme.txt (100%) rename {infrastructure => environment}/3_monitor/clear.sh (84%) rename {infrastructure => environment}/3_monitor/grafana_api_set.sh (100%) rename {infrastructure => environment}/3_monitor/grafana_recover.sh (100%) rename {infrastructure => environment}/3_monitor/init.sh (100%) rename {infrastructure => environment}/3_monitor/local_copy_pkg.sh (100%) rename {infrastructure => environment}/3_monitor/prometheus_get_node.py (100%) rename {infrastructure => environment}/3_monitor/start.sh (100%) rename {infrastructure => environment}/3_monitor/stop.sh (100%) rename {infrastructure => environment}/3_monitor/sysom-cec-status-dashboard.json (100%) rename {infrastructure => environment}/3_monitor/sysom-dashboard.json (100%) rename {infrastructure => environment}/3_monitor/sysom-migration-dashboard.json (100%) rename {infrastructure => environment}/3_monitor/sysom-netinfo-dashboard.json (100%) rename {infrastructure => environment}/3_monitor/sysom-prometheus.ini (56%) rename {infrastructure => environment}/clear_exclude (100%) rename {infrastructure => environment}/deploy_exclude (100%) delete mode 100644 microservice/sysom_vmcore/scripts/server_db_migrate.sh delete mode 100644 microservice/sysom_vul/scripts/server_db_migrate.sh rename {microservice => script/server}/clear_exclude (100%) rename {microservice => script/server}/deploy_exclude (100%) rename microservice/0_sysom_api/scripts/server_clear.sh => script/server/sysom_api/clear.sh (74%) rename microservice/sysom_migration/scripts/server_db_migrate.sh => script/server/sysom_api/db_migrate.sh (73%) rename microservice/0_sysom_api/scripts/server_init.sh => script/server/sysom_api/init.sh (84%) rename {microservice/0_sysom_api => script/server/sysom_api}/requirements.txt (100%) rename microservice/0_sysom_api/scripts/server_start.sh => script/server/sysom_api/start.sh (100%) rename microservice/0_sysom_api/scripts/server_stop.sh => script/server/sysom_api/stop.sh (100%) rename {microservice/0_sysom_api => script/server/sysom_api}/sysom-api.ini (44%) rename microservice/0_sysom_channel/scripts/server_clear.sh => script/server/sysom_channel/clear.sh (100%) rename microservice/0_sysom_channel/scripts/server_db_migrate.sh => script/server/sysom_channel/db_migrate.sh (73%) rename microservice/0_sysom_channel/scripts/server_init.sh => script/server/sysom_channel/init.sh (83%) rename {microservice/0_sysom_channel => script/server/sysom_channel}/requirements.txt (100%) rename microservice/0_sysom_channel/scripts/server_start.sh => script/server/sysom_channel/start.sh (100%) rename microservice/0_sysom_channel/scripts/server_stop.sh => script/server/sysom_channel/stop.sh (100%) rename {microservice/0_sysom_channel => script/server/sysom_channel}/sysom-channel.ini (42%) rename microservice/sysom_diagnosis/scripts/server_clear.sh => script/server/sysom_diagnosis/clear.sh (100%) rename microservice/sysom_hotfix/scripts/server_db_migrate.sh => script/server/sysom_diagnosis/db_migrate.sh (73%) rename microservice/0_sysom_api/apps/accounts/__init__.py => script/server/sysom_diagnosis/deploy.sh (100%) rename microservice/sysom_diagnosis/scripts/server_init.sh => script/server/sysom_diagnosis/init.sh (85%) rename {microservice => script/server}/sysom_diagnosis/requirements.txt (100%) rename microservice/sysom_diagnosis/scripts/server_start.sh => script/server/sysom_diagnosis/start.sh (100%) rename microservice/sysom_diagnosis/scripts/server_stop.sh => script/server/sysom_diagnosis/stop.sh (100%) rename {microservice => script/server}/sysom_diagnosis/sysom-diagnosis.ini (40%) rename microservice/sysom_hotfix/scripts/server_clear.sh => script/server/sysom_hotfix/clear.sh (100%) rename microservice/sysom_diagnosis/scripts/server_db_migrate.sh => script/server/sysom_hotfix/db_migrate.sh (73%) rename microservice/sysom_hotfix/scripts/server_init.sh => script/server/sysom_hotfix/init.sh (90%) rename {microservice => script/server}/sysom_hotfix/requirements.txt (100%) rename microservice/sysom_hotfix/scripts/server_start.sh => script/server/sysom_hotfix/start.sh (100%) rename microservice/sysom_hotfix/scripts/server_stop.sh => script/server/sysom_hotfix/stop.sh (100%) rename {microservice => script/server}/sysom_hotfix/sysom-hotfix.ini (39%) rename microservice/sysom_hotfix_builder/scripts/server_clear.sh => script/server/sysom_hotfix_builder/clear.sh (100%) rename microservice/sysom_hotfix_builder/scripts/server_init.sh => script/server/sysom_hotfix_builder/init.sh (88%) rename {microservice => script/server}/sysom_hotfix_builder/requirements.txt (100%) rename microservice/sysom_hotfix_builder/scripts/server_start.sh => script/server/sysom_hotfix_builder/start.sh (100%) rename microservice/sysom_hotfix_builder/scripts/server_stop.sh => script/server/sysom_hotfix_builder/stop.sh (100%) rename {microservice => script/server}/sysom_hotfix_builder/sysom-hotfix-builder.ini (45%) rename microservice/sysom_migration/scripts/server_clear.sh => script/server/sysom_migration/clear.sh (100%) rename microservice/0_sysom_api/scripts/server_db_migrate.sh => script/server/sysom_migration/db_migrate.sh (73%) rename microservice/sysom_migration/scripts/server_init.sh => script/server/sysom_migration/init.sh (92%) rename {microservice => script/server}/sysom_migration/requirements.txt (100%) rename microservice/0_sysom_api/apps/accounts/migrations/__init__.py => script/server/sysom_migration/start.sh (100%) rename microservice/0_sysom_api/apps/alarm/migrations/__init__.py => script/server/sysom_migration/stop.sh (100%) rename {microservice => script/server}/sysom_migration/sysom-migration.ini (40%) rename microservice/sysom_monitor_server/scripts/server_clear.sh => script/server/sysom_monitor_server/clear.sh (100%) rename microservice/sysom_monitor_server/scripts/server_db_migrate.sh => script/server/sysom_monitor_server/db_migrate.sh (73%) rename microservice/sysom_monitor_server/scripts/server_init.sh => script/server/sysom_monitor_server/init.sh (84%) rename {microservice => script/server}/sysom_monitor_server/requirements.txt (100%) rename microservice/sysom_monitor_server/scripts/server_start.sh => script/server/sysom_monitor_server/start.sh (100%) rename microservice/sysom_monitor_server/scripts/server_stop.sh => script/server/sysom_monitor_server/stop.sh (100%) rename {microservice => script/server}/sysom_monitor_server/sysom-monitor-server.ini (44%) rename microservice/sysom_vmcore/scripts/server_clear.sh => script/server/sysom_vmcore/clear.sh (91%) create mode 100644 script/server/sysom_vmcore/db_migrate.sh rename microservice/sysom_vmcore/scripts/server_init.sh => script/server/sysom_vmcore/init.sh (91%) rename {microservice/sysom_vmcore/scripts => script/server/sysom_vmcore}/parse_panic.py (100%) rename {microservice => script/server}/sysom_vmcore/requirements.txt (100%) rename microservice/sysom_vmcore/scripts/server_start.sh => script/server/sysom_vmcore/start.sh (100%) rename microservice/sysom_vmcore/scripts/server_stop.sh => script/server/sysom_vmcore/stop.sh (100%) rename {microservice => script/server}/sysom_vmcore/sysom-vmcore.ini (39%) rename {microservice/sysom_vmcore/scripts => script/server/sysom_vmcore}/vmcore_const.py (100%) rename microservice/sysom_vul/scripts/server_clear.sh => script/server/sysom_vul/clear.sh (100%) create mode 100644 script/server/sysom_vul/db_migrate.sh rename microservice/sysom_vul/scripts/server_init.sh => script/server/sysom_vul/init.sh (85%) rename {microservice => script/server}/sysom_vul/requirements.txt (100%) rename microservice/sysom_vul/scripts/server_start.sh => script/server/sysom_vul/start.sh (100%) rename microservice/sysom_vul/scripts/server_stop.sh => script/server/sysom_vul/stop.sh (100%) rename {microservice => script/server}/sysom_vul/sysom-vul.ini (39%) rename script/{clear.sh => sysom_clear.sh} (73%) rename script/{dbmigrate.sh => sysom_dbmigrate.sh} (100%) rename script/{deploy.sh => sysom_deploy.sh} (59%) rename {microservice => sysom_server}/.gitignore (100%) rename microservice/0_sysom_api/apps/common/__init__.py => sysom_server/clear_exclude (100%) create mode 100644 sysom_server/deploy_exclude rename {microservice/0_sysom_api => sysom_server/sysom_api}/.gitignore (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/__init__.py (100%) rename {microservice/0_sysom_api/apps/host => sysom_server/sysom_api/apps/accounts}/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/admin.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/apps.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/authentication.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/migrations/0001_initial.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/migrations/0002_user_allow_login.py (100%) rename {microservice/0_sysom_api/apps/host => sysom_server/sysom_api/apps/accounts}/migrations/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/models.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/permissions.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/serializer.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/tests.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/urls.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/accounts/views.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/admin.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/apps.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/migrations/0001_initial.py (100%) rename {microservice/0_sysom_api/apps/services => sysom_server/sysom_api/apps/alarm}/migrations/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/models.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/serializer.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/subscribe.json (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/urls.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/alarm/views.py (100%) rename {microservice/0_sysom_api/conf => sysom_server/sysom_api/apps/common}/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/common/common_model_viewset.py (100%) rename {microservice/sysom_diagnosis/apps/task => sysom_server/sysom_api/apps/host}/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/admin.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/apps.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/cec_api.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/heartbeat.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/migrations/0001_initial.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/migrations/0002_hostmodel_host_info.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/migrations/0003_alter_hostmodel_status.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/migrations/0004_auto_20221227_1705.py (100%) rename {microservice/sysom_diagnosis/apps/task => sysom_server/sysom_api/apps/host}/migrations/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/models.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/serializer.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/tests.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/urls.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/host/views.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/admin.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/apps.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/migrations/0001_initial.py (100%) rename {microservice/sysom_hotfix/apps/hotfix => sysom_server/sysom_api/apps/services/migrations}/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/models.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/serializer.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/tests.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/urls.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/apps/services/views.py (100%) rename {microservice/sysom_hotfix/apps/hotfix/migrations => sysom_server/sysom_api/conf}/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/conf/common.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/conf/develop.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/conf/product.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/conf/testing.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/config.yml (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/consumer/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/consumer/consumers.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/consumer/middleware.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/consumer/routing.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/authentications.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/base_model.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/decode/sysom_decode.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/excel.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/exception.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/paginations.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/renderers.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/response.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/ssh.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/lib/utils.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/manage.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/sysom/__init__.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/sysom/asgi.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/sysom/settings.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/sysom/urls.py (100%) rename {microservice/0_sysom_api => sysom_server/sysom_api}/sysom/wsgi.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/alembic.ini (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/alembic/README (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/alembic/env.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/alembic/script.py.mako (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/alembic/versions/1663292d0bd8_init_db.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/alembic/versions/3f54250e3fca_add_channel_params_table.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/__init__.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/crud.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/database.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/executor.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/models.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/routers/cec_status.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/routers/config.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/routers/file.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/routers/health.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/app/schemas.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/conf/common.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/conf/develop.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/conf/gunicorn.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/conf/product.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/conf/settings.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/conf/testing.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/config.yml (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/lib/channels/__init__.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/lib/channels/base.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/lib/channels/ssh.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/lib/ssh.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/lib/utils.py (100%) rename {microservice/0_sysom_channel => sysom_server/sysom_channel}/main.py (100%) rename {microservice/sysom_hotfix/conf => sysom_server/sysom_diagnosis/apps/task}/__init__.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/admin.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/apps.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/executor.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/filter.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/helper.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/migrations/0001_initial.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py (100%) rename {microservice/sysom_migration/apps/migration => sysom_server/sysom_diagnosis/apps/task/migrations}/__init__.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/models.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/seriaizer.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/tests.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/urls.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/apps/task/views.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/conf/common.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/conf/develop.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/conf/gunicorn.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/conf/product.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/conf/testing.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/config.yml (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/authentications.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/base_model.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/base_view.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/decode/sysom_decode.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/exception.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/paginations.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/renderers.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/response.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/lib/utils.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/manage.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/scripts/node_clear.sh (100%) rename {microservice => sysom_server}/sysom_diagnosis/scripts/node_init.sh (100%) rename {microservice => sysom_server}/sysom_diagnosis/scripts/node_update.sh (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/command (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/command_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/filecache (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/filecache_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/iofsstat (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/iofsstat_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/iohang (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/iohang_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/iolatency (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/iolatency_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/iosdiag_latency (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/iosdiag_latency_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/jitter (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/jitter_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/loadtask (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/loadtask_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/memgraph (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/memgraph_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/oomcheck (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/oomcheck_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/ossre (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/ossre_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/packetdrop (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/packetdrop_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/pingtrace (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/pingtrace_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/retran (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/retran_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/schedmoni (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/schedmoni_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/taskprofile (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/taskprofile_post (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/test (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/test.sh (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/tpython (100%) rename {microservice => sysom_server}/sysom_diagnosis/service_scripts/vmcore_analyse (100%) rename {microservice => sysom_server}/sysom_diagnosis/sysom_diagnosis/__init__.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/sysom_diagnosis/asgi.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/sysom_diagnosis/settings.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/sysom_diagnosis/urls.py (100%) rename {microservice => sysom_server}/sysom_diagnosis/sysom_diagnosis/wsgi.py (100%) rename {microservice/sysom_migration/apps/migration/migrations => sysom_server/sysom_hotfix/apps/hotfix}/__init__.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/admin.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/apps.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/function.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/migrations/0001_initial.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py (100%) rename {microservice/sysom_vmcore/apps/vmcore => sysom_server/sysom_hotfix/apps/hotfix/migrations}/__init__.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/models.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/serializer.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/urls.py (100%) rename {microservice => sysom_server}/sysom_hotfix/apps/hotfix/views.py (100%) rename {microservice/sysom_vmcore/apps/vmcore/migrations => sysom_server/sysom_hotfix/conf}/__init__.py (100%) rename {microservice => sysom_server}/sysom_hotfix/conf/common.py (100%) rename {microservice => sysom_server}/sysom_hotfix/conf/develop.py (100%) rename {microservice => sysom_server}/sysom_hotfix/conf/gunicorn.py (100%) rename {microservice => sysom_server}/sysom_hotfix/conf/product.py (100%) rename {microservice => sysom_server}/sysom_hotfix/conf/testing.py (100%) rename {microservice => sysom_server}/sysom_hotfix/config.yml (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/__init__.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/authentications.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/base_model.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/base_view.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/decode/sysom_decode.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/excel.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/exception.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/paginations.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/renderers.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/response.py (100%) rename {microservice => sysom_server}/sysom_hotfix/lib/utils.py (100%) rename {microservice => sysom_server}/sysom_hotfix/manage.py (100%) rename {microservice => sysom_server}/sysom_hotfix/sysom_hotfix/__init__.py (100%) rename {microservice => sysom_server}/sysom_hotfix/sysom_hotfix/asgi.py (100%) rename {microservice => sysom_server}/sysom_hotfix/sysom_hotfix/settings.py (100%) rename {microservice => sysom_server}/sysom_hotfix/sysom_hotfix/urls.py (100%) rename {microservice => sysom_server}/sysom_hotfix/sysom_hotfix/wsgi.py (100%) rename {microservice => sysom_server}/sysom_hotfix_builder/build_hotfix.sh (100%) rename {microservice => sysom_server}/sysom_hotfix_builder/build_rpm.sh (100%) rename {microservice => sysom_server}/sysom_hotfix_builder/builder.ini (100%) rename {microservice => sysom_server}/sysom_hotfix_builder/builder.py (100%) rename {microservice => sysom_server}/sysom_hotfix_builder/check_env.sh (100%) rename {microservice => sysom_server}/sysom_hotfix_builder/img_list.json (100%) rename {microservice/sysom_vul/apps/host => sysom_server/sysom_migration/apps/migration}/__init__.py (100%) rename {microservice => sysom_server}/sysom_migration/apps/migration/admin.py (100%) rename {microservice => sysom_server}/sysom_migration/apps/migration/apps.py (100%) rename {microservice => sysom_server}/sysom_migration/apps/migration/migrations/0001_initial.py (100%) rename {microservice/sysom_vul/apps/vul => sysom_server/sysom_migration/apps/migration/migrations}/__init__.py (100%) rename {microservice => sysom_server}/sysom_migration/apps/migration/models.py (100%) rename {microservice => sysom_server}/sysom_migration/apps/migration/tests.py (100%) rename {microservice => sysom_server}/sysom_migration/apps/migration/urls.py (100%) rename {microservice => sysom_server}/sysom_migration/apps/migration/views.py (100%) rename {microservice => sysom_server}/sysom_migration/conf/common.py (100%) rename {microservice => sysom_server}/sysom_migration/conf/develop.py (100%) rename {microservice => sysom_server}/sysom_migration/conf/gunicorn.py (100%) rename {microservice => sysom_server}/sysom_migration/conf/product.py (100%) rename {microservice => sysom_server}/sysom_migration/conf/testing.py (100%) rename {microservice => sysom_server}/sysom_migration/config.yml (100%) rename {microservice => sysom_server}/sysom_migration/lib/authentications.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/base_model.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/base_view.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/channel.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/decode/sysom_decode.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/exception.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/paginations.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/renderers.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/response.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/script.py (100%) rename {microservice => sysom_server}/sysom_migration/lib/utils.py (100%) rename {microservice => sysom_server}/sysom_migration/manage.py (100%) rename {microservice => sysom_server}/sysom_migration/sysom_migration/__init__.py (100%) rename {microservice => sysom_server}/sysom_migration/sysom_migration/asgi.py (100%) rename {microservice => sysom_server}/sysom_migration/sysom_migration/settings.py (100%) rename {microservice => sysom_server}/sysom_migration/sysom_migration/urls.py (100%) rename {microservice => sysom_server}/sysom_migration/sysom_migration/wsgi.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/.gitignore (100%) rename {microservice => sysom_server}/sysom_monitor_server/app/routeres/grafana.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/app/routeres/health.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/app/routeres/home.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/app/routeres/services.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/conf/common.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/conf/develop.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/conf/gunicorn.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/conf/product.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/conf/settings.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/conf/testing.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/config.yml (100%) rename {microservice => sysom_server}/sysom_monitor_server/main.py (100%) rename {microservice => sysom_server}/sysom_monitor_server/scripts/node_clear.sh (100%) rename {microservice => sysom_server}/sysom_monitor_server/scripts/node_init.sh (100%) rename {microservice => sysom_server}/sysom_monitor_server/scripts/node_update.sh (100%) rename {microservice/sysom_vul/apps/vul/migrations => sysom_server/sysom_vmcore/apps/vmcore}/__init__.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/admin.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/apps.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/migrations/0001_initial.py (100%) rename {microservice/sysom_vul/conf => sysom_server/sysom_vmcore/apps/vmcore/migrations}/__init__.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/models.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/serializer.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/tests.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/urls.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/views.py (100%) rename {microservice => sysom_server}/sysom_vmcore/apps/vmcore/vmcore.json (100%) rename {microservice => sysom_server}/sysom_vmcore/conf/common.py (100%) rename {microservice => sysom_server}/sysom_vmcore/conf/develop.py (100%) rename {microservice => sysom_server}/sysom_vmcore/conf/gunicorn.py (100%) rename {microservice => sysom_server}/sysom_vmcore/conf/product.py (100%) rename {microservice => sysom_server}/sysom_vmcore/conf/testing.py (100%) rename {microservice => sysom_server}/sysom_vmcore/config.yml (100%) rename {microservice => sysom_server}/sysom_vmcore/lib/authentications.py (100%) rename {microservice => sysom_server}/sysom_vmcore/lib/base_model.py (100%) rename {microservice => sysom_server}/sysom_vmcore/lib/decode/sysom_decode.py (100%) rename {microservice => sysom_server}/sysom_vmcore/lib/exception.py (100%) rename {microservice => sysom_server}/sysom_vmcore/lib/paginations.py (100%) rename {microservice => sysom_server}/sysom_vmcore/lib/renderers.py (100%) rename {microservice => sysom_server}/sysom_vmcore/lib/response.py (100%) rename {microservice => sysom_server}/sysom_vmcore/lib/utils.py (100%) rename {microservice => sysom_server}/sysom_vmcore/manage.py (100%) rename {microservice => sysom_server}/sysom_vmcore/scripts/node_clear.sh (100%) rename {microservice => sysom_server}/sysom_vmcore/scripts/node_init.sh (100%) rename {microservice => sysom_server}/sysom_vmcore/scripts/node_update.sh (100%) rename {microservice => sysom_server}/sysom_vmcore/scripts/vmcore_collect.py (100%) rename {microservice => sysom_server}/sysom_vmcore/sysom_vmcore/__init__.py (100%) rename {microservice => sysom_server}/sysom_vmcore/sysom_vmcore/asgi.py (100%) rename {microservice => sysom_server}/sysom_vmcore/sysom_vmcore/settings.py (100%) rename {microservice => sysom_server}/sysom_vmcore/sysom_vmcore/urls.py (100%) rename {microservice => sysom_server}/sysom_vmcore/sysom_vmcore/wsgi.py (100%) rename microservice/sysom_diagnosis/scripts/server_deploy.sh => sysom_server/sysom_vul/apps/host/__init__.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/host/app.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/host/models.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/host/urls.py (100%) rename microservice/sysom_migration/scripts/server_start.sh => sysom_server/sysom_vul/apps/vul/__init__.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/admin.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/apps.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/async_fetch.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/executor.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/migrations/0001_initial.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py (100%) rename microservice/sysom_migration/scripts/server_stop.sh => sysom_server/sysom_vul/apps/vul/migrations/__init__.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/models.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/serializer.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/ssh_pool.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/tests.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/urls.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/views.py (100%) rename {microservice => sysom_server}/sysom_vul/apps/vul/vul.py (100%) create mode 100644 sysom_server/sysom_vul/conf/__init__.py rename {microservice => sysom_server}/sysom_vul/conf/common.py (100%) rename {microservice => sysom_server}/sysom_vul/conf/develop.py (100%) rename {microservice => sysom_server}/sysom_vul/conf/gunicorn.py (100%) rename {microservice => sysom_server}/sysom_vul/conf/product.py (100%) rename {microservice => sysom_server}/sysom_vul/conf/testing.py (100%) rename {microservice => sysom_server}/sysom_vul/config.yml (100%) rename {microservice => sysom_server}/sysom_vul/lib/authentications.py (100%) rename {microservice => sysom_server}/sysom_vul/lib/base_model.py (100%) rename {microservice => sysom_server}/sysom_vul/lib/decode/sysom_decode.py (100%) rename {microservice => sysom_server}/sysom_vul/lib/exception.py (100%) rename {microservice => sysom_server}/sysom_vul/lib/paginations.py (100%) rename {microservice => sysom_server}/sysom_vul/lib/response.py (100%) rename {microservice => sysom_server}/sysom_vul/lib/utils.py (100%) rename {microservice => sysom_server}/sysom_vul/manage.py (100%) rename {microservice => sysom_server}/sysom_vul/sysom_vul/__init__.py (100%) rename {microservice => sysom_server}/sysom_vul/sysom_vul/asgi.py (100%) rename {microservice => sysom_server}/sysom_vul/sysom_vul/settings.py (100%) rename {microservice => sysom_server}/sysom_vul/sysom_vul/urls.py (100%) rename {microservice => sysom_server}/sysom_vul/sysom_vul/wsgi.py (100%) diff --git a/infrastructure/0_env/.pydistutils.cfg b/environment/0_env/.pydistutils.cfg similarity index 100% rename from infrastructure/0_env/.pydistutils.cfg rename to environment/0_env/.pydistutils.cfg diff --git a/infrastructure/0_env/clear.sh b/environment/0_env/clear.sh similarity index 100% rename from infrastructure/0_env/clear.sh rename to environment/0_env/clear.sh diff --git a/infrastructure/0_env/init.sh b/environment/0_env/init.sh similarity index 100% rename from infrastructure/0_env/init.sh rename to environment/0_env/init.sh diff --git a/infrastructure/0_env/pip.conf b/environment/0_env/pip.conf similarity index 100% rename from infrastructure/0_env/pip.conf rename to environment/0_env/pip.conf diff --git a/infrastructure/0_env/requirements.txt b/environment/0_env/requirements.txt similarity index 100% rename from infrastructure/0_env/requirements.txt rename to environment/0_env/requirements.txt diff --git a/infrastructure/0_env/start.sh b/environment/0_env/start.sh similarity index 100% rename from infrastructure/0_env/start.sh rename to environment/0_env/start.sh diff --git a/infrastructure/0_env/stop.sh b/environment/0_env/stop.sh similarity index 100% rename from infrastructure/0_env/stop.sh rename to environment/0_env/stop.sh diff --git a/infrastructure/1_sdk/.gitignore b/environment/1_sdk/.gitignore similarity index 100% rename from infrastructure/1_sdk/.gitignore rename to environment/1_sdk/.gitignore diff --git a/infrastructure/1_sdk/cec_base/__init__.py b/environment/1_sdk/cec_base/__init__.py similarity index 100% rename from infrastructure/1_sdk/cec_base/__init__.py rename to environment/1_sdk/cec_base/__init__.py diff --git a/infrastructure/1_sdk/cec_base/admin.py b/environment/1_sdk/cec_base/admin.py similarity index 100% rename from infrastructure/1_sdk/cec_base/admin.py rename to environment/1_sdk/cec_base/admin.py diff --git a/infrastructure/1_sdk/cec_base/base.py b/environment/1_sdk/cec_base/base.py similarity index 100% rename from infrastructure/1_sdk/cec_base/base.py rename to environment/1_sdk/cec_base/base.py diff --git a/infrastructure/1_sdk/cec_base/cec_client.py b/environment/1_sdk/cec_base/cec_client.py similarity index 100% rename from infrastructure/1_sdk/cec_base/cec_client.py rename to environment/1_sdk/cec_base/cec_client.py diff --git a/infrastructure/1_sdk/cec_base/cli.py b/environment/1_sdk/cec_base/cli.py similarity index 100% rename from infrastructure/1_sdk/cec_base/cli.py rename to environment/1_sdk/cec_base/cli.py diff --git a/infrastructure/1_sdk/cec_base/consumer.py b/environment/1_sdk/cec_base/consumer.py similarity index 100% rename from infrastructure/1_sdk/cec_base/consumer.py rename to environment/1_sdk/cec_base/consumer.py diff --git a/infrastructure/1_sdk/cec_base/event.py b/environment/1_sdk/cec_base/event.py similarity index 100% rename from infrastructure/1_sdk/cec_base/event.py rename to environment/1_sdk/cec_base/event.py diff --git a/infrastructure/1_sdk/cec_base/exceptions.py b/environment/1_sdk/cec_base/exceptions.py similarity index 100% rename from infrastructure/1_sdk/cec_base/exceptions.py rename to environment/1_sdk/cec_base/exceptions.py diff --git a/infrastructure/1_sdk/cec_base/meta.py b/environment/1_sdk/cec_base/meta.py similarity index 100% rename from infrastructure/1_sdk/cec_base/meta.py rename to environment/1_sdk/cec_base/meta.py diff --git a/infrastructure/1_sdk/cec_base/producer.py b/environment/1_sdk/cec_base/producer.py similarity index 100% rename from infrastructure/1_sdk/cec_base/producer.py rename to environment/1_sdk/cec_base/producer.py diff --git a/infrastructure/1_sdk/cec_base/url.py b/environment/1_sdk/cec_base/url.py similarity index 100% rename from infrastructure/1_sdk/cec_base/url.py rename to environment/1_sdk/cec_base/url.py diff --git a/infrastructure/1_sdk/cec_base/utils.py b/environment/1_sdk/cec_base/utils.py similarity index 100% rename from infrastructure/1_sdk/cec_base/utils.py rename to environment/1_sdk/cec_base/utils.py diff --git a/infrastructure/1_sdk/cec_redis/__init__.py b/environment/1_sdk/cec_redis/__init__.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/__init__.py rename to environment/1_sdk/cec_redis/__init__.py diff --git a/infrastructure/1_sdk/cec_redis/admin_static.py b/environment/1_sdk/cec_redis/admin_static.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/admin_static.py rename to environment/1_sdk/cec_redis/admin_static.py diff --git a/infrastructure/1_sdk/cec_redis/common.py b/environment/1_sdk/cec_redis/common.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/common.py rename to environment/1_sdk/cec_redis/common.py diff --git a/infrastructure/1_sdk/cec_redis/consume_status_storage.py b/environment/1_sdk/cec_redis/consume_status_storage.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/consume_status_storage.py rename to environment/1_sdk/cec_redis/consume_status_storage.py diff --git a/infrastructure/1_sdk/cec_redis/heartbeat.py b/environment/1_sdk/cec_redis/heartbeat.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/heartbeat.py rename to environment/1_sdk/cec_redis/heartbeat.py diff --git a/infrastructure/1_sdk/cec_redis/redis_admin.py b/environment/1_sdk/cec_redis/redis_admin.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/redis_admin.py rename to environment/1_sdk/cec_redis/redis_admin.py diff --git a/infrastructure/1_sdk/cec_redis/redis_consumer.py b/environment/1_sdk/cec_redis/redis_consumer.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/redis_consumer.py rename to environment/1_sdk/cec_redis/redis_consumer.py diff --git a/infrastructure/1_sdk/cec_redis/redis_producer.py b/environment/1_sdk/cec_redis/redis_producer.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/redis_producer.py rename to environment/1_sdk/cec_redis/redis_producer.py diff --git a/infrastructure/1_sdk/cec_redis/utils.py b/environment/1_sdk/cec_redis/utils.py similarity index 100% rename from infrastructure/1_sdk/cec_redis/utils.py rename to environment/1_sdk/cec_redis/utils.py diff --git a/infrastructure/1_sdk/channel_job/__init__.py b/environment/1_sdk/channel_job/__init__.py similarity index 100% rename from infrastructure/1_sdk/channel_job/__init__.py rename to environment/1_sdk/channel_job/__init__.py diff --git a/infrastructure/1_sdk/channel_job/exception.py b/environment/1_sdk/channel_job/exception.py similarity index 100% rename from infrastructure/1_sdk/channel_job/exception.py rename to environment/1_sdk/channel_job/exception.py diff --git a/infrastructure/1_sdk/channel_job/job.py b/environment/1_sdk/channel_job/job.py similarity index 100% rename from infrastructure/1_sdk/channel_job/job.py rename to environment/1_sdk/channel_job/job.py diff --git a/infrastructure/1_sdk/channel_job/model.py b/environment/1_sdk/channel_job/model.py similarity index 100% rename from infrastructure/1_sdk/channel_job/model.py rename to environment/1_sdk/channel_job/model.py diff --git a/infrastructure/1_sdk/clear.sh b/environment/1_sdk/clear.sh similarity index 100% rename from infrastructure/1_sdk/clear.sh rename to environment/1_sdk/clear.sh diff --git a/infrastructure/1_sdk/clogger/__init__.py b/environment/1_sdk/clogger/__init__.py similarity index 100% rename from infrastructure/1_sdk/clogger/__init__.py rename to environment/1_sdk/clogger/__init__.py diff --git a/infrastructure/1_sdk/clogger/clogger.py b/environment/1_sdk/clogger/clogger.py similarity index 100% rename from infrastructure/1_sdk/clogger/clogger.py rename to environment/1_sdk/clogger/clogger.py diff --git a/infrastructure/1_sdk/cmg_base/__init__.py b/environment/1_sdk/cmg_base/__init__.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/__init__.py rename to environment/1_sdk/cmg_base/__init__.py diff --git a/infrastructure/1_sdk/cmg_base/exceptions.py b/environment/1_sdk/cmg_base/exceptions.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/exceptions.py rename to environment/1_sdk/cmg_base/exceptions.py diff --git a/infrastructure/1_sdk/cmg_base/load_balancing_strategy.py b/environment/1_sdk/cmg_base/load_balancing_strategy.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/load_balancing_strategy.py rename to environment/1_sdk/cmg_base/load_balancing_strategy.py diff --git a/infrastructure/1_sdk/cmg_base/service_check.py b/environment/1_sdk/cmg_base/service_check.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/service_check.py rename to environment/1_sdk/cmg_base/service_check.py diff --git a/infrastructure/1_sdk/cmg_base/service_discovery.py b/environment/1_sdk/cmg_base/service_discovery.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/service_discovery.py rename to environment/1_sdk/cmg_base/service_discovery.py diff --git a/infrastructure/1_sdk/cmg_base/service_instance.py b/environment/1_sdk/cmg_base/service_instance.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/service_instance.py rename to environment/1_sdk/cmg_base/service_instance.py diff --git a/infrastructure/1_sdk/cmg_base/service_invoker.py b/environment/1_sdk/cmg_base/service_invoker.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/service_invoker.py rename to environment/1_sdk/cmg_base/service_invoker.py diff --git a/infrastructure/1_sdk/cmg_base/service_registry.py b/environment/1_sdk/cmg_base/service_registry.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/service_registry.py rename to environment/1_sdk/cmg_base/service_registry.py diff --git a/infrastructure/1_sdk/cmg_base/url.py b/environment/1_sdk/cmg_base/url.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/url.py rename to environment/1_sdk/cmg_base/url.py diff --git a/infrastructure/1_sdk/cmg_base/utils.py b/environment/1_sdk/cmg_base/utils.py similarity index 100% rename from infrastructure/1_sdk/cmg_base/utils.py rename to environment/1_sdk/cmg_base/utils.py diff --git a/infrastructure/1_sdk/cmg_redis/__init__.py b/environment/1_sdk/cmg_redis/__init__.py similarity index 100% rename from infrastructure/1_sdk/cmg_redis/__init__.py rename to environment/1_sdk/cmg_redis/__init__.py diff --git a/infrastructure/1_sdk/cmg_redis/common.py b/environment/1_sdk/cmg_redis/common.py similarity index 100% rename from infrastructure/1_sdk/cmg_redis/common.py rename to environment/1_sdk/cmg_redis/common.py diff --git a/infrastructure/1_sdk/cmg_redis/redis_service_discovery.py b/environment/1_sdk/cmg_redis/redis_service_discovery.py similarity index 100% rename from infrastructure/1_sdk/cmg_redis/redis_service_discovery.py rename to environment/1_sdk/cmg_redis/redis_service_discovery.py diff --git a/infrastructure/1_sdk/cmg_redis/redis_service_registry.py b/environment/1_sdk/cmg_redis/redis_service_registry.py similarity index 100% rename from infrastructure/1_sdk/cmg_redis/redis_service_registry.py rename to environment/1_sdk/cmg_redis/redis_service_registry.py diff --git a/infrastructure/1_sdk/cmg_redis/utils.py b/environment/1_sdk/cmg_redis/utils.py similarity index 100% rename from infrastructure/1_sdk/cmg_redis/utils.py rename to environment/1_sdk/cmg_redis/utils.py diff --git a/infrastructure/1_sdk/gcache_base/__init__.py b/environment/1_sdk/gcache_base/__init__.py similarity index 100% rename from infrastructure/1_sdk/gcache_base/__init__.py rename to environment/1_sdk/gcache_base/__init__.py diff --git a/infrastructure/1_sdk/gcache_base/exceptions.py b/environment/1_sdk/gcache_base/exceptions.py similarity index 100% rename from infrastructure/1_sdk/gcache_base/exceptions.py rename to environment/1_sdk/gcache_base/exceptions.py diff --git a/infrastructure/1_sdk/gcache_base/gcache.py b/environment/1_sdk/gcache_base/gcache.py similarity index 100% rename from infrastructure/1_sdk/gcache_base/gcache.py rename to environment/1_sdk/gcache_base/gcache.py diff --git a/infrastructure/1_sdk/gcache_base/url.py b/environment/1_sdk/gcache_base/url.py similarity index 100% rename from infrastructure/1_sdk/gcache_base/url.py rename to environment/1_sdk/gcache_base/url.py diff --git a/infrastructure/1_sdk/gcache_redis/__init__.py b/environment/1_sdk/gcache_redis/__init__.py similarity index 100% rename from infrastructure/1_sdk/gcache_redis/__init__.py rename to environment/1_sdk/gcache_redis/__init__.py diff --git a/infrastructure/1_sdk/gcache_redis/common.py b/environment/1_sdk/gcache_redis/common.py similarity index 100% rename from infrastructure/1_sdk/gcache_redis/common.py rename to environment/1_sdk/gcache_redis/common.py diff --git a/infrastructure/1_sdk/gcache_redis/redis_gcache.py b/environment/1_sdk/gcache_redis/redis_gcache.py similarity index 100% rename from infrastructure/1_sdk/gcache_redis/redis_gcache.py rename to environment/1_sdk/gcache_redis/redis_gcache.py diff --git a/infrastructure/1_sdk/gcache_redis/utils.py b/environment/1_sdk/gcache_redis/utils.py similarity index 100% rename from infrastructure/1_sdk/gcache_redis/utils.py rename to environment/1_sdk/gcache_redis/utils.py diff --git a/infrastructure/1_sdk/gclient_base/__init__.py b/environment/1_sdk/gclient_base/__init__.py similarity index 100% rename from infrastructure/1_sdk/gclient_base/__init__.py rename to environment/1_sdk/gclient_base/__init__.py diff --git a/infrastructure/1_sdk/gclient_base/exceptions.py b/environment/1_sdk/gclient_base/exceptions.py similarity index 100% rename from infrastructure/1_sdk/gclient_base/exceptions.py rename to environment/1_sdk/gclient_base/exceptions.py diff --git a/infrastructure/1_sdk/gclient_base/gclient.py b/environment/1_sdk/gclient_base/gclient.py similarity index 100% rename from infrastructure/1_sdk/gclient_base/gclient.py rename to environment/1_sdk/gclient_base/gclient.py diff --git a/infrastructure/1_sdk/gclient_base/url.py b/environment/1_sdk/gclient_base/url.py similarity index 100% rename from infrastructure/1_sdk/gclient_base/url.py rename to environment/1_sdk/gclient_base/url.py diff --git a/infrastructure/1_sdk/gclient_cmg/__init__.py b/environment/1_sdk/gclient_cmg/__init__.py similarity index 100% rename from infrastructure/1_sdk/gclient_cmg/__init__.py rename to environment/1_sdk/gclient_cmg/__init__.py diff --git a/infrastructure/1_sdk/gclient_cmg/cmg_gclient.py b/environment/1_sdk/gclient_cmg/cmg_gclient.py similarity index 100% rename from infrastructure/1_sdk/gclient_cmg/cmg_gclient.py rename to environment/1_sdk/gclient_cmg/cmg_gclient.py diff --git a/infrastructure/1_sdk/gclient_cmg/common.py b/environment/1_sdk/gclient_cmg/common.py similarity index 100% rename from infrastructure/1_sdk/gclient_cmg/common.py rename to environment/1_sdk/gclient_cmg/common.py diff --git a/infrastructure/1_sdk/gclient_http/__init__.py b/environment/1_sdk/gclient_http/__init__.py similarity index 100% rename from infrastructure/1_sdk/gclient_http/__init__.py rename to environment/1_sdk/gclient_http/__init__.py diff --git a/infrastructure/1_sdk/gclient_http/common.py b/environment/1_sdk/gclient_http/common.py similarity index 100% rename from infrastructure/1_sdk/gclient_http/common.py rename to environment/1_sdk/gclient_http/common.py diff --git a/infrastructure/1_sdk/gclient_http/http_gclient.py b/environment/1_sdk/gclient_http/http_gclient.py similarity index 100% rename from infrastructure/1_sdk/gclient_http/http_gclient.py rename to environment/1_sdk/gclient_http/http_gclient.py diff --git a/infrastructure/1_sdk/init.sh b/environment/1_sdk/init.sh similarity index 100% rename from infrastructure/1_sdk/init.sh rename to environment/1_sdk/init.sh diff --git a/infrastructure/1_sdk/redis_lua/__init__.py b/environment/1_sdk/redis_lua/__init__.py similarity index 100% rename from infrastructure/1_sdk/redis_lua/__init__.py rename to environment/1_sdk/redis_lua/__init__.py diff --git a/infrastructure/1_sdk/redis_lua/x_hdel.lua b/environment/1_sdk/redis_lua/x_hdel.lua similarity index 100% rename from infrastructure/1_sdk/redis_lua/x_hdel.lua rename to environment/1_sdk/redis_lua/x_hdel.lua diff --git a/infrastructure/1_sdk/redis_lua/x_hdrop_table.lua b/environment/1_sdk/redis_lua/x_hdrop_table.lua similarity index 100% rename from infrastructure/1_sdk/redis_lua/x_hdrop_table.lua rename to environment/1_sdk/redis_lua/x_hdrop_table.lua diff --git a/infrastructure/1_sdk/redis_lua/x_hexpire.lua b/environment/1_sdk/redis_lua/x_hexpire.lua similarity index 100% rename from infrastructure/1_sdk/redis_lua/x_hexpire.lua rename to environment/1_sdk/redis_lua/x_hexpire.lua diff --git a/infrastructure/1_sdk/redis_lua/x_hget.lua b/environment/1_sdk/redis_lua/x_hget.lua similarity index 100% rename from infrastructure/1_sdk/redis_lua/x_hget.lua rename to environment/1_sdk/redis_lua/x_hget.lua diff --git a/infrastructure/1_sdk/redis_lua/x_hgetall.lua b/environment/1_sdk/redis_lua/x_hgetall.lua similarity index 100% rename from infrastructure/1_sdk/redis_lua/x_hgetall.lua rename to environment/1_sdk/redis_lua/x_hgetall.lua diff --git a/infrastructure/1_sdk/redis_lua/x_hkeys.lua b/environment/1_sdk/redis_lua/x_hkeys.lua similarity index 100% rename from infrastructure/1_sdk/redis_lua/x_hkeys.lua rename to environment/1_sdk/redis_lua/x_hkeys.lua diff --git a/infrastructure/1_sdk/redis_lua/x_hlen.lua b/environment/1_sdk/redis_lua/x_hlen.lua similarity index 100% rename from infrastructure/1_sdk/redis_lua/x_hlen.lua rename to environment/1_sdk/redis_lua/x_hlen.lua diff --git a/infrastructure/1_sdk/redis_lua/x_hset.lua b/environment/1_sdk/redis_lua/x_hset.lua similarity index 100% rename from infrastructure/1_sdk/redis_lua/x_hset.lua rename to environment/1_sdk/redis_lua/x_hset.lua diff --git a/infrastructure/1_sdk/redis_lua/xreadis_hash_table.py b/environment/1_sdk/redis_lua/xreadis_hash_table.py similarity index 100% rename from infrastructure/1_sdk/redis_lua/xreadis_hash_table.py rename to environment/1_sdk/redis_lua/xreadis_hash_table.py diff --git a/infrastructure/1_sdk/setup_cec_base.py b/environment/1_sdk/setup_cec_base.py similarity index 100% rename from infrastructure/1_sdk/setup_cec_base.py rename to environment/1_sdk/setup_cec_base.py diff --git a/infrastructure/1_sdk/setup_cec_redis.py b/environment/1_sdk/setup_cec_redis.py similarity index 100% rename from infrastructure/1_sdk/setup_cec_redis.py rename to environment/1_sdk/setup_cec_redis.py diff --git a/infrastructure/1_sdk/setup_channel_job.py b/environment/1_sdk/setup_channel_job.py similarity index 100% rename from infrastructure/1_sdk/setup_channel_job.py rename to environment/1_sdk/setup_channel_job.py diff --git a/infrastructure/1_sdk/setup_clogger.py b/environment/1_sdk/setup_clogger.py similarity index 100% rename from infrastructure/1_sdk/setup_clogger.py rename to environment/1_sdk/setup_clogger.py diff --git a/infrastructure/1_sdk/setup_cmg_base.py b/environment/1_sdk/setup_cmg_base.py similarity index 100% rename from infrastructure/1_sdk/setup_cmg_base.py rename to environment/1_sdk/setup_cmg_base.py diff --git a/infrastructure/1_sdk/setup_cmg_redis.py b/environment/1_sdk/setup_cmg_redis.py similarity index 100% rename from infrastructure/1_sdk/setup_cmg_redis.py rename to environment/1_sdk/setup_cmg_redis.py diff --git a/infrastructure/1_sdk/setup_gcache_base.py b/environment/1_sdk/setup_gcache_base.py similarity index 100% rename from infrastructure/1_sdk/setup_gcache_base.py rename to environment/1_sdk/setup_gcache_base.py diff --git a/infrastructure/1_sdk/setup_gcache_redis.py b/environment/1_sdk/setup_gcache_redis.py similarity index 100% rename from infrastructure/1_sdk/setup_gcache_redis.py rename to environment/1_sdk/setup_gcache_redis.py diff --git a/infrastructure/1_sdk/setup_gclient_base.py b/environment/1_sdk/setup_gclient_base.py similarity index 100% rename from infrastructure/1_sdk/setup_gclient_base.py rename to environment/1_sdk/setup_gclient_base.py diff --git a/infrastructure/1_sdk/setup_gclient_cmg.py b/environment/1_sdk/setup_gclient_cmg.py similarity index 100% rename from infrastructure/1_sdk/setup_gclient_cmg.py rename to environment/1_sdk/setup_gclient_cmg.py diff --git a/infrastructure/1_sdk/setup_gclient_http.py b/environment/1_sdk/setup_gclient_http.py similarity index 100% rename from infrastructure/1_sdk/setup_gclient_http.py rename to environment/1_sdk/setup_gclient_http.py diff --git a/infrastructure/1_sdk/setup_redis_lua.py b/environment/1_sdk/setup_redis_lua.py similarity index 100% rename from infrastructure/1_sdk/setup_redis_lua.py rename to environment/1_sdk/setup_redis_lua.py diff --git a/infrastructure/1_sdk/setup_sysom_utils.py b/environment/1_sdk/setup_sysom_utils.py similarity index 100% rename from infrastructure/1_sdk/setup_sysom_utils.py rename to environment/1_sdk/setup_sysom_utils.py diff --git a/infrastructure/1_sdk/start.sh b/environment/1_sdk/start.sh similarity index 100% rename from infrastructure/1_sdk/start.sh rename to environment/1_sdk/start.sh diff --git a/infrastructure/1_sdk/stop.sh b/environment/1_sdk/stop.sh similarity index 100% rename from infrastructure/1_sdk/stop.sh rename to environment/1_sdk/stop.sh diff --git a/infrastructure/1_sdk/sysom_utils/__init__.py b/environment/1_sdk/sysom_utils/__init__.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/__init__.py rename to environment/1_sdk/sysom_utils/__init__.py diff --git a/infrastructure/1_sdk/sysom_utils/adddict.py b/environment/1_sdk/sysom_utils/adddict.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/adddict.py rename to environment/1_sdk/sysom_utils/adddict.py diff --git a/infrastructure/1_sdk/sysom_utils/config_parser.py b/environment/1_sdk/sysom_utils/config_parser.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/config_parser.py rename to environment/1_sdk/sysom_utils/config_parser.py diff --git a/infrastructure/1_sdk/sysom_utils/event_executor.py b/environment/1_sdk/sysom_utils/event_executor.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/event_executor.py rename to environment/1_sdk/sysom_utils/event_executor.py diff --git a/infrastructure/1_sdk/sysom_utils/framework.py b/environment/1_sdk/sysom_utils/framework.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/framework.py rename to environment/1_sdk/sysom_utils/framework.py diff --git a/infrastructure/1_sdk/sysom_utils/framework_plug_mag.py b/environment/1_sdk/sysom_utils/framework_plug_mag.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/framework_plug_mag.py rename to environment/1_sdk/sysom_utils/framework_plug_mag.py diff --git a/infrastructure/1_sdk/sysom_utils/framework_plugins.py b/environment/1_sdk/sysom_utils/framework_plugins.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/framework_plugins.py rename to environment/1_sdk/sysom_utils/framework_plugins.py diff --git a/infrastructure/1_sdk/sysom_utils/node_manager.py b/environment/1_sdk/sysom_utils/node_manager.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/node_manager.py rename to environment/1_sdk/sysom_utils/node_manager.py diff --git a/infrastructure/1_sdk/sysom_utils/yaml_concat.py b/environment/1_sdk/sysom_utils/yaml_concat.py similarity index 100% rename from infrastructure/1_sdk/sysom_utils/yaml_concat.py rename to environment/1_sdk/sysom_utils/yaml_concat.py diff --git a/infrastructure/1_sdk/test_cec_redis/__init__.py b/environment/1_sdk/test_cec_redis/__init__.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/__init__.py rename to environment/1_sdk/test_cec_redis/__init__.py diff --git a/infrastructure/1_sdk/test_cec_redis/test_heartbeat.py b/environment/1_sdk/test_cec_redis/test_heartbeat.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/test_heartbeat.py rename to environment/1_sdk/test_cec_redis/test_heartbeat.py diff --git a/infrastructure/1_sdk/test_cec_redis/test_redis_admin.py b/environment/1_sdk/test_cec_redis/test_redis_admin.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/test_redis_admin.py rename to environment/1_sdk/test_cec_redis/test_redis_admin.py diff --git a/infrastructure/1_sdk/test_cec_redis/test_redis_admin_async.py b/environment/1_sdk/test_cec_redis/test_redis_admin_async.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/test_redis_admin_async.py rename to environment/1_sdk/test_cec_redis/test_redis_admin_async.py diff --git a/infrastructure/1_sdk/test_cec_redis/test_redis_consumer.py b/environment/1_sdk/test_cec_redis/test_redis_consumer.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/test_redis_consumer.py rename to environment/1_sdk/test_cec_redis/test_redis_consumer.py diff --git a/infrastructure/1_sdk/test_cec_redis/test_redis_consumer_async.py b/environment/1_sdk/test_cec_redis/test_redis_consumer_async.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/test_redis_consumer_async.py rename to environment/1_sdk/test_cec_redis/test_redis_consumer_async.py diff --git a/infrastructure/1_sdk/test_cec_redis/test_redis_multi_consumer.py b/environment/1_sdk/test_cec_redis/test_redis_multi_consumer.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/test_redis_multi_consumer.py rename to environment/1_sdk/test_cec_redis/test_redis_multi_consumer.py diff --git a/infrastructure/1_sdk/test_cec_redis/test_redis_producer.py b/environment/1_sdk/test_cec_redis/test_redis_producer.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/test_redis_producer.py rename to environment/1_sdk/test_cec_redis/test_redis_producer.py diff --git a/infrastructure/1_sdk/test_cec_redis/test_utils.py b/environment/1_sdk/test_cec_redis/test_utils.py similarity index 100% rename from infrastructure/1_sdk/test_cec_redis/test_utils.py rename to environment/1_sdk/test_cec_redis/test_utils.py diff --git a/infrastructure/1_sdk/test_cmg_base/__init__.py b/environment/1_sdk/test_cmg_base/__init__.py similarity index 100% rename from infrastructure/1_sdk/test_cmg_base/__init__.py rename to environment/1_sdk/test_cmg_base/__init__.py diff --git a/infrastructure/1_sdk/test_cmg_base/test_health_check.py b/environment/1_sdk/test_cmg_base/test_health_check.py similarity index 100% rename from infrastructure/1_sdk/test_cmg_base/test_health_check.py rename to environment/1_sdk/test_cmg_base/test_health_check.py diff --git a/infrastructure/1_sdk/test_cmg_redis/__init__.py b/environment/1_sdk/test_cmg_redis/__init__.py similarity index 100% rename from infrastructure/1_sdk/test_cmg_redis/__init__.py rename to environment/1_sdk/test_cmg_redis/__init__.py diff --git a/infrastructure/1_sdk/test_cmg_redis/test_service_discovery.py b/environment/1_sdk/test_cmg_redis/test_service_discovery.py similarity index 100% rename from infrastructure/1_sdk/test_cmg_redis/test_service_discovery.py rename to environment/1_sdk/test_cmg_redis/test_service_discovery.py diff --git a/infrastructure/1_sdk/test_cmg_redis/test_service_registry.py b/environment/1_sdk/test_cmg_redis/test_service_registry.py similarity index 100% rename from infrastructure/1_sdk/test_cmg_redis/test_service_registry.py rename to environment/1_sdk/test_cmg_redis/test_service_registry.py diff --git a/infrastructure/2_local_services/clear.sh b/environment/2_local_services/clear.sh similarity index 100% rename from infrastructure/2_local_services/clear.sh rename to environment/2_local_services/clear.sh diff --git a/infrastructure/2_local_services/init.sh b/environment/2_local_services/init.sh similarity index 100% rename from infrastructure/2_local_services/init.sh rename to environment/2_local_services/init.sh diff --git a/infrastructure/2_local_services/nginx.conf b/environment/2_local_services/nginx.conf similarity index 100% rename from infrastructure/2_local_services/nginx.conf rename to environment/2_local_services/nginx.conf diff --git a/infrastructure/2_local_services/start.sh b/environment/2_local_services/start.sh similarity index 100% rename from infrastructure/2_local_services/start.sh rename to environment/2_local_services/start.sh diff --git a/infrastructure/2_local_services/stop.sh b/environment/2_local_services/stop.sh similarity index 100% rename from infrastructure/2_local_services/stop.sh rename to environment/2_local_services/stop.sh diff --git a/infrastructure/2_local_services/sysom-redis.ini b/environment/2_local_services/sysom-redis.ini similarity index 100% rename from infrastructure/2_local_services/sysom-redis.ini rename to environment/2_local_services/sysom-redis.ini diff --git a/infrastructure/2_local_services/sysom.conf b/environment/2_local_services/sysom.conf similarity index 100% rename from infrastructure/2_local_services/sysom.conf rename to environment/2_local_services/sysom.conf diff --git a/infrastructure/3_monitor/Readme.txt b/environment/3_monitor/Readme.txt similarity index 100% rename from infrastructure/3_monitor/Readme.txt rename to environment/3_monitor/Readme.txt diff --git a/infrastructure/3_monitor/clear.sh b/environment/3_monitor/clear.sh similarity index 84% rename from infrastructure/3_monitor/clear.sh rename to environment/3_monitor/clear.sh index 2cc36603..e4eb4d8a 100755 --- a/infrastructure/3_monitor/clear.sh +++ b/environment/3_monitor/clear.sh @@ -1,5 +1,6 @@ #!/bin/bash -x SERVICE_NAME=sysom-prometheus +RESOURCE_DIR=${ENVIRONMENT_HOME}/monitor del_cron() { sed -i '/prometheus/d' /var/spool/cron/root @@ -15,6 +16,8 @@ main() ###use supervisorctl update to stop and clear services### supervisorctl update + rm -rf ${RESOURCE_DIR} + del_cron } diff --git a/infrastructure/3_monitor/grafana_api_set.sh b/environment/3_monitor/grafana_api_set.sh similarity index 100% rename from infrastructure/3_monitor/grafana_api_set.sh rename to environment/3_monitor/grafana_api_set.sh diff --git a/infrastructure/3_monitor/grafana_recover.sh b/environment/3_monitor/grafana_recover.sh similarity index 100% rename from infrastructure/3_monitor/grafana_recover.sh rename to environment/3_monitor/grafana_recover.sh diff --git a/infrastructure/3_monitor/init.sh b/environment/3_monitor/init.sh similarity index 100% rename from infrastructure/3_monitor/init.sh rename to environment/3_monitor/init.sh diff --git a/infrastructure/3_monitor/local_copy_pkg.sh b/environment/3_monitor/local_copy_pkg.sh similarity index 100% rename from infrastructure/3_monitor/local_copy_pkg.sh rename to environment/3_monitor/local_copy_pkg.sh diff --git a/infrastructure/3_monitor/prometheus_get_node.py b/environment/3_monitor/prometheus_get_node.py similarity index 100% rename from infrastructure/3_monitor/prometheus_get_node.py rename to environment/3_monitor/prometheus_get_node.py diff --git a/infrastructure/3_monitor/start.sh b/environment/3_monitor/start.sh similarity index 100% rename from infrastructure/3_monitor/start.sh rename to environment/3_monitor/start.sh diff --git a/infrastructure/3_monitor/stop.sh b/environment/3_monitor/stop.sh similarity index 100% rename from infrastructure/3_monitor/stop.sh rename to environment/3_monitor/stop.sh diff --git a/infrastructure/3_monitor/sysom-cec-status-dashboard.json b/environment/3_monitor/sysom-cec-status-dashboard.json similarity index 100% rename from infrastructure/3_monitor/sysom-cec-status-dashboard.json rename to environment/3_monitor/sysom-cec-status-dashboard.json diff --git a/infrastructure/3_monitor/sysom-dashboard.json b/environment/3_monitor/sysom-dashboard.json similarity index 100% rename from infrastructure/3_monitor/sysom-dashboard.json rename to environment/3_monitor/sysom-dashboard.json diff --git a/infrastructure/3_monitor/sysom-migration-dashboard.json b/environment/3_monitor/sysom-migration-dashboard.json similarity index 100% rename from infrastructure/3_monitor/sysom-migration-dashboard.json rename to environment/3_monitor/sysom-migration-dashboard.json diff --git a/infrastructure/3_monitor/sysom-netinfo-dashboard.json b/environment/3_monitor/sysom-netinfo-dashboard.json similarity index 100% rename from infrastructure/3_monitor/sysom-netinfo-dashboard.json rename to environment/3_monitor/sysom-netinfo-dashboard.json diff --git a/infrastructure/3_monitor/sysom-prometheus.ini b/environment/3_monitor/sysom-prometheus.ini similarity index 56% rename from infrastructure/3_monitor/sysom-prometheus.ini rename to environment/3_monitor/sysom-prometheus.ini index 62c0cec5..f298737a 100644 --- a/infrastructure/3_monitor/sysom-prometheus.ini +++ b/environment/3_monitor/sysom-prometheus.ini @@ -1,6 +1,6 @@ [program:sysom-prometheus] -directory = /usr/local/sysom/server/environment/monitor/prometheus -command=/usr/local/sysom/server/environment/monitor/prometheus/prometheus +directory = /usr/local/sysom/environment/monitor/prometheus +command=/usr/local/sysom/environment/monitor/prometheus/prometheus startsecs=3 autostart=true autorestart=true diff --git a/infrastructure/clear_exclude b/environment/clear_exclude similarity index 100% rename from infrastructure/clear_exclude rename to environment/clear_exclude diff --git a/infrastructure/deploy_exclude b/environment/deploy_exclude similarity index 100% rename from infrastructure/deploy_exclude rename to environment/deploy_exclude diff --git a/microservice/sysom_vmcore/scripts/server_db_migrate.sh b/microservice/sysom_vmcore/scripts/server_db_migrate.sh deleted file mode 100644 index 89ba9eb6..00000000 --- a/microservice/sysom_vmcore/scripts/server_db_migrate.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) -VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -db_migrate() { - pushd ${SERVICE_HOME} - python manage.py migrate - popd -} - -source_virtualenv -db_migrate \ No newline at end of file diff --git a/microservice/sysom_vul/scripts/server_db_migrate.sh b/microservice/sysom_vul/scripts/server_db_migrate.sh deleted file mode 100644 index 89ba9eb6..00000000 --- a/microservice/sysom_vul/scripts/server_db_migrate.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) -VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - -db_migrate() { - pushd ${SERVICE_HOME} - python manage.py migrate - popd -} - -source_virtualenv -db_migrate \ No newline at end of file diff --git a/package.sh b/package.sh index 61f8cd4d..a50bbd96 100755 --- a/package.sh +++ b/package.sh @@ -16,8 +16,8 @@ check_cmd yarn check_cmd tar RELEASE=sysomRelease-$(date +"%Y%m%d%H%M%S") -MICROSERVICE_DIR=microservice -INFRASTRUCTURE_DIR=infrastructure +MICROSERVICE_DIR=sysom_server +ENVIRONMENT_DIR=environment WEBDIR=sysom_web CONFDIR=conf SCRIPTDIR=script @@ -29,7 +29,7 @@ yarn build popd || exit mkdir -p "${RELEASE}" -cp -r ${MICROSERVICE_DIR}/ ${INFRASTRUCTURE_DIR}/ ${TOOLSDIR}/ ${CONFDIR}/ "${RELEASE}"/ +cp -r ${MICROSERVICE_DIR}/ ${ENVIRONMENT_DIR}/ ${TOOLSDIR}/ ${CONFDIR}/ "${RELEASE}"/ cp -r ${WEBDIR}/dist/ "${RELEASE}"/${WEBDIR}/ mkdir -p "${RELEASE}"/${WEBDIR}/download/ cp ${SCRIPTDIR}/deploy/deploy.sh "${RELEASE}"/ diff --git a/script/deploy/clear.sh b/script/deploy/clear.sh index 85ffdcd2..6ccd78b9 100755 --- a/script/deploy/clear.sh +++ b/script/deploy/clear.sh @@ -18,7 +18,7 @@ export SERVER_HOME=${APP_HOME}/server del_app_home() { # Whether delete APP_HOME - if [ "`ls -A ${APP_HOME}`" = "" ]; then + if [ "`ls -A ${APP_HOME}`" = "script" ]; then rm -r ${APP_HOME} fi } @@ -38,21 +38,20 @@ del_conf() { fi } -stop_server() { - systemctl stop sysom-server.service - systemctl disable sysom-server.service - systemctl daemon-reload -} - -clear_server() { - bash -x ${SERVER_HOME}/init_scripts/server/clear.sh -} - - +BaseDir=$(dirname $(readlink -f "$0")) clear() { if [ -d "${APP_HOME}" ];then - stop_server - clear_server + if [ -f "script" ]; then + # release dir + pushd "script" + else + # source code dir + pushd `dirname ${BaseDir}` + fi + bash +x sysom.sh clear web + bash +x sysom.sh clear ms ALL + bash +x sysom.sh clear env ALL + popd del_conf del_app_home fi diff --git a/script/deploy/deploy.sh b/script/deploy/deploy.sh index 483833ab..0bd632dd 100755 --- a/script/deploy/deploy.sh +++ b/script/deploy/deploy.sh @@ -7,38 +7,27 @@ # Function: deploy sysom #***************************************************************# APP_NAME="sysom" -WEB_DIR="sysom_web" -MICROSERVICE_DIR="microservice" -INFRASTRUCTURE_DIR="infrastructure" -CONF_DIR="conf" -SCRIPT_DIR="script" APP_HOME=/usr/local/sysom SERVER_LOCAL_IP="" SERVER_PUBLIC_IP="" -SERVER_PORT="" SERVER_PORT=80 -if [ $# -lt 3 ] ; then - echo "USAGE: $0 INSTALL_DIR Internal_IP EXTERNAL_IP WEB_PORT" - echo "Or we use default install dir: /usr/local/sysom/" - echo "If WEB_PORT not set, use port 80 default" - echo "E.g.: $0 /usr/local/sysom 192.168.0.100 120.26.xx.xx 80" - exit 1 -else +if [ $# -gt 1 ]; then APP_HOME=$1 +fi +if [ $# -gt 2 ]; then SERVER_LOCAL_IP=$2 + SERVER_PUBLIC_IP=$2 +fi +if [ $# -gt 3 ]; then SERVER_PUBLIC_IP=$3 fi - if [ $# -eq 4 ] ; then SERVER_PORT=$4 fi export APP_NAME=${APP_NAME} export APP_HOME=${APP_HOME} -export SERVER_HOME=${APP_HOME}/server -export WEB_HOME=${APP_HOME}/web -export SCRIPT_HOME=${SERVER_HOME}/init_scripts export SERVER_LOCAL_IP=${SERVER_LOCAL_IP} export SERVER_PUBLIC_IP=${SERVER_PUBLIC_IP} export SERVER_PORT=${SERVER_PORT} @@ -50,45 +39,19 @@ if [ "$UID" -ne 0 ]; then exit 1 fi - -update_target() { - echo "INFO: copy project file..." - # WEB - if [ -d "${WEB_HOME}" ]; then - rm -rf ${WEB_HOME} - fi - - cp -r ${WEB_DIR} ${WEB_HOME} - - # SERVER - # if [ -d "${SERVER_HOME}" ]; then - # rm -rf ${SERVER_HOME} - # fi - mkdir -p ${SERVER_HOME} - if [ -d "${SERVER_HOME}/${MICROSERVICE_DIR}" ];then - rm -rf ${SERVER_HOME}/${MICROSERVICE_DIR} - fi - if [ -d "${SERVER_HOME}/${INFRASTRUCTURE_DIR}" ];then - rm -rf ${SERVER_HOME}/${INFRASTRUCTURE_DIR} - fi - cp -r ${MICROSERVICE_DIR} ${INFRASTRUCTURE_DIR} ${SERVER_HOME} - if [ ! -d "${SERVER_HOME}/conf" ];then - cp -r ${CONF_DIR} ${SERVER_HOME}/conf - fi - - - # SCRIPT - cp -r ${SCRIPT_DIR} ${SCRIPT_HOME} - cp tools/deploy/sysom-server.service /usr/lib/systemd/system/ -} - -start_script_server() { - bash -x ${SERVER_HOME}/init_scripts/server/init.sh -} - +BaseDir=$(dirname $(readlink -f "$0")) deploy() { - update_target - start_script_server + if [ -d "script" ]; then + # release dir + pushd "script" + else + # source code dir + pushd `dirname ${BaseDir}` + fi + bash +x sysom.sh deploy web + bash +x sysom.sh deploy env ALL + bash +x sysom.sh deploy ms ALL + popd } deploy \ No newline at end of file diff --git a/microservice/clear_exclude b/script/server/clear_exclude similarity index 100% rename from microservice/clear_exclude rename to script/server/clear_exclude diff --git a/microservice/deploy_exclude b/script/server/deploy_exclude similarity index 100% rename from microservice/deploy_exclude rename to script/server/deploy_exclude diff --git a/microservice/0_sysom_api/scripts/server_clear.sh b/script/server/sysom_api/clear.sh similarity index 74% rename from microservice/0_sysom_api/scripts/server_clear.sh rename to script/server/sysom_api/clear.sh index 07b92432..51fb859f 100644 --- a/microservice/0_sysom_api/scripts/server_clear.sh +++ b/script/server/sysom_api/clear.sh @@ -1,7 +1,7 @@ #!/bin/bash SERVICE_NAME=sysom-api clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini ###use supervisorctl update to stop and clear services### supervisorctl update } diff --git a/microservice/sysom_migration/scripts/server_db_migrate.sh b/script/server/sysom_api/db_migrate.sh similarity index 73% rename from microservice/sysom_migration/scripts/server_db_migrate.sh rename to script/server/sysom_api/db_migrate.sh index 89ba9eb6..071ddbef 100644 --- a/microservice/sysom_migration/scripts/server_db_migrate.sh +++ b/script/server/sysom_api/db_migrate.sh @@ -1,5 +1,6 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME source_virtualenv() { diff --git a/microservice/0_sysom_api/scripts/server_init.sh b/script/server/sysom_api/init.sh similarity index 84% rename from microservice/0_sysom_api/scripts/server_init.sh rename to script/server/sysom_api/init.sh index 6347540b..09176fb0 100644 --- a/microservice/0_sysom_api/scripts/server_init.sh +++ b/script/server/sysom_api/init.sh @@ -1,5 +1,7 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-api @@ -9,7 +11,7 @@ if [ "$UID" -ne 0 ]; then fi install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/0_sysom_api/requirements.txt b/script/server/sysom_api/requirements.txt similarity index 100% rename from microservice/0_sysom_api/requirements.txt rename to script/server/sysom_api/requirements.txt diff --git a/microservice/0_sysom_api/scripts/server_start.sh b/script/server/sysom_api/start.sh similarity index 100% rename from microservice/0_sysom_api/scripts/server_start.sh rename to script/server/sysom_api/start.sh diff --git a/microservice/0_sysom_api/scripts/server_stop.sh b/script/server/sysom_api/stop.sh similarity index 100% rename from microservice/0_sysom_api/scripts/server_stop.sh rename to script/server/sysom_api/stop.sh diff --git a/microservice/0_sysom_api/sysom-api.ini b/script/server/sysom_api/sysom-api.ini similarity index 44% rename from microservice/0_sysom_api/sysom-api.ini rename to script/server/sysom_api/sysom-api.ini index bddc95da..d3eebedb 100644 --- a/microservice/0_sysom_api/sysom-api.ini +++ b/script/server/sysom_api/sysom-api.ini @@ -1,7 +1,7 @@ [fcgi-program:sysom-api] socket=tcp://0.0.0.0:7001 -directory=/usr/local/sysom/server/microservice/0_sysom_api -command=/usr/local/sysom/server/environment/virtualenv/bin/daphne -u /run/daphne%(process_num)d.sock --fd 0 --access-log /var/log/sysom/sysom-api-access.log --proxy-headers sysom.asgi:application +directory=/usr/local/sysom/server/sysom_api +command=/usr/local/sysom/environment/virtualenv/bin/daphne -u /run/daphne%(process_num)d.sock --fd 0 --access-log /var/log/sysom/sysom-api-access.log --proxy-headers sysom.asgi:application numprocs=4 process_name=%(process_num)d autostart=true @@ -10,4 +10,4 @@ redirect_stderr=true stopasgroup=true stdout_logfile=/var/log/sysom/sysom-api.log stderr_logfile=/var/log/sysom/sysom-api-error.log -environment=PATH=/usr/local/sysom/server/virtualenv/bin:%(ENV_PATH)s +environment=PATH=/usr/local/sysom/virtualenv/bin:%(ENV_PATH)s diff --git a/microservice/0_sysom_channel/scripts/server_clear.sh b/script/server/sysom_channel/clear.sh similarity index 100% rename from microservice/0_sysom_channel/scripts/server_clear.sh rename to script/server/sysom_channel/clear.sh diff --git a/microservice/0_sysom_channel/scripts/server_db_migrate.sh b/script/server/sysom_channel/db_migrate.sh similarity index 73% rename from microservice/0_sysom_channel/scripts/server_db_migrate.sh rename to script/server/sysom_channel/db_migrate.sh index df481d7b..f530e1f8 100644 --- a/microservice/0_sysom_channel/scripts/server_db_migrate.sh +++ b/script/server/sysom_channel/db_migrate.sh @@ -1,5 +1,6 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME source_virtualenv() { diff --git a/microservice/0_sysom_channel/scripts/server_init.sh b/script/server/sysom_channel/init.sh similarity index 83% rename from microservice/0_sysom_channel/scripts/server_init.sh rename to script/server/sysom_channel/init.sh index 99f6a395..ec28fc04 100644 --- a/microservice/0_sysom_channel/scripts/server_init.sh +++ b/script/server/sysom_channel/init.sh @@ -1,5 +1,7 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-channel @@ -9,7 +11,7 @@ if [ "$UID" -ne 0 ]; then fi install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/0_sysom_channel/requirements.txt b/script/server/sysom_channel/requirements.txt similarity index 100% rename from microservice/0_sysom_channel/requirements.txt rename to script/server/sysom_channel/requirements.txt diff --git a/microservice/0_sysom_channel/scripts/server_start.sh b/script/server/sysom_channel/start.sh similarity index 100% rename from microservice/0_sysom_channel/scripts/server_start.sh rename to script/server/sysom_channel/start.sh diff --git a/microservice/0_sysom_channel/scripts/server_stop.sh b/script/server/sysom_channel/stop.sh similarity index 100% rename from microservice/0_sysom_channel/scripts/server_stop.sh rename to script/server/sysom_channel/stop.sh diff --git a/microservice/0_sysom_channel/sysom-channel.ini b/script/server/sysom_channel/sysom-channel.ini similarity index 42% rename from microservice/0_sysom_channel/sysom-channel.ini rename to script/server/sysom_channel/sysom-channel.ini index 5de0b95b..25d77461 100644 --- a/microservice/0_sysom_channel/sysom-channel.ini +++ b/script/server/sysom_channel/sysom-channel.ini @@ -1,9 +1,9 @@ [program:sysom-channel] -directory=/usr/local/sysom/server/microservice/0_sysom_channel -command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +directory=/usr/local/sysom/server/sysom_channel +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" +environment=PATH="/usr/local/sysom/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-channel-error.log stdout_logfile=/var/log/sysom/sysom-channel.log diff --git a/microservice/sysom_diagnosis/scripts/server_clear.sh b/script/server/sysom_diagnosis/clear.sh similarity index 100% rename from microservice/sysom_diagnosis/scripts/server_clear.sh rename to script/server/sysom_diagnosis/clear.sh diff --git a/microservice/sysom_hotfix/scripts/server_db_migrate.sh b/script/server/sysom_diagnosis/db_migrate.sh similarity index 73% rename from microservice/sysom_hotfix/scripts/server_db_migrate.sh rename to script/server/sysom_diagnosis/db_migrate.sh index 89ba9eb6..071ddbef 100644 --- a/microservice/sysom_hotfix/scripts/server_db_migrate.sh +++ b/script/server/sysom_diagnosis/db_migrate.sh @@ -1,5 +1,6 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME source_virtualenv() { diff --git a/microservice/0_sysom_api/apps/accounts/__init__.py b/script/server/sysom_diagnosis/deploy.sh similarity index 100% rename from microservice/0_sysom_api/apps/accounts/__init__.py rename to script/server/sysom_diagnosis/deploy.sh diff --git a/microservice/sysom_diagnosis/scripts/server_init.sh b/script/server/sysom_diagnosis/init.sh similarity index 85% rename from microservice/sysom_diagnosis/scripts/server_init.sh rename to script/server/sysom_diagnosis/init.sh index 2f0f210b..1dd0cfe9 100755 --- a/microservice/sysom_diagnosis/scripts/server_init.sh +++ b/script/server/sysom_diagnosis/init.sh @@ -1,5 +1,7 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-diagnosis @@ -9,7 +11,7 @@ source_virtualenv() { } install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/sysom_diagnosis/requirements.txt b/script/server/sysom_diagnosis/requirements.txt similarity index 100% rename from microservice/sysom_diagnosis/requirements.txt rename to script/server/sysom_diagnosis/requirements.txt diff --git a/microservice/sysom_diagnosis/scripts/server_start.sh b/script/server/sysom_diagnosis/start.sh similarity index 100% rename from microservice/sysom_diagnosis/scripts/server_start.sh rename to script/server/sysom_diagnosis/start.sh diff --git a/microservice/sysom_diagnosis/scripts/server_stop.sh b/script/server/sysom_diagnosis/stop.sh similarity index 100% rename from microservice/sysom_diagnosis/scripts/server_stop.sh rename to script/server/sysom_diagnosis/stop.sh diff --git a/microservice/sysom_diagnosis/sysom-diagnosis.ini b/script/server/sysom_diagnosis/sysom-diagnosis.ini similarity index 40% rename from microservice/sysom_diagnosis/sysom-diagnosis.ini rename to script/server/sysom_diagnosis/sysom-diagnosis.ini index 6a04f364..ac848752 100644 --- a/microservice/sysom_diagnosis/sysom-diagnosis.ini +++ b/script/server/sysom_diagnosis/sysom-diagnosis.ini @@ -1,9 +1,9 @@ [program:sysom-diagnosis] -directory=/usr/local/sysom/server/microservice/sysom_diagnosis -command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_diagnosis.wsgi:application +directory=/usr/local/sysom/server/sysom_diagnosis +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_diagnosis.wsgi:application startsecs=3 autostart=true autorestart=true stderr_logfile=/var/log/sysom/sysom-diagnosis-error.log stdout_logfile=/var/log/sysom/sysom-diagnosis.log -environment=PATH=/usr/local/sysom/server/virtualenv/bin:%(ENV_PATH)s \ No newline at end of file +environment=PATH=/usr/local/sysom/virtualenv/bin:%(ENV_PATH)s \ No newline at end of file diff --git a/microservice/sysom_hotfix/scripts/server_clear.sh b/script/server/sysom_hotfix/clear.sh similarity index 100% rename from microservice/sysom_hotfix/scripts/server_clear.sh rename to script/server/sysom_hotfix/clear.sh diff --git a/microservice/sysom_diagnosis/scripts/server_db_migrate.sh b/script/server/sysom_hotfix/db_migrate.sh similarity index 73% rename from microservice/sysom_diagnosis/scripts/server_db_migrate.sh rename to script/server/sysom_hotfix/db_migrate.sh index 89ba9eb6..071ddbef 100644 --- a/microservice/sysom_diagnosis/scripts/server_db_migrate.sh +++ b/script/server/sysom_hotfix/db_migrate.sh @@ -1,5 +1,6 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME source_virtualenv() { diff --git a/microservice/sysom_hotfix/scripts/server_init.sh b/script/server/sysom_hotfix/init.sh similarity index 90% rename from microservice/sysom_hotfix/scripts/server_init.sh rename to script/server/sysom_hotfix/init.sh index f4d8c17e..62da3a47 100644 --- a/microservice/sysom_hotfix/scripts/server_init.sh +++ b/script/server/sysom_hotfix/init.sh @@ -1,5 +1,7 @@ #! /bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-hotfix @@ -9,7 +11,7 @@ source_virtualenv() { } install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/sysom_hotfix/requirements.txt b/script/server/sysom_hotfix/requirements.txt similarity index 100% rename from microservice/sysom_hotfix/requirements.txt rename to script/server/sysom_hotfix/requirements.txt diff --git a/microservice/sysom_hotfix/scripts/server_start.sh b/script/server/sysom_hotfix/start.sh similarity index 100% rename from microservice/sysom_hotfix/scripts/server_start.sh rename to script/server/sysom_hotfix/start.sh diff --git a/microservice/sysom_hotfix/scripts/server_stop.sh b/script/server/sysom_hotfix/stop.sh similarity index 100% rename from microservice/sysom_hotfix/scripts/server_stop.sh rename to script/server/sysom_hotfix/stop.sh diff --git a/microservice/sysom_hotfix/sysom-hotfix.ini b/script/server/sysom_hotfix/sysom-hotfix.ini similarity index 39% rename from microservice/sysom_hotfix/sysom-hotfix.ini rename to script/server/sysom_hotfix/sysom-hotfix.ini index e2c211ee..d8b0a597 100644 --- a/microservice/sysom_hotfix/sysom-hotfix.ini +++ b/script/server/sysom_hotfix/sysom-hotfix.ini @@ -1,9 +1,9 @@ [program:sysom-hotfix] -directory = /usr/local/sysom/server/microservice/sysom_hotfix -command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_hotfix.wsgi:application +directory = /usr/local/sysom/server/sysom_hotfix +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_hotfix.wsgi:application startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" +environment=PATH="/usr/local/sysom/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-hotfix-error.log stdout_logfile=/var/log/sysom/sysom-hotfix.log diff --git a/microservice/sysom_hotfix_builder/scripts/server_clear.sh b/script/server/sysom_hotfix_builder/clear.sh similarity index 100% rename from microservice/sysom_hotfix_builder/scripts/server_clear.sh rename to script/server/sysom_hotfix_builder/clear.sh diff --git a/microservice/sysom_hotfix_builder/scripts/server_init.sh b/script/server/sysom_hotfix_builder/init.sh similarity index 88% rename from microservice/sysom_hotfix_builder/scripts/server_init.sh rename to script/server/sysom_hotfix_builder/init.sh index 685e2dcd..3a080d20 100644 --- a/microservice/sysom_hotfix_builder/scripts/server_init.sh +++ b/script/server/sysom_hotfix_builder/init.sh @@ -1,5 +1,7 @@ #! /bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-hotfix-builder NFS_SERVER_IP=${SERVER_LOCAL_IP} @@ -10,7 +12,7 @@ source_virtualenv() { } install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/sysom_hotfix_builder/requirements.txt b/script/server/sysom_hotfix_builder/requirements.txt similarity index 100% rename from microservice/sysom_hotfix_builder/requirements.txt rename to script/server/sysom_hotfix_builder/requirements.txt diff --git a/microservice/sysom_hotfix_builder/scripts/server_start.sh b/script/server/sysom_hotfix_builder/start.sh similarity index 100% rename from microservice/sysom_hotfix_builder/scripts/server_start.sh rename to script/server/sysom_hotfix_builder/start.sh diff --git a/microservice/sysom_hotfix_builder/scripts/server_stop.sh b/script/server/sysom_hotfix_builder/stop.sh similarity index 100% rename from microservice/sysom_hotfix_builder/scripts/server_stop.sh rename to script/server/sysom_hotfix_builder/stop.sh diff --git a/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini b/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini similarity index 45% rename from microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini rename to script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini index bd7427d0..3359d39b 100644 --- a/microservice/sysom_hotfix_builder/sysom-hotfix-builder.ini +++ b/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini @@ -1,9 +1,9 @@ [program:sysom-hotfix-builder] -directory = /usr/local/sysom/server/microservice/sysom_hotfix_builder -command=/usr/local/sysom/server/environment/virtualenv/bin/python3 builder.py +directory = /usr/local/sysom/server/sysom_hotfix_builder +command=/usr/local/sysom/environment/virtualenv/bin/python3 builder.py startsecs=3 autostart=true autorestart=true -environment=PATH=/usr/local/sysom/server/environment/virtualenv/bin:%(ENV_PATH)s +environment=PATH=/usr/local/sysom/environment/virtualenv/bin:%(ENV_PATH)s stderr_logfile=/var/log/sysom/sysom-hotfix-builder-error.log stdout_logfile=/var/log/sysom/sysom-hotfix-builder.log diff --git a/microservice/sysom_migration/scripts/server_clear.sh b/script/server/sysom_migration/clear.sh similarity index 100% rename from microservice/sysom_migration/scripts/server_clear.sh rename to script/server/sysom_migration/clear.sh diff --git a/microservice/0_sysom_api/scripts/server_db_migrate.sh b/script/server/sysom_migration/db_migrate.sh similarity index 73% rename from microservice/0_sysom_api/scripts/server_db_migrate.sh rename to script/server/sysom_migration/db_migrate.sh index 89ba9eb6..071ddbef 100644 --- a/microservice/0_sysom_api/scripts/server_db_migrate.sh +++ b/script/server/sysom_migration/db_migrate.sh @@ -1,5 +1,6 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME source_virtualenv() { diff --git a/microservice/sysom_migration/scripts/server_init.sh b/script/server/sysom_migration/init.sh similarity index 92% rename from microservice/sysom_migration/scripts/server_init.sh rename to script/server/sysom_migration/init.sh index d6a543a7..15f75d75 100644 --- a/microservice/sysom_migration/scripts/server_init.sh +++ b/script/server/sysom_migration/init.sh @@ -1,5 +1,7 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-migration ANCE_X86_PKG=ance-0.1.1-1.x86_64.rpm @@ -20,7 +22,7 @@ source_virtualenv() { } install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/sysom_migration/requirements.txt b/script/server/sysom_migration/requirements.txt similarity index 100% rename from microservice/sysom_migration/requirements.txt rename to script/server/sysom_migration/requirements.txt diff --git a/microservice/0_sysom_api/apps/accounts/migrations/__init__.py b/script/server/sysom_migration/start.sh similarity index 100% rename from microservice/0_sysom_api/apps/accounts/migrations/__init__.py rename to script/server/sysom_migration/start.sh diff --git a/microservice/0_sysom_api/apps/alarm/migrations/__init__.py b/script/server/sysom_migration/stop.sh similarity index 100% rename from microservice/0_sysom_api/apps/alarm/migrations/__init__.py rename to script/server/sysom_migration/stop.sh diff --git a/microservice/sysom_migration/sysom-migration.ini b/script/server/sysom_migration/sysom-migration.ini similarity index 40% rename from microservice/sysom_migration/sysom-migration.ini rename to script/server/sysom_migration/sysom-migration.ini index 1a08106f..064b1852 100644 --- a/microservice/sysom_migration/sysom-migration.ini +++ b/script/server/sysom_migration/sysom-migration.ini @@ -1,9 +1,9 @@ [program:sysom-migration] -directory = /usr/local/sysom/server/microservice/sysom_migration -command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_migration.wsgi:application +directory = /usr/local/sysom/server/sysom_migration +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_migration.wsgi:application startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" +environment=PATH="/usr/local/sysom/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-migration-error.log stdout_logfile=/var/log/sysom/sysom-migration.log diff --git a/microservice/sysom_monitor_server/scripts/server_clear.sh b/script/server/sysom_monitor_server/clear.sh similarity index 100% rename from microservice/sysom_monitor_server/scripts/server_clear.sh rename to script/server/sysom_monitor_server/clear.sh diff --git a/microservice/sysom_monitor_server/scripts/server_db_migrate.sh b/script/server/sysom_monitor_server/db_migrate.sh similarity index 73% rename from microservice/sysom_monitor_server/scripts/server_db_migrate.sh rename to script/server/sysom_monitor_server/db_migrate.sh index df481d7b..f530e1f8 100644 --- a/microservice/sysom_monitor_server/scripts/server_db_migrate.sh +++ b/script/server/sysom_monitor_server/db_migrate.sh @@ -1,5 +1,6 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME source_virtualenv() { diff --git a/microservice/sysom_monitor_server/scripts/server_init.sh b/script/server/sysom_monitor_server/init.sh similarity index 84% rename from microservice/sysom_monitor_server/scripts/server_init.sh rename to script/server/sysom_monitor_server/init.sh index f732901b..2e081666 100644 --- a/microservice/sysom_monitor_server/scripts/server_init.sh +++ b/script/server/sysom_monitor_server/init.sh @@ -1,5 +1,7 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-monitor-server @@ -9,7 +11,7 @@ if [ "$UID" -ne 0 ]; then fi install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/sysom_monitor_server/requirements.txt b/script/server/sysom_monitor_server/requirements.txt similarity index 100% rename from microservice/sysom_monitor_server/requirements.txt rename to script/server/sysom_monitor_server/requirements.txt diff --git a/microservice/sysom_monitor_server/scripts/server_start.sh b/script/server/sysom_monitor_server/start.sh similarity index 100% rename from microservice/sysom_monitor_server/scripts/server_start.sh rename to script/server/sysom_monitor_server/start.sh diff --git a/microservice/sysom_monitor_server/scripts/server_stop.sh b/script/server/sysom_monitor_server/stop.sh similarity index 100% rename from microservice/sysom_monitor_server/scripts/server_stop.sh rename to script/server/sysom_monitor_server/stop.sh diff --git a/microservice/sysom_monitor_server/sysom-monitor-server.ini b/script/server/sysom_monitor_server/sysom-monitor-server.ini similarity index 44% rename from microservice/sysom_monitor_server/sysom-monitor-server.ini rename to script/server/sysom_monitor_server/sysom-monitor-server.ini index 3eeb4f5d..cdb4aa60 100644 --- a/microservice/sysom_monitor_server/sysom-monitor-server.ini +++ b/script/server/sysom_monitor_server/sysom-monitor-server.ini @@ -1,9 +1,9 @@ [program:sysom-monitor-server] -directory=/usr/local/sysom/server/microservice/sysom_monitor_server -command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +directory=/usr/local/sysom/server/sysom_monitor_server +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" +environment=PATH="/usr/local/sysom/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-monitor-server-error.log stdout_logfile=/var/log/sysom/sysom-monitor-server.log diff --git a/microservice/sysom_vmcore/scripts/server_clear.sh b/script/server/sysom_vmcore/clear.sh similarity index 91% rename from microservice/sysom_vmcore/scripts/server_clear.sh rename to script/server/sysom_vmcore/clear.sh index 06921d22..116f1b28 100644 --- a/microservice/sysom_vmcore/scripts/server_clear.sh +++ b/script/server/sysom_vmcore/clear.sh @@ -1,8 +1,11 @@ #!/bin/bash SERVICE_NAME=sysom-vmcore + + clear_app() { sed -i '/vmcore/d' /var/spool/cron/root sed -i '/vmcore/d' /etc/exports + rm -rf ${SERVER_HOME}/vmcore exportfs -rv systemctl stop nfs-server systemctl stop rpcbind diff --git a/script/server/sysom_vmcore/db_migrate.sh b/script/server/sysom_vmcore/db_migrate.sh new file mode 100644 index 00000000..071ddbef --- /dev/null +++ b/script/server/sysom_vmcore/db_migrate.sh @@ -0,0 +1,18 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/sysom_vmcore/scripts/server_init.sh b/script/server/sysom_vmcore/init.sh similarity index 91% rename from microservice/sysom_vmcore/scripts/server_init.sh rename to script/server/sysom_vmcore/init.sh index 7d72113c..a3a15263 100644 --- a/microservice/sysom_vmcore/scripts/server_init.sh +++ b/script/server/sysom_vmcore/init.sh @@ -1,5 +1,7 @@ #! /bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME VIRTUALENV_PYTHON3=${VIRTUALENV_HOME}/bin/python3 SERVICE_NAME=sysom-vmcore @@ -12,7 +14,7 @@ source_virtualenv() { } install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/sysom_vmcore/scripts/parse_panic.py b/script/server/sysom_vmcore/parse_panic.py similarity index 100% rename from microservice/sysom_vmcore/scripts/parse_panic.py rename to script/server/sysom_vmcore/parse_panic.py diff --git a/microservice/sysom_vmcore/requirements.txt b/script/server/sysom_vmcore/requirements.txt similarity index 100% rename from microservice/sysom_vmcore/requirements.txt rename to script/server/sysom_vmcore/requirements.txt diff --git a/microservice/sysom_vmcore/scripts/server_start.sh b/script/server/sysom_vmcore/start.sh similarity index 100% rename from microservice/sysom_vmcore/scripts/server_start.sh rename to script/server/sysom_vmcore/start.sh diff --git a/microservice/sysom_vmcore/scripts/server_stop.sh b/script/server/sysom_vmcore/stop.sh similarity index 100% rename from microservice/sysom_vmcore/scripts/server_stop.sh rename to script/server/sysom_vmcore/stop.sh diff --git a/microservice/sysom_vmcore/sysom-vmcore.ini b/script/server/sysom_vmcore/sysom-vmcore.ini similarity index 39% rename from microservice/sysom_vmcore/sysom-vmcore.ini rename to script/server/sysom_vmcore/sysom-vmcore.ini index e07d1943..9d4263eb 100644 --- a/microservice/sysom_vmcore/sysom-vmcore.ini +++ b/script/server/sysom_vmcore/sysom-vmcore.ini @@ -1,9 +1,9 @@ [program:sysom-vmcore] -directory = /usr/local/sysom/server/microservice/sysom_vmcore -command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vmcore.wsgi:application +directory = /usr/local/sysom/server/sysom_vmcore +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vmcore.wsgi:application startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" +environment=PATH="/usr/local/sysom/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-vmcore-error.log stdout_logfile=/var/log/sysom/sysom-vmcore.log diff --git a/microservice/sysom_vmcore/scripts/vmcore_const.py b/script/server/sysom_vmcore/vmcore_const.py similarity index 100% rename from microservice/sysom_vmcore/scripts/vmcore_const.py rename to script/server/sysom_vmcore/vmcore_const.py diff --git a/microservice/sysom_vul/scripts/server_clear.sh b/script/server/sysom_vul/clear.sh similarity index 100% rename from microservice/sysom_vul/scripts/server_clear.sh rename to script/server/sysom_vul/clear.sh diff --git a/script/server/sysom_vul/db_migrate.sh b/script/server/sysom_vul/db_migrate.sh new file mode 100644 index 00000000..071ddbef --- /dev/null +++ b/script/server/sysom_vul/db_migrate.sh @@ -0,0 +1,18 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + python manage.py migrate + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/microservice/sysom_vul/scripts/server_init.sh b/script/server/sysom_vul/init.sh similarity index 85% rename from microservice/sysom_vul/scripts/server_init.sh rename to script/server/sysom_vul/init.sh index d7947298..a127295a 100644 --- a/microservice/sysom_vul/scripts/server_init.sh +++ b/script/server/sysom_vul/init.sh @@ -1,5 +1,7 @@ #!/bin/bash -SERVICE_HOME=$(dirname $(cd $(dirname ${BASH_SOURCE[0]}) && pwd)) +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME SERVICE_NAME=sysom-vul @@ -14,7 +16,7 @@ source_virtualenv() { } install_requirement() { - pushd ${SERVICE_HOME} + pushd ${SERVICE_SCRIPT_HOME} pip install -r requirements.txt popd } diff --git a/microservice/sysom_vul/requirements.txt b/script/server/sysom_vul/requirements.txt similarity index 100% rename from microservice/sysom_vul/requirements.txt rename to script/server/sysom_vul/requirements.txt diff --git a/microservice/sysom_vul/scripts/server_start.sh b/script/server/sysom_vul/start.sh similarity index 100% rename from microservice/sysom_vul/scripts/server_start.sh rename to script/server/sysom_vul/start.sh diff --git a/microservice/sysom_vul/scripts/server_stop.sh b/script/server/sysom_vul/stop.sh similarity index 100% rename from microservice/sysom_vul/scripts/server_stop.sh rename to script/server/sysom_vul/stop.sh diff --git a/microservice/sysom_vul/sysom-vul.ini b/script/server/sysom_vul/sysom-vul.ini similarity index 39% rename from microservice/sysom_vul/sysom-vul.ini rename to script/server/sysom_vul/sysom-vul.ini index 9e794178..efd23c60 100644 --- a/microservice/sysom_vul/sysom-vul.ini +++ b/script/server/sysom_vul/sysom-vul.ini @@ -1,9 +1,9 @@ [program:sysom-vul] -directory = /usr/local/sysom/server/microservice/sysom_vul -command=/usr/local/sysom/server/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vul.wsgi:application +directory = /usr/local/sysom/server/sysom_vul +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vul.wsgi:application startsecs=3 autostart=true autorestart=true -environment=PATH="/usr/local/sysom/server/environment/virtualenv/bin/" +environment=PATH="/usr/local/sysom/environment/virtualenv/bin/" stderr_logfile=/var/log/sysom/sysom-vul-error.log stdout_logfile=/var/log/sysom/sysom-vul.log diff --git a/script/sysom.sh b/script/sysom.sh index 120c58ba..5cd8d79c 100755 --- a/script/sysom.sh +++ b/script/sysom.sh @@ -14,9 +14,11 @@ then fi export SERVER_HOME=${APP_HOME}/server export NODE_HOME=${APP_HOME}/node -export MICROSERVICE_HOME=${SERVER_HOME}/microservice -export INFRASTRUCTURE_HOME=${SERVER_HOME}/infrastructure -export ENVIRONMENT_HOME=${SERVER_HOME}/environment +export SCRIPT_HOME=${APP_HOME}/script +export WEB_HOME=${APP_HOME}/web +export MICROSERVICE_HOME=${SERVER_HOME} +export MICROSERVICE_SCRIPT_HOME=${SCRIPT_HOME}/server +export ENVIRONMENT_HOME=${APP_HOME}/environment export GLOBAL_VIRTUALENV_HOME=${ENVIRONMENT_HOME}/virtualenv if [ "$SERVER_LOCAL_IP" == "" ] @@ -117,27 +119,27 @@ sub_help(){ } sub_deploy() { - bash +x ${BaseDir}/deploy.sh $@ + bash +x ${BaseDir}/sysom_deploy.sh $@ } sub_update() { - bash +x ${BaseDir}/update.sh $@ + bash +x ${BaseDir}/sysom_update.sh $@ } sub_start() { - bash +x ${BaseDir}/start.sh $@ + bash +x ${BaseDir}/sysom_start.sh $@ } sub_stop() { - bash +x ${BaseDir}/stop.sh $@ + bash +x ${BaseDir}/sysom_stop.sh $@ } sub_clear() { - bash +x ${BaseDir}/clear.sh $@ + bash +x ${BaseDir}/sysom_clear.sh $@ } sub_dbmigrate() { - bash +x ${BaseDir}/dbmigrate.sh $@ + bash +x ${BaseDir}/sysom_dbmigrate.sh $@ } sub_list() { diff --git a/script/clear.sh b/script/sysom_clear.sh similarity index 73% rename from script/clear.sh rename to script/sysom_clear.sh index c46a3af1..c0282333 100755 --- a/script/clear.sh +++ b/script/sysom_clear.sh @@ -13,16 +13,25 @@ green() { } -do_clear_infrastructure() { +do_clear_environment() { target=$1 targets=(${target//,/ }) for target in ${targets[*]} do - for service_dir in $(ls ${INFRASTRUCTURE_HOME} | grep ${target}) + for service_dir in $(ls ${ENVIRONMENT_HOME} | grep ${target}) do - target_dir=${INFRASTRUCTURE_HOME}/${service_dir} + target_dir=${ENVIRONMENT_HOME}/${service_dir} + if [ ! -d "${target_dir}" ]; then + continue + fi + pushd ${target_dir} + if [ ! -f "${target_dir}/clear.sh" ]; then + popd + continue + fi + green "$target_dir clear..................................." bash -x ${target_dir}/clear.sh if [ $? -ne 0 ];then @@ -38,9 +47,9 @@ do_clear_infrastructure() { done # rm infrastructure if the folder is empty - if [ "`ls ${INFRASTRUCTURE_HOME} | wc -l`" == "0" ] + if [ "`ls ${ENVIRONMENT_HOME} | wc -l`" == "0" ] then - rm -rf ${INFRASTRUCTURE_HOME} + rm -rf ${ENVIRONMENT_HOME} fi } @@ -49,31 +58,33 @@ do_clear_microservice() { targets=(${target//,/ }) for target in ${targets[*]} do - for service_dir in $(ls ${MICROSERVICE_HOME} | grep ${target}) + for service in $(ls ${MICROSERVICE_SCRIPT_HOME} | grep ${target}) do - target_dir=${MICROSERVICE_HOME}/${service_dir} - if [ ! -d "$target_dir" ] + target_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} + target_service_dir=${MICROSERVICE_HOME}/${service} + if [ ! -d "$target_script_dir" ] then continue fi - pushd ${target_dir} + pushd ${target_script_dir} - if [ ! -f ${target_dir}/scripts/server_clear.sh ] + if [ ! -f ${target_script_dir}/clear.sh ] then continue fi - green "$target_dir clear..................................." - bash -x ${target_dir}/scripts/server_clear.sh + green "$target_script_dir clear..................................." + bash -x ${target_script_dir}/clear.sh if [ $? -ne 0 ];then - red "$target_dir clear fail, please check..................................." + red "$target_script_dir clear fail, please check..................................." # red "sysom init failed, exit 1" # exit 1 fi popd # rm dir - rm -rf ${target_dir} + rm -rf ${target_script_dir} + rm -rf ${target_service_dir} done done @@ -114,31 +125,31 @@ sub_all() { echo "all" } -sub_infra() { - sub_infrastructure $@ +sub_env() { + sub_environment $@ } # -> env + local_services -sub_infrastructure() { +sub_environment() { target=$1 if [ "$target" == "ALL" ] then # Load clear_excludes clear_excludes="" - if [ -f "${INFRASTRUCTURE_HOME}/clear_exclude" ];then - clear_excludes=`cat ${INFRASTRUCTURE_HOME}/clear_exclude` + if [ -f "${ENVIRONMENT_HOME}/clear_exclude" ];then + clear_excludes=`cat ${ENVIRONMENT_HOME}/clear_exclude` fi # Clear all microservices - for infrastructure in $(ls -r $INFRASTRUCTURE_HOME) + for infrastructure in $(ls -r $ENVIRONMENT_HOME) do - if [[ $clear_excludes =~ $infrastructure ]] || [ ! -d "${INFRASTRUCTURE_HOME}/${infrastructure}" ];then + if [[ $clear_excludes =~ $infrastructure ]] || [ ! -d "${ENVIRONMENT_HOME}/${infrastructure}" ];then continue fi - do_clear_infrastructure $infrastructure + do_clear_environment $infrastructure done else # Clear specific microservices - do_clear_infrastructure $target + do_clear_environment $target fi } @@ -170,6 +181,11 @@ sub_microservice() { fi } +# Clear web +sub_web() { + rm -rf ${WEB_HOME} +} + subcommand=$1 case $subcommand in diff --git a/script/dbmigrate.sh b/script/sysom_dbmigrate.sh similarity index 100% rename from script/dbmigrate.sh rename to script/sysom_dbmigrate.sh diff --git a/script/deploy.sh b/script/sysom_deploy.sh similarity index 59% rename from script/deploy.sh rename to script/sysom_deploy.sh index 7a54c851..71378bb2 100755 --- a/script/deploy.sh +++ b/script/sysom_deploy.sh @@ -2,23 +2,9 @@ set -x ProgName=$(basename $0) BaseDir=$(dirname $(readlink -f "$0")) -# SYSOM_DATABASE_HOST=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a host | awk '{print $2}'` -# SYSOM_DATABASE_PORT=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a port | awk '{print $2}'` -# SYSOM_DATABASE_USER=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a user | awk '{print $2}'` -# SYSOM_DATABASE_PASSWORD=`cat $SYSOM_CONF | grep -Pzo '(?s)mysql.*n.*database:(.*?)\n' | grep -a password | awk '{print $2}'` +local_app_home=`dirname $BaseDir` SYSOM_DEPLOY_LOG=$LOG_HOME/sysom_deploy.log -# ###enable the service web menu### -# setup_web_menu_enable() -# { -# service=`echo $1 | grep sysom | awk -F"sysom_" '{print $NF}'` -# if [ "$service" != "" ] -# then -# mysql -h ${SYSOM_DATABASE_HOST} -P ${SYSOM_DATABASE_PORT} -u ${SYSOM_DATABASE_USER} -p${SYSOM_DATABASE_PASSWORD} \ -# -e "use sysom;insert into sys_service_info(service_name, created_at) values ('sysom_${service}', current_timestamp);" -# fi -# } - #################################################################################################################### # Helper functions #################################################################################################################### @@ -31,18 +17,6 @@ green() { printf '\33[1;32m%b\n\33[0m' "$1" } -generate_service_env() { - rm -f ${APP_HOME}/env - cat << EOF > ${APP_HOME}/env -APP_HOME=${APP_HOME} -SERVER_HOME=${APP_HOME}/server -NODE_HOME=${APP_HOME}/node -SERVER_LOCAL_IP=${SERVER_LOCAL_IP} -SERVER_PUBLIC_IP=${SERVER_PUBLIC_IP} -SERVER_PORT=${SERVER_PORT} -EOF -} - ensure_sysom_conf_exists() { if [ ! -f "$SYSOM_CONF" ] then @@ -50,6 +24,11 @@ ensure_sysom_conf_exists() { fi } +encure_scripts_exists() { + rm -rf $SCRIPT_HOME + cp -r $BaseDir $SCRIPT_HOME +} + update_global_config() { sed "s/SERVER_LOCAL_IP: 127.0.0.1/SERVER_LOCAL_IP: $SERVER_LOCAL_IP/g" -i ${SYSOM_CONF} sed "s/SERVER_PUBLIC_IP: 127.0.0.1/SERVER_PUBLIC_IP: $SERVER_PUBLIC_IP/g" -i ${SYSOM_CONF} @@ -82,49 +61,64 @@ update_global_config() { do_deploy_microservices() { mkdir -p ${MICROSERVICE_HOME} - local_microservice_dir=`dirname $BaseDir`/microservice + mkdir -p ${MICROSERVICE_SCRIPT_HOME} + local_microservice_dir=`dirname $BaseDir`/sysom_server + local_script_dir=server target=$1 targets=(${target//,/ }) for target in ${targets[*]} do - for service_dir in $(ls ${local_microservice_dir} | grep ${target}) + for service in $(ls ${local_microservice_dir} | grep ${target}) do - if [ "$local_microservice_dir" != "$MICROSERVICE_HOME" ] + local_service_dir=${local_microservice_dir}/${service} + local_service_deploy_script_dir=${local_script_dir}/${service} + service_dir=${MICROSERVICE_HOME}/${service} + service_deploy_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} + if [ "$local_app_home" != "$APP_HOME" ] then - # Copy microservice dir to deploy path if need - cp -r $local_microservice_dir/${service_dir} $MICROSERVICE_HOME/${service_dir} + rm -rf ${service_dir} + cp -r ${local_service_dir} ${service_dir} + rm -rf ${service_deploy_script_dir} + cp -r ${local_service_deploy_script_dir} ${service_deploy_script_dir} fi - target_dir=${MICROSERVICE_HOME}/${service_dir} - - pushd ${target_dir} - + ################################################################################################ + # Service conf initial + ################################################################################################ + pushd ${service_dir} # update microservice common.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} - if [ -f "${target_dir}/conf/common.py" ]; then - sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${target_dir}/conf/common.py + if [ -f "${service_dir}/conf/common.py" ]; then + sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${service_dir}/conf/common.py fi # update fastapi based microservice alembic/env.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} - if [ -f "${target_dir}/alembic/env.py" ]; then - sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${target_dir}/alembic/env.py + if [ -f "${service_dir}/alembic/env.py" ]; then + sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${service_dir}/alembic/env.py fi # update microservice gunicorn.py replace /var/log/sysom to ${LOG_HOME} - if [ -f "${target_dir}/conf/gunicorn.py" ]; then - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${target_dir}/conf/gunicorn.py + if [ -f "${targetservice_dir_dir}/conf/gunicorn.py" ]; then + sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_dir}/conf/gunicorn.py fi + popd + + ################################################################################################ + # Perform service initial script + ################################################################################################ + + pushd ${service_deploy_script_dir} # update *.ini replace /var/log/sysom to ${LOG_HOME} for ini_conf in $(ls sysom-*.ini) do - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${target_dir}/${ini_conf} + sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_deploy_script_dir}/${ini_conf} done - green "$target_dir initialing..................................." - if [ ! -f "${target_dir}/scripts/server_init.sh" ];then + green "$service_deploy_script_dir initialing..................................." + if [ ! -f "${service_deploy_script_dir}/init.sh" ];then popd continue fi - bash -x ${target_dir}/scripts/server_init.sh + bash -x ${service_deploy_script_dir}/init.sh if [ $? -ne 0 ];then - red "$target_dir initial fail, please check..................................." + red "$service_deploy_script_dir initial fail, please check..................................." red "sysom init failed, exit 1" exit 1 fi @@ -133,21 +127,21 @@ do_deploy_microservices() { done } -do_deploy_infrastructure() { - mkdir -p ${INFRASTRUCTURE_HOME} - local_infrastructure_dir=`dirname $BaseDir`/infrastructure +do_deploy_environment() { + mkdir -p ${ENVIRONMENT_HOME} + local_environment_dir=`dirname $BaseDir`/environment target=$1 targets=(${target//,/ }) for target in ${targets[*]} do - for service_dir in $(ls ${local_infrastructure_dir} | grep ${target}) + for env in $(ls ${local_environment_dir} | grep ${target}) do - if [ "$local_infrastructure_dir" != "$INFRASTRUCTURE_HOME" ] + if [ "$local_app_home" != "$APP_HOME" ] then - # Copy microservice dir to deploy path if need - cp -r $local_infrastructure_dir/${service_dir} $INFRASTRUCTURE_HOME/${service_dir} + # Copy env dir to deploy path if need + cp -r $local_environment_dir/${env} $ENVIRONMENT_HOME/${env} fi - target_dir=${INFRASTRUCTURE_HOME}/${service_dir} + target_dir=${ENVIRONMENT_HOME}/${env} pushd ${target_dir} green "$target_dir initialing..................................." @@ -190,32 +184,33 @@ sub_all() { echo "all" } -sub_infra() { - sub_infrastructure $@ +sub_env() { + sub_environment $@ } # -> env + local_services -sub_infrastructure() { +sub_environment() { target=$1 if [ "$target" == "ALL" ] then - local_infrastructure_dir=`dirname $BaseDir`/infrastructure + local_environment_dir=`dirname $BaseDir`/environment + # Load deploy_excludes deploy_excludes="" - if [ -f "${local_infrastructure_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_infrastructure_dir}/deploy_exclude` + if [ -f "${local_environment_dir}/deploy_exclude" ];then + deploy_excludes=`cat ${local_environment_dir}/deploy_exclude` fi # Deploy all microservices - for infrastructure in $(ls $local_infrastructure_dir) + for env in $(ls $local_environment_dir) do - if [[ $deploy_excludes =~ $infrastructure ]] || [ ! -d "${local_infrastructure_dir}/${infrastructure}" ];then + if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ];then continue fi - do_deploy_infrastructure $infrastructure + do_deploy_environment $env done else # Deploy specific microservices - do_deploy_infrastructure $target + do_deploy_environment $target fi } @@ -228,17 +223,20 @@ sub_microservice() { target=$1 if [ "$target" == "ALL" ] then - local_microservice_dir=`dirname $BaseDir`/microservice + local_microservice_deploy_dir=${local_app_home}/sysom_server + if [ ! -d ${local_microservice_deploy_dir} ]; then + local_microservice_deploy_dir=${local_app_home}/server + fi + # Load deploy_excludes deploy_excludes="" - echo "hhhhh ${local_microservice_dir}" - if [ -f "${local_microservice_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_microservice_dir}/deploy_exclude` + if [ -f "${local_microservice_deploy_dir}/deploy_exclude" ];then + deploy_excludes=`cat ${local_microservice_deploy_dir}/deploy_exclude` fi # Deploy all microservices - for microservice in $(ls $local_microservice_dir) + for microservice in $(ls $local_microservice_deploy_dir) do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_dir}/${microservice}" ];then + if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_deploy_dir}/${microservice}" ];then continue fi do_deploy_microservices ${microservice} @@ -249,6 +247,27 @@ sub_microservice() { fi } +# Deploy web +sub_web() { + local_web_home=${local_app_home}/sysom_web + if [ ! -d ${local_web_home} ]; then + local_web_home=${local_app_home}/web + fi + rm -rf ${WEB_HOME} + pushd ${local_web_home} + if [ -f "index.html" ] + then + # dist + cp -r ${local_web_home} ${WEB_HOME} + else + # source + yarn + yarn build + cp -r dist ${WEB_HOME} + fi + popd +} + subcommand=$1 case $subcommand in @@ -257,6 +276,7 @@ case $subcommand in ;; *) ensure_sysom_conf_exists + encure_scripts_exists update_global_config shift sub_${subcommand} $@ diff --git a/microservice/.gitignore b/sysom_server/.gitignore similarity index 100% rename from microservice/.gitignore rename to sysom_server/.gitignore diff --git a/microservice/0_sysom_api/apps/common/__init__.py b/sysom_server/clear_exclude similarity index 100% rename from microservice/0_sysom_api/apps/common/__init__.py rename to sysom_server/clear_exclude diff --git a/sysom_server/deploy_exclude b/sysom_server/deploy_exclude new file mode 100644 index 00000000..5e69798f --- /dev/null +++ b/sysom_server/deploy_exclude @@ -0,0 +1 @@ +sysom_hotfix_builder \ No newline at end of file diff --git a/microservice/0_sysom_api/.gitignore b/sysom_server/sysom_api/.gitignore similarity index 100% rename from microservice/0_sysom_api/.gitignore rename to sysom_server/sysom_api/.gitignore diff --git a/microservice/0_sysom_api/apps/__init__.py b/sysom_server/sysom_api/apps/__init__.py similarity index 100% rename from microservice/0_sysom_api/apps/__init__.py rename to sysom_server/sysom_api/apps/__init__.py diff --git a/microservice/0_sysom_api/apps/host/__init__.py b/sysom_server/sysom_api/apps/accounts/__init__.py similarity index 100% rename from microservice/0_sysom_api/apps/host/__init__.py rename to sysom_server/sysom_api/apps/accounts/__init__.py diff --git a/microservice/0_sysom_api/apps/accounts/admin.py b/sysom_server/sysom_api/apps/accounts/admin.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/admin.py rename to sysom_server/sysom_api/apps/accounts/admin.py diff --git a/microservice/0_sysom_api/apps/accounts/apps.py b/sysom_server/sysom_api/apps/accounts/apps.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/apps.py rename to sysom_server/sysom_api/apps/accounts/apps.py diff --git a/microservice/0_sysom_api/apps/accounts/authentication.py b/sysom_server/sysom_api/apps/accounts/authentication.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/authentication.py rename to sysom_server/sysom_api/apps/accounts/authentication.py diff --git a/microservice/0_sysom_api/apps/accounts/migrations/0001_initial.py b/sysom_server/sysom_api/apps/accounts/migrations/0001_initial.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/migrations/0001_initial.py rename to sysom_server/sysom_api/apps/accounts/migrations/0001_initial.py diff --git a/microservice/0_sysom_api/apps/accounts/migrations/0002_user_allow_login.py b/sysom_server/sysom_api/apps/accounts/migrations/0002_user_allow_login.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/migrations/0002_user_allow_login.py rename to sysom_server/sysom_api/apps/accounts/migrations/0002_user_allow_login.py diff --git a/microservice/0_sysom_api/apps/host/migrations/__init__.py b/sysom_server/sysom_api/apps/accounts/migrations/__init__.py similarity index 100% rename from microservice/0_sysom_api/apps/host/migrations/__init__.py rename to sysom_server/sysom_api/apps/accounts/migrations/__init__.py diff --git a/microservice/0_sysom_api/apps/accounts/models.py b/sysom_server/sysom_api/apps/accounts/models.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/models.py rename to sysom_server/sysom_api/apps/accounts/models.py diff --git a/microservice/0_sysom_api/apps/accounts/permissions.py b/sysom_server/sysom_api/apps/accounts/permissions.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/permissions.py rename to sysom_server/sysom_api/apps/accounts/permissions.py diff --git a/microservice/0_sysom_api/apps/accounts/serializer.py b/sysom_server/sysom_api/apps/accounts/serializer.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/serializer.py rename to sysom_server/sysom_api/apps/accounts/serializer.py diff --git a/microservice/0_sysom_api/apps/accounts/tests.py b/sysom_server/sysom_api/apps/accounts/tests.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/tests.py rename to sysom_server/sysom_api/apps/accounts/tests.py diff --git a/microservice/0_sysom_api/apps/accounts/urls.py b/sysom_server/sysom_api/apps/accounts/urls.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/urls.py rename to sysom_server/sysom_api/apps/accounts/urls.py diff --git a/microservice/0_sysom_api/apps/accounts/views.py b/sysom_server/sysom_api/apps/accounts/views.py similarity index 100% rename from microservice/0_sysom_api/apps/accounts/views.py rename to sysom_server/sysom_api/apps/accounts/views.py diff --git a/microservice/0_sysom_api/apps/alarm/__init__.py b/sysom_server/sysom_api/apps/alarm/__init__.py similarity index 100% rename from microservice/0_sysom_api/apps/alarm/__init__.py rename to sysom_server/sysom_api/apps/alarm/__init__.py diff --git a/microservice/0_sysom_api/apps/alarm/admin.py b/sysom_server/sysom_api/apps/alarm/admin.py similarity index 100% rename from microservice/0_sysom_api/apps/alarm/admin.py rename to sysom_server/sysom_api/apps/alarm/admin.py diff --git a/microservice/0_sysom_api/apps/alarm/apps.py b/sysom_server/sysom_api/apps/alarm/apps.py similarity index 100% rename from microservice/0_sysom_api/apps/alarm/apps.py rename to sysom_server/sysom_api/apps/alarm/apps.py diff --git a/microservice/0_sysom_api/apps/alarm/migrations/0001_initial.py b/sysom_server/sysom_api/apps/alarm/migrations/0001_initial.py similarity index 100% rename from microservice/0_sysom_api/apps/alarm/migrations/0001_initial.py rename to sysom_server/sysom_api/apps/alarm/migrations/0001_initial.py diff --git a/microservice/0_sysom_api/apps/services/migrations/__init__.py b/sysom_server/sysom_api/apps/alarm/migrations/__init__.py similarity index 100% rename from microservice/0_sysom_api/apps/services/migrations/__init__.py rename to sysom_server/sysom_api/apps/alarm/migrations/__init__.py diff --git a/microservice/0_sysom_api/apps/alarm/models.py b/sysom_server/sysom_api/apps/alarm/models.py similarity index 100% rename from microservice/0_sysom_api/apps/alarm/models.py rename to sysom_server/sysom_api/apps/alarm/models.py diff --git a/microservice/0_sysom_api/apps/alarm/serializer.py b/sysom_server/sysom_api/apps/alarm/serializer.py similarity index 100% rename from microservice/0_sysom_api/apps/alarm/serializer.py rename to sysom_server/sysom_api/apps/alarm/serializer.py diff --git a/microservice/0_sysom_api/apps/alarm/subscribe.json b/sysom_server/sysom_api/apps/alarm/subscribe.json similarity index 100% rename from microservice/0_sysom_api/apps/alarm/subscribe.json rename to sysom_server/sysom_api/apps/alarm/subscribe.json diff --git a/microservice/0_sysom_api/apps/alarm/urls.py b/sysom_server/sysom_api/apps/alarm/urls.py similarity index 100% rename from microservice/0_sysom_api/apps/alarm/urls.py rename to sysom_server/sysom_api/apps/alarm/urls.py diff --git a/microservice/0_sysom_api/apps/alarm/views.py b/sysom_server/sysom_api/apps/alarm/views.py similarity index 100% rename from microservice/0_sysom_api/apps/alarm/views.py rename to sysom_server/sysom_api/apps/alarm/views.py diff --git a/microservice/0_sysom_api/conf/__init__.py b/sysom_server/sysom_api/apps/common/__init__.py similarity index 100% rename from microservice/0_sysom_api/conf/__init__.py rename to sysom_server/sysom_api/apps/common/__init__.py diff --git a/microservice/0_sysom_api/apps/common/common_model_viewset.py b/sysom_server/sysom_api/apps/common/common_model_viewset.py similarity index 100% rename from microservice/0_sysom_api/apps/common/common_model_viewset.py rename to sysom_server/sysom_api/apps/common/common_model_viewset.py diff --git a/microservice/sysom_diagnosis/apps/task/__init__.py b/sysom_server/sysom_api/apps/host/__init__.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/__init__.py rename to sysom_server/sysom_api/apps/host/__init__.py diff --git a/microservice/0_sysom_api/apps/host/admin.py b/sysom_server/sysom_api/apps/host/admin.py similarity index 100% rename from microservice/0_sysom_api/apps/host/admin.py rename to sysom_server/sysom_api/apps/host/admin.py diff --git a/microservice/0_sysom_api/apps/host/apps.py b/sysom_server/sysom_api/apps/host/apps.py similarity index 100% rename from microservice/0_sysom_api/apps/host/apps.py rename to sysom_server/sysom_api/apps/host/apps.py diff --git a/microservice/0_sysom_api/apps/host/cec_api.py b/sysom_server/sysom_api/apps/host/cec_api.py similarity index 100% rename from microservice/0_sysom_api/apps/host/cec_api.py rename to sysom_server/sysom_api/apps/host/cec_api.py diff --git a/microservice/0_sysom_api/apps/host/heartbeat.py b/sysom_server/sysom_api/apps/host/heartbeat.py similarity index 100% rename from microservice/0_sysom_api/apps/host/heartbeat.py rename to sysom_server/sysom_api/apps/host/heartbeat.py diff --git a/microservice/0_sysom_api/apps/host/migrations/0001_initial.py b/sysom_server/sysom_api/apps/host/migrations/0001_initial.py similarity index 100% rename from microservice/0_sysom_api/apps/host/migrations/0001_initial.py rename to sysom_server/sysom_api/apps/host/migrations/0001_initial.py diff --git a/microservice/0_sysom_api/apps/host/migrations/0002_hostmodel_host_info.py b/sysom_server/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py similarity index 100% rename from microservice/0_sysom_api/apps/host/migrations/0002_hostmodel_host_info.py rename to sysom_server/sysom_api/apps/host/migrations/0002_hostmodel_host_info.py diff --git a/microservice/0_sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py b/sysom_server/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py similarity index 100% rename from microservice/0_sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py rename to sysom_server/sysom_api/apps/host/migrations/0003_alter_hostmodel_status.py diff --git a/microservice/0_sysom_api/apps/host/migrations/0004_auto_20221227_1705.py b/sysom_server/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py similarity index 100% rename from microservice/0_sysom_api/apps/host/migrations/0004_auto_20221227_1705.py rename to sysom_server/sysom_api/apps/host/migrations/0004_auto_20221227_1705.py diff --git a/microservice/sysom_diagnosis/apps/task/migrations/__init__.py b/sysom_server/sysom_api/apps/host/migrations/__init__.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/migrations/__init__.py rename to sysom_server/sysom_api/apps/host/migrations/__init__.py diff --git a/microservice/0_sysom_api/apps/host/models.py b/sysom_server/sysom_api/apps/host/models.py similarity index 100% rename from microservice/0_sysom_api/apps/host/models.py rename to sysom_server/sysom_api/apps/host/models.py diff --git a/microservice/0_sysom_api/apps/host/serializer.py b/sysom_server/sysom_api/apps/host/serializer.py similarity index 100% rename from microservice/0_sysom_api/apps/host/serializer.py rename to sysom_server/sysom_api/apps/host/serializer.py diff --git a/microservice/0_sysom_api/apps/host/tests.py b/sysom_server/sysom_api/apps/host/tests.py similarity index 100% rename from microservice/0_sysom_api/apps/host/tests.py rename to sysom_server/sysom_api/apps/host/tests.py diff --git a/microservice/0_sysom_api/apps/host/urls.py b/sysom_server/sysom_api/apps/host/urls.py similarity index 100% rename from microservice/0_sysom_api/apps/host/urls.py rename to sysom_server/sysom_api/apps/host/urls.py diff --git a/microservice/0_sysom_api/apps/host/views.py b/sysom_server/sysom_api/apps/host/views.py similarity index 100% rename from microservice/0_sysom_api/apps/host/views.py rename to sysom_server/sysom_api/apps/host/views.py diff --git a/microservice/0_sysom_api/apps/services/__init__.py b/sysom_server/sysom_api/apps/services/__init__.py similarity index 100% rename from microservice/0_sysom_api/apps/services/__init__.py rename to sysom_server/sysom_api/apps/services/__init__.py diff --git a/microservice/0_sysom_api/apps/services/admin.py b/sysom_server/sysom_api/apps/services/admin.py similarity index 100% rename from microservice/0_sysom_api/apps/services/admin.py rename to sysom_server/sysom_api/apps/services/admin.py diff --git a/microservice/0_sysom_api/apps/services/apps.py b/sysom_server/sysom_api/apps/services/apps.py similarity index 100% rename from microservice/0_sysom_api/apps/services/apps.py rename to sysom_server/sysom_api/apps/services/apps.py diff --git a/microservice/0_sysom_api/apps/services/migrations/0001_initial.py b/sysom_server/sysom_api/apps/services/migrations/0001_initial.py similarity index 100% rename from microservice/0_sysom_api/apps/services/migrations/0001_initial.py rename to sysom_server/sysom_api/apps/services/migrations/0001_initial.py diff --git a/microservice/sysom_hotfix/apps/hotfix/__init__.py b/sysom_server/sysom_api/apps/services/migrations/__init__.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/__init__.py rename to sysom_server/sysom_api/apps/services/migrations/__init__.py diff --git a/microservice/0_sysom_api/apps/services/models.py b/sysom_server/sysom_api/apps/services/models.py similarity index 100% rename from microservice/0_sysom_api/apps/services/models.py rename to sysom_server/sysom_api/apps/services/models.py diff --git a/microservice/0_sysom_api/apps/services/serializer.py b/sysom_server/sysom_api/apps/services/serializer.py similarity index 100% rename from microservice/0_sysom_api/apps/services/serializer.py rename to sysom_server/sysom_api/apps/services/serializer.py diff --git a/microservice/0_sysom_api/apps/services/tests.py b/sysom_server/sysom_api/apps/services/tests.py similarity index 100% rename from microservice/0_sysom_api/apps/services/tests.py rename to sysom_server/sysom_api/apps/services/tests.py diff --git a/microservice/0_sysom_api/apps/services/urls.py b/sysom_server/sysom_api/apps/services/urls.py similarity index 100% rename from microservice/0_sysom_api/apps/services/urls.py rename to sysom_server/sysom_api/apps/services/urls.py diff --git a/microservice/0_sysom_api/apps/services/views.py b/sysom_server/sysom_api/apps/services/views.py similarity index 100% rename from microservice/0_sysom_api/apps/services/views.py rename to sysom_server/sysom_api/apps/services/views.py diff --git a/microservice/sysom_hotfix/apps/hotfix/migrations/__init__.py b/sysom_server/sysom_api/conf/__init__.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/migrations/__init__.py rename to sysom_server/sysom_api/conf/__init__.py diff --git a/microservice/0_sysom_api/conf/common.py b/sysom_server/sysom_api/conf/common.py similarity index 100% rename from microservice/0_sysom_api/conf/common.py rename to sysom_server/sysom_api/conf/common.py diff --git a/microservice/0_sysom_api/conf/develop.py b/sysom_server/sysom_api/conf/develop.py similarity index 100% rename from microservice/0_sysom_api/conf/develop.py rename to sysom_server/sysom_api/conf/develop.py diff --git a/microservice/0_sysom_api/conf/product.py b/sysom_server/sysom_api/conf/product.py similarity index 100% rename from microservice/0_sysom_api/conf/product.py rename to sysom_server/sysom_api/conf/product.py diff --git a/microservice/0_sysom_api/conf/testing.py b/sysom_server/sysom_api/conf/testing.py similarity index 100% rename from microservice/0_sysom_api/conf/testing.py rename to sysom_server/sysom_api/conf/testing.py diff --git a/microservice/0_sysom_api/config.yml b/sysom_server/sysom_api/config.yml similarity index 100% rename from microservice/0_sysom_api/config.yml rename to sysom_server/sysom_api/config.yml diff --git a/microservice/0_sysom_api/consumer/__init__.py b/sysom_server/sysom_api/consumer/__init__.py similarity index 100% rename from microservice/0_sysom_api/consumer/__init__.py rename to sysom_server/sysom_api/consumer/__init__.py diff --git a/microservice/0_sysom_api/consumer/consumers.py b/sysom_server/sysom_api/consumer/consumers.py similarity index 100% rename from microservice/0_sysom_api/consumer/consumers.py rename to sysom_server/sysom_api/consumer/consumers.py diff --git a/microservice/0_sysom_api/consumer/middleware.py b/sysom_server/sysom_api/consumer/middleware.py similarity index 100% rename from microservice/0_sysom_api/consumer/middleware.py rename to sysom_server/sysom_api/consumer/middleware.py diff --git a/microservice/0_sysom_api/consumer/routing.py b/sysom_server/sysom_api/consumer/routing.py similarity index 100% rename from microservice/0_sysom_api/consumer/routing.py rename to sysom_server/sysom_api/consumer/routing.py diff --git a/microservice/0_sysom_api/lib/__init__.py b/sysom_server/sysom_api/lib/__init__.py similarity index 100% rename from microservice/0_sysom_api/lib/__init__.py rename to sysom_server/sysom_api/lib/__init__.py diff --git a/microservice/0_sysom_api/lib/authentications.py b/sysom_server/sysom_api/lib/authentications.py similarity index 100% rename from microservice/0_sysom_api/lib/authentications.py rename to sysom_server/sysom_api/lib/authentications.py diff --git a/microservice/0_sysom_api/lib/base_model.py b/sysom_server/sysom_api/lib/base_model.py similarity index 100% rename from microservice/0_sysom_api/lib/base_model.py rename to sysom_server/sysom_api/lib/base_model.py diff --git a/microservice/0_sysom_api/lib/decode/sysom_decode.py b/sysom_server/sysom_api/lib/decode/sysom_decode.py similarity index 100% rename from microservice/0_sysom_api/lib/decode/sysom_decode.py rename to sysom_server/sysom_api/lib/decode/sysom_decode.py diff --git a/microservice/0_sysom_api/lib/excel.py b/sysom_server/sysom_api/lib/excel.py similarity index 100% rename from microservice/0_sysom_api/lib/excel.py rename to sysom_server/sysom_api/lib/excel.py diff --git a/microservice/0_sysom_api/lib/exception.py b/sysom_server/sysom_api/lib/exception.py similarity index 100% rename from microservice/0_sysom_api/lib/exception.py rename to sysom_server/sysom_api/lib/exception.py diff --git a/microservice/0_sysom_api/lib/paginations.py b/sysom_server/sysom_api/lib/paginations.py similarity index 100% rename from microservice/0_sysom_api/lib/paginations.py rename to sysom_server/sysom_api/lib/paginations.py diff --git a/microservice/0_sysom_api/lib/renderers.py b/sysom_server/sysom_api/lib/renderers.py similarity index 100% rename from microservice/0_sysom_api/lib/renderers.py rename to sysom_server/sysom_api/lib/renderers.py diff --git a/microservice/0_sysom_api/lib/response.py b/sysom_server/sysom_api/lib/response.py similarity index 100% rename from microservice/0_sysom_api/lib/response.py rename to sysom_server/sysom_api/lib/response.py diff --git a/microservice/0_sysom_api/lib/ssh.py b/sysom_server/sysom_api/lib/ssh.py similarity index 100% rename from microservice/0_sysom_api/lib/ssh.py rename to sysom_server/sysom_api/lib/ssh.py diff --git a/microservice/0_sysom_api/lib/utils.py b/sysom_server/sysom_api/lib/utils.py similarity index 100% rename from microservice/0_sysom_api/lib/utils.py rename to sysom_server/sysom_api/lib/utils.py diff --git a/microservice/0_sysom_api/manage.py b/sysom_server/sysom_api/manage.py similarity index 100% rename from microservice/0_sysom_api/manage.py rename to sysom_server/sysom_api/manage.py diff --git a/microservice/0_sysom_api/sysom/__init__.py b/sysom_server/sysom_api/sysom/__init__.py similarity index 100% rename from microservice/0_sysom_api/sysom/__init__.py rename to sysom_server/sysom_api/sysom/__init__.py diff --git a/microservice/0_sysom_api/sysom/asgi.py b/sysom_server/sysom_api/sysom/asgi.py similarity index 100% rename from microservice/0_sysom_api/sysom/asgi.py rename to sysom_server/sysom_api/sysom/asgi.py diff --git a/microservice/0_sysom_api/sysom/settings.py b/sysom_server/sysom_api/sysom/settings.py similarity index 100% rename from microservice/0_sysom_api/sysom/settings.py rename to sysom_server/sysom_api/sysom/settings.py diff --git a/microservice/0_sysom_api/sysom/urls.py b/sysom_server/sysom_api/sysom/urls.py similarity index 100% rename from microservice/0_sysom_api/sysom/urls.py rename to sysom_server/sysom_api/sysom/urls.py diff --git a/microservice/0_sysom_api/sysom/wsgi.py b/sysom_server/sysom_api/sysom/wsgi.py similarity index 100% rename from microservice/0_sysom_api/sysom/wsgi.py rename to sysom_server/sysom_api/sysom/wsgi.py diff --git a/microservice/0_sysom_channel/alembic.ini b/sysom_server/sysom_channel/alembic.ini similarity index 100% rename from microservice/0_sysom_channel/alembic.ini rename to sysom_server/sysom_channel/alembic.ini diff --git a/microservice/0_sysom_channel/alembic/README b/sysom_server/sysom_channel/alembic/README similarity index 100% rename from microservice/0_sysom_channel/alembic/README rename to sysom_server/sysom_channel/alembic/README diff --git a/microservice/0_sysom_channel/alembic/env.py b/sysom_server/sysom_channel/alembic/env.py similarity index 100% rename from microservice/0_sysom_channel/alembic/env.py rename to sysom_server/sysom_channel/alembic/env.py diff --git a/microservice/0_sysom_channel/alembic/script.py.mako b/sysom_server/sysom_channel/alembic/script.py.mako similarity index 100% rename from microservice/0_sysom_channel/alembic/script.py.mako rename to sysom_server/sysom_channel/alembic/script.py.mako diff --git a/microservice/0_sysom_channel/alembic/versions/1663292d0bd8_init_db.py b/sysom_server/sysom_channel/alembic/versions/1663292d0bd8_init_db.py similarity index 100% rename from microservice/0_sysom_channel/alembic/versions/1663292d0bd8_init_db.py rename to sysom_server/sysom_channel/alembic/versions/1663292d0bd8_init_db.py diff --git a/microservice/0_sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py b/sysom_server/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py similarity index 100% rename from microservice/0_sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py rename to sysom_server/sysom_channel/alembic/versions/3f54250e3fca_add_channel_params_table.py diff --git a/microservice/0_sysom_channel/app/__init__.py b/sysom_server/sysom_channel/app/__init__.py similarity index 100% rename from microservice/0_sysom_channel/app/__init__.py rename to sysom_server/sysom_channel/app/__init__.py diff --git a/microservice/0_sysom_channel/app/crud.py b/sysom_server/sysom_channel/app/crud.py similarity index 100% rename from microservice/0_sysom_channel/app/crud.py rename to sysom_server/sysom_channel/app/crud.py diff --git a/microservice/0_sysom_channel/app/database.py b/sysom_server/sysom_channel/app/database.py similarity index 100% rename from microservice/0_sysom_channel/app/database.py rename to sysom_server/sysom_channel/app/database.py diff --git a/microservice/0_sysom_channel/app/executor.py b/sysom_server/sysom_channel/app/executor.py similarity index 100% rename from microservice/0_sysom_channel/app/executor.py rename to sysom_server/sysom_channel/app/executor.py diff --git a/microservice/0_sysom_channel/app/models.py b/sysom_server/sysom_channel/app/models.py similarity index 100% rename from microservice/0_sysom_channel/app/models.py rename to sysom_server/sysom_channel/app/models.py diff --git a/microservice/0_sysom_channel/app/routers/cec_status.py b/sysom_server/sysom_channel/app/routers/cec_status.py similarity index 100% rename from microservice/0_sysom_channel/app/routers/cec_status.py rename to sysom_server/sysom_channel/app/routers/cec_status.py diff --git a/microservice/0_sysom_channel/app/routers/config.py b/sysom_server/sysom_channel/app/routers/config.py similarity index 100% rename from microservice/0_sysom_channel/app/routers/config.py rename to sysom_server/sysom_channel/app/routers/config.py diff --git a/microservice/0_sysom_channel/app/routers/file.py b/sysom_server/sysom_channel/app/routers/file.py similarity index 100% rename from microservice/0_sysom_channel/app/routers/file.py rename to sysom_server/sysom_channel/app/routers/file.py diff --git a/microservice/0_sysom_channel/app/routers/health.py b/sysom_server/sysom_channel/app/routers/health.py similarity index 100% rename from microservice/0_sysom_channel/app/routers/health.py rename to sysom_server/sysom_channel/app/routers/health.py diff --git a/microservice/0_sysom_channel/app/schemas.py b/sysom_server/sysom_channel/app/schemas.py similarity index 100% rename from microservice/0_sysom_channel/app/schemas.py rename to sysom_server/sysom_channel/app/schemas.py diff --git a/microservice/0_sysom_channel/conf/common.py b/sysom_server/sysom_channel/conf/common.py similarity index 100% rename from microservice/0_sysom_channel/conf/common.py rename to sysom_server/sysom_channel/conf/common.py diff --git a/microservice/0_sysom_channel/conf/develop.py b/sysom_server/sysom_channel/conf/develop.py similarity index 100% rename from microservice/0_sysom_channel/conf/develop.py rename to sysom_server/sysom_channel/conf/develop.py diff --git a/microservice/0_sysom_channel/conf/gunicorn.py b/sysom_server/sysom_channel/conf/gunicorn.py similarity index 100% rename from microservice/0_sysom_channel/conf/gunicorn.py rename to sysom_server/sysom_channel/conf/gunicorn.py diff --git a/microservice/0_sysom_channel/conf/product.py b/sysom_server/sysom_channel/conf/product.py similarity index 100% rename from microservice/0_sysom_channel/conf/product.py rename to sysom_server/sysom_channel/conf/product.py diff --git a/microservice/0_sysom_channel/conf/settings.py b/sysom_server/sysom_channel/conf/settings.py similarity index 100% rename from microservice/0_sysom_channel/conf/settings.py rename to sysom_server/sysom_channel/conf/settings.py diff --git a/microservice/0_sysom_channel/conf/testing.py b/sysom_server/sysom_channel/conf/testing.py similarity index 100% rename from microservice/0_sysom_channel/conf/testing.py rename to sysom_server/sysom_channel/conf/testing.py diff --git a/microservice/0_sysom_channel/config.yml b/sysom_server/sysom_channel/config.yml similarity index 100% rename from microservice/0_sysom_channel/config.yml rename to sysom_server/sysom_channel/config.yml diff --git a/microservice/0_sysom_channel/lib/channels/__init__.py b/sysom_server/sysom_channel/lib/channels/__init__.py similarity index 100% rename from microservice/0_sysom_channel/lib/channels/__init__.py rename to sysom_server/sysom_channel/lib/channels/__init__.py diff --git a/microservice/0_sysom_channel/lib/channels/base.py b/sysom_server/sysom_channel/lib/channels/base.py similarity index 100% rename from microservice/0_sysom_channel/lib/channels/base.py rename to sysom_server/sysom_channel/lib/channels/base.py diff --git a/microservice/0_sysom_channel/lib/channels/ssh.py b/sysom_server/sysom_channel/lib/channels/ssh.py similarity index 100% rename from microservice/0_sysom_channel/lib/channels/ssh.py rename to sysom_server/sysom_channel/lib/channels/ssh.py diff --git a/microservice/0_sysom_channel/lib/ssh.py b/sysom_server/sysom_channel/lib/ssh.py similarity index 100% rename from microservice/0_sysom_channel/lib/ssh.py rename to sysom_server/sysom_channel/lib/ssh.py diff --git a/microservice/0_sysom_channel/lib/utils.py b/sysom_server/sysom_channel/lib/utils.py similarity index 100% rename from microservice/0_sysom_channel/lib/utils.py rename to sysom_server/sysom_channel/lib/utils.py diff --git a/microservice/0_sysom_channel/main.py b/sysom_server/sysom_channel/main.py similarity index 100% rename from microservice/0_sysom_channel/main.py rename to sysom_server/sysom_channel/main.py diff --git a/microservice/sysom_hotfix/conf/__init__.py b/sysom_server/sysom_diagnosis/apps/task/__init__.py similarity index 100% rename from microservice/sysom_hotfix/conf/__init__.py rename to sysom_server/sysom_diagnosis/apps/task/__init__.py diff --git a/microservice/sysom_diagnosis/apps/task/admin.py b/sysom_server/sysom_diagnosis/apps/task/admin.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/admin.py rename to sysom_server/sysom_diagnosis/apps/task/admin.py diff --git a/microservice/sysom_diagnosis/apps/task/apps.py b/sysom_server/sysom_diagnosis/apps/task/apps.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/apps.py rename to sysom_server/sysom_diagnosis/apps/task/apps.py diff --git a/microservice/sysom_diagnosis/apps/task/executor.py b/sysom_server/sysom_diagnosis/apps/task/executor.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/executor.py rename to sysom_server/sysom_diagnosis/apps/task/executor.py diff --git a/microservice/sysom_diagnosis/apps/task/filter.py b/sysom_server/sysom_diagnosis/apps/task/filter.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/filter.py rename to sysom_server/sysom_diagnosis/apps/task/filter.py diff --git a/microservice/sysom_diagnosis/apps/task/helper.py b/sysom_server/sysom_diagnosis/apps/task/helper.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/helper.py rename to sysom_server/sysom_diagnosis/apps/task/helper.py diff --git a/microservice/sysom_diagnosis/apps/task/migrations/0001_initial.py b/sysom_server/sysom_diagnosis/apps/task/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/migrations/0001_initial.py rename to sysom_server/sysom_diagnosis/apps/task/migrations/0001_initial.py diff --git a/microservice/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py b/sysom_server/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py rename to sysom_server/sysom_diagnosis/apps/task/migrations/0002_auto_20230110_1738.py diff --git a/microservice/sysom_migration/apps/migration/__init__.py b/sysom_server/sysom_diagnosis/apps/task/migrations/__init__.py similarity index 100% rename from microservice/sysom_migration/apps/migration/__init__.py rename to sysom_server/sysom_diagnosis/apps/task/migrations/__init__.py diff --git a/microservice/sysom_diagnosis/apps/task/models.py b/sysom_server/sysom_diagnosis/apps/task/models.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/models.py rename to sysom_server/sysom_diagnosis/apps/task/models.py diff --git a/microservice/sysom_diagnosis/apps/task/seriaizer.py b/sysom_server/sysom_diagnosis/apps/task/seriaizer.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/seriaizer.py rename to sysom_server/sysom_diagnosis/apps/task/seriaizer.py diff --git a/microservice/sysom_diagnosis/apps/task/tests.py b/sysom_server/sysom_diagnosis/apps/task/tests.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/tests.py rename to sysom_server/sysom_diagnosis/apps/task/tests.py diff --git a/microservice/sysom_diagnosis/apps/task/urls.py b/sysom_server/sysom_diagnosis/apps/task/urls.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/urls.py rename to sysom_server/sysom_diagnosis/apps/task/urls.py diff --git a/microservice/sysom_diagnosis/apps/task/views.py b/sysom_server/sysom_diagnosis/apps/task/views.py similarity index 100% rename from microservice/sysom_diagnosis/apps/task/views.py rename to sysom_server/sysom_diagnosis/apps/task/views.py diff --git a/microservice/sysom_diagnosis/conf/common.py b/sysom_server/sysom_diagnosis/conf/common.py similarity index 100% rename from microservice/sysom_diagnosis/conf/common.py rename to sysom_server/sysom_diagnosis/conf/common.py diff --git a/microservice/sysom_diagnosis/conf/develop.py b/sysom_server/sysom_diagnosis/conf/develop.py similarity index 100% rename from microservice/sysom_diagnosis/conf/develop.py rename to sysom_server/sysom_diagnosis/conf/develop.py diff --git a/microservice/sysom_diagnosis/conf/gunicorn.py b/sysom_server/sysom_diagnosis/conf/gunicorn.py similarity index 100% rename from microservice/sysom_diagnosis/conf/gunicorn.py rename to sysom_server/sysom_diagnosis/conf/gunicorn.py diff --git a/microservice/sysom_diagnosis/conf/product.py b/sysom_server/sysom_diagnosis/conf/product.py similarity index 100% rename from microservice/sysom_diagnosis/conf/product.py rename to sysom_server/sysom_diagnosis/conf/product.py diff --git a/microservice/sysom_diagnosis/conf/testing.py b/sysom_server/sysom_diagnosis/conf/testing.py similarity index 100% rename from microservice/sysom_diagnosis/conf/testing.py rename to sysom_server/sysom_diagnosis/conf/testing.py diff --git a/microservice/sysom_diagnosis/config.yml b/sysom_server/sysom_diagnosis/config.yml similarity index 100% rename from microservice/sysom_diagnosis/config.yml rename to sysom_server/sysom_diagnosis/config.yml diff --git a/microservice/sysom_diagnosis/lib/authentications.py b/sysom_server/sysom_diagnosis/lib/authentications.py similarity index 100% rename from microservice/sysom_diagnosis/lib/authentications.py rename to sysom_server/sysom_diagnosis/lib/authentications.py diff --git a/microservice/sysom_diagnosis/lib/base_model.py b/sysom_server/sysom_diagnosis/lib/base_model.py similarity index 100% rename from microservice/sysom_diagnosis/lib/base_model.py rename to sysom_server/sysom_diagnosis/lib/base_model.py diff --git a/microservice/sysom_diagnosis/lib/base_view.py b/sysom_server/sysom_diagnosis/lib/base_view.py similarity index 100% rename from microservice/sysom_diagnosis/lib/base_view.py rename to sysom_server/sysom_diagnosis/lib/base_view.py diff --git a/microservice/sysom_diagnosis/lib/decode/sysom_decode.py b/sysom_server/sysom_diagnosis/lib/decode/sysom_decode.py similarity index 100% rename from microservice/sysom_diagnosis/lib/decode/sysom_decode.py rename to sysom_server/sysom_diagnosis/lib/decode/sysom_decode.py diff --git a/microservice/sysom_diagnosis/lib/exception.py b/sysom_server/sysom_diagnosis/lib/exception.py similarity index 100% rename from microservice/sysom_diagnosis/lib/exception.py rename to sysom_server/sysom_diagnosis/lib/exception.py diff --git a/microservice/sysom_diagnosis/lib/paginations.py b/sysom_server/sysom_diagnosis/lib/paginations.py similarity index 100% rename from microservice/sysom_diagnosis/lib/paginations.py rename to sysom_server/sysom_diagnosis/lib/paginations.py diff --git a/microservice/sysom_diagnosis/lib/renderers.py b/sysom_server/sysom_diagnosis/lib/renderers.py similarity index 100% rename from microservice/sysom_diagnosis/lib/renderers.py rename to sysom_server/sysom_diagnosis/lib/renderers.py diff --git a/microservice/sysom_diagnosis/lib/response.py b/sysom_server/sysom_diagnosis/lib/response.py similarity index 100% rename from microservice/sysom_diagnosis/lib/response.py rename to sysom_server/sysom_diagnosis/lib/response.py diff --git a/microservice/sysom_diagnosis/lib/utils.py b/sysom_server/sysom_diagnosis/lib/utils.py similarity index 100% rename from microservice/sysom_diagnosis/lib/utils.py rename to sysom_server/sysom_diagnosis/lib/utils.py diff --git a/microservice/sysom_diagnosis/manage.py b/sysom_server/sysom_diagnosis/manage.py similarity index 100% rename from microservice/sysom_diagnosis/manage.py rename to sysom_server/sysom_diagnosis/manage.py diff --git a/microservice/sysom_diagnosis/scripts/node_clear.sh b/sysom_server/sysom_diagnosis/scripts/node_clear.sh similarity index 100% rename from microservice/sysom_diagnosis/scripts/node_clear.sh rename to sysom_server/sysom_diagnosis/scripts/node_clear.sh diff --git a/microservice/sysom_diagnosis/scripts/node_init.sh b/sysom_server/sysom_diagnosis/scripts/node_init.sh similarity index 100% rename from microservice/sysom_diagnosis/scripts/node_init.sh rename to sysom_server/sysom_diagnosis/scripts/node_init.sh diff --git a/microservice/sysom_diagnosis/scripts/node_update.sh b/sysom_server/sysom_diagnosis/scripts/node_update.sh similarity index 100% rename from microservice/sysom_diagnosis/scripts/node_update.sh rename to sysom_server/sysom_diagnosis/scripts/node_update.sh diff --git a/microservice/sysom_diagnosis/service_scripts/command b/sysom_server/sysom_diagnosis/service_scripts/command similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/command rename to sysom_server/sysom_diagnosis/service_scripts/command diff --git a/microservice/sysom_diagnosis/service_scripts/command_post b/sysom_server/sysom_diagnosis/service_scripts/command_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/command_post rename to sysom_server/sysom_diagnosis/service_scripts/command_post diff --git a/microservice/sysom_diagnosis/service_scripts/filecache b/sysom_server/sysom_diagnosis/service_scripts/filecache similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/filecache rename to sysom_server/sysom_diagnosis/service_scripts/filecache diff --git a/microservice/sysom_diagnosis/service_scripts/filecache_post b/sysom_server/sysom_diagnosis/service_scripts/filecache_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/filecache_post rename to sysom_server/sysom_diagnosis/service_scripts/filecache_post diff --git a/microservice/sysom_diagnosis/service_scripts/iofsstat b/sysom_server/sysom_diagnosis/service_scripts/iofsstat similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/iofsstat rename to sysom_server/sysom_diagnosis/service_scripts/iofsstat diff --git a/microservice/sysom_diagnosis/service_scripts/iofsstat_post b/sysom_server/sysom_diagnosis/service_scripts/iofsstat_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/iofsstat_post rename to sysom_server/sysom_diagnosis/service_scripts/iofsstat_post diff --git a/microservice/sysom_diagnosis/service_scripts/iohang b/sysom_server/sysom_diagnosis/service_scripts/iohang similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/iohang rename to sysom_server/sysom_diagnosis/service_scripts/iohang diff --git a/microservice/sysom_diagnosis/service_scripts/iohang_post b/sysom_server/sysom_diagnosis/service_scripts/iohang_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/iohang_post rename to sysom_server/sysom_diagnosis/service_scripts/iohang_post diff --git a/microservice/sysom_diagnosis/service_scripts/iolatency b/sysom_server/sysom_diagnosis/service_scripts/iolatency similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/iolatency rename to sysom_server/sysom_diagnosis/service_scripts/iolatency diff --git a/microservice/sysom_diagnosis/service_scripts/iolatency_post b/sysom_server/sysom_diagnosis/service_scripts/iolatency_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/iolatency_post rename to sysom_server/sysom_diagnosis/service_scripts/iolatency_post diff --git a/microservice/sysom_diagnosis/service_scripts/iosdiag_latency b/sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/iosdiag_latency rename to sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency diff --git a/microservice/sysom_diagnosis/service_scripts/iosdiag_latency_post b/sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/iosdiag_latency_post rename to sysom_server/sysom_diagnosis/service_scripts/iosdiag_latency_post diff --git a/microservice/sysom_diagnosis/service_scripts/jitter b/sysom_server/sysom_diagnosis/service_scripts/jitter similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/jitter rename to sysom_server/sysom_diagnosis/service_scripts/jitter diff --git a/microservice/sysom_diagnosis/service_scripts/jitter_post b/sysom_server/sysom_diagnosis/service_scripts/jitter_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/jitter_post rename to sysom_server/sysom_diagnosis/service_scripts/jitter_post diff --git a/microservice/sysom_diagnosis/service_scripts/loadtask b/sysom_server/sysom_diagnosis/service_scripts/loadtask similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/loadtask rename to sysom_server/sysom_diagnosis/service_scripts/loadtask diff --git a/microservice/sysom_diagnosis/service_scripts/loadtask_post b/sysom_server/sysom_diagnosis/service_scripts/loadtask_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/loadtask_post rename to sysom_server/sysom_diagnosis/service_scripts/loadtask_post diff --git a/microservice/sysom_diagnosis/service_scripts/memgraph b/sysom_server/sysom_diagnosis/service_scripts/memgraph similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/memgraph rename to sysom_server/sysom_diagnosis/service_scripts/memgraph diff --git a/microservice/sysom_diagnosis/service_scripts/memgraph_post b/sysom_server/sysom_diagnosis/service_scripts/memgraph_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/memgraph_post rename to sysom_server/sysom_diagnosis/service_scripts/memgraph_post diff --git a/microservice/sysom_diagnosis/service_scripts/oomcheck b/sysom_server/sysom_diagnosis/service_scripts/oomcheck similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/oomcheck rename to sysom_server/sysom_diagnosis/service_scripts/oomcheck diff --git a/microservice/sysom_diagnosis/service_scripts/oomcheck_post b/sysom_server/sysom_diagnosis/service_scripts/oomcheck_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/oomcheck_post rename to sysom_server/sysom_diagnosis/service_scripts/oomcheck_post diff --git a/microservice/sysom_diagnosis/service_scripts/ossre b/sysom_server/sysom_diagnosis/service_scripts/ossre similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/ossre rename to sysom_server/sysom_diagnosis/service_scripts/ossre diff --git a/microservice/sysom_diagnosis/service_scripts/ossre_post b/sysom_server/sysom_diagnosis/service_scripts/ossre_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/ossre_post rename to sysom_server/sysom_diagnosis/service_scripts/ossre_post diff --git a/microservice/sysom_diagnosis/service_scripts/packetdrop b/sysom_server/sysom_diagnosis/service_scripts/packetdrop similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/packetdrop rename to sysom_server/sysom_diagnosis/service_scripts/packetdrop diff --git a/microservice/sysom_diagnosis/service_scripts/packetdrop_post b/sysom_server/sysom_diagnosis/service_scripts/packetdrop_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/packetdrop_post rename to sysom_server/sysom_diagnosis/service_scripts/packetdrop_post diff --git a/microservice/sysom_diagnosis/service_scripts/pingtrace b/sysom_server/sysom_diagnosis/service_scripts/pingtrace similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/pingtrace rename to sysom_server/sysom_diagnosis/service_scripts/pingtrace diff --git a/microservice/sysom_diagnosis/service_scripts/pingtrace_post b/sysom_server/sysom_diagnosis/service_scripts/pingtrace_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/pingtrace_post rename to sysom_server/sysom_diagnosis/service_scripts/pingtrace_post diff --git a/microservice/sysom_diagnosis/service_scripts/retran b/sysom_server/sysom_diagnosis/service_scripts/retran similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/retran rename to sysom_server/sysom_diagnosis/service_scripts/retran diff --git a/microservice/sysom_diagnosis/service_scripts/retran_post b/sysom_server/sysom_diagnosis/service_scripts/retran_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/retran_post rename to sysom_server/sysom_diagnosis/service_scripts/retran_post diff --git a/microservice/sysom_diagnosis/service_scripts/schedmoni b/sysom_server/sysom_diagnosis/service_scripts/schedmoni similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/schedmoni rename to sysom_server/sysom_diagnosis/service_scripts/schedmoni diff --git a/microservice/sysom_diagnosis/service_scripts/schedmoni_post b/sysom_server/sysom_diagnosis/service_scripts/schedmoni_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/schedmoni_post rename to sysom_server/sysom_diagnosis/service_scripts/schedmoni_post diff --git a/microservice/sysom_diagnosis/service_scripts/taskprofile b/sysom_server/sysom_diagnosis/service_scripts/taskprofile similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/taskprofile rename to sysom_server/sysom_diagnosis/service_scripts/taskprofile diff --git a/microservice/sysom_diagnosis/service_scripts/taskprofile_post b/sysom_server/sysom_diagnosis/service_scripts/taskprofile_post similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/taskprofile_post rename to sysom_server/sysom_diagnosis/service_scripts/taskprofile_post diff --git a/microservice/sysom_diagnosis/service_scripts/test b/sysom_server/sysom_diagnosis/service_scripts/test similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/test rename to sysom_server/sysom_diagnosis/service_scripts/test diff --git a/microservice/sysom_diagnosis/service_scripts/test.sh b/sysom_server/sysom_diagnosis/service_scripts/test.sh similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/test.sh rename to sysom_server/sysom_diagnosis/service_scripts/test.sh diff --git a/microservice/sysom_diagnosis/service_scripts/tpython b/sysom_server/sysom_diagnosis/service_scripts/tpython similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/tpython rename to sysom_server/sysom_diagnosis/service_scripts/tpython diff --git a/microservice/sysom_diagnosis/service_scripts/vmcore_analyse b/sysom_server/sysom_diagnosis/service_scripts/vmcore_analyse similarity index 100% rename from microservice/sysom_diagnosis/service_scripts/vmcore_analyse rename to sysom_server/sysom_diagnosis/service_scripts/vmcore_analyse diff --git a/microservice/sysom_diagnosis/sysom_diagnosis/__init__.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/__init__.py similarity index 100% rename from microservice/sysom_diagnosis/sysom_diagnosis/__init__.py rename to sysom_server/sysom_diagnosis/sysom_diagnosis/__init__.py diff --git a/microservice/sysom_diagnosis/sysom_diagnosis/asgi.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/asgi.py similarity index 100% rename from microservice/sysom_diagnosis/sysom_diagnosis/asgi.py rename to sysom_server/sysom_diagnosis/sysom_diagnosis/asgi.py diff --git a/microservice/sysom_diagnosis/sysom_diagnosis/settings.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/settings.py similarity index 100% rename from microservice/sysom_diagnosis/sysom_diagnosis/settings.py rename to sysom_server/sysom_diagnosis/sysom_diagnosis/settings.py diff --git a/microservice/sysom_diagnosis/sysom_diagnosis/urls.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/urls.py similarity index 100% rename from microservice/sysom_diagnosis/sysom_diagnosis/urls.py rename to sysom_server/sysom_diagnosis/sysom_diagnosis/urls.py diff --git a/microservice/sysom_diagnosis/sysom_diagnosis/wsgi.py b/sysom_server/sysom_diagnosis/sysom_diagnosis/wsgi.py similarity index 100% rename from microservice/sysom_diagnosis/sysom_diagnosis/wsgi.py rename to sysom_server/sysom_diagnosis/sysom_diagnosis/wsgi.py diff --git a/microservice/sysom_migration/apps/migration/migrations/__init__.py b/sysom_server/sysom_hotfix/apps/hotfix/__init__.py similarity index 100% rename from microservice/sysom_migration/apps/migration/migrations/__init__.py rename to sysom_server/sysom_hotfix/apps/hotfix/__init__.py diff --git a/microservice/sysom_hotfix/apps/hotfix/admin.py b/sysom_server/sysom_hotfix/apps/hotfix/admin.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/admin.py rename to sysom_server/sysom_hotfix/apps/hotfix/admin.py diff --git a/microservice/sysom_hotfix/apps/hotfix/apps.py b/sysom_server/sysom_hotfix/apps/hotfix/apps.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/apps.py rename to sysom_server/sysom_hotfix/apps/hotfix/apps.py diff --git a/microservice/sysom_hotfix/apps/hotfix/function.py b/sysom_server/sysom_hotfix/apps/hotfix/function.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/function.py rename to sysom_server/sysom_hotfix/apps/hotfix/function.py diff --git a/microservice/sysom_hotfix/apps/hotfix/migrations/0001_initial.py b/sysom_server/sysom_hotfix/apps/hotfix/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/migrations/0001_initial.py rename to sysom_server/sysom_hotfix/apps/hotfix/migrations/0001_initial.py diff --git a/microservice/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py b/sysom_server/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py rename to sysom_server/sysom_hotfix/apps/hotfix/migrations/0002_auto_20230303_1601.py diff --git a/microservice/sysom_vmcore/apps/vmcore/__init__.py b/sysom_server/sysom_hotfix/apps/hotfix/migrations/__init__.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/__init__.py rename to sysom_server/sysom_hotfix/apps/hotfix/migrations/__init__.py diff --git a/microservice/sysom_hotfix/apps/hotfix/models.py b/sysom_server/sysom_hotfix/apps/hotfix/models.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/models.py rename to sysom_server/sysom_hotfix/apps/hotfix/models.py diff --git a/microservice/sysom_hotfix/apps/hotfix/serializer.py b/sysom_server/sysom_hotfix/apps/hotfix/serializer.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/serializer.py rename to sysom_server/sysom_hotfix/apps/hotfix/serializer.py diff --git a/microservice/sysom_hotfix/apps/hotfix/urls.py b/sysom_server/sysom_hotfix/apps/hotfix/urls.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/urls.py rename to sysom_server/sysom_hotfix/apps/hotfix/urls.py diff --git a/microservice/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py similarity index 100% rename from microservice/sysom_hotfix/apps/hotfix/views.py rename to sysom_server/sysom_hotfix/apps/hotfix/views.py diff --git a/microservice/sysom_vmcore/apps/vmcore/migrations/__init__.py b/sysom_server/sysom_hotfix/conf/__init__.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/migrations/__init__.py rename to sysom_server/sysom_hotfix/conf/__init__.py diff --git a/microservice/sysom_hotfix/conf/common.py b/sysom_server/sysom_hotfix/conf/common.py similarity index 100% rename from microservice/sysom_hotfix/conf/common.py rename to sysom_server/sysom_hotfix/conf/common.py diff --git a/microservice/sysom_hotfix/conf/develop.py b/sysom_server/sysom_hotfix/conf/develop.py similarity index 100% rename from microservice/sysom_hotfix/conf/develop.py rename to sysom_server/sysom_hotfix/conf/develop.py diff --git a/microservice/sysom_hotfix/conf/gunicorn.py b/sysom_server/sysom_hotfix/conf/gunicorn.py similarity index 100% rename from microservice/sysom_hotfix/conf/gunicorn.py rename to sysom_server/sysom_hotfix/conf/gunicorn.py diff --git a/microservice/sysom_hotfix/conf/product.py b/sysom_server/sysom_hotfix/conf/product.py similarity index 100% rename from microservice/sysom_hotfix/conf/product.py rename to sysom_server/sysom_hotfix/conf/product.py diff --git a/microservice/sysom_hotfix/conf/testing.py b/sysom_server/sysom_hotfix/conf/testing.py similarity index 100% rename from microservice/sysom_hotfix/conf/testing.py rename to sysom_server/sysom_hotfix/conf/testing.py diff --git a/microservice/sysom_hotfix/config.yml b/sysom_server/sysom_hotfix/config.yml similarity index 100% rename from microservice/sysom_hotfix/config.yml rename to sysom_server/sysom_hotfix/config.yml diff --git a/microservice/sysom_hotfix/lib/__init__.py b/sysom_server/sysom_hotfix/lib/__init__.py similarity index 100% rename from microservice/sysom_hotfix/lib/__init__.py rename to sysom_server/sysom_hotfix/lib/__init__.py diff --git a/microservice/sysom_hotfix/lib/authentications.py b/sysom_server/sysom_hotfix/lib/authentications.py similarity index 100% rename from microservice/sysom_hotfix/lib/authentications.py rename to sysom_server/sysom_hotfix/lib/authentications.py diff --git a/microservice/sysom_hotfix/lib/base_model.py b/sysom_server/sysom_hotfix/lib/base_model.py similarity index 100% rename from microservice/sysom_hotfix/lib/base_model.py rename to sysom_server/sysom_hotfix/lib/base_model.py diff --git a/microservice/sysom_hotfix/lib/base_view.py b/sysom_server/sysom_hotfix/lib/base_view.py similarity index 100% rename from microservice/sysom_hotfix/lib/base_view.py rename to sysom_server/sysom_hotfix/lib/base_view.py diff --git a/microservice/sysom_hotfix/lib/decode/sysom_decode.py b/sysom_server/sysom_hotfix/lib/decode/sysom_decode.py similarity index 100% rename from microservice/sysom_hotfix/lib/decode/sysom_decode.py rename to sysom_server/sysom_hotfix/lib/decode/sysom_decode.py diff --git a/microservice/sysom_hotfix/lib/excel.py b/sysom_server/sysom_hotfix/lib/excel.py similarity index 100% rename from microservice/sysom_hotfix/lib/excel.py rename to sysom_server/sysom_hotfix/lib/excel.py diff --git a/microservice/sysom_hotfix/lib/exception.py b/sysom_server/sysom_hotfix/lib/exception.py similarity index 100% rename from microservice/sysom_hotfix/lib/exception.py rename to sysom_server/sysom_hotfix/lib/exception.py diff --git a/microservice/sysom_hotfix/lib/paginations.py b/sysom_server/sysom_hotfix/lib/paginations.py similarity index 100% rename from microservice/sysom_hotfix/lib/paginations.py rename to sysom_server/sysom_hotfix/lib/paginations.py diff --git a/microservice/sysom_hotfix/lib/renderers.py b/sysom_server/sysom_hotfix/lib/renderers.py similarity index 100% rename from microservice/sysom_hotfix/lib/renderers.py rename to sysom_server/sysom_hotfix/lib/renderers.py diff --git a/microservice/sysom_hotfix/lib/response.py b/sysom_server/sysom_hotfix/lib/response.py similarity index 100% rename from microservice/sysom_hotfix/lib/response.py rename to sysom_server/sysom_hotfix/lib/response.py diff --git a/microservice/sysom_hotfix/lib/utils.py b/sysom_server/sysom_hotfix/lib/utils.py similarity index 100% rename from microservice/sysom_hotfix/lib/utils.py rename to sysom_server/sysom_hotfix/lib/utils.py diff --git a/microservice/sysom_hotfix/manage.py b/sysom_server/sysom_hotfix/manage.py similarity index 100% rename from microservice/sysom_hotfix/manage.py rename to sysom_server/sysom_hotfix/manage.py diff --git a/microservice/sysom_hotfix/sysom_hotfix/__init__.py b/sysom_server/sysom_hotfix/sysom_hotfix/__init__.py similarity index 100% rename from microservice/sysom_hotfix/sysom_hotfix/__init__.py rename to sysom_server/sysom_hotfix/sysom_hotfix/__init__.py diff --git a/microservice/sysom_hotfix/sysom_hotfix/asgi.py b/sysom_server/sysom_hotfix/sysom_hotfix/asgi.py similarity index 100% rename from microservice/sysom_hotfix/sysom_hotfix/asgi.py rename to sysom_server/sysom_hotfix/sysom_hotfix/asgi.py diff --git a/microservice/sysom_hotfix/sysom_hotfix/settings.py b/sysom_server/sysom_hotfix/sysom_hotfix/settings.py similarity index 100% rename from microservice/sysom_hotfix/sysom_hotfix/settings.py rename to sysom_server/sysom_hotfix/sysom_hotfix/settings.py diff --git a/microservice/sysom_hotfix/sysom_hotfix/urls.py b/sysom_server/sysom_hotfix/sysom_hotfix/urls.py similarity index 100% rename from microservice/sysom_hotfix/sysom_hotfix/urls.py rename to sysom_server/sysom_hotfix/sysom_hotfix/urls.py diff --git a/microservice/sysom_hotfix/sysom_hotfix/wsgi.py b/sysom_server/sysom_hotfix/sysom_hotfix/wsgi.py similarity index 100% rename from microservice/sysom_hotfix/sysom_hotfix/wsgi.py rename to sysom_server/sysom_hotfix/sysom_hotfix/wsgi.py diff --git a/microservice/sysom_hotfix_builder/build_hotfix.sh b/sysom_server/sysom_hotfix_builder/build_hotfix.sh similarity index 100% rename from microservice/sysom_hotfix_builder/build_hotfix.sh rename to sysom_server/sysom_hotfix_builder/build_hotfix.sh diff --git a/microservice/sysom_hotfix_builder/build_rpm.sh b/sysom_server/sysom_hotfix_builder/build_rpm.sh similarity index 100% rename from microservice/sysom_hotfix_builder/build_rpm.sh rename to sysom_server/sysom_hotfix_builder/build_rpm.sh diff --git a/microservice/sysom_hotfix_builder/builder.ini b/sysom_server/sysom_hotfix_builder/builder.ini similarity index 100% rename from microservice/sysom_hotfix_builder/builder.ini rename to sysom_server/sysom_hotfix_builder/builder.ini diff --git a/microservice/sysom_hotfix_builder/builder.py b/sysom_server/sysom_hotfix_builder/builder.py similarity index 100% rename from microservice/sysom_hotfix_builder/builder.py rename to sysom_server/sysom_hotfix_builder/builder.py diff --git a/microservice/sysom_hotfix_builder/check_env.sh b/sysom_server/sysom_hotfix_builder/check_env.sh similarity index 100% rename from microservice/sysom_hotfix_builder/check_env.sh rename to sysom_server/sysom_hotfix_builder/check_env.sh diff --git a/microservice/sysom_hotfix_builder/img_list.json b/sysom_server/sysom_hotfix_builder/img_list.json similarity index 100% rename from microservice/sysom_hotfix_builder/img_list.json rename to sysom_server/sysom_hotfix_builder/img_list.json diff --git a/microservice/sysom_vul/apps/host/__init__.py b/sysom_server/sysom_migration/apps/migration/__init__.py similarity index 100% rename from microservice/sysom_vul/apps/host/__init__.py rename to sysom_server/sysom_migration/apps/migration/__init__.py diff --git a/microservice/sysom_migration/apps/migration/admin.py b/sysom_server/sysom_migration/apps/migration/admin.py similarity index 100% rename from microservice/sysom_migration/apps/migration/admin.py rename to sysom_server/sysom_migration/apps/migration/admin.py diff --git a/microservice/sysom_migration/apps/migration/apps.py b/sysom_server/sysom_migration/apps/migration/apps.py similarity index 100% rename from microservice/sysom_migration/apps/migration/apps.py rename to sysom_server/sysom_migration/apps/migration/apps.py diff --git a/microservice/sysom_migration/apps/migration/migrations/0001_initial.py b/sysom_server/sysom_migration/apps/migration/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_migration/apps/migration/migrations/0001_initial.py rename to sysom_server/sysom_migration/apps/migration/migrations/0001_initial.py diff --git a/microservice/sysom_vul/apps/vul/__init__.py b/sysom_server/sysom_migration/apps/migration/migrations/__init__.py similarity index 100% rename from microservice/sysom_vul/apps/vul/__init__.py rename to sysom_server/sysom_migration/apps/migration/migrations/__init__.py diff --git a/microservice/sysom_migration/apps/migration/models.py b/sysom_server/sysom_migration/apps/migration/models.py similarity index 100% rename from microservice/sysom_migration/apps/migration/models.py rename to sysom_server/sysom_migration/apps/migration/models.py diff --git a/microservice/sysom_migration/apps/migration/tests.py b/sysom_server/sysom_migration/apps/migration/tests.py similarity index 100% rename from microservice/sysom_migration/apps/migration/tests.py rename to sysom_server/sysom_migration/apps/migration/tests.py diff --git a/microservice/sysom_migration/apps/migration/urls.py b/sysom_server/sysom_migration/apps/migration/urls.py similarity index 100% rename from microservice/sysom_migration/apps/migration/urls.py rename to sysom_server/sysom_migration/apps/migration/urls.py diff --git a/microservice/sysom_migration/apps/migration/views.py b/sysom_server/sysom_migration/apps/migration/views.py similarity index 100% rename from microservice/sysom_migration/apps/migration/views.py rename to sysom_server/sysom_migration/apps/migration/views.py diff --git a/microservice/sysom_migration/conf/common.py b/sysom_server/sysom_migration/conf/common.py similarity index 100% rename from microservice/sysom_migration/conf/common.py rename to sysom_server/sysom_migration/conf/common.py diff --git a/microservice/sysom_migration/conf/develop.py b/sysom_server/sysom_migration/conf/develop.py similarity index 100% rename from microservice/sysom_migration/conf/develop.py rename to sysom_server/sysom_migration/conf/develop.py diff --git a/microservice/sysom_migration/conf/gunicorn.py b/sysom_server/sysom_migration/conf/gunicorn.py similarity index 100% rename from microservice/sysom_migration/conf/gunicorn.py rename to sysom_server/sysom_migration/conf/gunicorn.py diff --git a/microservice/sysom_migration/conf/product.py b/sysom_server/sysom_migration/conf/product.py similarity index 100% rename from microservice/sysom_migration/conf/product.py rename to sysom_server/sysom_migration/conf/product.py diff --git a/microservice/sysom_migration/conf/testing.py b/sysom_server/sysom_migration/conf/testing.py similarity index 100% rename from microservice/sysom_migration/conf/testing.py rename to sysom_server/sysom_migration/conf/testing.py diff --git a/microservice/sysom_migration/config.yml b/sysom_server/sysom_migration/config.yml similarity index 100% rename from microservice/sysom_migration/config.yml rename to sysom_server/sysom_migration/config.yml diff --git a/microservice/sysom_migration/lib/authentications.py b/sysom_server/sysom_migration/lib/authentications.py similarity index 100% rename from microservice/sysom_migration/lib/authentications.py rename to sysom_server/sysom_migration/lib/authentications.py diff --git a/microservice/sysom_migration/lib/base_model.py b/sysom_server/sysom_migration/lib/base_model.py similarity index 100% rename from microservice/sysom_migration/lib/base_model.py rename to sysom_server/sysom_migration/lib/base_model.py diff --git a/microservice/sysom_migration/lib/base_view.py b/sysom_server/sysom_migration/lib/base_view.py similarity index 100% rename from microservice/sysom_migration/lib/base_view.py rename to sysom_server/sysom_migration/lib/base_view.py diff --git a/microservice/sysom_migration/lib/channel.py b/sysom_server/sysom_migration/lib/channel.py similarity index 100% rename from microservice/sysom_migration/lib/channel.py rename to sysom_server/sysom_migration/lib/channel.py diff --git a/microservice/sysom_migration/lib/decode/sysom_decode.py b/sysom_server/sysom_migration/lib/decode/sysom_decode.py similarity index 100% rename from microservice/sysom_migration/lib/decode/sysom_decode.py rename to sysom_server/sysom_migration/lib/decode/sysom_decode.py diff --git a/microservice/sysom_migration/lib/exception.py b/sysom_server/sysom_migration/lib/exception.py similarity index 100% rename from microservice/sysom_migration/lib/exception.py rename to sysom_server/sysom_migration/lib/exception.py diff --git a/microservice/sysom_migration/lib/paginations.py b/sysom_server/sysom_migration/lib/paginations.py similarity index 100% rename from microservice/sysom_migration/lib/paginations.py rename to sysom_server/sysom_migration/lib/paginations.py diff --git a/microservice/sysom_migration/lib/renderers.py b/sysom_server/sysom_migration/lib/renderers.py similarity index 100% rename from microservice/sysom_migration/lib/renderers.py rename to sysom_server/sysom_migration/lib/renderers.py diff --git a/microservice/sysom_migration/lib/response.py b/sysom_server/sysom_migration/lib/response.py similarity index 100% rename from microservice/sysom_migration/lib/response.py rename to sysom_server/sysom_migration/lib/response.py diff --git a/microservice/sysom_migration/lib/script.py b/sysom_server/sysom_migration/lib/script.py similarity index 100% rename from microservice/sysom_migration/lib/script.py rename to sysom_server/sysom_migration/lib/script.py diff --git a/microservice/sysom_migration/lib/utils.py b/sysom_server/sysom_migration/lib/utils.py similarity index 100% rename from microservice/sysom_migration/lib/utils.py rename to sysom_server/sysom_migration/lib/utils.py diff --git a/microservice/sysom_migration/manage.py b/sysom_server/sysom_migration/manage.py similarity index 100% rename from microservice/sysom_migration/manage.py rename to sysom_server/sysom_migration/manage.py diff --git a/microservice/sysom_migration/sysom_migration/__init__.py b/sysom_server/sysom_migration/sysom_migration/__init__.py similarity index 100% rename from microservice/sysom_migration/sysom_migration/__init__.py rename to sysom_server/sysom_migration/sysom_migration/__init__.py diff --git a/microservice/sysom_migration/sysom_migration/asgi.py b/sysom_server/sysom_migration/sysom_migration/asgi.py similarity index 100% rename from microservice/sysom_migration/sysom_migration/asgi.py rename to sysom_server/sysom_migration/sysom_migration/asgi.py diff --git a/microservice/sysom_migration/sysom_migration/settings.py b/sysom_server/sysom_migration/sysom_migration/settings.py similarity index 100% rename from microservice/sysom_migration/sysom_migration/settings.py rename to sysom_server/sysom_migration/sysom_migration/settings.py diff --git a/microservice/sysom_migration/sysom_migration/urls.py b/sysom_server/sysom_migration/sysom_migration/urls.py similarity index 100% rename from microservice/sysom_migration/sysom_migration/urls.py rename to sysom_server/sysom_migration/sysom_migration/urls.py diff --git a/microservice/sysom_migration/sysom_migration/wsgi.py b/sysom_server/sysom_migration/sysom_migration/wsgi.py similarity index 100% rename from microservice/sysom_migration/sysom_migration/wsgi.py rename to sysom_server/sysom_migration/sysom_migration/wsgi.py diff --git a/microservice/sysom_monitor_server/.gitignore b/sysom_server/sysom_monitor_server/.gitignore similarity index 100% rename from microservice/sysom_monitor_server/.gitignore rename to sysom_server/sysom_monitor_server/.gitignore diff --git a/microservice/sysom_monitor_server/app/routeres/grafana.py b/sysom_server/sysom_monitor_server/app/routeres/grafana.py similarity index 100% rename from microservice/sysom_monitor_server/app/routeres/grafana.py rename to sysom_server/sysom_monitor_server/app/routeres/grafana.py diff --git a/microservice/sysom_monitor_server/app/routeres/health.py b/sysom_server/sysom_monitor_server/app/routeres/health.py similarity index 100% rename from microservice/sysom_monitor_server/app/routeres/health.py rename to sysom_server/sysom_monitor_server/app/routeres/health.py diff --git a/microservice/sysom_monitor_server/app/routeres/home.py b/sysom_server/sysom_monitor_server/app/routeres/home.py similarity index 100% rename from microservice/sysom_monitor_server/app/routeres/home.py rename to sysom_server/sysom_monitor_server/app/routeres/home.py diff --git a/microservice/sysom_monitor_server/app/routeres/services.py b/sysom_server/sysom_monitor_server/app/routeres/services.py similarity index 100% rename from microservice/sysom_monitor_server/app/routeres/services.py rename to sysom_server/sysom_monitor_server/app/routeres/services.py diff --git a/microservice/sysom_monitor_server/conf/common.py b/sysom_server/sysom_monitor_server/conf/common.py similarity index 100% rename from microservice/sysom_monitor_server/conf/common.py rename to sysom_server/sysom_monitor_server/conf/common.py diff --git a/microservice/sysom_monitor_server/conf/develop.py b/sysom_server/sysom_monitor_server/conf/develop.py similarity index 100% rename from microservice/sysom_monitor_server/conf/develop.py rename to sysom_server/sysom_monitor_server/conf/develop.py diff --git a/microservice/sysom_monitor_server/conf/gunicorn.py b/sysom_server/sysom_monitor_server/conf/gunicorn.py similarity index 100% rename from microservice/sysom_monitor_server/conf/gunicorn.py rename to sysom_server/sysom_monitor_server/conf/gunicorn.py diff --git a/microservice/sysom_monitor_server/conf/product.py b/sysom_server/sysom_monitor_server/conf/product.py similarity index 100% rename from microservice/sysom_monitor_server/conf/product.py rename to sysom_server/sysom_monitor_server/conf/product.py diff --git a/microservice/sysom_monitor_server/conf/settings.py b/sysom_server/sysom_monitor_server/conf/settings.py similarity index 100% rename from microservice/sysom_monitor_server/conf/settings.py rename to sysom_server/sysom_monitor_server/conf/settings.py diff --git a/microservice/sysom_monitor_server/conf/testing.py b/sysom_server/sysom_monitor_server/conf/testing.py similarity index 100% rename from microservice/sysom_monitor_server/conf/testing.py rename to sysom_server/sysom_monitor_server/conf/testing.py diff --git a/microservice/sysom_monitor_server/config.yml b/sysom_server/sysom_monitor_server/config.yml similarity index 100% rename from microservice/sysom_monitor_server/config.yml rename to sysom_server/sysom_monitor_server/config.yml diff --git a/microservice/sysom_monitor_server/main.py b/sysom_server/sysom_monitor_server/main.py similarity index 100% rename from microservice/sysom_monitor_server/main.py rename to sysom_server/sysom_monitor_server/main.py diff --git a/microservice/sysom_monitor_server/scripts/node_clear.sh b/sysom_server/sysom_monitor_server/scripts/node_clear.sh similarity index 100% rename from microservice/sysom_monitor_server/scripts/node_clear.sh rename to sysom_server/sysom_monitor_server/scripts/node_clear.sh diff --git a/microservice/sysom_monitor_server/scripts/node_init.sh b/sysom_server/sysom_monitor_server/scripts/node_init.sh similarity index 100% rename from microservice/sysom_monitor_server/scripts/node_init.sh rename to sysom_server/sysom_monitor_server/scripts/node_init.sh diff --git a/microservice/sysom_monitor_server/scripts/node_update.sh b/sysom_server/sysom_monitor_server/scripts/node_update.sh similarity index 100% rename from microservice/sysom_monitor_server/scripts/node_update.sh rename to sysom_server/sysom_monitor_server/scripts/node_update.sh diff --git a/microservice/sysom_vul/apps/vul/migrations/__init__.py b/sysom_server/sysom_vmcore/apps/vmcore/__init__.py similarity index 100% rename from microservice/sysom_vul/apps/vul/migrations/__init__.py rename to sysom_server/sysom_vmcore/apps/vmcore/__init__.py diff --git a/microservice/sysom_vmcore/apps/vmcore/admin.py b/sysom_server/sysom_vmcore/apps/vmcore/admin.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/admin.py rename to sysom_server/sysom_vmcore/apps/vmcore/admin.py diff --git a/microservice/sysom_vmcore/apps/vmcore/apps.py b/sysom_server/sysom_vmcore/apps/vmcore/apps.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/apps.py rename to sysom_server/sysom_vmcore/apps/vmcore/apps.py diff --git a/microservice/sysom_vmcore/apps/vmcore/migrations/0001_initial.py b/sysom_server/sysom_vmcore/apps/vmcore/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/migrations/0001_initial.py rename to sysom_server/sysom_vmcore/apps/vmcore/migrations/0001_initial.py diff --git a/microservice/sysom_vul/conf/__init__.py b/sysom_server/sysom_vmcore/apps/vmcore/migrations/__init__.py similarity index 100% rename from microservice/sysom_vul/conf/__init__.py rename to sysom_server/sysom_vmcore/apps/vmcore/migrations/__init__.py diff --git a/microservice/sysom_vmcore/apps/vmcore/models.py b/sysom_server/sysom_vmcore/apps/vmcore/models.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/models.py rename to sysom_server/sysom_vmcore/apps/vmcore/models.py diff --git a/microservice/sysom_vmcore/apps/vmcore/serializer.py b/sysom_server/sysom_vmcore/apps/vmcore/serializer.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/serializer.py rename to sysom_server/sysom_vmcore/apps/vmcore/serializer.py diff --git a/microservice/sysom_vmcore/apps/vmcore/tests.py b/sysom_server/sysom_vmcore/apps/vmcore/tests.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/tests.py rename to sysom_server/sysom_vmcore/apps/vmcore/tests.py diff --git a/microservice/sysom_vmcore/apps/vmcore/urls.py b/sysom_server/sysom_vmcore/apps/vmcore/urls.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/urls.py rename to sysom_server/sysom_vmcore/apps/vmcore/urls.py diff --git a/microservice/sysom_vmcore/apps/vmcore/views.py b/sysom_server/sysom_vmcore/apps/vmcore/views.py similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/views.py rename to sysom_server/sysom_vmcore/apps/vmcore/views.py diff --git a/microservice/sysom_vmcore/apps/vmcore/vmcore.json b/sysom_server/sysom_vmcore/apps/vmcore/vmcore.json similarity index 100% rename from microservice/sysom_vmcore/apps/vmcore/vmcore.json rename to sysom_server/sysom_vmcore/apps/vmcore/vmcore.json diff --git a/microservice/sysom_vmcore/conf/common.py b/sysom_server/sysom_vmcore/conf/common.py similarity index 100% rename from microservice/sysom_vmcore/conf/common.py rename to sysom_server/sysom_vmcore/conf/common.py diff --git a/microservice/sysom_vmcore/conf/develop.py b/sysom_server/sysom_vmcore/conf/develop.py similarity index 100% rename from microservice/sysom_vmcore/conf/develop.py rename to sysom_server/sysom_vmcore/conf/develop.py diff --git a/microservice/sysom_vmcore/conf/gunicorn.py b/sysom_server/sysom_vmcore/conf/gunicorn.py similarity index 100% rename from microservice/sysom_vmcore/conf/gunicorn.py rename to sysom_server/sysom_vmcore/conf/gunicorn.py diff --git a/microservice/sysom_vmcore/conf/product.py b/sysom_server/sysom_vmcore/conf/product.py similarity index 100% rename from microservice/sysom_vmcore/conf/product.py rename to sysom_server/sysom_vmcore/conf/product.py diff --git a/microservice/sysom_vmcore/conf/testing.py b/sysom_server/sysom_vmcore/conf/testing.py similarity index 100% rename from microservice/sysom_vmcore/conf/testing.py rename to sysom_server/sysom_vmcore/conf/testing.py diff --git a/microservice/sysom_vmcore/config.yml b/sysom_server/sysom_vmcore/config.yml similarity index 100% rename from microservice/sysom_vmcore/config.yml rename to sysom_server/sysom_vmcore/config.yml diff --git a/microservice/sysom_vmcore/lib/authentications.py b/sysom_server/sysom_vmcore/lib/authentications.py similarity index 100% rename from microservice/sysom_vmcore/lib/authentications.py rename to sysom_server/sysom_vmcore/lib/authentications.py diff --git a/microservice/sysom_vmcore/lib/base_model.py b/sysom_server/sysom_vmcore/lib/base_model.py similarity index 100% rename from microservice/sysom_vmcore/lib/base_model.py rename to sysom_server/sysom_vmcore/lib/base_model.py diff --git a/microservice/sysom_vmcore/lib/decode/sysom_decode.py b/sysom_server/sysom_vmcore/lib/decode/sysom_decode.py similarity index 100% rename from microservice/sysom_vmcore/lib/decode/sysom_decode.py rename to sysom_server/sysom_vmcore/lib/decode/sysom_decode.py diff --git a/microservice/sysom_vmcore/lib/exception.py b/sysom_server/sysom_vmcore/lib/exception.py similarity index 100% rename from microservice/sysom_vmcore/lib/exception.py rename to sysom_server/sysom_vmcore/lib/exception.py diff --git a/microservice/sysom_vmcore/lib/paginations.py b/sysom_server/sysom_vmcore/lib/paginations.py similarity index 100% rename from microservice/sysom_vmcore/lib/paginations.py rename to sysom_server/sysom_vmcore/lib/paginations.py diff --git a/microservice/sysom_vmcore/lib/renderers.py b/sysom_server/sysom_vmcore/lib/renderers.py similarity index 100% rename from microservice/sysom_vmcore/lib/renderers.py rename to sysom_server/sysom_vmcore/lib/renderers.py diff --git a/microservice/sysom_vmcore/lib/response.py b/sysom_server/sysom_vmcore/lib/response.py similarity index 100% rename from microservice/sysom_vmcore/lib/response.py rename to sysom_server/sysom_vmcore/lib/response.py diff --git a/microservice/sysom_vmcore/lib/utils.py b/sysom_server/sysom_vmcore/lib/utils.py similarity index 100% rename from microservice/sysom_vmcore/lib/utils.py rename to sysom_server/sysom_vmcore/lib/utils.py diff --git a/microservice/sysom_vmcore/manage.py b/sysom_server/sysom_vmcore/manage.py similarity index 100% rename from microservice/sysom_vmcore/manage.py rename to sysom_server/sysom_vmcore/manage.py diff --git a/microservice/sysom_vmcore/scripts/node_clear.sh b/sysom_server/sysom_vmcore/scripts/node_clear.sh similarity index 100% rename from microservice/sysom_vmcore/scripts/node_clear.sh rename to sysom_server/sysom_vmcore/scripts/node_clear.sh diff --git a/microservice/sysom_vmcore/scripts/node_init.sh b/sysom_server/sysom_vmcore/scripts/node_init.sh similarity index 100% rename from microservice/sysom_vmcore/scripts/node_init.sh rename to sysom_server/sysom_vmcore/scripts/node_init.sh diff --git a/microservice/sysom_vmcore/scripts/node_update.sh b/sysom_server/sysom_vmcore/scripts/node_update.sh similarity index 100% rename from microservice/sysom_vmcore/scripts/node_update.sh rename to sysom_server/sysom_vmcore/scripts/node_update.sh diff --git a/microservice/sysom_vmcore/scripts/vmcore_collect.py b/sysom_server/sysom_vmcore/scripts/vmcore_collect.py similarity index 100% rename from microservice/sysom_vmcore/scripts/vmcore_collect.py rename to sysom_server/sysom_vmcore/scripts/vmcore_collect.py diff --git a/microservice/sysom_vmcore/sysom_vmcore/__init__.py b/sysom_server/sysom_vmcore/sysom_vmcore/__init__.py similarity index 100% rename from microservice/sysom_vmcore/sysom_vmcore/__init__.py rename to sysom_server/sysom_vmcore/sysom_vmcore/__init__.py diff --git a/microservice/sysom_vmcore/sysom_vmcore/asgi.py b/sysom_server/sysom_vmcore/sysom_vmcore/asgi.py similarity index 100% rename from microservice/sysom_vmcore/sysom_vmcore/asgi.py rename to sysom_server/sysom_vmcore/sysom_vmcore/asgi.py diff --git a/microservice/sysom_vmcore/sysom_vmcore/settings.py b/sysom_server/sysom_vmcore/sysom_vmcore/settings.py similarity index 100% rename from microservice/sysom_vmcore/sysom_vmcore/settings.py rename to sysom_server/sysom_vmcore/sysom_vmcore/settings.py diff --git a/microservice/sysom_vmcore/sysom_vmcore/urls.py b/sysom_server/sysom_vmcore/sysom_vmcore/urls.py similarity index 100% rename from microservice/sysom_vmcore/sysom_vmcore/urls.py rename to sysom_server/sysom_vmcore/sysom_vmcore/urls.py diff --git a/microservice/sysom_vmcore/sysom_vmcore/wsgi.py b/sysom_server/sysom_vmcore/sysom_vmcore/wsgi.py similarity index 100% rename from microservice/sysom_vmcore/sysom_vmcore/wsgi.py rename to sysom_server/sysom_vmcore/sysom_vmcore/wsgi.py diff --git a/microservice/sysom_diagnosis/scripts/server_deploy.sh b/sysom_server/sysom_vul/apps/host/__init__.py similarity index 100% rename from microservice/sysom_diagnosis/scripts/server_deploy.sh rename to sysom_server/sysom_vul/apps/host/__init__.py diff --git a/microservice/sysom_vul/apps/host/app.py b/sysom_server/sysom_vul/apps/host/app.py similarity index 100% rename from microservice/sysom_vul/apps/host/app.py rename to sysom_server/sysom_vul/apps/host/app.py diff --git a/microservice/sysom_vul/apps/host/models.py b/sysom_server/sysom_vul/apps/host/models.py similarity index 100% rename from microservice/sysom_vul/apps/host/models.py rename to sysom_server/sysom_vul/apps/host/models.py diff --git a/microservice/sysom_vul/apps/host/urls.py b/sysom_server/sysom_vul/apps/host/urls.py similarity index 100% rename from microservice/sysom_vul/apps/host/urls.py rename to sysom_server/sysom_vul/apps/host/urls.py diff --git a/microservice/sysom_migration/scripts/server_start.sh b/sysom_server/sysom_vul/apps/vul/__init__.py similarity index 100% rename from microservice/sysom_migration/scripts/server_start.sh rename to sysom_server/sysom_vul/apps/vul/__init__.py diff --git a/microservice/sysom_vul/apps/vul/admin.py b/sysom_server/sysom_vul/apps/vul/admin.py similarity index 100% rename from microservice/sysom_vul/apps/vul/admin.py rename to sysom_server/sysom_vul/apps/vul/admin.py diff --git a/microservice/sysom_vul/apps/vul/apps.py b/sysom_server/sysom_vul/apps/vul/apps.py similarity index 100% rename from microservice/sysom_vul/apps/vul/apps.py rename to sysom_server/sysom_vul/apps/vul/apps.py diff --git a/microservice/sysom_vul/apps/vul/async_fetch.py b/sysom_server/sysom_vul/apps/vul/async_fetch.py similarity index 100% rename from microservice/sysom_vul/apps/vul/async_fetch.py rename to sysom_server/sysom_vul/apps/vul/async_fetch.py diff --git a/microservice/sysom_vul/apps/vul/executor.py b/sysom_server/sysom_vul/apps/vul/executor.py similarity index 100% rename from microservice/sysom_vul/apps/vul/executor.py rename to sysom_server/sysom_vul/apps/vul/executor.py diff --git a/microservice/sysom_vul/apps/vul/migrations/0001_initial.py b/sysom_server/sysom_vul/apps/vul/migrations/0001_initial.py similarity index 100% rename from microservice/sysom_vul/apps/vul/migrations/0001_initial.py rename to sysom_server/sysom_vul/apps/vul/migrations/0001_initial.py diff --git a/microservice/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py b/sysom_server/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py similarity index 100% rename from microservice/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py rename to sysom_server/sysom_vul/apps/vul/migrations/0002_auto_20230324_1840.py diff --git a/microservice/sysom_migration/scripts/server_stop.sh b/sysom_server/sysom_vul/apps/vul/migrations/__init__.py similarity index 100% rename from microservice/sysom_migration/scripts/server_stop.sh rename to sysom_server/sysom_vul/apps/vul/migrations/__init__.py diff --git a/microservice/sysom_vul/apps/vul/models.py b/sysom_server/sysom_vul/apps/vul/models.py similarity index 100% rename from microservice/sysom_vul/apps/vul/models.py rename to sysom_server/sysom_vul/apps/vul/models.py diff --git a/microservice/sysom_vul/apps/vul/serializer.py b/sysom_server/sysom_vul/apps/vul/serializer.py similarity index 100% rename from microservice/sysom_vul/apps/vul/serializer.py rename to sysom_server/sysom_vul/apps/vul/serializer.py diff --git a/microservice/sysom_vul/apps/vul/ssh_pool.py b/sysom_server/sysom_vul/apps/vul/ssh_pool.py similarity index 100% rename from microservice/sysom_vul/apps/vul/ssh_pool.py rename to sysom_server/sysom_vul/apps/vul/ssh_pool.py diff --git a/microservice/sysom_vul/apps/vul/tests.py b/sysom_server/sysom_vul/apps/vul/tests.py similarity index 100% rename from microservice/sysom_vul/apps/vul/tests.py rename to sysom_server/sysom_vul/apps/vul/tests.py diff --git a/microservice/sysom_vul/apps/vul/urls.py b/sysom_server/sysom_vul/apps/vul/urls.py similarity index 100% rename from microservice/sysom_vul/apps/vul/urls.py rename to sysom_server/sysom_vul/apps/vul/urls.py diff --git a/microservice/sysom_vul/apps/vul/views.py b/sysom_server/sysom_vul/apps/vul/views.py similarity index 100% rename from microservice/sysom_vul/apps/vul/views.py rename to sysom_server/sysom_vul/apps/vul/views.py diff --git a/microservice/sysom_vul/apps/vul/vul.py b/sysom_server/sysom_vul/apps/vul/vul.py similarity index 100% rename from microservice/sysom_vul/apps/vul/vul.py rename to sysom_server/sysom_vul/apps/vul/vul.py diff --git a/sysom_server/sysom_vul/conf/__init__.py b/sysom_server/sysom_vul/conf/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/microservice/sysom_vul/conf/common.py b/sysom_server/sysom_vul/conf/common.py similarity index 100% rename from microservice/sysom_vul/conf/common.py rename to sysom_server/sysom_vul/conf/common.py diff --git a/microservice/sysom_vul/conf/develop.py b/sysom_server/sysom_vul/conf/develop.py similarity index 100% rename from microservice/sysom_vul/conf/develop.py rename to sysom_server/sysom_vul/conf/develop.py diff --git a/microservice/sysom_vul/conf/gunicorn.py b/sysom_server/sysom_vul/conf/gunicorn.py similarity index 100% rename from microservice/sysom_vul/conf/gunicorn.py rename to sysom_server/sysom_vul/conf/gunicorn.py diff --git a/microservice/sysom_vul/conf/product.py b/sysom_server/sysom_vul/conf/product.py similarity index 100% rename from microservice/sysom_vul/conf/product.py rename to sysom_server/sysom_vul/conf/product.py diff --git a/microservice/sysom_vul/conf/testing.py b/sysom_server/sysom_vul/conf/testing.py similarity index 100% rename from microservice/sysom_vul/conf/testing.py rename to sysom_server/sysom_vul/conf/testing.py diff --git a/microservice/sysom_vul/config.yml b/sysom_server/sysom_vul/config.yml similarity index 100% rename from microservice/sysom_vul/config.yml rename to sysom_server/sysom_vul/config.yml diff --git a/microservice/sysom_vul/lib/authentications.py b/sysom_server/sysom_vul/lib/authentications.py similarity index 100% rename from microservice/sysom_vul/lib/authentications.py rename to sysom_server/sysom_vul/lib/authentications.py diff --git a/microservice/sysom_vul/lib/base_model.py b/sysom_server/sysom_vul/lib/base_model.py similarity index 100% rename from microservice/sysom_vul/lib/base_model.py rename to sysom_server/sysom_vul/lib/base_model.py diff --git a/microservice/sysom_vul/lib/decode/sysom_decode.py b/sysom_server/sysom_vul/lib/decode/sysom_decode.py similarity index 100% rename from microservice/sysom_vul/lib/decode/sysom_decode.py rename to sysom_server/sysom_vul/lib/decode/sysom_decode.py diff --git a/microservice/sysom_vul/lib/exception.py b/sysom_server/sysom_vul/lib/exception.py similarity index 100% rename from microservice/sysom_vul/lib/exception.py rename to sysom_server/sysom_vul/lib/exception.py diff --git a/microservice/sysom_vul/lib/paginations.py b/sysom_server/sysom_vul/lib/paginations.py similarity index 100% rename from microservice/sysom_vul/lib/paginations.py rename to sysom_server/sysom_vul/lib/paginations.py diff --git a/microservice/sysom_vul/lib/response.py b/sysom_server/sysom_vul/lib/response.py similarity index 100% rename from microservice/sysom_vul/lib/response.py rename to sysom_server/sysom_vul/lib/response.py diff --git a/microservice/sysom_vul/lib/utils.py b/sysom_server/sysom_vul/lib/utils.py similarity index 100% rename from microservice/sysom_vul/lib/utils.py rename to sysom_server/sysom_vul/lib/utils.py diff --git a/microservice/sysom_vul/manage.py b/sysom_server/sysom_vul/manage.py similarity index 100% rename from microservice/sysom_vul/manage.py rename to sysom_server/sysom_vul/manage.py diff --git a/microservice/sysom_vul/sysom_vul/__init__.py b/sysom_server/sysom_vul/sysom_vul/__init__.py similarity index 100% rename from microservice/sysom_vul/sysom_vul/__init__.py rename to sysom_server/sysom_vul/sysom_vul/__init__.py diff --git a/microservice/sysom_vul/sysom_vul/asgi.py b/sysom_server/sysom_vul/sysom_vul/asgi.py similarity index 100% rename from microservice/sysom_vul/sysom_vul/asgi.py rename to sysom_server/sysom_vul/sysom_vul/asgi.py diff --git a/microservice/sysom_vul/sysom_vul/settings.py b/sysom_server/sysom_vul/sysom_vul/settings.py similarity index 100% rename from microservice/sysom_vul/sysom_vul/settings.py rename to sysom_server/sysom_vul/sysom_vul/settings.py diff --git a/microservice/sysom_vul/sysom_vul/urls.py b/sysom_server/sysom_vul/sysom_vul/urls.py similarity index 100% rename from microservice/sysom_vul/sysom_vul/urls.py rename to sysom_server/sysom_vul/sysom_vul/urls.py diff --git a/microservice/sysom_vul/sysom_vul/wsgi.py b/sysom_server/sysom_vul/sysom_vul/wsgi.py similarity index 100% rename from microservice/sysom_vul/sysom_vul/wsgi.py rename to sysom_server/sysom_vul/sysom_vul/wsgi.py -- Gitee From 3892c86525844c5c759d04e1073468cbcbf2fd14 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 11 May 2023 16:16:01 +0800 Subject: [PATCH 043/196] fix(diagnosis): Fix diagnostic microservice health check failure --- .../sysom_diagnosis/apps/task/urls.py | 1 - sysom_web/src/app.jsx | 42 ++++++++++--------- 2 files changed, 22 insertions(+), 21 deletions(-) diff --git a/sysom_server/sysom_diagnosis/apps/task/urls.py b/sysom_server/sysom_diagnosis/apps/task/urls.py index e4c02130..fac45c12 100644 --- a/sysom_server/sysom_diagnosis/apps/task/urls.py +++ b/sysom_server/sysom_diagnosis/apps/task/urls.py @@ -16,7 +16,6 @@ router.register('tasks', views.TaskAPIView) urlpatterns = [ path('api/v1/tasks/offline_import/', views.TaskAPIView.as_view({'post': 'offline_import'})), path('api/v1/tasks/host/', views.TaskAPIView.as_view({'get': 'get_host'})), - path('api/v1/', include(router.urls)), path('api/v2/tasks/', views.TaskAPIView.as_view({'post': 'create_task_v2'})), path('api/v1/tasks/health_check/', views.TaskAPIView.as_view({'get': 'health_check'})), path('api/v1/', include(router.urls)), diff --git a/sysom_web/src/app.jsx b/sysom_web/src/app.jsx index e2e88e5f..a9baac7c 100644 --- a/sysom_web/src/app.jsx +++ b/sysom_web/src/app.jsx @@ -154,28 +154,30 @@ export function patchRoutes({ routes }) { let diagnose = routes.find((item) => item.path == "/") .routes.find(item => item.name == "diagnose") - // Add forder - extraDiagnoseRoute.map(item => { - if (!_.keyBy(diagnose.routes, 'path')[item.path]) { - // Add forder if not exist - diagnose.routes = diagnose.routes.concat({ - ...item, - routes: [] - }) - } - }) + if (!!diagnose) { + // Add forder + extraDiagnoseRoute.map(item => { + if (!_.keyBy(diagnose.routes, 'path')[item.path]) { + // Add forder if not exist + diagnose.routes = diagnose.routes.concat({ + ...item, + routes: [] + }) + } + }) - //Add The extraDiagnoseRoute in it. - diagnose.routes.map(item => { - const new_routes = _.keyBy(extraDiagnoseRoute, 'path')[item.path]?.routes - if (item.routes && new_routes) { - item.routes = item.routes.concat(new_routes); + //Add The extraDiagnoseRoute in it. + diagnose.routes.map(item => { + const new_routes = _.keyBy(extraDiagnoseRoute, 'path')[item.path]?.routes + if (item.routes && new_routes) { + item.routes = item.routes.concat(new_routes); - } - if (!item.routes && new_routes) { - item.routes = new_routes - } - }) + } + if (!item.routes && new_routes) { + item.routes = new_routes + } + }) + } } import grafanaDash from './pages/Monitor/grafana' -- Gitee From 9c0025ef2794ccf82552c70e17410412e4f9c3b9 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 11 May 2023 16:35:09 +0800 Subject: [PATCH 044/196] refactor(deploy): Move local_services and monitor to "deps" dir --- .../0_local_services}/clear.sh | 0 .../0_local_services}/init.sh | 0 .../0_local_services}/nginx.conf | 0 .../0_local_services}/start.sh | 0 .../0_local_services}/stop.sh | 0 .../0_local_services}/sysom-redis.ini | 0 .../0_local_services}/sysom.conf | 0 .../3_monitor => deps/1_monitor}/Readme.txt | 0 .../3_monitor => deps/1_monitor}/clear.sh | 0 .../1_monitor}/grafana_api_set.sh | 0 .../1_monitor}/grafana_recover.sh | 0 .../3_monitor => deps/1_monitor}/init.sh | 0 .../1_monitor}/local_copy_pkg.sh | 0 .../1_monitor}/prometheus_get_node.py | 0 .../3_monitor => deps/1_monitor}/start.sh | 0 .../3_monitor => deps/1_monitor}/stop.sh | 0 .../sysom-cec-status-dashboard.json | 0 .../1_monitor}/sysom-dashboard.json | 0 .../1_monitor}/sysom-migration-dashboard.json | 0 .../1_monitor}/sysom-netinfo-dashboard.json | 0 .../1_monitor}/sysom-prometheus.ini | 0 package.sh | 3 +- script/deploy/clear.sh | 1 + script/deploy/deploy.sh | 1 + script/sysom.sh | 1 + script/sysom_clear.sh | 56 +++++++++++++++++++ script/sysom_deploy.sh | 51 +++++++++++++++++ 27 files changed, 112 insertions(+), 1 deletion(-) rename {environment/2_local_services => deps/0_local_services}/clear.sh (100%) rename {environment/2_local_services => deps/0_local_services}/init.sh (100%) rename {environment/2_local_services => deps/0_local_services}/nginx.conf (100%) rename {environment/2_local_services => deps/0_local_services}/start.sh (100%) rename {environment/2_local_services => deps/0_local_services}/stop.sh (100%) rename {environment/2_local_services => deps/0_local_services}/sysom-redis.ini (100%) rename {environment/2_local_services => deps/0_local_services}/sysom.conf (100%) rename {environment/3_monitor => deps/1_monitor}/Readme.txt (100%) rename {environment/3_monitor => deps/1_monitor}/clear.sh (100%) rename {environment/3_monitor => deps/1_monitor}/grafana_api_set.sh (100%) rename {environment/3_monitor => deps/1_monitor}/grafana_recover.sh (100%) rename {environment/3_monitor => deps/1_monitor}/init.sh (100%) rename {environment/3_monitor => deps/1_monitor}/local_copy_pkg.sh (100%) rename {environment/3_monitor => deps/1_monitor}/prometheus_get_node.py (100%) rename {environment/3_monitor => deps/1_monitor}/start.sh (100%) rename {environment/3_monitor => deps/1_monitor}/stop.sh (100%) rename {environment/3_monitor => deps/1_monitor}/sysom-cec-status-dashboard.json (100%) rename {environment/3_monitor => deps/1_monitor}/sysom-dashboard.json (100%) rename {environment/3_monitor => deps/1_monitor}/sysom-migration-dashboard.json (100%) rename {environment/3_monitor => deps/1_monitor}/sysom-netinfo-dashboard.json (100%) rename {environment/3_monitor => deps/1_monitor}/sysom-prometheus.ini (100%) diff --git a/environment/2_local_services/clear.sh b/deps/0_local_services/clear.sh similarity index 100% rename from environment/2_local_services/clear.sh rename to deps/0_local_services/clear.sh diff --git a/environment/2_local_services/init.sh b/deps/0_local_services/init.sh similarity index 100% rename from environment/2_local_services/init.sh rename to deps/0_local_services/init.sh diff --git a/environment/2_local_services/nginx.conf b/deps/0_local_services/nginx.conf similarity index 100% rename from environment/2_local_services/nginx.conf rename to deps/0_local_services/nginx.conf diff --git a/environment/2_local_services/start.sh b/deps/0_local_services/start.sh similarity index 100% rename from environment/2_local_services/start.sh rename to deps/0_local_services/start.sh diff --git a/environment/2_local_services/stop.sh b/deps/0_local_services/stop.sh similarity index 100% rename from environment/2_local_services/stop.sh rename to deps/0_local_services/stop.sh diff --git a/environment/2_local_services/sysom-redis.ini b/deps/0_local_services/sysom-redis.ini similarity index 100% rename from environment/2_local_services/sysom-redis.ini rename to deps/0_local_services/sysom-redis.ini diff --git a/environment/2_local_services/sysom.conf b/deps/0_local_services/sysom.conf similarity index 100% rename from environment/2_local_services/sysom.conf rename to deps/0_local_services/sysom.conf diff --git a/environment/3_monitor/Readme.txt b/deps/1_monitor/Readme.txt similarity index 100% rename from environment/3_monitor/Readme.txt rename to deps/1_monitor/Readme.txt diff --git a/environment/3_monitor/clear.sh b/deps/1_monitor/clear.sh similarity index 100% rename from environment/3_monitor/clear.sh rename to deps/1_monitor/clear.sh diff --git a/environment/3_monitor/grafana_api_set.sh b/deps/1_monitor/grafana_api_set.sh similarity index 100% rename from environment/3_monitor/grafana_api_set.sh rename to deps/1_monitor/grafana_api_set.sh diff --git a/environment/3_monitor/grafana_recover.sh b/deps/1_monitor/grafana_recover.sh similarity index 100% rename from environment/3_monitor/grafana_recover.sh rename to deps/1_monitor/grafana_recover.sh diff --git a/environment/3_monitor/init.sh b/deps/1_monitor/init.sh similarity index 100% rename from environment/3_monitor/init.sh rename to deps/1_monitor/init.sh diff --git a/environment/3_monitor/local_copy_pkg.sh b/deps/1_monitor/local_copy_pkg.sh similarity index 100% rename from environment/3_monitor/local_copy_pkg.sh rename to deps/1_monitor/local_copy_pkg.sh diff --git a/environment/3_monitor/prometheus_get_node.py b/deps/1_monitor/prometheus_get_node.py similarity index 100% rename from environment/3_monitor/prometheus_get_node.py rename to deps/1_monitor/prometheus_get_node.py diff --git a/environment/3_monitor/start.sh b/deps/1_monitor/start.sh similarity index 100% rename from environment/3_monitor/start.sh rename to deps/1_monitor/start.sh diff --git a/environment/3_monitor/stop.sh b/deps/1_monitor/stop.sh similarity index 100% rename from environment/3_monitor/stop.sh rename to deps/1_monitor/stop.sh diff --git a/environment/3_monitor/sysom-cec-status-dashboard.json b/deps/1_monitor/sysom-cec-status-dashboard.json similarity index 100% rename from environment/3_monitor/sysom-cec-status-dashboard.json rename to deps/1_monitor/sysom-cec-status-dashboard.json diff --git a/environment/3_monitor/sysom-dashboard.json b/deps/1_monitor/sysom-dashboard.json similarity index 100% rename from environment/3_monitor/sysom-dashboard.json rename to deps/1_monitor/sysom-dashboard.json diff --git a/environment/3_monitor/sysom-migration-dashboard.json b/deps/1_monitor/sysom-migration-dashboard.json similarity index 100% rename from environment/3_monitor/sysom-migration-dashboard.json rename to deps/1_monitor/sysom-migration-dashboard.json diff --git a/environment/3_monitor/sysom-netinfo-dashboard.json b/deps/1_monitor/sysom-netinfo-dashboard.json similarity index 100% rename from environment/3_monitor/sysom-netinfo-dashboard.json rename to deps/1_monitor/sysom-netinfo-dashboard.json diff --git a/environment/3_monitor/sysom-prometheus.ini b/deps/1_monitor/sysom-prometheus.ini similarity index 100% rename from environment/3_monitor/sysom-prometheus.ini rename to deps/1_monitor/sysom-prometheus.ini diff --git a/package.sh b/package.sh index a50bbd96..21d6549a 100755 --- a/package.sh +++ b/package.sh @@ -18,6 +18,7 @@ check_cmd tar RELEASE=sysomRelease-$(date +"%Y%m%d%H%M%S") MICROSERVICE_DIR=sysom_server ENVIRONMENT_DIR=environment +DEPS_DIR=deps WEBDIR=sysom_web CONFDIR=conf SCRIPTDIR=script @@ -29,7 +30,7 @@ yarn build popd || exit mkdir -p "${RELEASE}" -cp -r ${MICROSERVICE_DIR}/ ${ENVIRONMENT_DIR}/ ${TOOLSDIR}/ ${CONFDIR}/ "${RELEASE}"/ +cp -r ${MICROSERVICE_DIR}/ ${ENVIRONMENT_DIR}/ ${DEPS_DIR} ${TOOLSDIR}/ ${CONFDIR}/ "${RELEASE}"/ cp -r ${WEBDIR}/dist/ "${RELEASE}"/${WEBDIR}/ mkdir -p "${RELEASE}"/${WEBDIR}/download/ cp ${SCRIPTDIR}/deploy/deploy.sh "${RELEASE}"/ diff --git a/script/deploy/clear.sh b/script/deploy/clear.sh index 6ccd78b9..1371bc66 100755 --- a/script/deploy/clear.sh +++ b/script/deploy/clear.sh @@ -51,6 +51,7 @@ clear() { bash +x sysom.sh clear web bash +x sysom.sh clear ms ALL bash +x sysom.sh clear env ALL + bash +x sysom.sh clear deps ALL popd del_conf del_app_home diff --git a/script/deploy/deploy.sh b/script/deploy/deploy.sh index 0bd632dd..7f37d96b 100755 --- a/script/deploy/deploy.sh +++ b/script/deploy/deploy.sh @@ -50,6 +50,7 @@ deploy() { fi bash +x sysom.sh deploy web bash +x sysom.sh deploy env ALL + bash +x sysom.sh deploy deps ALL bash +x sysom.sh deploy ms ALL popd } diff --git a/script/sysom.sh b/script/sysom.sh index 5cd8d79c..ae2ae147 100755 --- a/script/sysom.sh +++ b/script/sysom.sh @@ -16,6 +16,7 @@ export SERVER_HOME=${APP_HOME}/server export NODE_HOME=${APP_HOME}/node export SCRIPT_HOME=${APP_HOME}/script export WEB_HOME=${APP_HOME}/web +export DEPS_HOME=${APP_HOME}/deps export MICROSERVICE_HOME=${SERVER_HOME} export MICROSERVICE_SCRIPT_HOME=${SCRIPT_HOME}/server export ENVIRONMENT_HOME=${APP_HOME}/environment diff --git a/script/sysom_clear.sh b/script/sysom_clear.sh index c0282333..91cda83c 100755 --- a/script/sysom_clear.sh +++ b/script/sysom_clear.sh @@ -101,6 +101,47 @@ do_clear_microservice() { fi } + +do_clear_deps() { + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]} + do + for dep in $(ls ${DEPS_HOME} | grep ${target}) + do + target_dir=${DEPS_HOME}/${dep} + if [ ! -d "${target_dir}" ]; then + continue + fi + + pushd ${target_dir} + + if [ ! -f "${target_dir}/clear.sh" ]; then + popd + continue + fi + + green "$target_dir clear..................................." + bash -x ${target_dir}/clear.sh + if [ $? -ne 0 ];then + red "$target_dir clear fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + popd + + # rm dir + rm -rf ${target_dir} + done + done + + # rm infrastructure if the folder is empty + if [ "`ls ${DEPS_HOME} | wc -l`" == "0" ] + then + rm -rf ${DEPS_HOME} + fi +} + #################################################################################################################### # Subcommands #################################################################################################################### @@ -181,6 +222,21 @@ sub_microservice() { fi } +sub_deps() { + target=$1 + if [ "$target" == "ALL" ] + then + # Deploy all deps + for dep in $(ls -r $DEPS_HOME) + do + do_clear_deps ${dep} + done + else + # Deploy specific dep + do_clear_deps $target + fi +} + # Clear web sub_web() { rm -rf ${WEB_HOME} diff --git a/script/sysom_deploy.sh b/script/sysom_deploy.sh index 71378bb2..5babd0d3 100755 --- a/script/sysom_deploy.sh +++ b/script/sysom_deploy.sh @@ -159,6 +159,36 @@ do_deploy_environment() { done } +do_deploy_deps() { + mkdir -p ${DEPS_HOME} + local_deps_dir=`dirname $BaseDir`/deps + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]} + do + for dep in $(ls ${local_deps_dir} | grep ${target}) + do + if [ "$local_app_home" != "$APP_HOME" ] + then + # Copy dep dir to deploy path if need + cp -r $local_deps_dir/${dep} $DEPS_HOME/${dep} + fi + target_dir=${DEPS_HOME}/${dep} + pushd ${target_dir} + + green "$target_dir initialing..................................." + bash -x ${target_dir}/init.sh + if [ $? -ne 0 ];then + red "$target_dir initial fail, please check..................................." + red "sysom init failed, exit 1" + exit 1 + fi + popd + done + + done +} + #################################################################################################################### # Subcommands @@ -218,6 +248,10 @@ sub_ms() { sub_microservice $@ } +sub_server() { + sub_microservice $@ +} + # All microservices sub_microservice() { target=$1 @@ -247,6 +281,23 @@ sub_microservice() { fi } +sub_deps() { + target=$1 + if [ "$target" == "ALL" ] + then + local_deps_deploy_dir=${local_app_home}/deps + + # Deploy all microservices + for dep in $(ls $local_deps_deploy_dir) + do + do_deploy_deps ${dep} + done + else + # Deploy specific microservices + do_deploy_deps $target + fi +} + # Deploy web sub_web() { local_web_home=${local_app_home}/sysom_web -- Gitee From f3cb4ebd053173c2ee3cfc63341296db65e52fc3 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 11 May 2023 17:49:36 +0800 Subject: [PATCH 045/196] fix(deploy): Add missing deps --- deps/0_local_services/init.sh | 5 +++++ environment/0_env/init.sh | 3 +++ script/server/sysom_hotfix_builder/init.sh | 4 ++++ 3 files changed, 12 insertions(+) diff --git a/deps/0_local_services/init.sh b/deps/0_local_services/init.sh index a06901ca..341006fe 100755 --- a/deps/0_local_services/init.sh +++ b/deps/0_local_services/init.sh @@ -11,6 +11,7 @@ REDIS_PKG=redis-5.0.14.tar.gz usr_local_redis=0 setup_database() { + rpm -q --quiet mariadb-server || yum install -y mariadb-server echo "INFO: begin create db..." systemctl restart mariadb.service @@ -24,6 +25,7 @@ setup_database() { mysql -uroot -e "flush privileges;" } setup_nginx() { + rpm -q --quiet nginx || yum install -y nginx mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak cp nginx.conf /etc/nginx/ cp sysom.conf /etc/nginx/conf.d/ @@ -39,11 +41,14 @@ init_conf() { } setup_redis() { + rpm -q --quiet redis || yum install -y redis ###we need redis version >= 5.0.0, check redis version### redis_version=`yum list all | grep "^redis.x86_64" | awk '{print $2}' | awk -F"." '{print $1}'` echo ${redis_version} if [ $redis_version -lt 5 ] then + rpm -q --quiet gcc || yum install -y gcc + rpm -q --quiet make || yum install -y make echo "redis version in yum repo is less than 5.0.0, we will compile redis(5.0.14) and install it." if [ ! -e ${REDIS_PKG} ] then diff --git a/environment/0_env/init.sh b/environment/0_env/init.sh index 946cd247..4111531d 100755 --- a/environment/0_env/init.sh +++ b/environment/0_env/init.sh @@ -40,6 +40,9 @@ touch_env_rpms() { sed -i '/epel\/8\/Everything/{n;s/enabled=0/enabled=1/;}' /etc/yum.repos.d/AnolisOS-DDE.repo fi rpm -q --quiet python3 || yum install -y python3 + rpm -q --quiet supervisor || yum install -y supervisor + rpm -q --quiet wget || yum install -y wget + rpm -q --quiet cronie || yum install -y cronie } check_requirements() { diff --git a/script/server/sysom_hotfix_builder/init.sh b/script/server/sysom_hotfix_builder/init.sh index 3a080d20..8ee738b6 100644 --- a/script/server/sysom_hotfix_builder/init.sh +++ b/script/server/sysom_hotfix_builder/init.sh @@ -52,6 +52,10 @@ start_app() { } deploy() { + rpm -q --quiet gcc || yum install -y gcc + rpm -q --quiet make || yum install -y make + rpm -q --quiet nfs-utils || yum install -y nfs-utils + rpm -q --quiet rpcbind || yum install -y rpcbind source_virtualenv install_requirement init_conf -- Gitee From cdc72ca6e5f9e914e39982ea17fff0bc4b38f25f Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 11 May 2023 18:40:40 +0800 Subject: [PATCH 046/196] fix(monitor_server): Add missing NodeDispatcherPlugin config --- sysom_server/sysom_monitor_server/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sysom_server/sysom_monitor_server/main.py b/sysom_server/sysom_monitor_server/main.py index 4618f7cf..e6ed12c2 100644 --- a/sysom_server/sysom_monitor_server/main.py +++ b/sysom_server/sysom_monitor_server/main.py @@ -1,7 +1,7 @@ from fastapi import FastAPI from conf.settings import * from app.routeres import home, services, health, grafana -from sysom_utils import CmgPlugin, SysomFramework +from sysom_utils import CmgPlugin, SysomFramework, NodeDispatcherPlugin app = FastAPI() @@ -13,6 +13,7 @@ app.include_router(health.router, prefix="/api/v1/monitor/health") @app.on_event("startup") async def on_start(): SysomFramework.load_plugin_cls(CmgPlugin) \ + .load_plugin_cls(NodeDispatcherPlugin) \ .enable_channel_job() \ .start() -- Gitee From 858e5d449a036259cc2a303ace453886253b6221 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 11 May 2023 19:41:54 +0800 Subject: [PATCH 047/196] refactor(deps): Move the files generated during deps deployment to the deps directory --- deps/0_local_services/clear.sh | 1 + deps/0_local_services/init.sh | 6 +++--- deps/0_local_services/sysom-redis.ini | 4 ++-- deps/1_monitor/clear.sh | 2 +- deps/1_monitor/init.sh | 2 +- deps/1_monitor/sysom-prometheus.ini | 4 ++-- 6 files changed, 10 insertions(+), 9 deletions(-) diff --git a/deps/0_local_services/clear.sh b/deps/0_local_services/clear.sh index 8e8b2f74..c885feb8 100755 --- a/deps/0_local_services/clear.sh +++ b/deps/0_local_services/clear.sh @@ -2,6 +2,7 @@ SERVICE_NAME=sysom-redis clear_local_redis() { rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + rm -rf ${DEPS_HOME}/redis ###use supervisorctl update to stop and clear services### supervisorctl update } diff --git a/deps/0_local_services/init.sh b/deps/0_local_services/init.sh index 341006fe..98c4e49f 100755 --- a/deps/0_local_services/init.sh +++ b/deps/0_local_services/init.sh @@ -63,9 +63,9 @@ setup_redis() { tar -zxvf ${REDIS_PKG} pushd ${REDIS_DIR} make - mkdir -p ${ENVIRONMENT_HOME}/redis - cp redis.conf ${ENVIRONMENT_HOME}/redis/ - cp src/redis-server ${ENVIRONMENT_HOME}/redis/ + mkdir -p ${DEPS_HOME}/redis + cp redis.conf ${DEPS_HOME}/redis/ + cp src/redis-server ${DEPS_HOME}/redis/ if [ $? -ne 0 ] then echo "redis compile or install error, exit 1" diff --git a/deps/0_local_services/sysom-redis.ini b/deps/0_local_services/sysom-redis.ini index b0f35b7e..8cebfcd8 100644 --- a/deps/0_local_services/sysom-redis.ini +++ b/deps/0_local_services/sysom-redis.ini @@ -1,6 +1,6 @@ [program:sysom-redis] -directory = /usr/local/sysom/server/redis -command=/usr/local/sysom/server/redis/redis-server /usr/local/sysom/server/redis/redis.conf +directory = /usr/local/sysom/deps/redis +command=/usr/local/sysom/deps/redis/redis-server /usr/local/sysom/deps/redis/redis.conf startsecs=3 autostart=true autorestart=true diff --git a/deps/1_monitor/clear.sh b/deps/1_monitor/clear.sh index e4eb4d8a..0dcf411a 100755 --- a/deps/1_monitor/clear.sh +++ b/deps/1_monitor/clear.sh @@ -1,6 +1,6 @@ #!/bin/bash -x SERVICE_NAME=sysom-prometheus -RESOURCE_DIR=${ENVIRONMENT_HOME}/monitor +RESOURCE_DIR=${DEPS_HOME}/monitor del_cron() { sed -i '/prometheus/d' /var/spool/cron/root diff --git a/deps/1_monitor/init.sh b/deps/1_monitor/init.sh index 888ef7f8..c9320e3f 100755 --- a/deps/1_monitor/init.sh +++ b/deps/1_monitor/init.sh @@ -1,5 +1,5 @@ #!/bin/bash -x -RESOURCE_DIR=${ENVIRONMENT_HOME}/monitor +RESOURCE_DIR=${DEPS_HOME}/monitor ARCH=`uname -m` GRAFANA_PKG=grafana-9.2.2-1.${ARCH}.rpm PROMETHEUS_VER=2.29.1 diff --git a/deps/1_monitor/sysom-prometheus.ini b/deps/1_monitor/sysom-prometheus.ini index f298737a..8cd518c6 100644 --- a/deps/1_monitor/sysom-prometheus.ini +++ b/deps/1_monitor/sysom-prometheus.ini @@ -1,6 +1,6 @@ [program:sysom-prometheus] -directory = /usr/local/sysom/environment/monitor/prometheus -command=/usr/local/sysom/environment/monitor/prometheus/prometheus +directory = /usr/local/sysom/deps/monitor/prometheus +command=/usr/local/sysom/deps/monitor/prometheus/prometheus startsecs=3 autostart=true autorestart=true -- Gitee From 15141b46093b734d113fa08f40ae8aff5d63b302 Mon Sep 17 00:00:00 2001 From: ydzhang Date: Thu, 11 May 2023 22:09:39 +0800 Subject: [PATCH 048/196] Realize function of using Redis cache of deleting hotfix job. This commit contains the following changes: 1. Added Cache Object for redis operation 2. Added Redis Global settings in common.py 3. Realized the function that we use redis to sync the deletion of hotfix job, if one hotfix job is waiting and deleted. In case of running in builder as a ghost, we use redis to transform the deleted key of hotfix. When build find the deletion of this key, this hotfix job will be ignored. change the coding of RedisCache and remove building status change when building specific kernel hotfix This commit contains the following changes: 1. Move Cache object into lib.function 2. fix the bug when building one hotfix will invoke building state one more time --- .../sysom_hotfix/apps/hotfix/function.py | 89 +++++++++++++++ .../sysom_hotfix/apps/hotfix/views.py | 13 ++- sysom_server/sysom_hotfix/conf/common.py | 4 + sysom_server/sysom_hotfix_builder/builder.ini | 8 +- sysom_server/sysom_hotfix_builder/builder.py | 101 ++++++++++++------ 5 files changed, 180 insertions(+), 35 deletions(-) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/function.py b/sysom_server/sysom_hotfix/apps/hotfix/function.py index c93e9fe1..bbda5d75 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/function.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/function.py @@ -2,6 +2,8 @@ import os import re from clogger import logger from django.conf import settings +import threading +import redis from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel from cec_base.producer import dispatch_producer, Producer @@ -244,5 +246,92 @@ class FunctionClass(): logger.error(str(e)) return None +class CECListener(): + + def __init__(self, cec_url, listen_topic) -> None: + try: + logger.info("Server CECListener init ...") + print(">>>>>>>>>>>>>>>>init CECListener....") + self.cec_url = cec_url + self.listen_topic = listen_topic + self.sync_key = "sync" # this key is to tag this message for sync job + self.rpm_key = "rpm_name" # this key is to tag this message for sync rpm name + self.thread_runner = threading.Thread(target=self.listener, name="hotfix_server_listener") + except Exception as e: + return None + + def run(self): + logger.info("Server CEC Listener start...") + self.thread_runner.start() + + def listener(self): + with dispatch_admin(self.cec_url) as admin: + if not admin.is_topic_exist(self.listen_topic): + admin.create_topic(self.listen_topic) + + consumer_id = Consumer.generate_consumer_id() + # one server just need one group id and one consumer is enough + self.consumer = dispatch_consumer(self.cec_url, self.listen_topic, + consumer_id=consumer_id, + group_id="server_listener") + logger.info("Server CECListener init finished...") + for event in self.consumer: + try: + parameters = event.value + hotfix_id = parameters["hotfix_id"] + if self.sync_key in parameters: + if self.rpm_key in parameters: + + self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) + else: + self.update_hotfix_job_status(hotfix_id, parameters["status"]) + except Exception as e: + logger.error(str(e)) + finally: + logger.info("ack one msg from builder...") + self.consumer.ack(event=event) + + def update_hotfix_job_status(self, hotfix_id, status): + hotfix = HotfixModel.objects.filter(id=hotfix_id).first() + logger.info("ack one job status of : %s from builder of hotfix id : %s " % (status, str(hotfix_id))) + if hotfix is None: + logger.error("%s : hotfix job is not exist filtered by hotfix id : %s" % (sys._getframe().f_code.co_name, str(hotfix_id))) + return None + if status == "waiting": + hotfix.building_status=0 + elif status == "building": + hotfix.building_status=1 + elif status == "failed": + hotfix.building_status=2 + elif status == "success": + hotfix.building_status=3 + else: + logger.error("%s : hotfix job status update failed. status : %s is not supported" % (sys._getframe().f_code.co_name, status)) + hotfix.save() + return hotfix_id + + def sync_hotfix_job_rpm_name(self, hotfix_id, rpm_name): + try: + logger.info("get rpm_name of %s from builfer..." % rpm_name) + hotfix = HotfixModel.objects.filter(id=hotfix_id).first() + rpm_package = rpm_name + hotfix.rpm_package = rpm_package + hotfix.save() + except Exception as e: + logger.error("%s : Exception raised..." % sys._getframe().f_code.co_name) + return None + + +class RedisCache(): + def __init__(self): + host = settings.REDIS_HOST + port = settings.REDIS_PORT + pool = redis.ConnectionPool(host=host, port=port) + self.conn = redis.Redis(connection_pool=pool) + + def get(self,key): + return self.conn.get(key) + def set(self,key,value): + self.conn.set(key,value) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 1e8c6302..2f2afa00 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -32,8 +32,7 @@ from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer from cec_base.event import Event from django.http import HttpResponse, FileResponse -from apps.hotfix.function import FunctionClass - +from lib.function import FunctionClass, RedisCache class SaveUploadFile(APIView): authentication_classes = [] @@ -248,8 +247,14 @@ class HotfixAPIView(GenericViewSet, if hotfix is None: return other_response(message="can not delete this hotfix", result={"msg":"Hotfix not found"}, code=400) else: - if hotfix.building_status == 0 or hotfix.building_status == 1: - return other_response(message="can not delete this hotfix:this hotfix status is waiting or running..", code=400) + if hotfix.building_status == 1: + return other_response(message="can not delete this hotfix:this hotfix status is running..", code=400) + try: + cache = RedisCache() + cache.set(hotfix.id,hotfix.id) + except Exception as e: + logger.error(str(e)) + return other_response(message="communication to redis of delete pool failed...", code=400) hotfix.deleted_at=human_datetime() hotfix.delete() return success(result={}, message="invoke delete_hotfix") diff --git a/sysom_server/sysom_hotfix/conf/common.py b/sysom_server/sysom_hotfix/conf/common.py index 7b9539c4..d1e79d9c 100644 --- a/sysom_server/sysom_hotfix/conf/common.py +++ b/sysom_server/sysom_hotfix/conf/common.py @@ -63,6 +63,10 @@ DATABASES = { } } + +REDIS_HOST = "127.0.0.1" +REDIS_PORT = "6379" + ROOT_URLCONF = 'sysom_hotfix.urls' WSGI_APPLICATION = 'sysom_hotfix.wsgi.application' diff --git a/sysom_server/sysom_hotfix_builder/builder.ini b/sysom_server/sysom_hotfix_builder/builder.ini index 09f762fe..02aeff21 100644 --- a/sysom_server/sysom_hotfix_builder/builder.ini +++ b/sysom_server/sysom_hotfix_builder/builder.ini @@ -10,4 +10,10 @@ job_status_topic = hotfix_building_status [builder] hotfix_base = /hotfix_build/hotfix nfs_dir_home = /usr/local/sysom/server/builder/hotfix -package_repo = /hotfix/packages \ No newline at end of file +package_repo = /hotfix/packages +tmpdir = /hotfix_tmpdir +src_rpm_tmpdir = /tmp/hotfix_src + +[redis] +host = localhost +port = 6379 diff --git a/sysom_server/sysom_hotfix_builder/builder.py b/sysom_server/sysom_hotfix_builder/builder.py index c0e9507e..1083e8c0 100644 --- a/sysom_server/sysom_hotfix_builder/builder.py +++ b/sysom_server/sysom_hotfix_builder/builder.py @@ -18,7 +18,7 @@ import re import sys import subprocess import configparser -import shutil +import redis from cec_base.consumer import Consumer, dispatch_consumer from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer, Producer @@ -106,10 +106,47 @@ class ServerConnector(): "hotfix_id" : id, "status" : status }) + + + def sync_rpm_name(self, hotfix_id, rpm_name): + logger.info("produce rpm_name ") + self.cec.produce(self.cec_produce_topic, { + "hotfix_id" : hotfix_id, + "sync" : True, + "rpm_name" : rpm_name + }) + +""" +Object : Cache +Function : initalize the Redis Cache Object +""" +class Cache(): + def __init__(self, con : configparser.ConfigParser): + redis_conn = dict(con.items('redis')) + host = redis_conn['host'] + port = redis_conn['port'] + pool = redis.ConnectionPool(host=host, port=port) + self.conn = redis.Redis(connection_pool=pool) + + def get(self,key): + return self.conn.get(key) + + +class HotfixBuilder(): + + def __init__(self, con : configparser.ConfigParser): + # read init parameters from builder.ini + # cec + cec = dict(con.items('cec')) + cec_url = cec['cec_url'] + cec_hotfix_topic = cec['cec_hotfix_topic'] + hotfix_building_status_topic = cec['job_status_topic'] class HotfixBuilder(): def __init__(self, nfs_dir_home, hotfix_base, cec_url, cec_topic, server_ip, username, password, packages_repo): + cache = Cache(con) + self.cache = cache self.nfs_dir_home = nfs_dir_home self.hotfix_base = hotfix_base self.cec_url = cec_url @@ -381,8 +418,6 @@ class HotfixBuilder(): self.fd.write("Hotfix name : %s\n" % hotfix_name) self.fd.write("=========================================================\n") - self.connector.change_building_status(hotfix_id, "building") - self.check_kernel_source(git_repo, source_code_repo, kernel_version, is_src_package) image = self.get_building_image(kernel_version) @@ -460,8 +495,6 @@ class HotfixBuilder(): log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) self.fd = open(log_file_path, "a") - self.connector.change_building_status(hotfix_id, "building") - self.fd.write("=========================================================\n") self.fd.write("Created Hotfix Building Task ... \n") self.fd.write("Kernel Version: %s\n" % kernel_version) @@ -530,13 +563,16 @@ class HotfixBuilder(): description = self.extract_description_from_patch(local_patch) - # run the build hotfix script + # run the build hotfix script, if is_src_package: + # use src.rpm cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -c {} -v {} -r {} 2>&1 1 >> {} ".format( self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image, self.hotfix_base, local_patch, kernel_version, description, self.hotfix_base, hotfix_name, log_file_path, kernel_config, vmlinux, source_code_path, log_file_path ) else: + # use git branch for source management + git_branch = src_origin cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -c {} -v {} -r {} -t {} 2>&1 1 >> {}".format( self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image, self.hotfix_base, local_patch, kernel_version, description, self.hotfix_base, hotfix_name, log_file_path, kernel_config, vmlinux, source_code_repo, git_branch, log_file_path @@ -597,30 +633,34 @@ class HotfixBuilder(): log_file = parameters['log_file'] self.hotfix_id = parameters['hotfix_id'] log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) - - if parameters['arch'] == self.local_arch: - # this operation aims to clear the previous log if rebuild - with open(log_file_path, "w") as f: - f.write("=========================================================\n") - f.write("==========*******Sysom Hotfix Building System*******==============\n") - f.write("=========================================================\n") - - self.connector.change_building_status(self.hotfix_id, "building") - - # for each run, update the repo - cmd = "chmod +x check_env.sh && ./check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) - with os.popen(cmd) as process: - output = process.read() - - customize = parameters['customize'] - # before build one job, clear the tmpdir - self.clear_tmpdir() - if not customize: - self.build_supported_kernel(parameters) - else: - self.build_customize_kernel(parameters) - logger.info(log_file) - self.fd.close() + is_deleted = self.cache.get(str(self.hotfix_id)) + # if this hotfix_id is not exist in the deleted pool + if not is_deleted: + if parameters['arch'] == self.local_arch: + # this operation aims to clear the previous log if rebuild + with open(log_file_path, "w") as f: + f.write("=========================================================\n") + f.write("==========*******Sysom Hotfix Building System*******==============\n") + f.write("=========================================================\n") + + self.change_building_status("building") + + # for each run, update the repo + cmd = "chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) + with os.popen(cmd) as process: + output = process.read() + + customize = parameters['customize'] + # before build one job, clear the tmpdir + self.clear_tmpdir() + if not customize: + self.build_supported_kernel(parameters) + else: + self.build_customize_kernel(parameters) + logger.info(log_file) + self.fd.close() + else: + logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id) except Exception as e: logger.error(str(e)) self.connector.change_building_status(self.hotfix_id, "failed") @@ -629,6 +669,7 @@ class HotfixBuilder(): self.fd.close() consumer.ack(event) + if __name__ == "__main__": config_file = "builder.ini" -- Gitee From d4af711b563f9cc0fcd1da7b9753f97cce6f6392 Mon Sep 17 00:00:00 2001 From: huangtuq Date: Fri, 12 May 2023 14:34:52 +0800 Subject: [PATCH 049/196] set dashboard readonly Signed-off-by: huangtuq --- deps/1_monitor/sysom-dashboard.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deps/1_monitor/sysom-dashboard.json b/deps/1_monitor/sysom-dashboard.json index 892aaf50..e2f20ed8 100644 --- a/deps/1_monitor/sysom-dashboard.json +++ b/deps/1_monitor/sysom-dashboard.json @@ -47,7 +47,7 @@ } ] }, - "editable": true, + "editable": false, "gnetId": 1860, "graphTooltip": 0, "id": null, -- Gitee From 011f579120fe31bbefa1bd27f079e48bad0c86ca Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 12 May 2023 14:45:26 +0800 Subject: [PATCH 050/196] feat(monitor): Monitor service add api for prometheus to get targets --- deps/1_monitor/init.sh | 17 +++++------------ .../app/routeres/prometheus.py | 19 +++++++++++++++++++ sysom_server/sysom_monitor_server/main.py | 3 ++- 3 files changed, 26 insertions(+), 13 deletions(-) create mode 100644 sysom_server/sysom_monitor_server/app/routeres/prometheus.py diff --git a/deps/1_monitor/init.sh b/deps/1_monitor/init.sh index c9320e3f..ac381439 100755 --- a/deps/1_monitor/init.sh +++ b/deps/1_monitor/init.sh @@ -75,10 +75,11 @@ add_auto_discovery() cat << EOF >> prometheus.yml - job_name: 'auto_discovery' - file_sd_configs: - - files: - - "${RESOURCE_DIR}/prometheus/node/node.json" - refresh_interval: 10s + metrics_path: '/metrics' + scheme: 'http' + http_sd_configs: + - url: 'http://localhost/api/v1/monitor/prometheus/sd_config' + refresh_interval: 10s - job_name: "cec_monitor" metrics_path: "/api/v1/channel/cec_status/metrics" static_configs: @@ -86,7 +87,6 @@ add_auto_discovery() EOF popd - cp prometheus_get_node.py ${RESOURCE_DIR}/prometheus/ } init_conf() { @@ -164,12 +164,6 @@ configure_grafana() fi } -configure_cron() -{ - echo "* * * * * ${VIRTUALENV_PYTHON3} ${RESOURCE_DIR}/prometheus/prometheus_get_node.py ${SERVER_PORT}" >> /var/spool/cron/root - echo "* * * * * sleep 30;${VIRTUALENV_PYTHON3} ${RESOURCE_DIR}/prometheus/prometheus_get_node.py ${SERVER_PORT}" >> /var/spool/cron/root -} - main() { echo "perpare download resource packages: grafana, prometheus, node_exporter" @@ -182,7 +176,6 @@ main() start_prometheus_service configure_grafana - configure_cron # install_and_config_influxdb } diff --git a/sysom_server/sysom_monitor_server/app/routeres/prometheus.py b/sysom_server/sysom_monitor_server/app/routeres/prometheus.py new file mode 100644 index 00000000..e178c794 --- /dev/null +++ b/sysom_server/sysom_monitor_server/app/routeres/prometheus.py @@ -0,0 +1,19 @@ +import aiohttp +from fastapi import APIRouter, Response +from sysom_utils import SysomFramework + +router = APIRouter() +g_client = SysomFramework.gclient("sysom_api") + + +@router.get("/sd_config") +async def search_grafana(response: Response): + res = g_client.get("/api/v1/host/").json() + return [ + { + "targets": [f"{item['ip']}:9100" for item in res["data"]], + "labels": { + "source": "sysom_monitor_server" + } + } + ] diff --git a/sysom_server/sysom_monitor_server/main.py b/sysom_server/sysom_monitor_server/main.py index e6ed12c2..fb290cef 100644 --- a/sysom_server/sysom_monitor_server/main.py +++ b/sysom_server/sysom_monitor_server/main.py @@ -1,12 +1,13 @@ from fastapi import FastAPI from conf.settings import * -from app.routeres import home, services, health, grafana +from app.routeres import home, services, health, grafana, prometheus from sysom_utils import CmgPlugin, SysomFramework, NodeDispatcherPlugin app = FastAPI() app.include_router(home.router, prefix="/api/v1/monitor/home") app.include_router(grafana.router, prefix="/api/v1/monitor/grafana") +app.include_router(prometheus.router, prefix="/api/v1/monitor/prometheus") app.include_router(services.router, prefix="/api/v1/monitor/services") app.include_router(health.router, prefix="/api/v1/monitor/health") -- Gitee From 154d5d43dd046323d1bdd40a90efeb26adead23f Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 12 May 2023 16:41:20 +0800 Subject: [PATCH 051/196] fix(redis_lua): lua script lost after redis restart --- environment/1_sdk/redis_lua/xreadis_hash_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/environment/1_sdk/redis_lua/xreadis_hash_table.py b/environment/1_sdk/redis_lua/xreadis_hash_table.py index 8eeb7adf..92092368 100644 --- a/environment/1_sdk/redis_lua/xreadis_hash_table.py +++ b/environment/1_sdk/redis_lua/xreadis_hash_table.py @@ -37,7 +37,7 @@ class XRedisHashTable: def _get_or_cache_script_sha(self, name: str): target_hash = self._script_hashes.get(name, "") - if not target_hash or self._redis.script_exists(target_hash)[0]: + if not target_hash or not self._redis.script_exists(target_hash)[0]: # Need load script target_hash = self._load_lua_scripts_from_file(name) self._script_hashes[name] = target_hash -- Gitee From 793b189b1e179f4feee3ecbb98f18b59752709d5 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 12 May 2023 17:13:32 +0800 Subject: [PATCH 052/196] fix(sysom_framwork): default_channel_job_executor is initialized without a start --- environment/1_sdk/sysom_utils/framework.py | 1 + 1 file changed, 1 insertion(+) diff --git a/environment/1_sdk/sysom_utils/framework.py b/environment/1_sdk/sysom_utils/framework.py index b4ea13e8..1cef8713 100644 --- a/environment/1_sdk/sysom_utils/framework.py +++ b/environment/1_sdk/sysom_utils/framework.py @@ -70,6 +70,7 @@ class SysomFramework: cls._config.get_local_channel_job_url(), cls.gclient("sysom_channel") ) + default_channel_job_executor.start() return cls @classmethod -- Gitee From 39e9ba6992262d75db9229d2b719651a36b5cea7 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 12 May 2023 17:55:03 +0800 Subject: [PATCH 053/196] fix(deploy): Clear failed in release mode --- script/deploy/clear.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/deploy/clear.sh b/script/deploy/clear.sh index 1371bc66..c36671a2 100755 --- a/script/deploy/clear.sh +++ b/script/deploy/clear.sh @@ -41,7 +41,7 @@ del_conf() { BaseDir=$(dirname $(readlink -f "$0")) clear() { if [ -d "${APP_HOME}" ];then - if [ -f "script" ]; then + if [ -d "script" ]; then # release dir pushd "script" else -- Gitee From 138d9aea3cc9a5cdf1d321dd2b87f36e9a06a8ee Mon Sep 17 00:00:00 2001 From: wb-672209 Date: Thu, 27 Apr 2023 11:13:58 +0800 Subject: [PATCH 054/196] =?UTF-8?q?=E8=87=AA=E5=AE=9A=E4=B9=89=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E7=89=88=E6=9C=AC=E8=AE=BE=E8=AE=A1=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=E9=83=A8=E5=88=86=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 去掉代码中的无用注释 修改KernelVerison中的请求体中的数据来源 新增git_rule\source_devel\source_debuginfo三个请求体内字段 --- sysom_web/config/routes.js | 21 ++++- sysom_web/src/locales/zh-CN/menu.js | 4 +- sysom_web/src/locales/zh-CN/pages.js | 5 + .../VersionConfig}/KernelVersion.jsx | 92 +++++++++++++++---- .../hotfix/Version/VersionConfig/index.jsx | 19 ++++ .../VersionCustomize}/KernelConfig.jsx | 8 +- .../VersionCustomize}/index.jsx | 16 +--- .../hotfix/components/KernelConfigForm.jsx | 49 ++++++++-- .../hotfix/components/VersionConfigForm.jsx | 4 +- sysom_web/src/pages/hotfix/service.js | 27 ++++++ 10 files changed, 199 insertions(+), 46 deletions(-) rename sysom_web/src/pages/hotfix/{VersionConf => Version/VersionConfig}/KernelVersion.jsx (63%) create mode 100644 sysom_web/src/pages/hotfix/Version/VersionConfig/index.jsx rename sysom_web/src/pages/hotfix/{VersionConf => Version/VersionCustomize}/KernelConfig.jsx (95%) rename sysom_web/src/pages/hotfix/{VersionConf => Version/VersionCustomize}/index.jsx (59%) diff --git a/sysom_web/config/routes.js b/sysom_web/config/routes.js index 2449f0bd..d207487a 100644 --- a/sysom_web/config/routes.js +++ b/sysom_web/config/routes.js @@ -265,9 +265,24 @@ export default [ component: './hotfix/HotfixLog' }, { - path: '/hotfix/verion_config', - name: 'versionconf', - component: './hotfix/VersionConf' + path: '/hotfix/version', + name: 'version', + routes: [ + { + path: '/hotfix/version/config', + name: 'config', + component: './hotfix/Version/VersionConfig' + }, + { + path: '/hotfix/version/config', + redirect: '/hotfix/Version/VersionConfig', + }, + { + path: '/hotfix/version/customize', + name: 'customize', + component: './hotfix/Version/VersionCustomize', + }, + ] } ] }, diff --git a/sysom_web/src/locales/zh-CN/menu.js b/sysom_web/src/locales/zh-CN/menu.js index d5f542d4..0d3c7fde 100644 --- a/sysom_web/src/locales/zh-CN/menu.js +++ b/sysom_web/src/locales/zh-CN/menu.js @@ -101,6 +101,8 @@ export default { 'menu.hotfix': '热补丁中心', 'menu.hotfix.make': '热补丁制作', 'menu.hotfix.formal': '热补丁列表', - 'menu.hotfix.versionconf': '自定义内核版本配置', + 'menu.hotfix.version': '自定义内核版本配置', 'menu.hotfix.selfdefine': '自定义热补丁制作', + 'menu.hotfix.version.config': '操作系统配置', + 'menu.hotfix.version.customize': '内核版本配置', }; \ No newline at end of file diff --git a/sysom_web/src/locales/zh-CN/pages.js b/sysom_web/src/locales/zh-CN/pages.js index c4b345a8..e839e485 100644 --- a/sysom_web/src/locales/zh-CN/pages.js +++ b/sysom_web/src/locales/zh-CN/pages.js @@ -178,6 +178,7 @@ export default { 'pages.hotfix.building_status': '构建状态', 'pages.hotfix.formal': '转正式包', 'pages.hotfix.os_type': '操作系统名', + 'pages.hotfix.kernel_repo_git': '源码仓库地址', 'pages.hotfix.patch_name': 'patch文件名称', 'pages.hotfix.upload': '文件上传', 'pages.hotfix.download': '下载', @@ -189,6 +190,10 @@ export default { 'pages.hotfix.title.versionconf': '自定义内核版本配置', 'pages.hotfix.image': '构建镜像', 'pages.hotfix.hotfix_name': "热补丁名称", + 'pages.hotfix.version_repo_devel': 'devel仓库地址', + 'pages.hotfix.version_repo_debuginfo': 'debuginfo仓库地址', + 'pages.hotfix.synch_information': '同步信息', + 'pages.hotfix.synchronization': '同步', 'pages.account.account_list': '账号列表', 'pages.account.username': '用户名', 'pages.account.password': '密码', diff --git a/sysom_web/src/pages/hotfix/VersionConf/KernelVersion.jsx b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx similarity index 63% rename from sysom_web/src/pages/hotfix/VersionConf/KernelVersion.jsx rename to sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx index 308735af..0eb34b18 100644 --- a/sysom_web/src/pages/hotfix/VersionConf/KernelVersion.jsx +++ b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx @@ -2,13 +2,14 @@ import React, { useRef, useState, useEffect } from 'react'; import { useIntl, FormattedMessage } from 'umi'; import { PageContainer } from '@ant-design/pro-layout'; import ProTable from '@ant-design/pro-table'; -import { Popconfirm, message, Upload, Button, Select, Form, Switch} from 'antd'; -import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeOsType } from '../service'; -import { ProForm, ModalForm, ProFormText, ProFormTextArea, ProFormSelect } from '@ant-design/pro-form'; +import { Popconfirm, message, Upload, Button, Select, Form, Switch, Tag} from 'antd'; +import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeOsType, PostWhetherSyncRequest } from '../../service'; +import { ProForm, ModalForm, ProFormText, ProFormTextArea, ProFormSelect, ProFormSwitch } from '@ant-design/pro-form'; import { async } from '@antv/x6/lib/registry/marker/async'; import { PropertySafetyFilled } from '@ant-design/icons'; -import KernelConfigForm from '../components/KernelConfigForm' +import KernelConfigForm from '../../components/KernelConfigForm' import ProCard from '@ant-design/pro-card'; +import CompoundedSpace from 'antd/lib/space'; const { Divider } = ProCard; @@ -36,9 +37,6 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { const oslistRef = useRef(); const intl = useIntl(); const [count, setCount] = useState(0); - useEffect(()=>{ - getOSTypeList(); - },[]); const onPostTask = () => { oslistRef.current.reload(); } @@ -47,6 +45,8 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { title: , dataIndex: 'os_type', key: 'os_type', + width: 200, + ellipsis: true, dataIndex: 'os_type', tooltip: '操作系统名，请为您该系列的操作系统类型命名' }, @@ -62,12 +62,46 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { valueType: 'input', tooltip: '输入该类操作系统构建热补丁时使用的镜像,如不填写则使用默认提供的Anolis镜像' }, + { + title: , + dataIndex: 'source_devel', + valueType: 'input', + // tooltip: '该操作系统类型的源码仓库地址', + }, + { + title: , + dataIndex: 'source_debuginfo', + valueType: 'input', + // tooltip: '该操作系统类型的源码仓库地址', + }, + { + title: , + dataIndex: 'sync_status', + valueType: 'input', + width: 180, + render: (_, record) => { + if(record.sync_status === 2){ + return <>同步成功
{record.synced_at} + }else if(record.sync_status === 1){ + return <>同步失败
{record.synced_at} + }else{ + return <>- + } + } + }, { title: , key: 'option', dataIndex: 'option', + width: 160, valueType: 'option', render: (_, record) => [ + + { + await PostWhetherSyncRequest(record.id); + oslistRef.current.reload(); + }}> + , { if (record.id == undefined) { @@ -98,11 +132,15 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { onCancel: () => console.log('取消'), }} onFinish={async (values) => { + console.log(values,"ppooo"); const data = { id: record.id, os_type_name: values.os_type_name, git_repo_link: values.git_repo_link, - image: values.building_image + image: values.building_image, + git_rule: values.git_rule, + source_devel: values.source_devel, + source_debuginfo: values.source_debuginfo } await postChangeOsType(data); message.success('提交成功'); @@ -127,13 +165,21 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { placeholder="请输入源码包地址" initialValue={record.source_repo} /> : - + <> + + + } { placeholder="请输入构建镜像" initialValue={record.image} /> + + ], @@ -150,7 +210,7 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { ]; return ( <> - + { + const actionRef = useRef(); + const refNetListTable = useRef(); + const intl = useIntl(); + + return ( + + + + ); +}; + +export default KernelVersionList; \ No newline at end of file diff --git a/sysom_web/src/pages/hotfix/VersionConf/KernelConfig.jsx b/sysom_web/src/pages/hotfix/Version/VersionCustomize/KernelConfig.jsx similarity index 95% rename from sysom_web/src/pages/hotfix/VersionConf/KernelConfig.jsx rename to sysom_web/src/pages/hotfix/Version/VersionCustomize/KernelConfig.jsx index ce656043..8c62a67b 100644 --- a/sysom_web/src/pages/hotfix/VersionConf/KernelConfig.jsx +++ b/sysom_web/src/pages/hotfix/Version/VersionCustomize/KernelConfig.jsx @@ -3,9 +3,9 @@ import { useIntl, FormattedMessage } from 'umi'; import { PageContainer } from '@ant-design/pro-layout'; import ProTable from '@ant-design/pro-table'; import { Popconfirm, message, Upload, Button, Select, Form} from 'antd'; -import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeKernelVersion } from '../service'; +import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeKernelVersion } from '../../service'; import { ProForm, ModalForm, ProFormText, ProFormTextArea, ProFormSelect } from '@ant-design/pro-form'; -import VersionConfigForm from '../components/VersionConfigForm' +import VersionConfigForm from '../../components/VersionConfigForm' import ProCard from '@ant-design/pro-card'; const { Divider } = ProCard; @@ -133,7 +133,7 @@ const VersionConfigList = React.forwardRef((props, ref) => { placeholder="请输入名称" initialValue={record.kernel_version} /> - + @@ -144,7 +144,7 @@ const VersionConfigList = React.forwardRef((props, ref) => { ]; return ( <> - + { const actionRef = useRef(); - const refNetListTable = useRef(); const intl = useIntl(); const [dataostype, setDataOsType] = useState([]); - const [dataoptionlist, setDataOptionList] = useState([]); - - const callback = (count) => { - const dr = [{label: count[0].os_type, value: count[0].os_type}]; - setDataOsType(dr); - } useEffect(()=>{ DataOptionList(); @@ -29,14 +21,12 @@ const KernelVersionConfigList = () => { arr.push({label: i.os_type,value: i.os_type}) }) } - setDataOptionList({arr:arr}); + setDataOsType({arr:arr}); } return ( - -
- + ); }; diff --git a/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx b/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx index 4ed28ed5..66698949 100644 --- a/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx +++ b/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx @@ -10,18 +10,22 @@ import { Switch } from 'antd'; export default (props, ref) => { const kernelformRef = useRef(); const [readonlyone, setReadonlyOne] = useState(false); + const [synchronization, setSynchronization] = useState(false); const { loading, error, run } = useRequest(submitOSType, { manual: true, onSuccess: (result, params) => { kernelformRef.current.resetFields(); props?.onSuccess?.(result, params); - props.parentBack(params); }, }); const KernelConfigChange = (e) => { setReadonlyOne(e); } + const ConfigSynchronizationChange = (e) => { + setSynchronization(e); + + } return ( { tooltip="该操作系统类型的源码包仓库地址" label="源码包地址" /> : - + <> + + + } { tooltip="输入该类操作系统构建热补丁时使用的镜像,如不填写则使用默认提供的Anolis镜像" label="构建镜像" /> + + { unCheckedChildren="否" onChange={KernelConfigChange} /> + diff --git a/sysom_web/src/pages/hotfix/components/VersionConfigForm.jsx b/sysom_web/src/pages/hotfix/components/VersionConfigForm.jsx index c53c8460..3f376121 100644 --- a/sysom_web/src/pages/hotfix/components/VersionConfigForm.jsx +++ b/sysom_web/src/pages/hotfix/components/VersionConfigForm.jsx @@ -9,6 +9,7 @@ import { submitKernelVersion, getOSTypeList } from '../service' export default (props) => { const formRef = useRef(); const [veroptionList,setVerOptionList] = useState([]); + const [dataostype, setDataOsType] = useState([]); const { loading, error, run } = useRequest(submitKernelVersion, { manual: true, onSuccess: (result, params) => { @@ -16,7 +17,6 @@ export default (props) => { props?.onSuccess?.(result, params); }, }); - return ( { Date: Mon, 15 May 2023 16:50:47 +0800 Subject: [PATCH 055/196] fix(diagnosis): Ensure that diagnostic tasks are properly finished 1. First, the asynchronous interface of channel_job is used uniformly for communication, guaranteeing that each task will time out within the specified time; 2. Secondly, a check is made at the start of the diagnostic microservice to mark tasks that have not finished for more than 10 minutes as timeouts --- .../sysom_diagnosis/apps/task/executor.py | 59 ++++++---- .../sysom_diagnosis/apps/task/helper.py | 103 +++++++++--------- .../sysom_diagnosis/apps/task/views.py | 5 +- 3 files changed, 93 insertions(+), 74 deletions(-) diff --git a/sysom_server/sysom_diagnosis/apps/task/executor.py b/sysom_server/sysom_diagnosis/apps/task/executor.py index 47230ebd..66e6e558 100644 --- a/sysom_server/sysom_diagnosis/apps/task/executor.py +++ b/sysom_server/sysom_diagnosis/apps/task/executor.py @@ -1,22 +1,19 @@ from clogger import logger -import functools from apps.task.models import JobModel from django.conf import settings -from cec_base.cec_client import MultiConsumer, CecAsyncConsumeTask from cec_base.event import Event from cec_base.consumer import Consumer -from sysom_utils import ConfigParser +from sysom_utils import AsyncEventExecutor, CecAsyncConsumeTask, ConfigParser +from asgiref.sync import sync_to_async +from datetime import datetime, timedelta +from django.db.models import Q from .helper import DiagnosisHelper -class DiagnosisTaskExecutor(MultiConsumer): +class DiagnosisTaskExecutor(AsyncEventExecutor): def __init__(self, config: ConfigParser): - super().__init__( - settings.SYSOM_CEC_PRODUCER_URL, - custom_callback=self.on_receive_event - ) - self._config = config + super().__init__(settings.SYSOM_CEC_PRODUCER_URL, callback=self.process_event) self.append_group_consume_task( settings.SYSOM_CEC_DIAGNOSIS_TASK_DISPATCH_TOPIC, settings.SYSOM_CEC_DIAGNOSIS_CONSUMER_GROUP, @@ -24,11 +21,10 @@ class DiagnosisTaskExecutor(MultiConsumer): ensure_topic_exist=True ) - def on_receive_event(self, event: Event, task: CecAsyncConsumeTask): - """Process received event""" + async def process_event(self, event: Event, task: CecAsyncConsumeTask): try: if task.topic_name == settings.SYSOM_CEC_DIAGNOSIS_TASK_DISPATCH_TOPIC: - self._process_task_dispatch_event(event) + await self._process_task_dispatch_event(event) else: # Unexpected logger.error("Receive unknown topic event, unexpected!!") @@ -40,14 +36,17 @@ class DiagnosisTaskExecutor(MultiConsumer): ################################################################################################ # 事件处理 ################################################################################################ - def _process_task_dispatch_event(self, event: Event): + async def _process_task_dispatch_event(self, event: Event): """Process diagnosis task dispatch event { "task_id": "xxx" } """ try: - self._execute_diagnosis_task_by_id(event.value["task_id"]) + if isinstance(event.value, dict): + await self._execute_diagnosis_task_by_id(event.value["task_id"]) + else: + raise Exception("expect event.value is dict") except Exception as exc: logger.exception( f"Diagnosis process task dispatch event error: {str(exc)}") @@ -56,19 +55,33 @@ class DiagnosisTaskExecutor(MultiConsumer): # 诊断任务执行 ################################################################################################ - def _execute_diagnosis_task_by_id(self, task_id: str): - instance = JobModel.objects.get(task_id=task_id) - self._execute_diagnosis_task_by_model(instance) + async def _execute_diagnosis_task_by_id(self, task_id: str): + instance = await sync_to_async(JobModel.objects.get)(task_id=task_id) + await self._execute_diagnosis_task_by_model(instance) - def _execute_diagnosis_task_by_model(self, instance: JobModel): + async def _execute_diagnosis_task_by_model(self, instance: JobModel): # 1. Preprocess - res = DiagnosisHelper.preprocess(instance) + res = await DiagnosisHelper.preprocess(instance) # 2. Execute and Postprocess if res: - res = DiagnosisHelper.execute( - instance, - functools.partial(DiagnosisHelper.postprocess, instance) - ) + job_result = await DiagnosisHelper.execute(instance) + await DiagnosisHelper.postprocess(instance, job_result) # 3. TODO: produce task execute result to cec + + def start(self): + super().start() + + # Check and mark timeout tasks + expire_minutes_ago = datetime.now() - timedelta(minutes=10) + instances = JobModel.objects.filter( + Q(created_at__lte=expire_minutes_ago) & + Q(status="Running") + ) + for instance in instances: + instance.code = 1 + instance.status = "Fail" + instance.result = "Diagnosis execute task timeout" + instance.err_msg = "Diagnosis execute task timeout" + instance.save() diff --git a/sysom_server/sysom_diagnosis/apps/task/helper.py b/sysom_server/sysom_diagnosis/apps/task/helper.py index f16231aa..a912a218 100644 --- a/sysom_server/sysom_diagnosis/apps/task/helper.py +++ b/sysom_server/sysom_diagnosis/apps/task/helper.py @@ -4,11 +4,11 @@ import subprocess from clogger import logger import tempfile import ast -from typing import Callable +from typing import List from apps.task.models import JobModel from django.conf import settings -from django.db import connection from lib.utils import uuid_8 +from asgiref.sync import sync_to_async from channel_job.job import default_channel_job_executor, JobResult @@ -16,17 +16,15 @@ class DiagnosisHelper: """A helper class used to perform diagnosis task""" @staticmethod - def _update_job(instance: JobModel, **kwargs): + async def _update_job(instance: JobModel, **kwargs): """Update JobModel object""" try: if isinstance(kwargs.get("result", ""), dict): kwargs['result'] = json.dumps(kwargs.get('result', "")) instance.__dict__.update(**kwargs) - instance.save() + await sync_to_async(instance.save)() except Exception as e: raise e - finally: - connection.close() @staticmethod def init(data: dict, user: dict) -> JobModel: @@ -78,9 +76,30 @@ class DiagnosisHelper: return JobModel.objects.create(**task_params) @staticmethod - def preprocess(instance: JobModel) -> bool: + async def run_subprocess(cmd: List[str]) -> dict: + resp = subprocess.run(cmd, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + return { + "stdout": resp.stdout.decode("utf-8"), + "stderr": resp.stderr.decode("utf-8"), + "returncode": resp.returncode + } + # proc = await asyncio.create_subprocess_shell( + # cmd, + # stdout=asyncio.subprocess.PIPE, + # stderr=asyncio.subprocess.PIPE + # ) + # stdout, stderr = await proc.communicate() + # return { + # "stdout": stdout.decode("utf-8"), + # "stderr": stderr.decode("utf-8"), + # "returncode": proc.returncode + # } + + @staticmethod + async def preprocess(instance: JobModel) -> bool: """"Perform diagnosis preprocessing - + { "commands":[ { @@ -111,25 +130,23 @@ class DiagnosisHelper: raise Exception( "Can not find script file, please check service name") try: - command_list = [service_path, json.dumps(params)] - resp = subprocess.run(command_list, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + resp = await DiagnosisHelper.run_subprocess([service_path, json.dumps(params)]) except Exception as exc: raise Exception( f"Execute preprocess script error: {str(exc)}" ) from exc # 3. If the preprocessing script executes with an error - if resp.returncode != 0: + if resp["returncode"] != 0: raise (Exception( - f"Execute preprocess script error: {str(resp.stderr.decode('utf-8'))}" + f"Execute preprocess script error: {resp['stderr']}" )) # 4. If the preprocessing script executes successfully, # take out the processing result - stdout = resp.stdout.decode('utf-8') + stdout = resp["stdout"] resp = ast.literal_eval(stdout) - # + # resp_scripts = resp.get("commands") # 5. If the preprocessing result not contains 'commands', it's a not expect bug @@ -140,22 +157,22 @@ class DiagnosisHelper: # 6. If the preprocessing script executes successfully, the parameters are compliant # and the Job instance is updated - DiagnosisHelper._update_job( + await DiagnosisHelper._update_job( instance, command=json.dumps(resp_scripts), status="Running" ) success = True except Exception as exc: logger.exception( f"Diagnosis preprocess error: {str(exc)}") - DiagnosisHelper._update_job( + await DiagnosisHelper._update_job( instance, result="Diagnosis preprocess error", status="Fail", code=1, err_msg=f"Diagnosis preprocess error: {str(exc)}") return success @staticmethod - def execute(instance: JobModel, result_callback: Callable[[JobResult], None]) -> bool: + async def execute(instance: JobModel) -> JobResult: """Execute diagnosis task""" - success = False + job_result = JobResult(code=1, err_msg="Diagnosis execute task error") try: resp_scripts = json.loads(instance.command) params = instance.params @@ -179,28 +196,20 @@ class DiagnosisHelper: }, "timeout": 1000 * 60 * 10, # 10 minutes } - # 前 n - 1 个命令同步执行 - if idx < len(resp_scripts) - 1: - job_result = default_channel_job_executor.dispatch_job(**job_params) \ - .execute() - if job_result.code != 0: - raise Exception( - f"Task execute failed: {job_result.err_msg}") - else: - # 最后一个命令异步执行 - default_channel_job_executor.dispatch_job(**job_params) \ - .execute_async_with_callback(result_callback) - success = True + job_result = await default_channel_job_executor.dispatch_job(**job_params).execute_async() + if job_result.code != 0: + raise Exception( + f"Task execute failed: {job_result.err_msg}") except Exception as exc: - logger.exception( - f"Diagnosis execute task error: {str(exc)}") - DiagnosisHelper._update_job( + job_result.err_msg = f"Diagnosis execute task error: {str(exc)}" + logger.exception(job_result.err_msg) + await DiagnosisHelper._update_job( instance, result="Diagnosis execute task error", status="Fail", - code=1, err_msg=f"Diagnosis execute task error: {str(exc)}") - return success + code=1, err_msg=job_result.err_msg) + return job_result @staticmethod - def postprocess(instance: JobModel, job_result: JobResult): + async def postprocess(instance: JobModel, job_result: JobResult): """Perform diagnosis postprocessing JobResult -> { "code": 0, => 0 表示成功，1表示失败 @@ -222,7 +231,7 @@ class DiagnosisHelper: code = job_result.code err_msg = job_result.err_msg if code != 0: - DiagnosisHelper._update_job( + await DiagnosisHelper._update_job( instance, status="Fail", code=code, result=job_result.result, err_msg=err_msg) return @@ -247,41 +256,37 @@ class DiagnosisHelper: # 创建一个临时文件，用于暂存中间结果 with tempfile.NamedTemporaryFile(mode="w") as tmp_file: - command_list = [service_post_path, - tmp_file.name, instance.task_id] try: # 将要传递的中间结果写入到临时文件当中 tmp_file.write(job_result.result) tmp_file.flush() - resp = subprocess.run(command_list, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + resp = await DiagnosisHelper.run_subprocess([service_post_path, tmp_file.name, instance.task_id]) except Exception as exc: raise Exception( f"Execute postprocess script error: {str(exc)}" ) - if resp.returncode != 0: + if resp["returncode"] != 0: raise (Exception( - f"Execute postprocess script error: {str(resp.stderr.decode('utf-8'))}" + f"Execute postprocess script error: {str(resp['stderr'])}" )) - stdout = resp.stdout - result = json.loads(stdout.decode('utf-8').strip()) + result = json.loads(resp["stdout"].strip()) code = result.get("code", 1) if code != 0: err_msg = result.get("err_msg", "Postprocess error") # 后处理脚本认为诊断出错 - DiagnosisHelper._update_job( + await DiagnosisHelper._update_job( instance, err_msg=err_msg, status="Fail") else: rel_result = result.get("result", {}) # 后处理脚本执行结束，更新任务状态 - DiagnosisHelper._update_job( + await DiagnosisHelper._update_job( instance, result=rel_result, status="Success" ) pass except Exception as exc: logger.exception( f"Diagnosis postprocess error: {str(exc)}") - DiagnosisHelper._update_job( + await DiagnosisHelper._update_job( instance, result="Diagnosis postprocess error", status="Fail", code=1, err_msg=f"Diagnosis postprocess error: {str(exc)}") diff --git a/sysom_server/sysom_diagnosis/apps/task/views.py b/sysom_server/sysom_diagnosis/apps/task/views.py index dac33026..124dd123 100644 --- a/sysom_server/sysom_diagnosis/apps/task/views.py +++ b/sysom_server/sysom_diagnosis/apps/task/views.py @@ -15,6 +15,7 @@ from lib.base_view import CommonModelViewSet from lib.response import success, not_found, ErrorResponse, other_response from lib.authentications import TokenAuthentication from channel_job.job import JobResult +from asgiref.sync import async_to_sync from .helper import DiagnosisHelper @@ -129,7 +130,7 @@ class TaskAPIView(CommonModelViewSet, data, getattr(request, 'user')) # 3. postprocess - DiagnosisHelper.postprocess( + async_to_sync(DiagnosisHelper.postprocess)( instance, job_result=JobResult(code=0, result=offline_log)) return success({ "task_id": instance.task_id @@ -147,5 +148,5 @@ class TaskAPIView(CommonModelViewSet, if res.status_code == 200: return success(result=res.json().get('data', [])) else: - return success() + return ErrorResponse(msg=f"Get host failed, status_code={res.status_code}") -- Gitee From dd65189c0813629777f688701519e82244699102 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 15 May 2023 19:11:41 +0800 Subject: [PATCH 056/196] fix(sdk): Add lock for dispatch_* To prevent errors caused by concurrent calls to dispatch_*, you need to add a lock to the dispatch_* call --- environment/1_sdk/cec_base/consumer.py | 39 +++++++++--------- environment/1_sdk/cec_base/producer.py | 35 ++++++++-------- .../1_sdk/cmg_base/service_discovery.py | 39 +++++++++--------- .../1_sdk/cmg_base/service_registry.py | 37 +++++++++-------- environment/1_sdk/gcache_base/gcache.py | 40 ++++++++++--------- environment/1_sdk/gclient_base/gclient.py | 37 +++++++++-------- 6 files changed, 122 insertions(+), 105 deletions(-) diff --git a/environment/1_sdk/cec_base/consumer.py b/environment/1_sdk/cec_base/consumer.py index 970b6ee1..00530215 100644 --- a/environment/1_sdk/cec_base/consumer.py +++ b/environment/1_sdk/cec_base/consumer.py @@ -9,6 +9,7 @@ Description: import functools import importlib import uuid +from threading import Lock from abc import ABCMeta, abstractmethod from enum import Enum from typing import List @@ -61,9 +62,8 @@ class Consumer(Connectable, metaclass=ABCMeta): default_batch_consume_limit(int): Default batch consume limit """ - proto_dict = { - - } + proto_dict = {} + proto_lock = Lock() def __init__(self, topic_name: str, consumer_id: str = "", group_id: str = "", start_from_now: bool = True, **kwargs): @@ -305,22 +305,23 @@ def dispatch_consumer(url: str, topic_name: str, consumer_id: str = "", ... , start_from_now=False) """ cec_url = CecUrl.parse(url) - if cec_url.proto not in Consumer.proto_dict: - # Check if dynamic import is possible - target_module = f"cec_{cec_url.proto}.{cec_url.proto}_consumer" - try: - module = importlib.import_module(target_module) - Consumer.register( - cec_url.proto, - getattr(module, f'{cec_url.proto.capitalize()}Consumer') - ) - except ModuleNotFoundError as exc: - logger.error( - f"Try to auto import module {target_module} failed." - ) - raise CecProtoNotExistsException( - f"Proto '{cec_url.proto}' not exists in Cec-base-Consumer." - ) from exc + with Consumer.proto_lock: + if cec_url.proto not in Consumer.proto_dict: + # Check if dynamic import is possible + target_module = f"cec_{cec_url.proto}.{cec_url.proto}_consumer" + try: + module = importlib.import_module(target_module) + Consumer.register( + cec_url.proto, + getattr(module, f'{cec_url.proto.capitalize()}Consumer') + ) + except ModuleNotFoundError as exc: + logger.error( + f"Try to auto import module {target_module} failed." + ) + raise CecProtoNotExistsException( + f"Proto '{cec_url.proto}' not exists in Cec-base-Consumer." + ) from exc consumer_instance = Consumer.proto_dict[cec_url.proto]( cec_url, topic_name=topic_name, diff --git a/environment/1_sdk/cec_base/producer.py b/environment/1_sdk/cec_base/producer.py index e45fb32b..520e0690 100644 --- a/environment/1_sdk/cec_base/producer.py +++ b/environment/1_sdk/cec_base/producer.py @@ -7,6 +7,7 @@ File producer.py Description: """ import importlib +from threading import Lock from abc import ABCMeta, abstractmethod from typing import Callable, Union from clogger import logger @@ -24,6 +25,7 @@ class Producer(Connectable, metaclass=ABCMeta): """ proto_dict = {} + proto_lock = Lock() @abstractmethod def produce(self, topic_name: str, message_value: Union[bytes, str, dict], @@ -124,22 +126,23 @@ def dispatch_producer(url: str, **kwargs) -> Producer: ..."redis://localhost:6379?password=123456") """ cec_url = CecUrl.parse(url) - if cec_url.proto not in Producer.proto_dict: - # Check if dynamic import is possible - target_module = f"cec_{cec_url.proto}.{cec_url.proto}_producer" - try: - module = importlib.import_module(target_module) - Producer.register( - cec_url.proto, - getattr(module, f'{cec_url.proto.capitalize()}Producer') - ) - except ModuleNotFoundError as exc: - logger.error( - f"Try to auto import module {target_module} failed." - ) - raise CecProtoNotExistsException( - f"Proto '{cec_url.proto}' not exists in Cec-base-Producer." - ) from exc + with Producer.proto_lock: + if cec_url.proto not in Producer.proto_dict: + # Check if dynamic import is possible + target_module = f"cec_{cec_url.proto}.{cec_url.proto}_producer" + try: + module = importlib.import_module(target_module) + Producer.register( + cec_url.proto, + getattr(module, f'{cec_url.proto.capitalize()}Producer') + ) + except ModuleNotFoundError as exc: + logger.error( + f"Try to auto import module {target_module} failed." + ) + raise CecProtoNotExistsException( + f"Proto '{cec_url.proto}' not exists in Cec-base-Producer." + ) from exc producer_instance = Producer.proto_dict[cec_url.proto](cec_url, **kwargs) logger.info( f"Cec-base-Producer dispatch one producer instance success. " diff --git a/environment/1_sdk/cmg_base/service_discovery.py b/environment/1_sdk/cmg_base/service_discovery.py index 3b375091..2abb9903 100644 --- a/environment/1_sdk/cmg_base/service_discovery.py +++ b/environment/1_sdk/cmg_base/service_discovery.py @@ -7,6 +7,7 @@ File service_discovery.py Description: """ import importlib +from threading import Lock from abc import ABCMeta, abstractmethod from clogger import logger from typing import List @@ -24,6 +25,7 @@ class ServiceDiscovery(metaclass=ABCMeta): """ proto_dict = {} + proto_lock = Lock() def __init__(self, fetch_interval: int): self.fetch_interval = fetch_interval @@ -108,24 +110,25 @@ def dispatch_service_discovery(url: str, **kwargs) -> ServiceDiscovery: ..."redis://localhost:6379?password=123456") """ cmg_url = CmgUrl.parse(url) - if cmg_url.proto not in ServiceDiscovery.proto_dict: - # Check if dynamic import is possible - target_module = f"cmg_{cmg_url.proto}.{cmg_url.proto}_service_discovery" - try: - module = importlib.import_module(target_module) - ServiceDiscovery.protocol_register( - cmg_url.proto, - getattr(module, - f'{cmg_url.proto.capitalize()}ServiceDiscovery') - ) - except ModuleNotFoundError as exc: - logger.error( - f"Try to auto import module {target_module} failed." - ) - raise CmgProtoNotExistsException( - f"Proto '{cmg_url.proto}' not exists in Cmg-base" - "-ServiceDiscovery." - ) from exc + with ServiceDiscovery.proto_lock: + if cmg_url.proto not in ServiceDiscovery.proto_dict: + # Check if dynamic import is possible + target_module = f"cmg_{cmg_url.proto}.{cmg_url.proto}_service_discovery" + try: + module = importlib.import_module(target_module) + ServiceDiscovery.protocol_register( + cmg_url.proto, + getattr(module, + f'{cmg_url.proto.capitalize()}ServiceDiscovery') + ) + except ModuleNotFoundError as exc: + logger.error( + f"Try to auto import module {target_module} failed." + ) + raise CmgProtoNotExistsException( + f"Proto '{cmg_url.proto}' not exists in Cmg-base" + "-ServiceDiscovery." + ) from exc service_registry_instance = ServiceDiscovery.proto_dict[cmg_url.proto]( cmg_url, **kwargs) logger.info( diff --git a/environment/1_sdk/cmg_base/service_registry.py b/environment/1_sdk/cmg_base/service_registry.py index a1ed9065..4fc17c27 100644 --- a/environment/1_sdk/cmg_base/service_registry.py +++ b/environment/1_sdk/cmg_base/service_registry.py @@ -7,6 +7,7 @@ File service_registry.py Description: """ import importlib +from threading import Lock from abc import ABCMeta, abstractmethod from clogger import logger from .service_instance import ServiceInstance @@ -22,6 +23,7 @@ class ServiceRegistry(metaclass=ABCMeta): """ proto_dict = {} + proto_lock = Lock() @abstractmethod def register(self, service_instance: ServiceInstance): @@ -83,23 +85,24 @@ def dispatch_service_registry(url: str, **kwargs) -> ServiceRegistry: ..."redis://localhost:6379?password=123456") """ cmg_url = CmgUrl.parse(url) - if cmg_url.proto not in ServiceRegistry.proto_dict: - # Check if dynamic import is possible - target_module = f"cmg_{cmg_url.proto}.{cmg_url.proto}_service_registry" - try: - module = importlib.import_module(target_module) - ServiceRegistry.protocol_register( - cmg_url.proto, - getattr(module, f'{cmg_url.proto.capitalize()}ServiceRegistry') - ) - except ModuleNotFoundError as exc: - logger.error( - f"Try to auto import module {target_module} failed." - ) - raise CmgProtoNotExistsException( - f"Proto '{cmg_url.proto}' not exists in Cmg-base" - "-ServiceRegistry." - ) from exc + with ServiceRegistry.proto_lock: + if cmg_url.proto not in ServiceRegistry.proto_dict: + # Check if dynamic import is possible + target_module = f"cmg_{cmg_url.proto}.{cmg_url.proto}_service_registry" + try: + module = importlib.import_module(target_module) + ServiceRegistry.protocol_register( + cmg_url.proto, + getattr(module, f'{cmg_url.proto.capitalize()}ServiceRegistry') + ) + except ModuleNotFoundError as exc: + logger.error( + f"Try to auto import module {target_module} failed." + ) + raise CmgProtoNotExistsException( + f"Proto '{cmg_url.proto}' not exists in Cmg-base" + "-ServiceRegistry." + ) from exc service_registry_instance = ServiceRegistry.proto_dict[cmg_url.proto]( cmg_url, **kwargs) logger.info( diff --git a/environment/1_sdk/gcache_base/gcache.py b/environment/1_sdk/gcache_base/gcache.py index c6b0a947..fe8aa8cf 100644 --- a/environment/1_sdk/gcache_base/gcache.py +++ b/environment/1_sdk/gcache_base/gcache.py @@ -9,6 +9,7 @@ Description: import importlib from abc import ABCMeta, abstractmethod from typing import Union, Optional +from threading import Lock from clogger import logger from .exceptions import GCacheProtoAlreadyExistsException, \ GCacheProtoNotExistsException, GCacheException @@ -22,6 +23,8 @@ class GCache(metaclass=ABCMeta): proto_dict = {} + proto_lock = Lock() + @abstractmethod def store(self, key: str, value: Union[int, float, dict, str], expire: int = -1) -> bool: @@ -142,25 +145,26 @@ def dispatch_g_cache(cache_name: str, url: str, **kwargs) -> GCache: ..."redis://localhost:6379?password=123456") """ cmg_url = GCacheUrl.parse(url) - if cmg_url.proto not in GCache.proto_dict: - # Check if dynamic import is possible - target_module = f"gcache_{cmg_url.proto}.{cmg_url.proto}_gcache" - try: - module = importlib.import_module(target_module) - GCache.protocol_register( - cmg_url.proto, - getattr(module, f'{cmg_url.proto.capitalize()}GCache') - ) - except ModuleNotFoundError as exc: - logger.error( - f"Try to auto import module {target_module} failed." - ) - raise GCacheProtoNotExistsException( - f"Proto '{cmg_url.proto}' not exists in GCache-base" - "-GCache." - ) from exc + with GCache.proto_lock: + if cmg_url.proto not in GCache.proto_dict: + # Check if dynamic import is possible + target_module = f"gcache_{cmg_url.proto}.{cmg_url.proto}_gcache" + try: + module = importlib.import_module(target_module) + GCache.protocol_register( + cmg_url.proto, + getattr(module, f'{cmg_url.proto.capitalize()}GCache') + ) + except ModuleNotFoundError as exc: + logger.error( + f"Try to auto import module {target_module} failed." + ) + raise GCacheProtoNotExistsException( + f"Proto '{cmg_url.proto}' not exists in GCache-base" + "-GCache." + ) from exc g_cache_instance = GCache.proto_dict[cmg_url.proto](cache_name, - cmg_url, **kwargs) + cmg_url, **kwargs) logger.info( f"GCache-base-GCache dispatch one GCache instance " f"success. proto={cmg_url.proto}, url={url}" diff --git a/environment/1_sdk/gclient_base/gclient.py b/environment/1_sdk/gclient_base/gclient.py index 90ac7014..df710aa0 100644 --- a/environment/1_sdk/gclient_base/gclient.py +++ b/environment/1_sdk/gclient_base/gclient.py @@ -8,6 +8,7 @@ Description: """ import requests import importlib +from threading import Lock from clogger import logger from abc import ABCMeta, abstractmethod from .exceptions import GClientProtoAlreadyExistsException, \ @@ -17,6 +18,7 @@ from .url import GClientUrl class GClient(metaclass=ABCMeta): proto_dict = {} + proto_lock = Lock() @abstractmethod def get(self, path: str, params=None, @@ -101,23 +103,24 @@ def dispatch_g_client(url: str, **kwargs) -> GClient: ..."http://localhost:6379") """ g_client_url = GClientUrl.parse(url) - if g_client_url.proto not in GClient.proto_dict: - # Check if dynamic import is possible - target_module = f"gclient_{g_client_url.proto}.{g_client_url.proto}_gclient" - try: - module = importlib.import_module(target_module) - GClient.protocol_register( - g_client_url.proto, - getattr(module, f'{g_client_url.proto.capitalize()}GClient') - ) - except ModuleNotFoundError as exc: - logger.error( - f"Try to auto import module {target_module} failed." - ) - raise GClientProtoNotExistsException( - f"Proto '{g_client_url.proto}' not exists in GClient-base" - "-GClient." - ) from exc + with GClient.proto_lock: + if g_client_url.proto not in GClient.proto_dict: + # Check if dynamic import is possible + target_module = f"gclient_{g_client_url.proto}.{g_client_url.proto}_gclient" + try: + module = importlib.import_module(target_module) + GClient.protocol_register( + g_client_url.proto, + getattr(module, f'{g_client_url.proto.capitalize()}GClient') + ) + except ModuleNotFoundError as exc: + logger.error( + f"Try to auto import module {target_module} failed." + ) + raise GClientProtoNotExistsException( + f"Proto '{g_client_url.proto}' not exists in GClient-base" + "-GClient." + ) from exc g_client_instance = GClient.proto_dict[g_client_url.proto](g_client_url, **kwargs) logger.info( -- Gitee From 6c6b0c0c3b7737addf63123890dd3c545654f2be Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 16 May 2023 12:52:18 +0800 Subject: [PATCH 057/196] refactor(channel): Auto scan models defined for alembic migrate --- conf/config.yml | 2 +- sysom_server/sysom_channel/alembic.ini | 2 +- sysom_server/sysom_channel/alembic/env.py | 26 ++++++++++++++--------- sysom_server/sysom_channel/conf/common.py | 2 +- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/conf/config.yml b/conf/config.yml index 9cb24e9d..3ce52a57 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -59,7 +59,7 @@ sysom_server: sysom_service: path: - root_path: !concat [*global_root_path, "/server/microservice"] + root_path: !concat [*global_root_path, "/server"] framework: gclient: protocol: cmg diff --git a/sysom_server/sysom_channel/alembic.ini b/sysom_server/sysom_channel/alembic.ini index 272b3062..f6ab9feb 100644 --- a/sysom_server/sysom_channel/alembic.ini +++ b/sysom_server/sysom_channel/alembic.ini @@ -52,7 +52,7 @@ version_path_separator = os # Use os.pathsep. Default configuration used for ne # are written from script.py.mako # output_encoding = utf-8 -sqlalchemy.url = mariadb+pymysql://sysom:sysom_admin@127.0.0.1/sysom +sqlalchemy.url = "" [post_write_hooks] diff --git a/sysom_server/sysom_channel/alembic/env.py b/sysom_server/sysom_channel/alembic/env.py index 177ac6e3..517ae502 100644 --- a/sysom_server/sysom_channel/alembic/env.py +++ b/sysom_server/sysom_channel/alembic/env.py @@ -1,22 +1,29 @@ +import inspect +import app.models as models from logging.config import fileConfig - from sqlalchemy import engine_from_config from sqlalchemy import pool from app.models import Base from alembic import context -from pathlib import Path -from sysom_utils import ConfigParser +from conf.settings import YAML_CONFIG ################################################################## # Load yaml config first ################################################################## -BASE_DIR = Path(__file__).resolve().parent -YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" -YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR.parent}/config.yml" - -YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) mysql_config = YAML_CONFIG.get_server_config().db.mysql +################################################################## +# Scan models +################################################################## +service_tables = [] +for name, data in inspect.getmembers(models, inspect.isclass): + if data.__module__ != "app.models": + continue + if "__tablename__" in data.__dict__: + service_tables.append(data.__dict__["__tablename__"]) + elif "__table__" in data.__dict__: + service_tables.append(data.__dict__["__table__"]) + # this is the Alembic Config object, which provides # access to the values within the .ini file in use. config = context.config @@ -46,9 +53,8 @@ target_metadata = Base.metadata # my_important_option = config.get_main_option("my_important_option") # ... etc. - def include_object(object, name, type_, reflected, compare_to): - if type_ == "table" and name not in ["sys_channel_setting", "sys_channel_params"]: + if type_ == "table" and name not in service_tables: return False return True diff --git a/sysom_server/sysom_channel/conf/common.py b/sysom_server/sysom_channel/conf/common.py index 47f42045..d9ff187a 100644 --- a/sysom_server/sysom_channel/conf/common.py +++ b/sysom_server/sysom_channel/conf/common.py @@ -16,7 +16,7 @@ BASE_DIR = Path(__file__).resolve().parent.parent ################################################################## # Load yaml config first ################################################################## -YAML_GLOBAL_CONFIG_PATH = f"/etc/sysom/config.yml" +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) -- Gitee From 74a1bcbfcd99b39bfcbb5fe4d17e948ffaac2ed9 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 16 May 2023 13:54:08 +0800 Subject: [PATCH 058/196] refactor(framework): Move logger init to SysomFramework.init --- environment/1_sdk/sysom_utils/framework.py | 9 +++++++++ sysom_server/sysom_api/conf/common.py | 10 ---------- sysom_server/sysom_channel/conf/common.py | 9 --------- sysom_server/sysom_diagnosis/conf/common.py | 10 ---------- sysom_server/sysom_hotfix/conf/common.py | 17 ++++------------- sysom_server/sysom_migration/conf/common.py | 12 ++---------- .../sysom_monitor_server/conf/common.py | 12 ------------ sysom_server/sysom_vmcore/conf/common.py | 15 +++------------ sysom_server/sysom_vul/conf/common.py | 18 ++++-------------- 9 files changed, 22 insertions(+), 90 deletions(-) diff --git a/environment/1_sdk/sysom_utils/framework.py b/environment/1_sdk/sysom_utils/framework.py index 1cef8713..8e605633 100644 --- a/environment/1_sdk/sysom_utils/framework.py +++ b/environment/1_sdk/sysom_utils/framework.py @@ -11,6 +11,7 @@ from typing import Optional, Dict, Type from gcache_base import dispatch_g_cache, GCache from channel_job.job import default_channel_job_executor from gclient_base import dispatch_g_client, GClient +from clogger import logger from .config_parser import ConfigParser from .framework_plug_mag import FrameworkPlugMag, FrameworkPluginBase @@ -27,7 +28,15 @@ class SysomFramework: def init(cls, config: ConfigParser): cls._config = config cls._framework_plug_mag = FrameworkPlugMag(config) + cls.init_logger(config) return cls + + @classmethod + def init_logger(cls, config: ConfigParser): + log_format = config.get_server_config().logger.format + log_level = config.get_server_config().logger.level + logger.set_format(log_format) + logger.set_level(log_level) @classmethod def gcache(cls, cache_name: str) -> GCache: diff --git a/sysom_server/sysom_api/conf/common.py b/sysom_server/sysom_api/conf/common.py index ad4a65ab..be8da5e0 100644 --- a/sysom_server/sysom_api/conf/common.py +++ b/sysom_server/sysom_api/conf/common.py @@ -1,4 +1,3 @@ -from clogger import logger import os import datetime from pathlib import Path @@ -152,12 +151,3 @@ SYSOM_CEC_API_HOST_TOPIC = \ ################################################################## # channl_job SDK 需要的url SYSOM_HOST_CEC_URL = YAML_CONFIG.get_local_channel_job_url() - - -################################################################## -# Config settings -################################################################## -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) diff --git a/sysom_server/sysom_channel/conf/common.py b/sysom_server/sysom_channel/conf/common.py index d9ff187a..49bfca61 100644 --- a/sysom_server/sysom_channel/conf/common.py +++ b/sysom_server/sysom_channel/conf/common.py @@ -6,7 +6,6 @@ Email mfeng@linux.alibaba.com File common.py Description: """ -from clogger import logger import os from pathlib import Path from sysom_utils import ConfigParser, CecTarget, SysomFramework @@ -66,11 +65,3 @@ SYSOM_CEC_CHANNEL_TOPIC = cec_config.topics.SYSOM_CEC_CHANNEL_TOPIC SYSOM_CEC_CHANNEL_CONSUMER_GROUP = cec_config.consumer_group # 通道模块用于投递执行结果的主题 SYSOM_CEC_CHANNEL_RESULT_TOPIC = cec_config.topics.SYSOM_CEC_CHANNEL_RESULT_TOPIC - -################################################################## -# Logging config -################################################################## -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) diff --git a/sysom_server/sysom_diagnosis/conf/common.py b/sysom_server/sysom_diagnosis/conf/common.py index 5fa3117f..c5ca3bb9 100644 --- a/sysom_server/sysom_diagnosis/conf/common.py +++ b/sysom_server/sysom_diagnosis/conf/common.py @@ -1,4 +1,3 @@ -from clogger import logger import os from pathlib import Path from sysom_utils import ConfigParser, CecTarget, SysomFramework @@ -115,12 +114,3 @@ REST_FRAMEWORK = { 'UNICODE_JSON': True, 'EXCEPTION_HANDLER': 'lib.exception.exception_handler' } - -################################################################## -# Config settings -################################################################## -# Config log format -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) diff --git a/sysom_server/sysom_hotfix/conf/common.py b/sysom_server/sysom_hotfix/conf/common.py index d1e79d9c..cd2a38f2 100644 --- a/sysom_server/sysom_hotfix/conf/common.py +++ b/sysom_server/sysom_hotfix/conf/common.py @@ -1,9 +1,7 @@ import os import datetime -import sys from pathlib import Path -from sysom_utils import ConfigParser, CecTarget -from clogger import logger +from sysom_utils import ConfigParser, CecTarget, SysomFramework BASE_DIR = Path(__file__).resolve().parent.parent @@ -16,6 +14,8 @@ YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) +SysomFramework.init(YAML_CONFIG) + ########################################################################################## # Django Config ########################################################################################## @@ -121,13 +121,4 @@ HOST_URL = 'http://127.0.0.1:7001/api/v1/host/' SYSOM_CEC_URL = YAML_CONFIG.get_cec_url(CecTarget.PRODUCER) SYSOM_CEC_HOTFIX_TOPIC = "hotfix_job" # channl_job SDK 需要的url -CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() - -################################################################## -# Config settings -################################################################## -# Config log format -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) +CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() \ No newline at end of file diff --git a/sysom_server/sysom_migration/conf/common.py b/sysom_server/sysom_migration/conf/common.py index c805ae13..eb6c1415 100644 --- a/sysom_server/sysom_migration/conf/common.py +++ b/sysom_server/sysom_migration/conf/common.py @@ -11,7 +11,7 @@ https://docs.djangoproject.com/en/3.2/ref/settings/ """ from pathlib import Path from clogger import logger -from sysom_utils import ConfigParser, CecTarget +from sysom_utils import ConfigParser, CecTarget, SysomFramework # Build paths inside the project like this: BASE_DIR / 'subdir'. @@ -25,6 +25,7 @@ YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) +SysomFramework.init(YAML_CONFIG) ########################################################################################## # Django Config @@ -168,12 +169,3 @@ REST_FRAMEWORK = { 'UNICODE_JSON': True, 'EXCEPTION_HANDLER': 'lib.exception.exception_handler' } - -################################################################## -# Config settings -################################################################## -# Config log format -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) diff --git a/sysom_server/sysom_monitor_server/conf/common.py b/sysom_server/sysom_monitor_server/conf/common.py index f3800c6c..807d89e8 100644 --- a/sysom_server/sysom_monitor_server/conf/common.py +++ b/sysom_server/sysom_monitor_server/conf/common.py @@ -6,8 +6,6 @@ Email mfeng@linux.alibaba.com File common.py Description: """ -from clogger import logger -import sys from pathlib import Path from sysom_utils import ConfigParser, SysomFramework @@ -29,13 +27,3 @@ SysomFramework.init(YAML_CONFIG) ################################################################## # channl_job SDK 需要的url CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() - - -################################################################## -# Logger settings -################################################################## -# Config log format -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) diff --git a/sysom_server/sysom_vmcore/conf/common.py b/sysom_server/sysom_vmcore/conf/common.py index 73be5e25..3ce2167c 100644 --- a/sysom_server/sysom_vmcore/conf/common.py +++ b/sysom_server/sysom_vmcore/conf/common.py @@ -9,11 +9,9 @@ https://docs.djangoproject.com/en/3.2/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.2/ref/settings/ """ -from clogger import logger -import sys import os from pathlib import Path -from sysom_utils import ConfigParser +from sysom_utils import ConfigParser, SysomFramework # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -26,6 +24,8 @@ YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) +SysomFramework.init(YAML_CONFIG) + ################################################################## # Cec settings ################################################################## @@ -156,12 +156,3 @@ REST_FRAMEWORK = { 'UNICODE_JSON': True, 'EXCEPTION_HANDLER': 'lib.exception.exception_handler' } - -################################################################## -# Config settings -################################################################## -# Config log format -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) diff --git a/sysom_server/sysom_vul/conf/common.py b/sysom_server/sysom_vul/conf/common.py index 9a1d96a0..e91d9d78 100644 --- a/sysom_server/sysom_vul/conf/common.py +++ b/sysom_server/sysom_vul/conf/common.py @@ -9,11 +9,9 @@ https://docs.djangoproject.com/en/3.2/topics/settings/ For the full list of settings and their values, see https://docs.djangoproject.com/en/3.2/ref/settings/ """ -import sys import os from pathlib import Path -from clogger import logger -from sysom_utils.config_parser import ConfigParser, CecTarget +from sysom_utils import ConfigParser, CecTarget, SysomFramework # Build paths inside the project like this: BASE_DIR / 'subdir'. BASE_DIR = Path(__file__).resolve().parent.parent @@ -25,6 +23,8 @@ YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) +SysomFramework.init(YAML_CONFIG) + # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = YAML_CONFIG.get_server_config().jwt.get("SECRET_KEY", "") @@ -108,7 +108,6 @@ USE_L10N = True USE_TZ = True - # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.2/howto/static-files/ @@ -158,17 +157,8 @@ CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() ################################################################## # Scan cve ################################################################## -INTERVAL_TIME = 60 * 10 #　second +INTERVAL_TIME = 60 * 10 # 　second HOST_SERVICE_NAME = 'host' HOST_SERVICE_PORT = 7001 SERVICE_URL = 'http://127.0.0.1' HOST_API_VERSION_ = '/api/v1/' - -################################################################## -# Config settings -################################################################## -# Config log format -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) -- Gitee From 07d3ba30207cc68060f205fb64210280cddd1240 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 16 May 2023 14:08:57 +0800 Subject: [PATCH 059/196] reafctor(channel): Move db dialect and engine config to config.yml --- conf/config.yml | 2 ++ sysom_server/sysom_channel/alembic/env.py | 7 ++----- sysom_server/sysom_channel/conf/common.py | 2 +- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/conf/config.yml b/conf/config.yml index 3ce52a57..b10e4045 100644 --- a/conf/config.yml +++ b/conf/config.yml @@ -19,6 +19,8 @@ sysom_server: username: password: mysql: + dialect: mariadb + engine: pymysql host: localhost port: 3306 user: sysom diff --git a/sysom_server/sysom_channel/alembic/env.py b/sysom_server/sysom_channel/alembic/env.py index 517ae502..8fcb7206 100644 --- a/sysom_server/sysom_channel/alembic/env.py +++ b/sysom_server/sysom_channel/alembic/env.py @@ -5,7 +5,7 @@ from sqlalchemy import engine_from_config from sqlalchemy import pool from app.models import Base from alembic import context -from conf.settings import YAML_CONFIG +from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL ################################################################## # Load yaml config first @@ -36,10 +36,7 @@ if config.config_file_name is not None: # Update mysql config according config.yml config.set_main_option( "sqlalchemy.url", - ( - f"mariadb+pymysql://{mysql_config.user}:{mysql_config.password}@" - f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" - ) + SQLALCHEMY_DATABASE_URL ) # add your model's MetaData object here diff --git a/sysom_server/sysom_channel/conf/common.py b/sysom_server/sysom_channel/conf/common.py index 49bfca61..32be2027 100644 --- a/sysom_server/sysom_channel/conf/common.py +++ b/sysom_server/sysom_channel/conf/common.py @@ -30,7 +30,7 @@ SysomFramework.init(YAML_CONFIG) # fastapi config ################################################################## SQLALCHEMY_DATABASE_URL = ( - f"mariadb+pymysql://{mysql_config.user}:{mysql_config.password}@" + f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" ) -- Gitee From be4ffed01ba6344ff03eac12765f90c20495c6e8 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 16 May 2023 16:36:55 +0800 Subject: [PATCH 060/196] feat(channel): Support Support for non-root users to manage nodes 1. Support for managing nodes with non-root users, require target user can run all command without password, you can edit "/etc/sudoers", add the following line. (replace to target username) ALL=(ALL) NOPASSWD: ALL 2. When channel_job sends a command, it supports specifying that the command does not require root privileges, and when the username is not root, sudo bash -c "" will not be used to wrapper command --- environment/1_sdk/channel_job/model.py | 5 +++ sysom_server/sysom_api/apps/host/heartbeat.py | 1 + sysom_server/sysom_channel/app/executor.py | 20 ++++++------ .../sysom_channel/app/routers/file.py | 32 +++++++++++++++---- sysom_server/sysom_channel/lib/ssh.py | 15 +++++++-- 5 files changed, 53 insertions(+), 20 deletions(-) diff --git a/environment/1_sdk/channel_job/model.py b/environment/1_sdk/channel_job/model.py index 61028493..ce37e13e 100644 --- a/environment/1_sdk/channel_job/model.py +++ b/environment/1_sdk/channel_job/model.py @@ -15,6 +15,7 @@ class JobEntry: CHANNEL_PARAMS_TIMEOUT = "__channel_params_timeout" CHANNEL_PARAMS_AUTO_RETRY = "__channel_params_auto_retry" CHANNEL_PARAMS_RETURN_AS_STREAM = "__channel_params_return_as_stream" + CHANNEL_PARAMS_NO_NEED_SUDO = "__channel_params_no_need_sudo" def __init__(self, channel_type: str = "ssh", channel_opt: str = "cmd", params: dict = {}, echo: dict = {}, @@ -28,6 +29,7 @@ class JobEntry: self.listen_topic = listen_topic self.timeout = kwargs.get("timeout", 1000) self.auto_retry = kwargs.get("auto_retry", False) + self.no_need_sudo = kwargs.get("no_need_sudo", False) # This field indicates whether the execution results need to # be returned as streamed self.return_as_stream = False @@ -41,6 +43,9 @@ class JobEntry: self.params[self.CHANNEL_PARAMS_AUTO_RETRY] = self.auto_retry if self.CHANNEL_PARAMS_RETURN_AS_STREAM not in self.params: self.params[self.CHANNEL_PARAMS_RETURN_AS_STREAM] = self.return_as_stream + if self.CHANNEL_PARAMS_NO_NEED_SUDO not in self.params and \ + self.no_need_sudo: + self.params[self.CHANNEL_PARAMS_NO_NEED_SUDO] = self.no_need_sudo result = { "channel": self.channel_type, "type": self.channel_opt, diff --git a/sysom_server/sysom_api/apps/host/heartbeat.py b/sysom_server/sysom_api/apps/host/heartbeat.py index 0694dc2d..fc034f89 100644 --- a/sysom_server/sysom_api/apps/host/heartbeat.py +++ b/sysom_server/sysom_api/apps/host/heartbeat.py @@ -110,6 +110,7 @@ class HeartBeatProcess(Process): } params['timeout'] = self._heartbeat_interval * 1000 params['auto_retry'] = True + params['no_need_sudo'] = True try: self._channel_job.dispatch_job(**params)\ diff --git a/sysom_server/sysom_channel/app/executor.py b/sysom_server/sysom_channel/app/executor.py index f4696857..44996d69 100644 --- a/sysom_server/sysom_channel/app/executor.py +++ b/sysom_server/sysom_channel/app/executor.py @@ -24,11 +24,7 @@ from app.crud import update_or_create_channel_params, get_channel_params_by_inst from app.database import SessionLocal from app import schemas from lib.channels.base import ChannelResult, ChannelException, ChannelCode - - -CHANNEL_PARAMS_TIMEOUT = "__channel_params_timeout" -CHANNEL_PARAMS_AUTO_RETRY = "__channel_params_auto_retry" -CHANNEL_PARAMS_RETURN_AS_STREAM = "__channel_params_return_as_stream" +from channel_job import JobEntry class ChannelListener(MultiConsumer): @@ -133,9 +129,11 @@ class ChannelListener(MultiConsumer): self._producer.flush() params = task.get("params", {}).copy() instance = params.get("instance", "") - timeout = params.pop(CHANNEL_PARAMS_TIMEOUT, None) - auto_retry = params.pop(CHANNEL_PARAMS_AUTO_RETRY, False) - return_as_stream = params.pop(CHANNEL_PARAMS_RETURN_AS_STREAM, False) + timeout = params.pop(JobEntry.CHANNEL_PARAMS_TIMEOUT, None) + auto_retry = params.pop(JobEntry.CHANNEL_PARAMS_AUTO_RETRY, False) + no_need_sudo = params.pop(JobEntry.CHANNEL_PARAMS_NO_NEED_SUDO, False) + return_as_stream = params.pop( + JobEntry.CHANNEL_PARAMS_RETURN_AS_STREAM, False) # Get params from sys_channel_params table if instance exists with SessionLocal() as db: @@ -148,6 +146,7 @@ class ChannelListener(MultiConsumer): res = await self._get_channel(channel_type)(**params).run_command_auto_retry_async( timeout=timeout, auto_retry=auto_retry, + no_need_sudo=no_need_sudo, on_data_received=on_data_received if return_as_stream else None ) return res @@ -156,8 +155,8 @@ class ChannelListener(MultiConsumer): """init opt""" params = task.get("params", {}).copy() instance = params.get("instance", "") - timeout = params.pop(CHANNEL_PARAMS_TIMEOUT, None) - auto_retry = params.pop(CHANNEL_PARAMS_AUTO_RETRY, False) + timeout = params.pop(JobEntry.CHANNEL_PARAMS_TIMEOUT, None) + auto_retry = params.pop(JobEntry.CHANNEL_PARAMS_AUTO_RETRY, False) res = await self._get_channel(channel_type).initial_async( **params, timeout=timeout, auto_retry=auto_retry ) @@ -165,6 +164,7 @@ class ChannelListener(MultiConsumer): # Save params after init channel success if res.code == 0: with SessionLocal() as db: + params.pop("password", "") update_or_create_channel_params(db, schemas.ChannelParams( instance=instance, params=json.dumps(params) diff --git a/sysom_server/sysom_channel/app/routers/file.py b/sysom_server/sysom_channel/app/routers/file.py index b04bdacb..9f46a835 100644 --- a/sysom_server/sysom_channel/app/routers/file.py +++ b/sysom_server/sysom_channel/app/routers/file.py @@ -6,12 +6,15 @@ Email mfeng@linux.alibaba.com File file.py Description: """ +import json from fastapi import APIRouter, File, UploadFile, Form from conf.settings import * import aiofiles import asyncio from lib.ssh import AsyncSSH -from app.schemas import RequestParamGetFileFromNode +from app.schemas import RequestParamGetFileFromNode, ChannelParams +from app.crud import get_channel_params_by_instance +from app.database import SessionLocal from starlette.responses import FileResponse, JSONResponse from starlette.background import BackgroundTask @@ -38,11 +41,18 @@ async def send_file_to_node( # 2. Initiate N job to pull uploaded files tasks = [] instances = target_instances.split(";") - for instance in instances: - tasks.append(AsyncSSH(instance).send_file_to_remote_async( - tmp_file.name, - target_path - )) + with SessionLocal() as db: + for instance in instances: + params = {} + params_instance = get_channel_params_by_instance(db, instance) + if params_instance is not None: + params = json.loads( + ChannelParams.from_orm(params_instance).params + ) + tasks.append(AsyncSSH(instance, **params).send_file_to_remote_async( + tmp_file.name, + target_path + )) # 3. Wait all scp task finish scp_result = await asyncio.gather(*tasks) @@ -70,7 +80,15 @@ async def get_file_from_node( param: RequestParamGetFileFromNode ): async with aiofiles.tempfile.NamedTemporaryFile(dir=TMP_DOWNLOAD_DIR, delete=False) as tmp_file: - err = await AsyncSSH(param.target_instance).get_file_from_remote_async(tmp_file.name, param.remote_path) + params = {} + with SessionLocal() as db: + params_instance = get_channel_params_by_instance(db, param.target_instance) + if params_instance is not None: + params = json.loads( + ChannelParams.from_orm(params_instance).params + ) + err = await AsyncSSH(param.target_instance, **params) \ + .get_file_from_remote_async(tmp_file.name, param.remote_path) tmp_file.flush() if err is None: return FileResponse( diff --git a/sysom_server/sysom_channel/lib/ssh.py b/sysom_server/sysom_channel/lib/ssh.py index d0fde7cd..81411dec 100644 --- a/sysom_server/sysom_channel/lib/ssh.py +++ b/sysom_server/sysom_channel/lib/ssh.py @@ -98,7 +98,7 @@ class AsyncSSH: command = f'mkdir -p -m 700 ~/.ssh && \ echo {public_key!r} >> ~/.ssh/authorized_keys && \ chmod 600 ~/.ssh/authorized_keys' - res = await self.run_command_async(command, timeout=timeout) + res = await self.run_command_async(command, timeout=timeout, no_need_sudo=True) if res.code != 0: raise ChannelException( f'Init {self._hostname} failed: {res.err_msg}', @@ -113,18 +113,23 @@ class AsyncSSH: self, command: str, timeout: Optional[int] = DEFAULT_CONNENT_TIMEOUT, on_data_received: Optional[Callable[[str, asyncssh.DataType], None]] = None, + **kwargs ) -> ChannelResult: return syncify(self.run_command_async, raise_sync_error=False)( command=command, timeout=timeout, - on_data_received=on_data_received + on_data_received=on_data_received, + **kwargs ) async def run_command_async( self, command: str, timeout: Optional[int] = DEFAULT_CONNENT_TIMEOUT, on_data_received: Optional[Callable[[str, asyncssh.DataType], None]] = None, + no_need_sudo: bool = False, ) -> ChannelResult: + if self.connect_args.get("username", "root") != "root" and not no_need_sudo: + command = f'sudo bash -c "{command}"' channel_result = ChannelResult( code=1, result="SSH Channel: Run command error", err_msg="SSH Channel: Run command error") try: @@ -184,7 +189,11 @@ class AsyncSSH: try: async with asyncssh.connect(self._hostname, **self.connect_args) as conn: if mode == "push": - await conn.run(f"mkdir -p {os.path.dirname(remote_path)}") + username = self.connect_args.get("username", "root") + if username != "root": + await conn.run(f"sudo mkdir -p {os.path.dirname(remote_path)} && sudo chown -R {username} {os.path.dirname(remote_path)}") + else: + await conn.run(f"mkdir -p {os.path.dirname(remote_path)}") await asyncssh.scp(local_path, (conn, remote_path)) else: await asyncssh.scp((conn, remote_path), local_path) -- Gitee From 571dbeb656e959e9edcb9959f46baa2fb1e7e7ea Mon Sep 17 00:00:00 2001 From: wb-672209 Date: Thu, 4 May 2023 15:26:40 +0800 Subject: [PATCH 061/196] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E4=B8=AD=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为前端页面增加《同步中》的kernelverison同步状态显示 去掉页面中的无用注释 --- .../pages/hotfix/Version/VersionConfig/KernelVersion.jsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx index 0eb34b18..f76f6f48 100644 --- a/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx +++ b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx @@ -66,13 +66,11 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { title: , dataIndex: 'source_devel', valueType: 'input', - // tooltip: '该操作系统类型的源码仓库地址', }, { title: , dataIndex: 'source_debuginfo', valueType: 'input', - // tooltip: '该操作系统类型的源码仓库地址', }, { title: , @@ -84,6 +82,8 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { return <>同步成功
{record.synced_at} }else if(record.sync_status === 1){ return <>同步失败
{record.synced_at} + }else if(record.sync_status === 0){ + return <>同步中
{record.synced_at} }else{ return <>- } @@ -130,9 +130,8 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { modalProps={{ destroyOnClose: true, onCancel: () => console.log('取消'), - }} + }} onFinish={async (values) => { - console.log(values,"ppooo"); const data = { id: record.id, os_type_name: values.os_type_name, -- Gitee From e2ad41b6f770a4931f0a2492b2328b333ec74cf6 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 17 May 2023 11:40:41 +0800 Subject: [PATCH 062/196] refactor(sysom_api): Use uvicorn to replace daphne --- environment/0_env/requirements.txt | 2 +- script/server/sysom_api/init.sh | 2 +- script/server/sysom_api/requirements.txt | 2 +- script/server/sysom_api/sysom-api.ini | 9 ++----- sysom_server/sysom_api/apps/host/heartbeat.py | 24 +++++++++---------- sysom_server/sysom_api/conf/gunicorn.py | 23 ++++++++++++++++++ sysom_server/sysom_api/config.yml | 1 + sysom_server/sysom_channel/conf/gunicorn.py | 7 +++++- sysom_server/sysom_channel/config.yml | 1 + 9 files changed, 48 insertions(+), 23 deletions(-) create mode 100644 sysom_server/sysom_api/conf/gunicorn.py diff --git a/environment/0_env/requirements.txt b/environment/0_env/requirements.txt index 012fd9a0..bc63b2c9 100644 --- a/environment/0_env/requirements.txt +++ b/environment/0_env/requirements.txt @@ -31,7 +31,7 @@ PyMySQL==1.0.2 requests==2.27.1 redis==4.3.4 schedule==1.1.0 -# uvicorn==0.16.0 +uvicorn[standard]==0.16.0 # wheel==0.37.1 # xlwt==1.3.0 # xlrd==2.0.1 diff --git a/script/server/sysom_api/init.sh b/script/server/sysom_api/init.sh index 09176fb0..423a187a 100644 --- a/script/server/sysom_api/init.sh +++ b/script/server/sysom_api/init.sh @@ -35,7 +35,7 @@ init_conf() { start_app() { ###if supervisor service started, we need use "supervisorctl update" to start new conf#### supervisorctl update - supervisorctl status ${SERVICE_NAME}:0 + supervisorctl status ${SERVICE_NAME} if [ $? -eq 0 ] then echo "${SERVICE_NAME} service start success..." diff --git a/script/server/sysom_api/requirements.txt b/script/server/sysom_api/requirements.txt index f57e6c2e..e45be157 100644 --- a/script/server/sysom_api/requirements.txt +++ b/script/server/sysom_api/requirements.txt @@ -6,8 +6,8 @@ django-apscheduler==0.6.2 django-redis==5.2.0 djangorestframework==3.14.0 drf-yasg==1.21.4 -daphne==3.0.2 paramiko==2.12.0 +uvicorn[standard]==0.16.0 pandas>=1.1.5 python-multipart==0.0.5 prometheus_client==0.16.0 diff --git a/script/server/sysom_api/sysom-api.ini b/script/server/sysom_api/sysom-api.ini index d3eebedb..2e7febed 100644 --- a/script/server/sysom_api/sysom-api.ini +++ b/script/server/sysom_api/sysom-api.ini @@ -1,13 +1,8 @@ -[fcgi-program:sysom-api] -socket=tcp://0.0.0.0:7001 +[program:sysom-api] directory=/usr/local/sysom/server/sysom_api -command=/usr/local/sysom/environment/virtualenv/bin/daphne -u /run/daphne%(process_num)d.sock --fd 0 --access-log /var/log/sysom/sysom-api-access.log --proxy-headers sysom.asgi:application -numprocs=4 -process_name=%(process_num)d +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom.asgi:application autostart=true autorestart=true -redirect_stderr=true -stopasgroup=true stdout_logfile=/var/log/sysom/sysom-api.log stderr_logfile=/var/log/sysom/sysom-api-error.log environment=PATH=/usr/local/sysom/virtualenv/bin:%(ENV_PATH)s diff --git a/sysom_server/sysom_api/apps/host/heartbeat.py b/sysom_server/sysom_api/apps/host/heartbeat.py index fc034f89..687e534c 100644 --- a/sysom_server/sysom_api/apps/host/heartbeat.py +++ b/sysom_server/sysom_api/apps/host/heartbeat.py @@ -58,16 +58,16 @@ class HeartBeatProcess(Process): # 2. Get total process num and idx # If get failed, retry it, maximum of 10 retries - for _ in range(10): - total_process_num, cur_idx, err_msg = get_total_process_num_and_idx() - if cur_idx != -1: - break - time.sleep(1) - if cur_idx != -1: - self._total_process_num = total_process_num - self._current_process_idx = cur_idx - else: - logger.error(err_msg) + # for _ in range(10): + # total_process_num, cur_idx, err_msg = get_total_process_num_and_idx() + # if cur_idx != -1: + # break + # time.sleep(1) + # if cur_idx != -1: + # self._total_process_num = total_process_num + # self._current_process_idx = cur_idx + # else: + # logger.error(err_msg) def _register_task(self): queryset = HostModel.objects.filter(status__in=[0, 2]) @@ -76,8 +76,8 @@ class HeartBeatProcess(Process): return for idx, instance in enumerate(queryset): # Each process handles part of the heartbeat task. - if idx % self._total_process_num == self._current_process_idx: - self._send_host_heart_beat(instance) + # if idx % self._total_process_num == self._current_process_idx: + self._send_host_heart_beat(instance) def run(self) -> None: logger.info(f'守护进程PID: {self.pid}') diff --git a/sysom_server/sysom_api/conf/gunicorn.py b/sysom_server/sysom_api/conf/gunicorn.py new file mode 100644 index 00000000..53a97adc --- /dev/null +++ b/sysom_server/sysom_api/conf/gunicorn.py @@ -0,0 +1,23 @@ +''' +API Service Gunicorn Settings +''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + +workers = 2 # 指定工作进程数 + +threads = 3 + +bind = f'{bind}:{port}' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/sysom-api-access.log' + +loglevel = 'error' + +proc_name = 'api_service' diff --git a/sysom_server/sysom_api/config.yml b/sysom_server/sysom_api/config.yml index 03feb2c9..a146e36a 100644 --- a/sysom_server/sysom_api/config.yml +++ b/sysom_server/sysom_api/config.yml @@ -16,6 +16,7 @@ sysom_service: service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 + bind: 127.0.0.1 port: 7001 heartbeat: HEARTBEAT_INTERVAL: 20 diff --git a/sysom_server/sysom_channel/conf/gunicorn.py b/sysom_server/sysom_channel/conf/gunicorn.py index 41957101..abda4abc 100644 --- a/sysom_server/sysom_channel/conf/gunicorn.py +++ b/sysom_server/sysom_channel/conf/gunicorn.py @@ -1,11 +1,16 @@ ''' Channel Service Gunicorn Settings ''' +from conf.settings import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + workers = 2 # 指定工作进程数 threads = 3 -bind = '0.0.0.0:7003' +bind = f'{bind}:{port}' worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 diff --git a/sysom_server/sysom_channel/config.yml b/sysom_server/sysom_channel/config.yml index b05b9b45..9e46dc46 100644 --- a/sysom_server/sysom_channel/config.yml +++ b/sysom_server/sysom_channel/config.yml @@ -12,6 +12,7 @@ sysom_service: service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 + bind: 127.0.0.1 port: 7003 framework: gcache: -- Gitee From 17f0540cfd67a9580fa7ed9823f691df6fc8dc29 Mon Sep 17 00:00:00 2001 From: wb-672209 Date: Thu, 27 Apr 2023 11:13:58 +0800 Subject: [PATCH 063/196] =?UTF-8?q?=E8=87=AA=E5=AE=9A=E4=B9=89=E5=86=85?= =?UTF-8?q?=E6=A0=B8=E7=89=88=E6=9C=AC=E8=AE=BE=E8=AE=A1=E5=89=8D=E7=AB=AF?= =?UTF-8?q?=E9=83=A8=E5=88=86=E5=BC=80=E5=8F=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 去掉代码中的无用注释 修改KernelVerison中的请求体中的数据来源 新增git_rule\source_devel\source_debuginfo三个请求体内字段 --- sysom_web/config/routes.js | 21 ++++- sysom_web/src/locales/zh-CN/menu.js | 4 +- sysom_web/src/locales/zh-CN/pages.js | 5 + .../VersionConfig}/KernelVersion.jsx | 92 +++++++++++++++---- .../hotfix/Version/VersionConfig/index.jsx | 19 ++++ .../VersionCustomize}/KernelConfig.jsx | 8 +- .../VersionCustomize}/index.jsx | 16 +--- .../hotfix/components/KernelConfigForm.jsx | 49 ++++++++-- .../hotfix/components/VersionConfigForm.jsx | 4 +- sysom_web/src/pages/hotfix/service.js | 27 ++++++ 10 files changed, 199 insertions(+), 46 deletions(-) rename sysom_web/src/pages/hotfix/{VersionConf => Version/VersionConfig}/KernelVersion.jsx (63%) create mode 100644 sysom_web/src/pages/hotfix/Version/VersionConfig/index.jsx rename sysom_web/src/pages/hotfix/{VersionConf => Version/VersionCustomize}/KernelConfig.jsx (95%) rename sysom_web/src/pages/hotfix/{VersionConf => Version/VersionCustomize}/index.jsx (59%) diff --git a/sysom_web/config/routes.js b/sysom_web/config/routes.js index 2449f0bd..d207487a 100644 --- a/sysom_web/config/routes.js +++ b/sysom_web/config/routes.js @@ -265,9 +265,24 @@ export default [ component: './hotfix/HotfixLog' }, { - path: '/hotfix/verion_config', - name: 'versionconf', - component: './hotfix/VersionConf' + path: '/hotfix/version', + name: 'version', + routes: [ + { + path: '/hotfix/version/config', + name: 'config', + component: './hotfix/Version/VersionConfig' + }, + { + path: '/hotfix/version/config', + redirect: '/hotfix/Version/VersionConfig', + }, + { + path: '/hotfix/version/customize', + name: 'customize', + component: './hotfix/Version/VersionCustomize', + }, + ] } ] }, diff --git a/sysom_web/src/locales/zh-CN/menu.js b/sysom_web/src/locales/zh-CN/menu.js index d5f542d4..0d3c7fde 100644 --- a/sysom_web/src/locales/zh-CN/menu.js +++ b/sysom_web/src/locales/zh-CN/menu.js @@ -101,6 +101,8 @@ export default { 'menu.hotfix': '热补丁中心', 'menu.hotfix.make': '热补丁制作', 'menu.hotfix.formal': '热补丁列表', - 'menu.hotfix.versionconf': '自定义内核版本配置', + 'menu.hotfix.version': '自定义内核版本配置', 'menu.hotfix.selfdefine': '自定义热补丁制作', + 'menu.hotfix.version.config': '操作系统配置', + 'menu.hotfix.version.customize': '内核版本配置', }; \ No newline at end of file diff --git a/sysom_web/src/locales/zh-CN/pages.js b/sysom_web/src/locales/zh-CN/pages.js index c4b345a8..e839e485 100644 --- a/sysom_web/src/locales/zh-CN/pages.js +++ b/sysom_web/src/locales/zh-CN/pages.js @@ -178,6 +178,7 @@ export default { 'pages.hotfix.building_status': '构建状态', 'pages.hotfix.formal': '转正式包', 'pages.hotfix.os_type': '操作系统名', + 'pages.hotfix.kernel_repo_git': '源码仓库地址', 'pages.hotfix.patch_name': 'patch文件名称', 'pages.hotfix.upload': '文件上传', 'pages.hotfix.download': '下载', @@ -189,6 +190,10 @@ export default { 'pages.hotfix.title.versionconf': '自定义内核版本配置', 'pages.hotfix.image': '构建镜像', 'pages.hotfix.hotfix_name': "热补丁名称", + 'pages.hotfix.version_repo_devel': 'devel仓库地址', + 'pages.hotfix.version_repo_debuginfo': 'debuginfo仓库地址', + 'pages.hotfix.synch_information': '同步信息', + 'pages.hotfix.synchronization': '同步', 'pages.account.account_list': '账号列表', 'pages.account.username': '用户名', 'pages.account.password': '密码', diff --git a/sysom_web/src/pages/hotfix/VersionConf/KernelVersion.jsx b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx similarity index 63% rename from sysom_web/src/pages/hotfix/VersionConf/KernelVersion.jsx rename to sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx index 308735af..0eb34b18 100644 --- a/sysom_web/src/pages/hotfix/VersionConf/KernelVersion.jsx +++ b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx @@ -2,13 +2,14 @@ import React, { useRef, useState, useEffect } from 'react'; import { useIntl, FormattedMessage } from 'umi'; import { PageContainer } from '@ant-design/pro-layout'; import ProTable from '@ant-design/pro-table'; -import { Popconfirm, message, Upload, Button, Select, Form, Switch} from 'antd'; -import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeOsType } from '../service'; -import { ProForm, ModalForm, ProFormText, ProFormTextArea, ProFormSelect } from '@ant-design/pro-form'; +import { Popconfirm, message, Upload, Button, Select, Form, Switch, Tag} from 'antd'; +import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeOsType, PostWhetherSyncRequest } from '../../service'; +import { ProForm, ModalForm, ProFormText, ProFormTextArea, ProFormSelect, ProFormSwitch } from '@ant-design/pro-form'; import { async } from '@antv/x6/lib/registry/marker/async'; import { PropertySafetyFilled } from '@ant-design/icons'; -import KernelConfigForm from '../components/KernelConfigForm' +import KernelConfigForm from '../../components/KernelConfigForm' import ProCard from '@ant-design/pro-card'; +import CompoundedSpace from 'antd/lib/space'; const { Divider } = ProCard; @@ -36,9 +37,6 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { const oslistRef = useRef(); const intl = useIntl(); const [count, setCount] = useState(0); - useEffect(()=>{ - getOSTypeList(); - },[]); const onPostTask = () => { oslistRef.current.reload(); } @@ -47,6 +45,8 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { title: , dataIndex: 'os_type', key: 'os_type', + width: 200, + ellipsis: true, dataIndex: 'os_type', tooltip: '操作系统名，请为您该系列的操作系统类型命名' }, @@ -62,12 +62,46 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { valueType: 'input', tooltip: '输入该类操作系统构建热补丁时使用的镜像,如不填写则使用默认提供的Anolis镜像' }, + { + title: , + dataIndex: 'source_devel', + valueType: 'input', + // tooltip: '该操作系统类型的源码仓库地址', + }, + { + title: , + dataIndex: 'source_debuginfo', + valueType: 'input', + // tooltip: '该操作系统类型的源码仓库地址', + }, + { + title: , + dataIndex: 'sync_status', + valueType: 'input', + width: 180, + render: (_, record) => { + if(record.sync_status === 2){ + return <>同步成功
{record.synced_at} + }else if(record.sync_status === 1){ + return <>同步失败
{record.synced_at} + }else{ + return <>- + } + } + }, { title: , key: 'option', dataIndex: 'option', + width: 160, valueType: 'option', render: (_, record) => [ + + { + await PostWhetherSyncRequest(record.id); + oslistRef.current.reload(); + }}> + , { if (record.id == undefined) { @@ -98,11 +132,15 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { onCancel: () => console.log('取消'), }} onFinish={async (values) => { + console.log(values,"ppooo"); const data = { id: record.id, os_type_name: values.os_type_name, git_repo_link: values.git_repo_link, - image: values.building_image + image: values.building_image, + git_rule: values.git_rule, + source_devel: values.source_devel, + source_debuginfo: values.source_debuginfo } await postChangeOsType(data); message.success('提交成功'); @@ -127,13 +165,21 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { placeholder="请输入源码包地址" initialValue={record.source_repo} /> : - + <> + + + } { placeholder="请输入构建镜像" initialValue={record.image} /> + + ], @@ -150,7 +210,7 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { ]; return ( <> - + { + const actionRef = useRef(); + const refNetListTable = useRef(); + const intl = useIntl(); + + return ( + + + + ); +}; + +export default KernelVersionList; \ No newline at end of file diff --git a/sysom_web/src/pages/hotfix/VersionConf/KernelConfig.jsx b/sysom_web/src/pages/hotfix/Version/VersionCustomize/KernelConfig.jsx similarity index 95% rename from sysom_web/src/pages/hotfix/VersionConf/KernelConfig.jsx rename to sysom_web/src/pages/hotfix/Version/VersionCustomize/KernelConfig.jsx index ce656043..8c62a67b 100644 --- a/sysom_web/src/pages/hotfix/VersionConf/KernelConfig.jsx +++ b/sysom_web/src/pages/hotfix/Version/VersionCustomize/KernelConfig.jsx @@ -3,9 +3,9 @@ import { useIntl, FormattedMessage } from 'umi'; import { PageContainer } from '@ant-design/pro-layout'; import ProTable from '@ant-design/pro-table'; import { Popconfirm, message, Upload, Button, Select, Form} from 'antd'; -import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeKernelVersion } from '../service'; +import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeKernelVersion } from '../../service'; import { ProForm, ModalForm, ProFormText, ProFormTextArea, ProFormSelect } from '@ant-design/pro-form'; -import VersionConfigForm from '../components/VersionConfigForm' +import VersionConfigForm from '../../components/VersionConfigForm' import ProCard from '@ant-design/pro-card'; const { Divider } = ProCard; @@ -133,7 +133,7 @@ const VersionConfigList = React.forwardRef((props, ref) => { placeholder="请输入名称" initialValue={record.kernel_version} /> - + @@ -144,7 +144,7 @@ const VersionConfigList = React.forwardRef((props, ref) => { ]; return ( <> - + { const actionRef = useRef(); - const refNetListTable = useRef(); const intl = useIntl(); const [dataostype, setDataOsType] = useState([]); - const [dataoptionlist, setDataOptionList] = useState([]); - - const callback = (count) => { - const dr = [{label: count[0].os_type, value: count[0].os_type}]; - setDataOsType(dr); - } useEffect(()=>{ DataOptionList(); @@ -29,14 +21,12 @@ const KernelVersionConfigList = () => { arr.push({label: i.os_type,value: i.os_type}) }) } - setDataOptionList({arr:arr}); + setDataOsType({arr:arr}); } return ( - -
- + ); }; diff --git a/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx b/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx index 4ed28ed5..66698949 100644 --- a/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx +++ b/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx @@ -10,18 +10,22 @@ import { Switch } from 'antd'; export default (props, ref) => { const kernelformRef = useRef(); const [readonlyone, setReadonlyOne] = useState(false); + const [synchronization, setSynchronization] = useState(false); const { loading, error, run } = useRequest(submitOSType, { manual: true, onSuccess: (result, params) => { kernelformRef.current.resetFields(); props?.onSuccess?.(result, params); - props.parentBack(params); }, }); const KernelConfigChange = (e) => { setReadonlyOne(e); } + const ConfigSynchronizationChange = (e) => { + setSynchronization(e); + + } return ( { tooltip="该操作系统类型的源码包仓库地址" label="源码包地址" /> : - + <> + + + } { tooltip="输入该类操作系统构建热补丁时使用的镜像,如不填写则使用默认提供的Anolis镜像" label="构建镜像" /> + + { unCheckedChildren="否" onChange={KernelConfigChange} /> + diff --git a/sysom_web/src/pages/hotfix/components/VersionConfigForm.jsx b/sysom_web/src/pages/hotfix/components/VersionConfigForm.jsx index c53c8460..3f376121 100644 --- a/sysom_web/src/pages/hotfix/components/VersionConfigForm.jsx +++ b/sysom_web/src/pages/hotfix/components/VersionConfigForm.jsx @@ -9,6 +9,7 @@ import { submitKernelVersion, getOSTypeList } from '../service' export default (props) => { const formRef = useRef(); const [veroptionList,setVerOptionList] = useState([]); + const [dataostype, setDataOsType] = useState([]); const { loading, error, run } = useRequest(submitKernelVersion, { manual: true, onSuccess: (result, params) => { @@ -16,7 +17,6 @@ export default (props) => { props?.onSuccess?.(result, params); }, }); - return ( { Date: Thu, 4 May 2023 15:26:40 +0800 Subject: [PATCH 064/196] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=90=8C=E6=AD=A5?= =?UTF-8?q?=E4=B8=AD=E7=8A=B6=E6=80=81?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 为前端页面增加《同步中》的kernelverison同步状态显示 去掉页面中的无用注释 --- .../pages/hotfix/Version/VersionConfig/KernelVersion.jsx | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx index 0eb34b18..f76f6f48 100644 --- a/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx +++ b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx @@ -66,13 +66,11 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { title: , dataIndex: 'source_devel', valueType: 'input', - // tooltip: '该操作系统类型的源码仓库地址', }, { title: , dataIndex: 'source_debuginfo', valueType: 'input', - // tooltip: '该操作系统类型的源码仓库地址', }, { title: , @@ -84,6 +82,8 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { return <>同步成功
{record.synced_at} }else if(record.sync_status === 1){ return <>同步失败
{record.synced_at} + }else if(record.sync_status === 0){ + return <>同步中
{record.synced_at} }else{ return <>- } @@ -130,9 +130,8 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { modalProps={{ destroyOnClose: true, onCancel: () => console.log('取消'), - }} + }} onFinish={async (values) => { - console.log(values,"ppooo"); const data = { id: record.id, os_type_name: values.os_type_name, -- Gitee From 9242ba53a3683d9e225b28f699e8757b69bb6611 Mon Sep 17 00:00:00 2001 From: wb-cp908446 Date: Thu, 4 May 2023 17:46:46 +0800 Subject: [PATCH 065/196] develop function of kernel sync function This commit update the function of customize kernel version auto sync. For user who config their customize kernel version, our program now will sync their kernel repo for each kernel version auto. --- .../sysom_hotfix/apps/hotfix/function.py | 2 +- .../migrations/0003_auto_20230504_1712.py | 38 +++ .../sysom_hotfix/apps/hotfix/models.py | 5 + sysom_server/sysom_hotfix/apps/hotfix/urls.py | 1 + .../sysom_hotfix/apps/hotfix/views.py | 219 +++++++++++++++++- sysom_server/sysom_hotfix/conf/common.py | 15 +- 6 files changed, 275 insertions(+), 5 deletions(-) create mode 100644 sysom_server/sysom_hotfix/apps/hotfix/migrations/0003_auto_20230504_1712.py diff --git a/sysom_server/sysom_hotfix/apps/hotfix/function.py b/sysom_server/sysom_hotfix/apps/hotfix/function.py index bbda5d75..495b3202 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/function.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/function.py @@ -43,7 +43,7 @@ class FunctionClass(): if kernel_version is not None: objects = objects.filter(kernel_version=kernel_version) if patch_file is not None: - objects = objects.filter(patch_file=patch_file) + objects = objects.filter(patch_file__contains=patch_file) if hotfix_name is not None: objects = objects.filter(hotfix_name=hotfix_name) return objects diff --git a/sysom_server/sysom_hotfix/apps/hotfix/migrations/0003_auto_20230504_1712.py b/sysom_server/sysom_hotfix/apps/hotfix/migrations/0003_auto_20230504_1712.py new file mode 100644 index 00000000..41b137b0 --- /dev/null +++ b/sysom_server/sysom_hotfix/apps/hotfix/migrations/0003_auto_20230504_1712.py @@ -0,0 +1,38 @@ +# Generated by Django 3.2.16 on 2023-05-04 09:12 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('hotfix', '0002_auto_20230303_1601'), + ] + + operations = [ + migrations.AddField( + model_name='ostypemodel', + name='git_rule', + field=models.CharField(default='', max_length=255, verbose_name='该系列的源码git正则规则'), + ), + migrations.AddField( + model_name='ostypemodel', + name='source_debuginfo', + field=models.CharField(default='', max_length=255, verbose_name='该系列的debuginfo仓库地址'), + ), + migrations.AddField( + model_name='ostypemodel', + name='source_devel', + field=models.CharField(default='', max_length=255, verbose_name='该系列的devel仓库地址'), + ), + migrations.AddField( + model_name='ostypemodel', + name='sync_status', + field=models.IntegerField(blank=True, null=True, verbose_name='同步状态'), + ), + migrations.AddField( + model_name='ostypemodel', + name='synced_at', + field=models.CharField(blank=True, max_length=255, null=True, verbose_name='同步时间'), + ), + ] diff --git a/sysom_server/sysom_hotfix/apps/hotfix/models.py b/sysom_server/sysom_hotfix/apps/hotfix/models.py index 60cc370e..12045152 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/models.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/models.py @@ -36,7 +36,12 @@ class OSTypeModel(BaseModel): os_type = models.CharField(max_length=25, verbose_name="操作系统类型") source_repo = models.CharField(default="",max_length=255, verbose_name="源码仓库地址") image = models.CharField(max_length=255, verbose_name="该系列的构建容器镜像") + git_rule = models.CharField(max_length=255, default="", verbose_name="该系列的源码git正则规则") + source_devel = models.CharField(max_length=255, default="", verbose_name="该系列的devel仓库地址") + source_debuginfo = models.CharField(max_length=255, default="", verbose_name="该系列的debuginfo仓库地址") + sync_status = models.IntegerField(null=True, blank=True, verbose_name="同步状态") src_pkg_mark = models.BooleanField(default=0, verbose_name="是否使用src包") + synced_at = models.CharField(max_length=255,null=True, blank=True, verbose_name="同步时间") class Meta: db_table = "sys_hotfix_ostype" diff --git a/sysom_server/sysom_hotfix/apps/hotfix/urls.py b/sysom_server/sysom_hotfix/apps/hotfix/urls.py index de5e4673..0269846a 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/urls.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/urls.py @@ -19,6 +19,7 @@ urlpatterns = [ path('api/v1/hotfix/insert_building_log/', views.HotfixAPIView.as_view({'post': 'insert_building_log'})), path('api/v1/hotfix/get_build_log/', views.HotfixAPIView.as_view({'get': 'get_build_log'})), path('api/v1/hotfix/sync_building_log/', views.HotfixAPIView.as_view({'post': 'sync_build_log'})), + path('api/v1/hotfix/sync_kernel/', views.HotfixAPIView.as_view({'post': 'sync_kernel_by_os_type'})), path('api/v1/hotfix/update_hotfix_name/', views.HotfixAPIView.as_view({'post': 'update_hotfix_name'})), path('api/v1/hotfix/download_hotfix/', views.HotfixAPIView.as_view({'get': 'download_hotfix_file'})), path('api/v1/hotfix/create_os_type_relation/', views.HotfixAPIView.as_view({'post': 'insert_os_type_relation'})), diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 2f2afa00..8ee2d5b9 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -1,7 +1,9 @@ +import threading from clogger import logger import re import os import time +import urllib from typing import Any from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -32,7 +34,9 @@ from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer from cec_base.event import Event from django.http import HttpResponse, FileResponse -from lib.function import FunctionClass, RedisCache +from apps.hotfix.function import FunctionClass +from bs4 import BeautifulSoup +from git import repo class SaveUploadFile(APIView): authentication_classes = [] @@ -366,6 +370,11 @@ class HotfixAPIView(GenericViewSet, os_type = request.data["os_type"] source_repo = request.data["source_repo"] src_pkg_mark = request.data["src_pkg_mark"] + git_rule = request.data.get("git_rule", None) + source_devel = request.data["source_devel"] + source_debuginfo = request.data["source_debuginfo"] + sync_conf = request.data.get("sync_conf", None) + logger.info(sync_conf) try: image = request.data['image'] except Exception as e: @@ -378,8 +387,18 @@ class HotfixAPIView(GenericViewSet, os_type = os_type, source_repo = source_repo, image = image, - src_pkg_mark = src_pkg_mark + src_pkg_mark = src_pkg_mark, + git_rule = git_rule, + source_devel = source_devel, + source_debuginfo = source_debuginfo ) + if sync_conf: + if os_type_object is None: + return other_response(msg="can not find the OS type record", code=400) + if os_type_object.sync_status == 0: + return other_response(msg="Synchronizing, please wait for synchronization to complete", code=400) + thread_runner = threading.Thread(target=self.sync_kernel, name="sync_kernel",args=(os_type_object.id,)) + thread_runner.start() else: return other_response(message="same OS Type found in record..", code=400) except Exception as e: @@ -483,14 +502,210 @@ class HotfixAPIView(GenericViewSet, os_type_object.os_type = request.data['os_type'] os_type_object.source_repo = request.data['source_repo'] os_type_object.image = request.data['image'] + os_type_object.source_devel = request.data['source_devel'] + os_type_object.source_debuginfo = request.data['source_debuginfo'] src_pkg_mark = request.data.get("src_pkg_mark", None) if src_pkg_mark: os_type_object.src_pkg_mark = request.data["src_pkg_mark"] + else: + os_type_object.git_rule = request.data['git_rule'] os_type_object.save() except Exception as e: return other_response(msg=str(e), code=400) return success(result={"msg":"successfully update os_type object"},message="invoke update_ostype") + def update_ostype_sync_status(self, id, status): + try: + os_type_object = OSTypeModel.objects.all().filter(id=id).first() + if os_type_object is None: + return other_response(msg="can not find the OS type record", code=400) + if status == 2: + print(human_datetime()) + os_type_object.synced_at = human_datetime() + os_type_object.sync_status = status + os_type_object.save() + except Exception as e: + return other_response(msg=str(e), code=400) + return success(result={"msg":"successfully update os_type object"},message="invoke update_ostype") + + def sync_kernel_by_os_type(self, request): + id=request.data.get("id") + os_type_object = OSTypeModel.objects.all().filter(id=id).first() + if os_type_object is None: + return other_response(msg="can not find the OS type record", code=400) + if os_type_object.sync_status == 0: + return other_response(msg="Synchronizing, please wait for synchronization to complete", code=400) + thread_runner=threading.Thread(target=self.sync_kernel, name="sync_kernel",args=(id,)) + thread_runner.start() + print(id) + return success(result={"msg":"success trigger sync kernel"}) + + def sync_kernel(self, id): + try: + os_type_object = OSTypeModel.objects.all().filter(id=id).first() + src_pkg_mark = os_type_object.src_pkg_mark + if src_pkg_mark: + return self.sync_source(id, os_type_object.os_type, os_type_object.source_repo, os_type_object.source_devel, os_type_object.source_debuginfo) + else: + return self.sync_git(id, os_type_object.os_type, os_type_object.source_repo, os_type_object.source_devel, os_type_object.source_debuginfo, os_type_object.git_rule) + except Exception as e: + logger.error(e) + return other_response(msg=str(e), code=400) + + def sync_git(self, id, os_type, git_repo, source_devel, source_debuginfo, git_rule): + try: + self.update_ostype_sync_status(id=id, status=0) + source_devel_list = source_devel.split(",") + x86_devel_source = list(filter(lambda x: "x86_64" in x, source_devel_list))[0] + arm_devel_source = list(filter(lambda x: "aarch64" in x, source_devel_list))[0] + x86_source_devel_lists = self.get_git_devel_list(x86_devel_source) + arm_source_devel_lists = self.get_git_devel_list(arm_devel_source) + source_debuginfo_list = source_debuginfo.split(",") + x86_debuginfo_source = list(filter(lambda x: "x86_64" in x, source_debuginfo_list))[0] + arm_debuginfo_source = list(filter(lambda x: "aarch64" in x, source_debuginfo_list))[0] + self.insert_kernel_version_relation_git(os_type, git_repo, git_rule, x86_devel_source, x86_source_devel_lists, x86_debuginfo_source) + self.insert_kernel_version_relation_git(os_type, git_repo, git_rule, arm_devel_source, arm_source_devel_lists, arm_debuginfo_source) + self.update_ostype_sync_status(id=id, status=2) + logger.info("successfully sync kernel") + return success(result={"msg":"successfully sync os_type"}) + except Exception as e: + logger.error(e) + self.update_ostype_sync_status(id=id, status=1) + return other_response(msg=str(e), code=400) + + def insert_kernel_version_relation_git(self, os_type, git_repo, git_rule, x86_devel_source, source_devel_list, x86_debuginfo_source): + for kernel_devel_rpm in source_devel_list: + version = ".".join(kernel_devel_rpm.replace("kernel-devel-", '').split(".")[:-1]) + matchobj = re.match(git_rule, version) + if matchobj: + branch = matchobj.group() + if not branch: + logger.error("branch of version {} not found".format(version)) + continue + source = git_repo + '/' + branch + x86_debuginfo_link = self.get_debuginfo_rpm(x86_debuginfo_source, version) + x86_debuginfo_link = x86_debuginfo_source + x86_debuginfo_link + x86_develinfo_link = x86_devel_source + kernel_devel_rpm + self.insert_kernel_version_relation_internal(kernel_version=version,os_type=os_type, + source=source, devel_link=x86_develinfo_link,debuginfo_link=x86_debuginfo_link) + + def sync_source(self, id, os_type, source_repo, source_devel, source_debuginfo): + try: + self.update_ostype_sync_status(id=id, status=0) + source_lists = self.get_source_list(source_repo) + for kernel_rpm in source_lists: + version = ".".join(kernel_rpm.replace("kernel-", '').split(".")[:-1]).replace(".src", "") + kernel_url = source_repo + kernel_rpm + source_devel_list = source_devel.split(",") + x86_devel = list(filter(lambda x: "x86_64" in x, source_devel_list))[0] + source_debuginfo_list = source_debuginfo.split(",") + x86_debuginfo = list(filter(lambda x: "x86_64" in x, source_debuginfo_list))[0] + aarch64_devel = list(filter(lambda x: "aarch64" in x, source_devel_list))[0] + aarch64_debuginfo = list(filter(lambda x: "aarch64" in x, source_debuginfo_list))[0] + x86_devel_link = self.get_devel_rpm(x86_devel, version) + x86_devel_link = x86_devel+ x86_devel_link + x86_debuginfo_link = self.get_debuginfo_rpm(x86_debuginfo, version) + x86_debuginfo_link = x86_debuginfo + x86_debuginfo_link + aarch64devel_link = self.get_devel_rpm(aarch64_devel, version) + aarch64devel_link = aarch64_devel + aarch64devel_link + aarch64debuginfo_link = self.get_debuginfo_rpm(aarch64_debuginfo, version) + aarch64debuginfo_link = aarch64_debuginfo + aarch64debuginfo_link + self.insert_kernel_version_relation_internal(kernel_version=version+".x86_64",os_type=os_type, + source=kernel_url, devel_link=aarch64devel_link,debuginfo_link=aarch64debuginfo_link) + self.insert_kernel_version_relation_internal(kernel_version=version+".aarch64",os_type=os_type, + source=kernel_url, devel_link=x86_devel_link,debuginfo_link=x86_debuginfo_link) + self.update_ostype_sync_status(id=id, status=2) + logger.info("msg:successfully sync kernel") + return success(result={"msg":"successfully sync os_type"}) + except Exception as e: + logger.error(e) + self.update_ostype_sync_status(id=id, status=1) + return other_response(msg=str(e), code=400) + + + def insert_kernel_version_relation_internal(self,kernel_version, os_type, source, devel_link, debuginfo_link): + logger.info("start kernel is :%s" %kernel_version) + kernel_object = KernelVersionModel.objects.all().filter(kernel_version=kernel_version).first() + if kernel_object is None: + logger.info("start insert kernel is :%s" %kernel_version) + kernel_object = KernelVersionModel.objects.create( + kernel_version = kernel_version, + os_type=os_type, + source = source, + devel_link = devel_link, + debuginfo_link = debuginfo_link + ) + else: + logger.info("same kernel version found in record...") + return other_response(msg="same kernel version found in record...") + + def get_source_list(self, url): + try: + response = urllib.request.urlopen(url) + html = response.read() + soup = BeautifulSoup(html, "html.parser") + linklist = soup.tbody.select('a') + linkrpmlists = [] + for kernel_rpm in linklist: + kernel_rpm = kernel_rpm.get_text().strip() + if re.findall('^kernel-\d', kernel_rpm): + linkrpmlists.append(kernel_rpm) + logger.info(linkrpmlists) + return linkrpmlists + except Exception as e: + logger.error(e) + return [] + + def get_git_devel_list(self, url): + try: + logger.info(url) + response = urllib.request.urlopen(url) + html = response.read() + soup = BeautifulSoup(html, "html.parser") + linklist = soup.tbody.select('a') + linkrpmlists = [] + for kernel_rpm in linklist: + kernel_rpm = kernel_rpm.get_text().strip() + if re.findall('^kernel-devel-\d', kernel_rpm): + linkrpmlists.append(kernel_rpm) + # logger.info(linkrpmlists) + return linkrpmlists + except Exception as e: + logger.error(e) + return [] + + def get_devel_rpm(self, source_devel, version): + try: + # logger.info("source_devel is :%s" % source_devel) + # logger.info("version is :%s" % version) + response = urllib.request.urlopen(source_devel) + html = response.read() + soup = BeautifulSoup(html, "html.parser") + linklist = soup.tbody.select('a') + for kernel_rpm in linklist: + kernel_rpm = kernel_rpm.get_text().strip() + if re.findall('^kernel-devel-\d', kernel_rpm) and version in kernel_rpm: + return kernel_rpm + return None + except Exception as e: + logger.error(e) + return None + + def get_debuginfo_rpm(self, source_debuginfo, version): + try: + response = urllib.request.urlopen(source_debuginfo) + html = response.read() + soup = BeautifulSoup(html, "html.parser") + linklist = soup.tbody.select('a') + for kernel_rpm in linklist: + kernel_rpm = kernel_rpm.get_text().strip() + if re.findall('^kernel-debuginfo-\d', kernel_rpm) and version in kernel_rpm: + return kernel_rpm + return None + except Exception as e: + logger.error(e) + return None + class HealthViewset(CommonModelViewSet): def health_check(self, request, *args, **kwargs): return success(result={}) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/conf/common.py b/sysom_server/sysom_hotfix/conf/common.py index cd2a38f2..d30b8329 100644 --- a/sysom_server/sysom_hotfix/conf/common.py +++ b/sysom_server/sysom_hotfix/conf/common.py @@ -109,7 +109,7 @@ MEDIA_ROOT = os.path.join(BASE_DIR, 'uploads') # Hotfix Platform settings ################################################################## HOTFIX_FILE_STORAGE_REPO = "/usr/local/sysom/server/hotfix_builder/hotfix-nfs" - +HOTFIX_BASE = "/hotfix_build/hotfix" ################################################################## # Server Host settings ################################################################## @@ -119,6 +119,17 @@ HOST_URL = 'http://127.0.0.1:7001/api/v1/host/' # Cec settings ################################################################## SYSOM_CEC_URL = YAML_CONFIG.get_cec_url(CecTarget.PRODUCER) +CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() SYSOM_CEC_HOTFIX_TOPIC = "hotfix_job" # channl_job SDK 需要的url -CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() \ No newline at end of file +CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() +SYSOM_CEC_HOTFIX_SERVER_MSG_TOPIC = "hotfix_msg" + +################################################################## +# Config settings +################################################################## +# Config log format +log_format = YAML_CONFIG.get_server_config().logger.format +log_level = YAML_CONFIG.get_server_config().logger.level +logger.set_format(log_format) +logger.set_level(log_level) -- Gitee From 007754e659a32d964c75eb234fd4b7834c22c096 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 17 May 2023 14:19:27 +0800 Subject: [PATCH 066/196] refactor(framework): Move service's bind and port define to config 1. Move all services's bind and port config to service/config.yml 2. Use Uvincorn work to deploy services --- script/server/sysom_diagnosis/sysom-diagnosis.ini | 2 +- script/server/sysom_hotfix/sysom-hotfix.ini | 2 +- script/server/sysom_migration/sysom-migration.ini | 2 +- script/server/sysom_vmcore/sysom-vmcore.ini | 2 +- script/server/sysom_vul/sysom-vul.ini | 2 +- sysom_server/sysom_diagnosis/conf/gunicorn.py | 9 +++++++-- sysom_server/sysom_diagnosis/config.yml | 1 + sysom_server/sysom_hotfix/conf/gunicorn.py | 9 +++++++-- sysom_server/sysom_hotfix/config.yml | 1 + sysom_server/sysom_migration/conf/gunicorn.py | 9 +++++++-- sysom_server/sysom_migration/config.yml | 1 + sysom_server/sysom_monitor_server/conf/gunicorn.py | 7 ++++++- sysom_server/sysom_monitor_server/config.yml | 1 + sysom_server/sysom_vmcore/conf/gunicorn.py | 9 +++++++-- sysom_server/sysom_vmcore/config.yml | 1 + sysom_server/sysom_vul/conf/gunicorn.py | 9 +++++++-- sysom_server/sysom_vul/config.yml | 1 + 17 files changed, 52 insertions(+), 16 deletions(-) diff --git a/script/server/sysom_diagnosis/sysom-diagnosis.ini b/script/server/sysom_diagnosis/sysom-diagnosis.ini index ac848752..25579a64 100644 --- a/script/server/sysom_diagnosis/sysom-diagnosis.ini +++ b/script/server/sysom_diagnosis/sysom-diagnosis.ini @@ -1,6 +1,6 @@ [program:sysom-diagnosis] directory=/usr/local/sysom/server/sysom_diagnosis -command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_diagnosis.wsgi:application +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_diagnosis.asgi:application startsecs=3 autostart=true autorestart=true diff --git a/script/server/sysom_hotfix/sysom-hotfix.ini b/script/server/sysom_hotfix/sysom-hotfix.ini index d8b0a597..e3f1b324 100644 --- a/script/server/sysom_hotfix/sysom-hotfix.ini +++ b/script/server/sysom_hotfix/sysom-hotfix.ini @@ -1,6 +1,6 @@ [program:sysom-hotfix] directory = /usr/local/sysom/server/sysom_hotfix -command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_hotfix.wsgi:application +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_hotfix.asgi:application startsecs=3 autostart=true autorestart=true diff --git a/script/server/sysom_migration/sysom-migration.ini b/script/server/sysom_migration/sysom-migration.ini index 064b1852..feb252b8 100644 --- a/script/server/sysom_migration/sysom-migration.ini +++ b/script/server/sysom_migration/sysom-migration.ini @@ -1,6 +1,6 @@ [program:sysom-migration] directory = /usr/local/sysom/server/sysom_migration -command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_migration.wsgi:application +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_migration.asgi:application startsecs=3 autostart=true autorestart=true diff --git a/script/server/sysom_vmcore/sysom-vmcore.ini b/script/server/sysom_vmcore/sysom-vmcore.ini index 9d4263eb..fcdb24cc 100644 --- a/script/server/sysom_vmcore/sysom-vmcore.ini +++ b/script/server/sysom_vmcore/sysom-vmcore.ini @@ -1,6 +1,6 @@ [program:sysom-vmcore] directory = /usr/local/sysom/server/sysom_vmcore -command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vmcore.wsgi:application +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vmcore.asgi:application startsecs=3 autostart=true autorestart=true diff --git a/script/server/sysom_vul/sysom-vul.ini b/script/server/sysom_vul/sysom-vul.ini index efd23c60..18ef6a2c 100644 --- a/script/server/sysom_vul/sysom-vul.ini +++ b/script/server/sysom_vul/sysom-vul.ini @@ -1,6 +1,6 @@ [program:sysom-vul] directory = /usr/local/sysom/server/sysom_vul -command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vul.wsgi:application +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py sysom_vul.asgi:application startsecs=3 autostart=true autorestart=true diff --git a/sysom_server/sysom_diagnosis/conf/gunicorn.py b/sysom_server/sysom_diagnosis/conf/gunicorn.py index 70d0098d..2150d2e6 100644 --- a/sysom_server/sysom_diagnosis/conf/gunicorn.py +++ b/sysom_server/sysom_diagnosis/conf/gunicorn.py @@ -1,13 +1,18 @@ ''' Diagnosis Service Gunicorn Settings ''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + workers = 2 # 指定工作进程数 threads = 3 -bind = '0.0.0.0:7002' +bind = f'{bind}:{port}' -# worker_class = 'gevent' # 工作模式线程, 默认为sync模式 +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) diff --git a/sysom_server/sysom_diagnosis/config.yml b/sysom_server/sysom_diagnosis/config.yml index 799f69de..33504e0f 100644 --- a/sysom_server/sysom_diagnosis/config.yml +++ b/sysom_server/sysom_diagnosis/config.yml @@ -17,6 +17,7 @@ sysom_service: service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 + bind: 127.0.0.1 port: 7002 framework: gcache: diff --git a/sysom_server/sysom_hotfix/conf/gunicorn.py b/sysom_server/sysom_hotfix/conf/gunicorn.py index 07db94d5..f0bd34ab 100644 --- a/sysom_server/sysom_hotfix/conf/gunicorn.py +++ b/sysom_server/sysom_hotfix/conf/gunicorn.py @@ -1,13 +1,18 @@ ''' Hotfix Service Gunicorn Settings ''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + workers = 2 # 指定工作进程数 threads = 3 -bind = '0.0.0.0:7007' +bind = f'{bind}:{port}' -# worker_class = 'gevent' # 工作模式线程, 默认为sync模式 +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) diff --git a/sysom_server/sysom_hotfix/config.yml b/sysom_server/sysom_hotfix/config.yml index 4e312518..08d3e700 100644 --- a/sysom_server/sysom_hotfix/config.yml +++ b/sysom_server/sysom_hotfix/config.yml @@ -16,6 +16,7 @@ sysom_service: service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 + bind: 127.0.0.1 port: 7007 framework: gcache: diff --git a/sysom_server/sysom_migration/conf/gunicorn.py b/sysom_server/sysom_migration/conf/gunicorn.py index 52639f43..c22a7284 100644 --- a/sysom_server/sysom_migration/conf/gunicorn.py +++ b/sysom_server/sysom_migration/conf/gunicorn.py @@ -1,13 +1,18 @@ ''' Migration Service Gunicorn Settings ''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + workers = 4 # 指定工作进程数 threads = 8 -bind = '0.0.0.0:7006' +bind = f'{bind}:{port}' -# worker_class = 'gevent' # 工作模式线程, 默认为sync模式 +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) diff --git a/sysom_server/sysom_migration/config.yml b/sysom_server/sysom_migration/config.yml index 863b560a..6ac34c92 100644 --- a/sysom_server/sysom_migration/config.yml +++ b/sysom_server/sysom_migration/config.yml @@ -16,6 +16,7 @@ sysom_service: service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 + bind: 127.0.0.1 port: 7006 framework: gcache: diff --git a/sysom_server/sysom_monitor_server/conf/gunicorn.py b/sysom_server/sysom_monitor_server/conf/gunicorn.py index f846eba5..895be76a 100644 --- a/sysom_server/sysom_monitor_server/conf/gunicorn.py +++ b/sysom_server/sysom_monitor_server/conf/gunicorn.py @@ -1,11 +1,16 @@ ''' Monitor server Service Gunicorn Settings ''' +from conf.settings import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + workers = 2 # 指定工作进程数 threads = 3 -bind = '0.0.0.0:7009' +bind = f'{bind}:{port}' worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 diff --git a/sysom_server/sysom_monitor_server/config.yml b/sysom_server/sysom_monitor_server/config.yml index 4d7da764..b7a62e13 100644 --- a/sysom_server/sysom_monitor_server/config.yml +++ b/sysom_server/sysom_monitor_server/config.yml @@ -18,6 +18,7 @@ sysom_service: service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 + bind: 127.0.0.1 port: 7009 framework: gcache: diff --git a/sysom_server/sysom_vmcore/conf/gunicorn.py b/sysom_server/sysom_vmcore/conf/gunicorn.py index 3274a317..849725cb 100644 --- a/sysom_server/sysom_vmcore/conf/gunicorn.py +++ b/sysom_server/sysom_vmcore/conf/gunicorn.py @@ -1,13 +1,18 @@ ''' Vmcore Service Gunicorn Settings ''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + workers = 2 # 指定工作进程数 threads = 3 -bind = '0.0.0.0:7004' +bind = f'{bind}:{port}' -# worker_class = 'gevent' # 工作模式线程, 默认为sync模式 +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) diff --git a/sysom_server/sysom_vmcore/config.yml b/sysom_server/sysom_vmcore/config.yml index 9871ebdc..1a4f6ab8 100644 --- a/sysom_server/sysom_vmcore/config.yml +++ b/sysom_server/sysom_vmcore/config.yml @@ -16,6 +16,7 @@ sysom_service: service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 + bind: 127.0.0.1 port: 7004 framework: gcache: diff --git a/sysom_server/sysom_vul/conf/gunicorn.py b/sysom_server/sysom_vul/conf/gunicorn.py index 15162c5c..765f9c15 100644 --- a/sysom_server/sysom_vul/conf/gunicorn.py +++ b/sysom_server/sysom_vul/conf/gunicorn.py @@ -1,15 +1,20 @@ ''' Vul Service Gunicorn Settings ''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + workers = 2 # 指定工作进程数 threads = 3 -bind = '0.0.0.0:7005' +bind = f'{bind}:{port}' timeout = 90 -# worker_class = 'gevent' # 工作模式线程, 默认为sync模式 +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) diff --git a/sysom_server/sysom_vul/config.yml b/sysom_server/sysom_vul/config.yml index 1f34f5fe..9a04927f 100644 --- a/sysom_server/sysom_vul/config.yml +++ b/sysom_server/sysom_vul/config.yml @@ -16,6 +16,7 @@ sysom_service: service_dir: *SERVICE_NAME protocol: http host: 127.0.0.1 + bind: 127.0.0.1 port: 7005 framework: gcache: -- Gitee From 04e1e3ca8f87a0e7ef9ac64a064fc0b5f38e8c4f Mon Sep 17 00:00:00 2001 From: wb-cp908446 Date: Fri, 5 May 2023 15:04:29 +0800 Subject: [PATCH 067/196] Bug fix: Remove the invalid assignment and the useless code. Sync os_type and update with human_datetime remove logger from common file --- sysom_server/sysom_hotfix/apps/hotfix/views.py | 14 +++++--------- sysom_server/sysom_hotfix/conf/common.py | 8 -------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 8ee2d5b9..2b75e148 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -437,7 +437,7 @@ class HotfixAPIView(GenericViewSet, queryset = OSTypeModel.objects.all().filter(deleted_at=None) response = serializer.OSTypeSerializer(queryset, many=True) except Exception as e: - print(str(e)) + logger.error(e) return other_response(message=str(e), result={"msg":"get_os_type_relation failed"}, code=400) return success(result=response.data, message="get_os_type_relation") @@ -446,7 +446,7 @@ class HotfixAPIView(GenericViewSet, queryset = KernelVersionModel.objects.all().filter(deleted_at=None) response = serializer.KernelSerializer(queryset, many=True) except Exception as e: - print(str(e)) + logger.error(e) return other_response(message=str(e), result={"msg":"get_kernel_relation failed"}, code=400) return success(result=response.data, message="get_kernel_relation") @@ -519,9 +519,7 @@ class HotfixAPIView(GenericViewSet, os_type_object = OSTypeModel.objects.all().filter(id=id).first() if os_type_object is None: return other_response(msg="can not find the OS type record", code=400) - if status == 2: - print(human_datetime()) - os_type_object.synced_at = human_datetime() + os_type_object.synced_at = human_datetime() os_type_object.sync_status = status os_type_object.save() except Exception as e: @@ -537,7 +535,6 @@ class HotfixAPIView(GenericViewSet, return other_response(msg="Synchronizing, please wait for synchronization to complete", code=400) thread_runner=threading.Thread(target=self.sync_kernel, name="sync_kernel",args=(id,)) thread_runner.start() - print(id) return success(result={"msg":"success trigger sync kernel"}) def sync_kernel(self, id): @@ -577,6 +574,7 @@ class HotfixAPIView(GenericViewSet, for kernel_devel_rpm in source_devel_list: version = ".".join(kernel_devel_rpm.replace("kernel-devel-", '').split(".")[:-1]) matchobj = re.match(git_rule, version) + branch = None if matchobj: branch = matchobj.group() if not branch: @@ -668,7 +666,7 @@ class HotfixAPIView(GenericViewSet, kernel_rpm = kernel_rpm.get_text().strip() if re.findall('^kernel-devel-\d', kernel_rpm): linkrpmlists.append(kernel_rpm) - # logger.info(linkrpmlists) + logger.info(linkrpmlists) return linkrpmlists except Exception as e: logger.error(e) @@ -676,8 +674,6 @@ class HotfixAPIView(GenericViewSet, def get_devel_rpm(self, source_devel, version): try: - # logger.info("source_devel is :%s" % source_devel) - # logger.info("version is :%s" % version) response = urllib.request.urlopen(source_devel) html = response.read() soup = BeautifulSoup(html, "html.parser") diff --git a/sysom_server/sysom_hotfix/conf/common.py b/sysom_server/sysom_hotfix/conf/common.py index d30b8329..4651c16c 100644 --- a/sysom_server/sysom_hotfix/conf/common.py +++ b/sysom_server/sysom_hotfix/conf/common.py @@ -125,11 +125,3 @@ SYSOM_CEC_HOTFIX_TOPIC = "hotfix_job" CHANNEL_JOB_URL = YAML_CONFIG.get_local_channel_job_url() SYSOM_CEC_HOTFIX_SERVER_MSG_TOPIC = "hotfix_msg" -################################################################## -# Config settings -################################################################## -# Config log format -log_format = YAML_CONFIG.get_server_config().logger.format -log_level = YAML_CONFIG.get_server_config().logger.level -logger.set_format(log_format) -logger.set_level(log_level) -- Gitee From b223652b0527e4273669effaff56fdae3db831e1 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 17 May 2023 16:19:22 +0800 Subject: [PATCH 068/196] feat(template): Support one-click microservice creation Just used ./sysom.sh create server to create new microservice --- script/sysom.sh | 4 + script/sysom_create.sh | 141 ++++++++++++++++++ template/README.md | 71 +++++++++ template/fastapi/scripts/clear.sh | 8 + template/fastapi/scripts/db_migrate.sh | 18 +++ template/fastapi/scripts/init.sh | 54 +++++++ template/fastapi/scripts/requirements.txt | 19 +++ template/fastapi/scripts/start.sh | 7 + template/fastapi/scripts/stop.sh | 7 + template/fastapi/scripts/template.ini | 9 ++ template/fastapi/service/alembic.ini | 102 +++++++++++++ template/fastapi/service/alembic/README | 1 + template/fastapi/service/alembic/env.py | 110 ++++++++++++++ .../fastapi/service/alembic/script.py.mako | 24 +++ template/fastapi/service/app/__init__.py | 8 + template/fastapi/service/app/crud.py | 30 ++++ template/fastapi/service/app/database.py | 20 +++ template/fastapi/service/app/models.py | 22 +++ .../fastapi/service/app/routers/health.py | 21 +++ template/fastapi/service/app/schemas.py | 22 +++ template/fastapi/service/conf/common.py | 33 ++++ template/fastapi/service/conf/develop.py | 15 ++ template/fastapi/service/conf/gunicorn.py | 23 +++ template/fastapi/service/conf/product.py | 15 ++ template/fastapi/service/conf/settings.py | 19 +++ template/fastapi/service/conf/testing.py | 14 ++ template/fastapi/service/config.yml | 35 +++++ template/fastapi/service/lib/README.md | 1 + template/fastapi/service/main.py | 46 ++++++ 29 files changed, 899 insertions(+) create mode 100644 script/sysom_create.sh create mode 100644 template/README.md create mode 100644 template/fastapi/scripts/clear.sh create mode 100644 template/fastapi/scripts/db_migrate.sh create mode 100644 template/fastapi/scripts/init.sh create mode 100644 template/fastapi/scripts/requirements.txt create mode 100644 template/fastapi/scripts/start.sh create mode 100644 template/fastapi/scripts/stop.sh create mode 100644 template/fastapi/scripts/template.ini create mode 100644 template/fastapi/service/alembic.ini create mode 100644 template/fastapi/service/alembic/README create mode 100644 template/fastapi/service/alembic/env.py create mode 100644 template/fastapi/service/alembic/script.py.mako create mode 100644 template/fastapi/service/app/__init__.py create mode 100644 template/fastapi/service/app/crud.py create mode 100644 template/fastapi/service/app/database.py create mode 100644 template/fastapi/service/app/models.py create mode 100644 template/fastapi/service/app/routers/health.py create mode 100644 template/fastapi/service/app/schemas.py create mode 100644 template/fastapi/service/conf/common.py create mode 100644 template/fastapi/service/conf/develop.py create mode 100644 template/fastapi/service/conf/gunicorn.py create mode 100644 template/fastapi/service/conf/product.py create mode 100644 template/fastapi/service/conf/settings.py create mode 100644 template/fastapi/service/conf/testing.py create mode 100644 template/fastapi/service/config.yml create mode 100644 template/fastapi/service/lib/README.md create mode 100644 template/fastapi/service/main.py diff --git a/script/sysom.sh b/script/sysom.sh index ae2ae147..3e141e37 100755 --- a/script/sysom.sh +++ b/script/sysom.sh @@ -143,6 +143,10 @@ sub_dbmigrate() { bash +x ${BaseDir}/sysom_dbmigrate.sh $@ } +sub_create() { + bash +x ${BaseDir}/sysom_create.sh $@ +} + sub_list() { echo "list" } diff --git a/script/sysom_create.sh b/script/sysom_create.sh new file mode 100644 index 00000000..d3b40edc --- /dev/null +++ b/script/sysom_create.sh @@ -0,0 +1,141 @@ +#!/bin/bash -x +set -x +ProgName=$(basename $0) +BaseDir=$(dirname $(readlink -f "$0")) + +#################################################################################################################### +# Helper functions +#################################################################################################################### + +red() { + printf '\33[1;31m%b\n\33[0m' "$1" +} + +green() { + printf '\33[1;32m%b\n\33[0m' "$1" +} + +check_port_conflict() { + port=$1 + + # scan all service + local_microservice_dir=`dirname $BaseDir`/sysom_server + for service in $(ls ${local_microservice_dir}) + do + service_config_dir=${local_microservice_dir}/${service}/config.yml + if [ ! -f "${service_config_dir}" ]; then + continue + fi + _service_port=`cat $service_config_dir | grep 'sysom_service:' -A 6 | awk '/port/ {print $2}'` + if [ "${port}" == "${_service_port}" ]; then + red "Error: ${port} already used by ${service}" + exit 1 + fi + done +} + +check_service_name_conflict() { + service_name=$1 + local_microservice_dir=`dirname $BaseDir`/sysom_server + for service in $(ls ${local_microservice_dir}) + do + if [ "${service_name}" == "${service}" ];then + red "Error: ${service} already exists" + exit 1 + fi + done +} + +do_fill_template() { + service_name_shorter=$3 + service_name=sysom_$3 + service_middle=sysom-$3 + + for file in `ls $1` + do + if [ -d $1"/"$file ] + then + do_fill_template $1"/"$file $2 $3 + else + target_file=$1"/"$file + sed "s/\$TEMPLATE_SERVICE_NAME_SHORTER/$service_name_shorter/g" -i $target_file + sed "s/\$TEMPLATE_SERVICE_NAME_MIDDLE/$service_middle/g" -i $target_file + sed "s/\$TEMPLATE_SERVICE_NAME/$service_name/g" -i $target_file + sed "s/\$TEMPLATE_SERVICE_PORT/$2/g" -i $target_file + fi + done +} + + +#################################################################################################################### +# Subcommands +#################################################################################################################### + +sub_help(){ + echo "Usage: $ProgName [options]" + echo "Subcommands:" + echo " server Create one new server(microservice)" + echo " Example: $ProgName server demo" + echo "" + echo "For help with each subcommand run:" + echo "$ProgName -h|--help" + echo "" +} + +sub_server() { + sub_microservice $@ +} + +sub_service() { + sub_microservice $@ +} + +sub_microservice() { + name=$1 + service_port=$2 + service_name_shorter=$name + service_name=sysom_$name + service_middle=sysom-$name + + green $service_port + + # Check port conflict + check_port_conflict $service_port + + # Check service_name conflict + check_service_name_conflict $service_name + + # Copy template file to sysom_service and script/server + target_script_dir=`dirname $BaseDir`/script/server/${service_name} + target_service_dir=`dirname $BaseDir`/sysom_server/${service_name} + pushd `dirname $BaseDir`/template/fastapi + cp -r scripts $target_script_dir + cp -r service $target_service_dir + popd + + # Fill service and port info to template + do_fill_template $target_script_dir $service_port $name + do_fill_template $target_service_dir $service_port $name + + # rename supervisor conf + pushd $target_script_dir + mv template.ini $service_middle.ini + popd +} + + +subcommand=$1 +case $subcommand in + "" | "-h" | "--help") + sub_help + ;; + *) + shift + sub_${subcommand} $@ + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac \ No newline at end of file diff --git a/template/README.md b/template/README.md new file mode 100644 index 00000000..d39fa57a --- /dev/null +++ b/template/README.md @@ -0,0 +1,71 @@ +# 微服务模板 + +## 1. 快速新建微服务 +```bash +# 在 sysom/script 下调用如下命令 +./sysom.sh create server <微服务名称> <微服务监听端口> +``` +> 注意： +> - 微服务名称不用带 sysom_ 前缀，会自动补充； +> - 微服务的名称不能和现有的微服务冲突 => 如果冲突创建过程会报错提示 +> - 微服务监听端口不能和现有的微服务冲突 => 如果冲突创建过程会报错提示 + +比如创建一个名称为 demo 的微服务可以执行下列命令： +```bash +./sysom.sh create server demo 7010 +``` + +执行后的目录结果如下： +``` +├── conf +│ └── config.yml +├── cookie +├── deps +├── docker +├── docs +├── environment +├── LICENSE +├── package_rpm_offline.sh +├── package_rpm_online.sh +├── package.sh +├── README.md +├── republish.sh +├── script +│ ├── deploy + ├── server + │ ├── clear_exclude + │ ├── clear.sh + │ ├── deploy_exclude + │ ├── init.sh + │ ├── sysom_api + │ ├── sysom_channel + │ ├── sysom_demo => 新增 demo 微服务的部署脚本 + │ ├── sysom_diagnosis + │ ├── sysom_hotfix + │ ├── sysom_hotfix_builder + │ ├── sysom_migration + │ ├── sysom_monitor_server + │ ├── sysom_vmcore + │ └── sysom_vul +│ ├── sysom_clear.sh +│ ├── sysom_create.sh +│ ├── sysom_dbmigrate.sh +│ ├── sysom_deploy.sh +│ └── sysom.sh +├── sysom_server +│ ├── clear_exclude +│ ├── conf +│ ├── deploy_exclude +│ ├── sysom_api +│ ├── sysom_channel +│ ├── sysom_demo => 新增 demo 微服务目录 +│ ├── sysom_diagnosis +│ ├── sysom_hotfix +│ ├── sysom_hotfix_builder +│ ├── sysom_migration +│ ├── sysom_monitor_server +│ ├── sysom_vmcore +│ └── sysom_vul +├── sysom_web +├── template +``` \ No newline at end of file diff --git a/template/fastapi/scripts/clear.sh b/template/fastapi/scripts/clear.sh new file mode 100644 index 00000000..71e088b3 --- /dev/null +++ b/template/fastapi/scripts/clear.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} +clear_app diff --git a/template/fastapi/scripts/db_migrate.sh b/template/fastapi/scripts/db_migrate.sh new file mode 100644 index 00000000..f530e1f8 --- /dev/null +++ b/template/fastapi/scripts/db_migrate.sh @@ -0,0 +1,18 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + alembic upgrade head + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/template/fastapi/scripts/init.sh b/template/fastapi/scripts/init.sh new file mode 100644 index 00000000..a93249c3 --- /dev/null +++ b/template/fastapi/scripts/init.sh @@ -0,0 +1,54 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME +SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +install_requirement() { + pushd ${SERVICE_SCRIPT_HOME} + pip install -r requirements.txt + popd +} + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +init_conf() { + pushd ${SERVICE_HOME} + alembic upgrade head + popd + + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini +} + +start_app() { + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ] + then + echo "supervisorctl start ${SERVICE_NAME} success..." + return 0 + fi + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 +} + +deploy() { + source_virtualenv + install_requirement + init_conf + start_app +} + +deploy diff --git a/template/fastapi/scripts/requirements.txt b/template/fastapi/scripts/requirements.txt new file mode 100644 index 00000000..8c2a17df --- /dev/null +++ b/template/fastapi/scripts/requirements.txt @@ -0,0 +1,19 @@ +clogger==0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +sysom_utils>=0.0.1 +alembic==1.7.7 +anyio==3.6.2 +asyncer==0.0.2 +asyncssh==2.12.0 +fastapi==0.83.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +uvicorn==0.16.0 +gunicorn==20.1.0 +python-multipart==0.0.5 +###################################################################### +# Add your custom python requirements here +###################################################################### \ No newline at end of file diff --git a/template/fastapi/scripts/start.sh b/template/fastapi/scripts/start.sh new file mode 100644 index 00000000..e6c8d1fa --- /dev/null +++ b/template/fastapi/scripts/start.sh @@ -0,0 +1,7 @@ +#!/bin/bash +SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE +start_app() { + supervisorctl start $SERVICE_NAME +} + +start_app diff --git a/template/fastapi/scripts/stop.sh b/template/fastapi/scripts/stop.sh new file mode 100644 index 00000000..4b514ad1 --- /dev/null +++ b/template/fastapi/scripts/stop.sh @@ -0,0 +1,7 @@ +#!/bin/bash +SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE +stop_app() { + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/template/fastapi/scripts/template.ini b/template/fastapi/scripts/template.ini new file mode 100644 index 00000000..7cd748ed --- /dev/null +++ b/template/fastapi/scripts/template.ini @@ -0,0 +1,9 @@ +[program:$TEMPLATE_SERVICE_NAME_MIDDLE] +directory=/usr/local/sysom/server/$TEMPLATE_SERVICE_NAME +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +startsecs=3 +autostart=true +autorestart=true +environment=PATH="/usr/local/sysom/environment/virtualenv/bin/" +stderr_logfile=/var/log/sysom/$TEMPLATE_SERVICE_NAME_MIDDLE-error.log +stdout_logfile=/var/log/sysom/$TEMPLATE_SERVICE_NAME_MIDDLE.log diff --git a/template/fastapi/service/alembic.ini b/template/fastapi/service/alembic.ini new file mode 100644 index 00000000..f6ab9feb --- /dev/null +++ b/template/fastapi/service/alembic.ini @@ -0,0 +1,102 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = "" + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/template/fastapi/service/alembic/README b/template/fastapi/service/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/template/fastapi/service/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/template/fastapi/service/alembic/env.py b/template/fastapi/service/alembic/env.py new file mode 100644 index 00000000..8fcb7206 --- /dev/null +++ b/template/fastapi/service/alembic/env.py @@ -0,0 +1,110 @@ +import inspect +import app.models as models +from logging.config import fileConfig +from sqlalchemy import engine_from_config +from sqlalchemy import pool +from app.models import Base +from alembic import context +from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL + +################################################################## +# Load yaml config first +################################################################## +mysql_config = YAML_CONFIG.get_server_config().db.mysql + +################################################################## +# Scan models +################################################################## +service_tables = [] +for name, data in inspect.getmembers(models, inspect.isclass): + if data.__module__ != "app.models": + continue + if "__tablename__" in data.__dict__: + service_tables.append(data.__dict__["__tablename__"]) + elif "__table__" in data.__dict__: + service_tables.append(data.__dict__["__table__"]) + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Update mysql config according config.yml +config.set_main_option( + "sqlalchemy.url", + SQLALCHEMY_DATABASE_URL +) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def include_object(object, name, type_, reflected, compare_to): + if type_ == "table" and name not in service_tables: + return False + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + include_object=include_object, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, + include_object=include_object + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/template/fastapi/service/alembic/script.py.mako b/template/fastapi/service/alembic/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/template/fastapi/service/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/template/fastapi/service/app/__init__.py b/template/fastapi/service/app/__init__.py new file mode 100644 index 00000000..3fb0576a --- /dev/null +++ b/template/fastapi/service/app/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py +Description: +""" \ No newline at end of file diff --git a/template/fastapi/service/app/crud.py b/template/fastapi/service/app/crud.py new file mode 100644 index 00000000..6e0e26c3 --- /dev/null +++ b/template/fastapi/service/app/crud.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File crud.py +Description: +""" +from typing import Optional +from sqlalchemy.orm import Session +from app import models, schemas + +################################################################################################ +# Define database crud here +################################################################################################ + +# def get_person_by_name(db: Session, name: str) -> Optional[models.Person]: +# return db.query(models.Person).filter(models.Person.name == name).first() + +# def create_person(db: Session, person: schemas.Person) -> models.Person: +# person = models.Person(**person.dict()) +# db.add(person) +# db.commit() +# db.refresh(person) +# return person + +# def del_person_by_id(db: Session, person_id: int): +# person = db.get(models.Person, person_id) +# db.delete(person) +# db.commit() diff --git a/template/fastapi/service/app/database.py b/template/fastapi/service/app/database.py new file mode 100644 index 00000000..d1500828 --- /dev/null +++ b/template/fastapi/service/app/database.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File database.py +Description: +""" +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from conf.settings import SQLALCHEMY_DATABASE_URL + +engine = create_engine( + SQLALCHEMY_DATABASE_URL, connect_args={} +) + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() \ No newline at end of file diff --git a/template/fastapi/service/app/models.py b/template/fastapi/service/app/models.py new file mode 100644 index 00000000..60fa74d4 --- /dev/null +++ b/template/fastapi/service/app/models.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File models.py +Description: +""" +from sqlalchemy import Column, Integer, String +from app.database import Base + + +########################################################################### +# Define databse model here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/sql-databases/ +# class Person(Base): +# __tablename__ = "sys_person" +# id = Column(Integer, primary_key=True) +# name = Column(String(254), unique=True) +# age = Column(Integer) \ No newline at end of file diff --git a/template/fastapi/service/app/routers/health.py b/template/fastapi/service/app/routers/health.py new file mode 100644 index 00000000..9b6f43fc --- /dev/null +++ b/template/fastapi/service/app/routers/health.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/04/17 19:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get("/check") +async def get_channel_config(): + return { + "code": 0, + "err_msg": "", + "data": "" + } diff --git a/template/fastapi/service/app/schemas.py b/template/fastapi/service/app/schemas.py new file mode 100644 index 00000000..aa413a9d --- /dev/null +++ b/template/fastapi/service/app/schemas.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File schemas.py +Description: +""" +from pydantic import BaseModel + +########################################################################### +# Define schemas here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/response-model/ +# class Person(BaseModel): +# id: int +# name: str +# age: int + +# class Config: +# orm_mode = True \ No newline at end of file diff --git a/template/fastapi/service/conf/common.py b/template/fastapi/service/conf/common.py new file mode 100644 index 00000000..1b38077e --- /dev/null +++ b/template/fastapi/service/conf/common.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" +from pathlib import Path +from sysom_utils import ConfigParser, SysomFramework + +BASE_DIR = Path(__file__).resolve().parent.parent + +################################################################## +# Load yaml config first +################################################################## +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" + +YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) + +mysql_config = YAML_CONFIG.get_server_config().db.mysql +service_config = YAML_CONFIG.get_service_config() + +SysomFramework.init(YAML_CONFIG) + +################################################################## +# fastapi config +################################################################## +SQLALCHEMY_DATABASE_URL = ( + f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" + f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" +) \ No newline at end of file diff --git a/template/fastapi/service/conf/develop.py b/template/fastapi/service/conf/develop.py new file mode 100644 index 00000000..bb64bf49 --- /dev/null +++ b/template/fastapi/service/conf/develop.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File develoop.py +Description: +""" +from .common import * + +''' +开发环境配置项 +''' + +DEBUG = True \ No newline at end of file diff --git a/template/fastapi/service/conf/gunicorn.py b/template/fastapi/service/conf/gunicorn.py new file mode 100644 index 00000000..edca8d44 --- /dev/null +++ b/template/fastapi/service/conf/gunicorn.py @@ -0,0 +1,23 @@ +''' +Channel Service Gunicorn Settings +''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + +workers = 2 # 指定工作进程数 + +threads = 3 + +bind = f'{bind}:{port}' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/$TEMPLATE_SERVICE_NAME_MIDDLE-access.log' + +loglevel = 'error' + +proc_name = 'channel_service' diff --git a/template/fastapi/service/conf/product.py b/template/fastapi/service/conf/product.py new file mode 100644 index 00000000..c748de1d --- /dev/null +++ b/template/fastapi/service/conf/product.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File product.py +Description: +""" +from .common import * + +''' +生产环境配置项 +''' + +DEBUG = False diff --git a/template/fastapi/service/conf/settings.py b/template/fastapi/service/conf/settings.py new file mode 100644 index 00000000..b9156df0 --- /dev/null +++ b/template/fastapi/service/conf/settings.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File settings.py +Description: +""" +import os + +env = os.environ.get("env", "product") + + +if env == "develop": + from .develop import * +elif env == "testing": + from .testing import * +elif env == "product": + from .product import * \ No newline at end of file diff --git a/template/fastapi/service/conf/testing.py b/template/fastapi/service/conf/testing.py new file mode 100644 index 00000000..864f53a2 --- /dev/null +++ b/template/fastapi/service/conf/testing.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File testing.py +Description: +""" +from .common import * + +''' +测试环境配置项 +''' +DEBUG = True diff --git a/template/fastapi/service/config.yml b/template/fastapi/service/config.yml new file mode 100644 index 00000000..ba929083 --- /dev/null +++ b/template/fastapi/service/config.yml @@ -0,0 +1,35 @@ +vars: + SERVICE_NAME: &SERVICE_NAME $TEMPLATE_SERVICE_NAME + SERVICE_CONSUMER_GROUP: + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] + +sysom_server: + cec: + consumer_group: SYSOM_CEC_CHANNEL_CONSUMER_GROUP + +sysom_service: + service_name: *SERVICE_NAME + service_dir: *SERVICE_NAME + protocol: http + host: 127.0.0.1 + bind: 127.0.0.1 + port: $TEMPLATE_SERVICE_PORT + framework: + gcache: + protocol: redis + node_dispatch: + cmg: + tags: + - $TEMPLATE_SERVICE_NAME_SHORTER + - FastApi + # Metadata of service + metadata: + check: + type: http + url: "/api/v1/$TEMPLATE_SERVICE_NAME_SHORTER/health/check" + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false + diff --git a/template/fastapi/service/lib/README.md b/template/fastapi/service/lib/README.md new file mode 100644 index 00000000..3ec74424 --- /dev/null +++ b/template/fastapi/service/lib/README.md @@ -0,0 +1 @@ +The current directory holds the public libraries or utils needed for microservices \ No newline at end of file diff --git a/template/fastapi/service/main.py b/template/fastapi/service/main.py new file mode 100644 index 00000000..19041020 --- /dev/null +++ b/template/fastapi/service/main.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File ssh.py +Description: +""" +from clogger import logger +from fastapi import FastAPI +from app.routers import health +from conf.settings import YAML_CONFIG +from sysom_utils import CmgPlugin, SysomFramework + + +app = FastAPI() + +app.include_router(health.router, prefix="/api/v1/$TEMPLATE_SERVICE_NAME_SHORTER/health") + + +############################################################################# +# Write your API interface here, or add to app/routes +############################################################################# + + + +def init_framwork(): + SysomFramework\ + .init(YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() + logger.info("SysomFramework init finished!") + + +@app.on_event("startup") +async def on_start(): + init_framwork() + + ############################################################################# + # Perform some microservice initialization operations over here + ############################################################################# + + +@app.on_event("shutdown") +async def on_shutdown(): + pass \ No newline at end of file -- Gitee From 3cf3fb911813452d7256b89ec7ace87fc50eed52 Mon Sep 17 00:00:00 2001 From: wb-cp908446 Date: Mon, 8 May 2023 17:58:17 +0800 Subject: [PATCH 069/196] This commit use shell rule provided by user to generate branch of kernel version --- .../sysom_hotfix/apps/hotfix/views.py | 46 +++++++++++++------ 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 2b75e148..09e69e11 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -4,6 +4,7 @@ import re import os import time import urllib +import subprocess from typing import Any from drf_yasg import openapi from drf_yasg.utils import swagger_auto_schema @@ -560,8 +561,8 @@ class HotfixAPIView(GenericViewSet, source_debuginfo_list = source_debuginfo.split(",") x86_debuginfo_source = list(filter(lambda x: "x86_64" in x, source_debuginfo_list))[0] arm_debuginfo_source = list(filter(lambda x: "aarch64" in x, source_debuginfo_list))[0] - self.insert_kernel_version_relation_git(os_type, git_repo, git_rule, x86_devel_source, x86_source_devel_lists, x86_debuginfo_source) - self.insert_kernel_version_relation_git(os_type, git_repo, git_rule, arm_devel_source, arm_source_devel_lists, arm_debuginfo_source) + self.insert_kernel_version_relation_git(os_type, git_rule, x86_devel_source, x86_source_devel_lists, x86_debuginfo_source) + self.insert_kernel_version_relation_git(os_type, git_rule, arm_devel_source, arm_source_devel_lists, arm_debuginfo_source) self.update_ostype_sync_status(id=id, status=2) logger.info("successfully sync kernel") return success(result={"msg":"successfully sync os_type"}) @@ -570,22 +571,37 @@ class HotfixAPIView(GenericViewSet, self.update_ostype_sync_status(id=id, status=1) return other_response(msg=str(e), code=400) - def insert_kernel_version_relation_git(self, os_type, git_repo, git_rule, x86_devel_source, source_devel_list, x86_debuginfo_source): + def insert_kernel_version_relation_git(self, os_type, git_rule, devel_source, source_devel_list, debuginfo_source): for kernel_devel_rpm in source_devel_list: version = ".".join(kernel_devel_rpm.replace("kernel-devel-", '').split(".")[:-1]) - matchobj = re.match(git_rule, version) - branch = None - if matchobj: - branch = matchobj.group() - if not branch: - logger.error("branch of version {} not found".format(version)) - continue - source = git_repo + '/' + branch - x86_debuginfo_link = self.get_debuginfo_rpm(x86_debuginfo_source, version) - x86_debuginfo_link = x86_debuginfo_source + x86_debuginfo_link - x86_develinfo_link = x86_devel_source + kernel_devel_rpm + if not git_rule: + source = "" + else: + branch = self.git_branch_by_git_rule(git_rule, version) + if not branch: + continue + source = branch + debuginfo_link = self.get_debuginfo_rpm(debuginfo_source, version) + debuginfo_link = debuginfo_source + debuginfo_link + develinfo_link = devel_source + kernel_devel_rpm self.insert_kernel_version_relation_internal(kernel_version=version,os_type=os_type, - source=source, devel_link=x86_develinfo_link,debuginfo_link=x86_debuginfo_link) + source=source, devel_link=develinfo_link,debuginfo_link=debuginfo_link) + + def git_branch_by_git_rule(self, git_rule, version): + branch = None + # cmd = git_rule %version + cmd = "kernel_version=%s\n"%version +"version=%s\n"%git_rule %"kernel_version" + "echo $version" + logger.info("The command executed is: {}".format(cmd)) + p = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE) + while p.poll() is None: + if p.wait() is not 0: + logger.info(f"执行失败") + break + else: + branch=p.stdout.read().decode("utf-8").strip("\r\n") + if not branch: + logger.error("branch of version {} not found".format(version)) + return branch def sync_source(self, id, os_type, source_repo, source_devel, source_debuginfo): try: -- Gitee From c9120512a63eea49a2355894b188b14ca9b864fe Mon Sep 17 00:00:00 2001 From: Wardenjohn Date: Tue, 9 May 2023 21:13:54 +0800 Subject: [PATCH 070/196] upload branch generate template This commit upload two template of script for user to fulfill their rule of generating the branch rule. This script uses kernel version as input, and branch/tag is output. This script will act on each kernel version to generate the branch of each kernel version --- sysom_server/sysom_hotfix/template/template.py | 17 +++++++++++++++++ sysom_server/sysom_hotfix/template/template.sh | 17 +++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100755 sysom_server/sysom_hotfix/template/template.py create mode 100755 sysom_server/sysom_hotfix/template/template.sh diff --git a/sysom_server/sysom_hotfix/template/template.py b/sysom_server/sysom_hotfix/template/template.py new file mode 100755 index 00000000..bb89767a --- /dev/null +++ b/sysom_server/sysom_hotfix/template/template.py @@ -0,0 +1,17 @@ +#!/usr/bin/python +""" +# Please follow the input of this script +# input : kernel_version : eg. 4.19.91-26.an7.x86_64 +# output : branch generated from the logic given by user from the given kernel_version +# User who just fulfill the branch generating logic in function "generate_branch" will be enough +""" +import os +import sys + +def generate_branch(kernel_version): + # finish your branch generating logic from kernel version here ... + return kernel_version + +if __name__ == "__main__": + kernel_version = sys.argv[1] + print(generate_branch(kernel_version)) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/template/template.sh b/sysom_server/sysom_hotfix/template/template.sh new file mode 100755 index 00000000..d5c2c239 --- /dev/null +++ b/sysom_server/sysom_hotfix/template/template.sh @@ -0,0 +1,17 @@ +#!/usr/bin/sh +# Please follow the input of this script +# input : kernel_version : eg. 4.19.91-26.an7.x86_64 +# output : branch generated from the logic given by user from the given kernel_version +# User who just fulfill the branch generating logic in function "generate_branch" will be enough +kernel_version="$1" +branch="" + +generate_branch() { + # finish your branch generating logic here ... + a=${kernel_version##*-} + b=${a%%.*} + branch="Version_release_${b}" +} + +generate_branch +echo $branch \ No newline at end of file -- Gitee From ef28fbcfed1c2509729d3e3738d225a0b7e52d46 Mon Sep 17 00:00:00 2001 From: wb-672209 Date: Wed, 10 May 2023 18:22:24 +0800 Subject: [PATCH 071/196] =?UTF-8?q?=E6=AD=A3=E5=88=99=E8=A7=84=E5=88=99?= =?UTF-8?q?=E6=94=B9=E6=88=90=E4=B8=8A=E4=BC=A0=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 修改原本通过输入命令的形式来配置分支规则 但是考虑到这种做法门槛较高，因此改成上传规则脚本 用户编写分支规则脚本，上传以后后台可以调用该脚本用于生成每一个内核版本的分支 --- sysom_web/src/locales/en-US/pages.js | 85 ++++++++++++------- sysom_web/src/locales/zh-CN/pages.js | 1 + .../Version/VersionConfig/KernelVersion.jsx | 35 +++++++- .../hotfix/components/KernelConfigForm.jsx | 19 +++-- sysom_web/src/pages/hotfix/service.js | 26 +++++- 5 files changed, 121 insertions(+), 45 deletions(-) diff --git a/sysom_web/src/locales/en-US/pages.js b/sysom_web/src/locales/en-US/pages.js index 14f0a305..b3da93bb 100644 --- a/sysom_web/src/locales/en-US/pages.js +++ b/sysom_web/src/locales/en-US/pages.js @@ -1,35 +1,4 @@ export default { - // 'pages.language': 'English', - // 'pages.layouts.userLayout.title': 'SYSOM', - // 'pages.login.accountLogin.tab': 'Account Login', - // 'pages.login.accountLogin.errorMessage': 'Incorrect username/password(admin/ant.design)', - // 'pages.login.failure': 'Login failed, please try again!', - // 'pages.login.success': 'Login successful!', - // 'pages.login.username.placeholder': 'Username: admin or user', - // 'pages.login.username.required': 'Please input your username!', - // 'pages.login.password.placeholder': 'Password: ant.design', - // 'pages.login.password.required': 'Please input your password!', - // 'pages.login.phoneLogin.tab': 'Phone Login', - // 'pages.login.phoneLogin.errorMessage': 'Verification Code Error', - // 'pages.login.phoneNumber.placeholder': 'Phone Number', - // 'pages.login.phoneNumber.required': 'Please input your phone number!', - // 'pages.login.phoneNumber.invalid': 'Phone number is invalid!', - // 'pages.login.captcha.placeholder': 'Verification Code', - // 'pages.login.captcha.required': 'Please input verification code!', - // 'pages.login.phoneLogin.getVerificationCode': 'Get Code', - // 'pages.getCaptchaSecondText': 'sec(s)', - // 'pages.login.rememberMe': 'Remember me', - // 'pages.login.forgotPassword': 'Forgot Password ?', - // 'pages.login.submit': 'Login', - // 'pages.login.loginWith': 'Login with :', - // 'pages.login.registerAccount': 'Register Account', - // 'pages.welcome.advancedComponent': 'Advanced Component', - // 'pages.welcome.link': 'Welcome', - // 'pages.welcome.advancedLayout': 'Advanced Layout', - // 'pages.welcome.alertMessage': 'Faster and stronger heavy-duty components have been released.', - // 'pages.admin.subPage.title': 'This page can only be viewed by Admin', - // 'pages.admin.subPage.alertMessage': 'Umi ui is now released, welcome to use npm run ui to start the experience.', - 'pages.language': '中文', 'pages.layouts.userLayout.title': 'System Operation&Maintenance', 'pages.login.accountLogin.tab': 'Account Login', @@ -192,6 +161,60 @@ export default { 'pages.click.enter': 'click to enter', 'pages.login': 'Login', 'pages.changepassword': 'Change Password', + 'pages.hotfix.title': 'Hot patch list', + 'pages.hotfix.created_at': 'Creation time', + 'pages.hotfix.kernel_version': 'Kernel version', + 'pages.hotfix.creator': 'Founder', + 'pages.hotfix.progress': 'Progress', + 'pages.hotfix.patch_path': 'Patch path', + 'pages.hotfix.delete_hotfix_not_exist': 'The hotfixID to be deleted does not exist', + 'pages.hotfix.failed.formal': 'A hot patch that fails to be built cannot be transferred to an official package', + 'pages.hotfix.operation': 'Operation', + 'pages.hotfix.delete': 'Delete', + 'pages.hotfix.building_status': 'Construction state', + 'pages.hotfix.formal': 'Transfer of formal packet', + 'pages.hotfix.os_type': 'Operating system name', + 'pages.hotfix.kernel_repo_git': 'Source repository address', + 'pages.hotfix.git_rule': 'Regular rule', + 'pages.hotfix.patch_name': 'Patch file name', + 'pages.hotfix.upload': 'File upload', + 'pages.hotfix.download': 'Download', + 'pages.hotfix.kernel_repo_location': 'Source git repository address', + 'pages.hotfix.kernel_devel_location': 'Devel link', + 'pages.hotfix.kernel_debuginfo_location': 'Debuginfo link', + 'pages.hotfix.kernel_repo_branch': 'Git tags/branches', + 'pages.hotfix.title.os_type': 'Operating system configuration', + 'pages.hotfix.title.versionconf': 'Customize kernel version configuration', + 'pages.hotfix.image': 'Build image', + 'pages.hotfix.hotfix_name': "Hot patch name", + 'pages.hotfix.version_repo_devel': 'Devel warehouse address', + 'pages.hotfix.version_repo_debuginfo': 'Debuginfo Warehouse address', + 'pages.hotfix.synch_information': 'Synchronous information', + 'pages.hotfix.synchronization': 'Synchronization', + 'pages.account.account_list': 'Account list', + 'pages.account.username': 'User name', + 'pages.account.password': 'Password', + 'pages.account.is_admin': 'Administrator or not', + 'pages.account.allow_login': 'Allow login', + 'pages.account.account_create_time': 'Creation time', + 'pages.account.description': 'Introduction', + 'pages.account.action': 'Operation', + 'pages.account.delete': 'Delete', + 'pages.account.edit': 'Edit', + 'pages.account.edit_modal_title': 'Edit user information', + 'pages.account.required_password': 'User password must be filled!', + 'pages.account.required_username': 'User name must be filled!', + 'pages.account.is_delete_account': 'Are you sure to cancel this account?', + 'pages.account.yes': 'Confirm', + 'pages.account.think': 'Let me think again', + 'pages.account.input_username': 'Please enter the user name', + 'pages.account.input_description': 'Please enter a brief introduction', + 'pages.account.input_password': 'Please enter password', + 'pages.account.deleting': 'Deleting', + 'pages.account.delete_success': 'Deleted successfully', + 'pages.account.create_success': 'Successfully created', + 'pages.account.create_account_modal': 'Create a user', + 'pages.account.add_account': 'Add an account', 'pages.hostTable.batchimport': 'Batch import', 'pages.hostTable.importexcel': 'Import excel file', 'pages.hostTable.temdownload': 'Template download', diff --git a/sysom_web/src/locales/zh-CN/pages.js b/sysom_web/src/locales/zh-CN/pages.js index e839e485..8fc6ca7a 100644 --- a/sysom_web/src/locales/zh-CN/pages.js +++ b/sysom_web/src/locales/zh-CN/pages.js @@ -179,6 +179,7 @@ export default { 'pages.hotfix.formal': '转正式包', 'pages.hotfix.os_type': '操作系统名', 'pages.hotfix.kernel_repo_git': '源码仓库地址', + 'pages.hotfix.git_rule': '正则规则', 'pages.hotfix.patch_name': 'patch文件名称', 'pages.hotfix.upload': '文件上传', 'pages.hotfix.download': '下载', diff --git a/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx index f76f6f48..d7ac5ae2 100644 --- a/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx +++ b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx @@ -3,10 +3,10 @@ import { useIntl, FormattedMessage } from 'umi'; import { PageContainer } from '@ant-design/pro-layout'; import ProTable from '@ant-design/pro-table'; import { Popconfirm, message, Upload, Button, Select, Form, Switch, Tag} from 'antd'; -import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeOsType, PostWhetherSyncRequest } from '../../service'; +import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeOsType, PostWhetherSyncRequest, downloadHotfixFile } from '../../service'; import { ProForm, ModalForm, ProFormText, ProFormTextArea, ProFormSelect, ProFormSwitch } from '@ant-design/pro-form'; import { async } from '@antv/x6/lib/registry/marker/async'; -import { PropertySafetyFilled } from '@ant-design/icons'; +import { PropertySafetyFilled, DownloadOutlined } from '@ant-design/icons'; import KernelConfigForm from '../../components/KernelConfigForm' import ProCard from '@ant-design/pro-card'; import CompoundedSpace from 'antd/lib/space'; @@ -33,6 +33,22 @@ const handleDelOSType = async (record) => { } } +const downloadHotfix = async (record) => { + const res = await downloadHotfixFile(record.id); + if (res) { + const url = window.URL.createObjectURL(res.data); + const link = document.getElementById('downloadDiv'); //创建a标签 + link.style.display = 'none'; + link.href = url; // 设置a标签路径 + link.download = res.response.headers.get('content-disposition').split("attachment;filename=")[1]; //设置文件名 + document.body.appendChild(link); + link.click(); + URL.revokeObjectURL(link.href); // 释放 URL对象 + document.body.removeChild(link); + console.log(res.response.headers.get('content-disposition').split("attachment;filename=")[1]) + } +} + const OSTypeConfigList = React.forwardRef((props, ref) => { const oslistRef = useRef(); const intl = useIntl(); @@ -45,7 +61,7 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { title: , dataIndex: 'os_type', key: 'os_type', - width: 200, + width: 100, ellipsis: true, dataIndex: 'os_type', tooltip: '操作系统名，请为您该系列的操作系统类型命名' @@ -56,6 +72,19 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { valueType: 'input', tooltip: '该操作系统类型的源码仓库地址', }, + { + title: , + dataIndex: 'git_rule', + width: 100, + valueType: 'input', + render: (_, record) => { + return + ] } ]; return ( diff --git a/sysom_web/src/pages/hotfix/service.js b/sysom_web/src/pages/hotfix/service.js index 34ba6a49..8ed25703 100644 --- a/sysom_web/src/pages/hotfix/service.js +++ b/sysom_web/src/pages/hotfix/service.js @@ -340,4 +340,20 @@ export async function PostWhetherSyncRequest(params, options) { params: params, ...(options || {}), }); +} + +export async function postOneclickDeployment(params, options){ + const token = localStorage.getItem('token'); + return request('/api/v1/hotfix/oneclick_deploy/', { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': token, + }, + data: { + "kernel_version": params.kernel_version, + "rpm_package": params.rpm_package, + }, + ...(options || {}), + }) } \ No newline at end of file -- Gitee From 67c25e84a3de06ba12843ddded73eb3e135c7397 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 6 Jun 2023 20:57:29 +0800 Subject: [PATCH 091/196] feat(script): Add install.sh and uninstall.sh --- deps/0_mysql/{deploy.sh => install.sh} | 4 +- deps/0_mysql/uninstall.sh | 9 ++ deps/1_redis/{deploy.sh => install.sh} | 4 +- deps/1_redis/uninstall.sh | 9 ++ deps/2_nginx/deploy.sh | 18 ---- deps/2_nginx/init.sh | 8 +- deps/2_nginx/install.sh | 13 +++ deps/2_nginx/uninstall.sh | 9 ++ deps/3_prometheus/clear.sh | 1 - deps/3_prometheus/{deploy.sh => install.sh} | 4 +- deps/3_prometheus/uninstall.sh | 9 ++ deps/4_grafana/clear.sh | 3 +- deps/4_grafana/{deploy.sh => install.sh} | 4 +- deps/4_grafana/uninstall.sh | 9 ++ environment/0_env/clear.sh | 12 +-- environment/0_env/{deploy.sh => install.sh} | 4 +- environment/0_env/uninstall.sh | 9 ++ environment/1_sdk/clear.sh | 9 ++ environment/1_sdk/{deploy.sh => install.sh} | 4 +- environment/1_sdk/uninstall.sh | 8 ++ script/deploy/clear.sh | 6 +- script/deploy/deploy.sh | 10 +-- .../sysom_api/{deploy.sh => install.sh} | 4 +- script/server/sysom_api/uninstall.sh | 9 ++ .../sysom_channel/{deploy.sh => install.sh} | 4 +- script/server/sysom_channel/uninstall.sh | 9 ++ .../sysom_diagnosis/{deploy.sh => install.sh} | 4 +- script/server/sysom_diagnosis/uninstall.sh | 9 ++ .../sysom_hotfix/{deploy.sh => install.sh} | 4 +- script/server/sysom_hotfix/uninstall.sh | 9 ++ .../{deploy.sh => install.sh} | 4 +- .../server/sysom_hotfix_builder/uninstall.sh | 9 ++ .../sysom_migration/{deploy.sh => install.sh} | 4 +- script/server/sysom_migration/uninstall.sh | 9 ++ .../{deploy.sh => install.sh} | 4 +- .../server/sysom_monitor_server/uninstall.sh | 9 ++ .../sysom_vmcore/{deploy.sh => install.sh} | 4 +- script/server/sysom_vmcore/uninstall.sh | 9 ++ .../sysom_vul/{deploy.sh => install.sh} | 4 +- script/server/sysom_vul/uninstall.sh | 9 ++ script/sysom_clear.sh | 5 +- script/sysom_init.sh | 14 ++-- script/sysom_install.sh | 82 ++++++++++--------- script/sysom_start.sh | 28 ++----- script/sysom_stop.sh | 28 ++----- script/sysom_uninstall.sh | 54 ++++++------ 46 files changed, 318 insertions(+), 176 deletions(-) rename deps/0_mysql/{deploy.sh => install.sh} (85%) create mode 100644 deps/0_mysql/uninstall.sh rename deps/1_redis/{deploy.sh => install.sh} (98%) create mode 100644 deps/1_redis/uninstall.sh delete mode 100644 deps/2_nginx/deploy.sh create mode 100644 deps/2_nginx/install.sh create mode 100644 deps/2_nginx/uninstall.sh rename deps/3_prometheus/{deploy.sh => install.sh} (97%) create mode 100644 deps/3_prometheus/uninstall.sh rename deps/4_grafana/{deploy.sh => install.sh} (95%) create mode 100644 deps/4_grafana/uninstall.sh rename environment/0_env/{deploy.sh => install.sh} (98%) create mode 100644 environment/0_env/uninstall.sh rename environment/1_sdk/{deploy.sh => install.sh} (96%) create mode 100644 environment/1_sdk/uninstall.sh rename script/server/sysom_api/{deploy.sh => install.sh} (95%) create mode 100644 script/server/sysom_api/uninstall.sh rename script/server/sysom_channel/{deploy.sh => install.sh} (95%) create mode 100644 script/server/sysom_channel/uninstall.sh rename script/server/sysom_diagnosis/{deploy.sh => install.sh} (95%) create mode 100644 script/server/sysom_diagnosis/uninstall.sh rename script/server/sysom_hotfix/{deploy.sh => install.sh} (96%) create mode 100644 script/server/sysom_hotfix/uninstall.sh rename script/server/sysom_hotfix_builder/{deploy.sh => install.sh} (97%) create mode 100644 script/server/sysom_hotfix_builder/uninstall.sh rename script/server/sysom_migration/{deploy.sh => install.sh} (98%) create mode 100644 script/server/sysom_migration/uninstall.sh rename script/server/sysom_monitor_server/{deploy.sh => install.sh} (95%) create mode 100644 script/server/sysom_monitor_server/uninstall.sh rename script/server/sysom_vmcore/{deploy.sh => install.sh} (96%) create mode 100644 script/server/sysom_vmcore/uninstall.sh rename script/server/sysom_vul/{deploy.sh => install.sh} (95%) create mode 100644 script/server/sysom_vul/uninstall.sh diff --git a/deps/0_mysql/deploy.sh b/deps/0_mysql/install.sh similarity index 85% rename from deps/0_mysql/deploy.sh rename to deps/0_mysql/install.sh index ea06e3b3..6c27fc11 100644 --- a/deps/0_mysql/deploy.sh +++ b/deps/0_mysql/install.sh @@ -1,9 +1,9 @@ #!/bin/bash BaseDir=$(dirname $(readlink -f "$0")) -deploy_app() { +install_app() { rpm -q --quiet mariadb-server || yum install -y mariadb-server systemctl enable mariadb.service } -deploy_app +install_app diff --git a/deps/0_mysql/uninstall.sh b/deps/0_mysql/uninstall.sh new file mode 100644 index 00000000..452f9c5e --- /dev/null +++ b/deps/0_mysql/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing, don't erase mysql + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/deps/1_redis/deploy.sh b/deps/1_redis/install.sh similarity index 98% rename from deps/1_redis/deploy.sh rename to deps/1_redis/install.sh index d0e4cad7..b35c09ed 100644 --- a/deps/1_redis/deploy.sh +++ b/deps/1_redis/install.sh @@ -41,8 +41,8 @@ setup_redis() { fi } -deploy_app() { +install_app() { setup_redis } -deploy_app +install_app diff --git a/deps/1_redis/uninstall.sh b/deps/1_redis/uninstall.sh new file mode 100644 index 00000000..aca97ee6 --- /dev/null +++ b/deps/1_redis/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing, don't erase redis + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/deps/2_nginx/deploy.sh b/deps/2_nginx/deploy.sh deleted file mode 100644 index 49b6418d..00000000 --- a/deps/2_nginx/deploy.sh +++ /dev/null @@ -1,18 +0,0 @@ -#!/bin/bash -BaseDir=$(dirname $(readlink -f "$0")) - -setup_nginx() { - rpm -q --quiet nginx || yum install -y nginx - mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak - cp nginx.conf /etc/nginx/ - cp sysom.conf /etc/nginx/conf.d/ - ###change the install dir base on param $1### - sed -i "s;SERVER_PORT;${SERVER_PORT};g" /etc/nginx/conf.d/sysom.conf - sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/nginx/conf.d/sysom.conf -} - -deploy_app() { - setup_nginx -} - -deploy_app diff --git a/deps/2_nginx/init.sh b/deps/2_nginx/init.sh index ab07d6e7..2346eba5 100644 --- a/deps/2_nginx/init.sh +++ b/deps/2_nginx/init.sh @@ -2,8 +2,12 @@ BaseDir=$(dirname $(readlink -f "$0")) init_app() { - # do nothing - echo "" + mv /etc/nginx/nginx.conf /etc/nginx/nginx.conf.bak + cp nginx.conf /etc/nginx/ + cp sysom.conf /etc/nginx/conf.d/ + ###change the install dir base on param $1### + sed -i "s;SERVER_PORT;${SERVER_PORT};g" /etc/nginx/conf.d/sysom.conf + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/nginx/conf.d/sysom.conf } init_app diff --git a/deps/2_nginx/install.sh b/deps/2_nginx/install.sh new file mode 100644 index 00000000..9ac72eb5 --- /dev/null +++ b/deps/2_nginx/install.sh @@ -0,0 +1,13 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +setup_nginx() { + rpm -q --quiet nginx || yum install -y nginx + systemctl enable nginx +} + +install_app() { + setup_nginx +} + +install_app diff --git a/deps/2_nginx/uninstall.sh b/deps/2_nginx/uninstall.sh new file mode 100644 index 00000000..8f990713 --- /dev/null +++ b/deps/2_nginx/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing, don't erase nginx + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/deps/3_prometheus/clear.sh b/deps/3_prometheus/clear.sh index c1b1f27b..c3321706 100644 --- a/deps/3_prometheus/clear.sh +++ b/deps/3_prometheus/clear.sh @@ -5,7 +5,6 @@ SERVICE_NAME=sysom-prometheus clear_app() { rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini supervisorctl update - rm -rf $DEPS_HOME/prometheus } # Stop first diff --git a/deps/3_prometheus/deploy.sh b/deps/3_prometheus/install.sh similarity index 97% rename from deps/3_prometheus/deploy.sh rename to deps/3_prometheus/install.sh index 18f084b3..76886c07 100644 --- a/deps/3_prometheus/deploy.sh +++ b/deps/3_prometheus/install.sh @@ -34,8 +34,8 @@ install_prometheus() popd } -deploy_app() { +install_app() { install_prometheus } -deploy_app +install_app diff --git a/deps/3_prometheus/uninstall.sh b/deps/3_prometheus/uninstall.sh new file mode 100644 index 00000000..1d36a0bf --- /dev/null +++ b/deps/3_prometheus/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + rm -rf ${DEPS_HOME}/prometheus + rm -f ${DEPS_HOME}/prometheus*.tar.gz +} + +uninstall_app \ No newline at end of file diff --git a/deps/4_grafana/clear.sh b/deps/4_grafana/clear.sh index df04c5fa..7608bf85 100644 --- a/deps/4_grafana/clear.sh +++ b/deps/4_grafana/clear.sh @@ -3,7 +3,8 @@ BaseDir=$(dirname $(readlink -f "$0")) SERVICE_NAME=sysom-prometheus clear_app() { - rpm -e grafana + # do nothing + echo "" } bash -x grafana_recover.sh diff --git a/deps/4_grafana/deploy.sh b/deps/4_grafana/install.sh similarity index 95% rename from deps/4_grafana/deploy.sh rename to deps/4_grafana/install.sh index 7b2a7350..2b24b55f 100644 --- a/deps/4_grafana/deploy.sh +++ b/deps/4_grafana/install.sh @@ -22,8 +22,8 @@ install_grafana() popd } -deploy_app() { +install_app() { install_grafana } -deploy_app +install_app diff --git a/deps/4_grafana/uninstall.sh b/deps/4_grafana/uninstall.sh new file mode 100644 index 00000000..2fce17ac --- /dev/null +++ b/deps/4_grafana/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + bash -x grafana_recover.sh + rm -rf ${DEPS_HOME}/grafana*.rpm +} + +uninstall_app \ No newline at end of file diff --git a/environment/0_env/clear.sh b/environment/0_env/clear.sh index 6f9baa9c..9670f969 100755 --- a/environment/0_env/clear.sh +++ b/environment/0_env/clear.sh @@ -1,9 +1,9 @@ #!/bin/bash +x - -VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME - -remove_virtualenv() { - rm -rf ${VIRTUALENV_HOME} +BaseDir=$(dirname $(readlink -f "$0")) +clear_app() { + # do nothing + echo "" + } -remove_virtualenv \ No newline at end of file +clear_app \ No newline at end of file diff --git a/environment/0_env/deploy.sh b/environment/0_env/install.sh similarity index 98% rename from environment/0_env/deploy.sh rename to environment/0_env/install.sh index 9e36e17e..b4eff2d3 100644 --- a/environment/0_env/deploy.sh +++ b/environment/0_env/install.sh @@ -81,11 +81,11 @@ touch_virtualenv() { } -deploy_app() { +install_app() { check_selinux_status touch_env_rpms touch_virtualenv check_requirements } -deploy_app +install_app diff --git a/environment/0_env/uninstall.sh b/environment/0_env/uninstall.sh new file mode 100644 index 00000000..1295a922 --- /dev/null +++ b/environment/0_env/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +uninstall_app() { + rm -rf ${VIRTUALENV_HOME} +} + +uninstall_app \ No newline at end of file diff --git a/environment/1_sdk/clear.sh b/environment/1_sdk/clear.sh index e69de29b..9670f969 100755 --- a/environment/1_sdk/clear.sh +++ b/environment/1_sdk/clear.sh @@ -0,0 +1,9 @@ +#!/bin/bash +x +BaseDir=$(dirname $(readlink -f "$0")) +clear_app() { + # do nothing + echo "" + +} + +clear_app \ No newline at end of file diff --git a/environment/1_sdk/deploy.sh b/environment/1_sdk/install.sh similarity index 96% rename from environment/1_sdk/deploy.sh rename to environment/1_sdk/install.sh index ccbbebbc..bd1212b7 100755 --- a/environment/1_sdk/deploy.sh +++ b/environment/1_sdk/install.sh @@ -26,9 +26,9 @@ install_sdk() { popd } -deploy_app() { +install_app() { source_virtualenv install_sdk } -deploy_app +install_app diff --git a/environment/1_sdk/uninstall.sh b/environment/1_sdk/uninstall.sh new file mode 100644 index 00000000..63952f36 --- /dev/null +++ b/environment/1_sdk/uninstall.sh @@ -0,0 +1,8 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + echo "uninstall sdk success" +} + +uninstall_app \ No newline at end of file diff --git a/script/deploy/clear.sh b/script/deploy/clear.sh index c36671a2..a2190696 100755 --- a/script/deploy/clear.sh +++ b/script/deploy/clear.sh @@ -48,10 +48,8 @@ clear() { # source code dir pushd `dirname ${BaseDir}` fi - bash +x sysom.sh clear web - bash +x sysom.sh clear ms ALL - bash +x sysom.sh clear env ALL - bash +x sysom.sh clear deps ALL + bash +x sysom.sh clear ALL + bash +x sysom.sh uninstall ALL popd del_conf del_app_home diff --git a/script/deploy/deploy.sh b/script/deploy/deploy.sh index df2164c7..a507e3a0 100755 --- a/script/deploy/deploy.sh +++ b/script/deploy/deploy.sh @@ -48,14 +48,8 @@ deploy() { # source code dir pushd `dirname ${BaseDir}` fi - bash +x sysom.sh deploy web - bash +x sysom.sh deploy env ALL - bash +x sysom.sh deploy deps ALL - bash +x sysom.sh deploy ms ALL - bash +x sysom.sh init web - bash +x sysom.sh init env ALL - bash +x sysom.sh init deps ALL - bash +x sysom.sh init ms ALL + bash +x sysom.sh install ALL + bash +x sysom.sh init ALL popd } diff --git a/script/server/sysom_api/deploy.sh b/script/server/sysom_api/install.sh similarity index 95% rename from script/server/sysom_api/deploy.sh rename to script/server/sysom_api/install.sh index 263fdef3..287efb40 100644 --- a/script/server/sysom_api/deploy.sh +++ b/script/server/sysom_api/install.sh @@ -21,9 +21,9 @@ source_virtualenv() { source ${VIRTUALENV_HOME}/bin/activate || exit 1 } -deploy_app() { +install_app() { source_virtualenv install_requirement } -deploy_app +install_app diff --git a/script/server/sysom_api/uninstall.sh b/script/server/sysom_api/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_api/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/server/sysom_channel/deploy.sh b/script/server/sysom_channel/install.sh similarity index 95% rename from script/server/sysom_channel/deploy.sh rename to script/server/sysom_channel/install.sh index 335a32c2..feb99a2e 100644 --- a/script/server/sysom_channel/deploy.sh +++ b/script/server/sysom_channel/install.sh @@ -21,9 +21,9 @@ source_virtualenv() { source ${VIRTUALENV_HOME}/bin/activate || exit 1 } -deploy_app() { +install_app() { source_virtualenv install_requirement } -deploy_app +install_app diff --git a/script/server/sysom_channel/uninstall.sh b/script/server/sysom_channel/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_channel/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/server/sysom_diagnosis/deploy.sh b/script/server/sysom_diagnosis/install.sh similarity index 95% rename from script/server/sysom_diagnosis/deploy.sh rename to script/server/sysom_diagnosis/install.sh index 99d75f99..d897e24c 100644 --- a/script/server/sysom_diagnosis/deploy.sh +++ b/script/server/sysom_diagnosis/install.sh @@ -21,9 +21,9 @@ source_virtualenv() { source ${VIRTUALENV_HOME}/bin/activate || exit 1 } -deploy_app() { +install_app() { source_virtualenv install_requirement } -deploy_app +install_app diff --git a/script/server/sysom_diagnosis/uninstall.sh b/script/server/sysom_diagnosis/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_diagnosis/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/server/sysom_hotfix/deploy.sh b/script/server/sysom_hotfix/install.sh similarity index 96% rename from script/server/sysom_hotfix/deploy.sh rename to script/server/sysom_hotfix/install.sh index 25aac716..7c470db3 100644 --- a/script/server/sysom_hotfix/deploy.sh +++ b/script/server/sysom_hotfix/install.sh @@ -21,11 +21,11 @@ source_virtualenv() { source ${VIRTUALENV_HOME}/bin/activate || exit 1 } -deploy_app() { +install_app() { rpm -q --quiet nfs-utils || yum install -y nfs-utils rpm -q --quiet rpcbind || yum install -y rpcbind source_virtualenv install_requirement } -deploy_app +install_app diff --git a/script/server/sysom_hotfix/uninstall.sh b/script/server/sysom_hotfix/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_hotfix/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/deploy.sh b/script/server/sysom_hotfix_builder/install.sh similarity index 97% rename from script/server/sysom_hotfix_builder/deploy.sh rename to script/server/sysom_hotfix_builder/install.sh index f5084f28..050d16fd 100644 --- a/script/server/sysom_hotfix_builder/deploy.sh +++ b/script/server/sysom_hotfix_builder/install.sh @@ -27,7 +27,7 @@ install_package() { rpm -q --quiet docker git || yum install -y docker git || echo "Warngin : Docker is not installed in this machine!" } -deploy_app() { +install_app() { rpm -q --quiet gcc || yum install -y gcc rpm -q --quiet make || yum install -y make rpm -q --quiet nfs-utils || yum install -y nfs-utils @@ -37,4 +37,4 @@ deploy_app() { install_package } -deploy_app +install_app diff --git a/script/server/sysom_hotfix_builder/uninstall.sh b/script/server/sysom_hotfix_builder/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_hotfix_builder/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/server/sysom_migration/deploy.sh b/script/server/sysom_migration/install.sh similarity index 98% rename from script/server/sysom_migration/deploy.sh rename to script/server/sysom_migration/install.sh index cb0719fc..a2bf460d 100644 --- a/script/server/sysom_migration/deploy.sh +++ b/script/server/sysom_migration/install.sh @@ -51,10 +51,10 @@ check_or_download_ance() { popd } -deploy_app() { +install_app() { source_virtualenv install_requirement check_or_download_ance } -deploy_app +install_app diff --git a/script/server/sysom_migration/uninstall.sh b/script/server/sysom_migration/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_migration/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/server/sysom_monitor_server/deploy.sh b/script/server/sysom_monitor_server/install.sh similarity index 95% rename from script/server/sysom_monitor_server/deploy.sh rename to script/server/sysom_monitor_server/install.sh index 3a13e509..f5e24ecb 100644 --- a/script/server/sysom_monitor_server/deploy.sh +++ b/script/server/sysom_monitor_server/install.sh @@ -21,9 +21,9 @@ source_virtualenv() { source ${VIRTUALENV_HOME}/bin/activate || exit 1 } -deploy_app() { +install_app() { source_virtualenv install_requirement } -deploy_app +install_app diff --git a/script/server/sysom_monitor_server/uninstall.sh b/script/server/sysom_monitor_server/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_monitor_server/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/server/sysom_vmcore/deploy.sh b/script/server/sysom_vmcore/install.sh similarity index 96% rename from script/server/sysom_vmcore/deploy.sh rename to script/server/sysom_vmcore/install.sh index 3b410a15..0da76949 100644 --- a/script/server/sysom_vmcore/deploy.sh +++ b/script/server/sysom_vmcore/install.sh @@ -21,11 +21,11 @@ source_virtualenv() { source ${VIRTUALENV_HOME}/bin/activate || exit 1 } -deploy_app() { +install_app() { rpm -q --quiet nfs-utils || yum install -y nfs-utils rpm -q --quiet rpcbind || yum install -y rpcbind source_virtualenv install_requirement } -deploy_app +install_app diff --git a/script/server/sysom_vmcore/uninstall.sh b/script/server/sysom_vmcore/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_vmcore/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/server/sysom_vul/deploy.sh b/script/server/sysom_vul/install.sh similarity index 95% rename from script/server/sysom_vul/deploy.sh rename to script/server/sysom_vul/install.sh index 63aa5ce9..8e7de040 100644 --- a/script/server/sysom_vul/deploy.sh +++ b/script/server/sysom_vul/install.sh @@ -21,9 +21,9 @@ source_virtualenv() { source ${VIRTUALENV_HOME}/bin/activate || exit 1 } -deploy_app() { +install_app() { source_virtualenv install_requirement } -deploy_app +install_app diff --git a/script/server/sysom_vul/uninstall.sh b/script/server/sysom_vul/uninstall.sh new file mode 100644 index 00000000..0aac04c0 --- /dev/null +++ b/script/server/sysom_vul/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/script/sysom_clear.sh b/script/sysom_clear.sh index c0173753..48b34e9f 100755 --- a/script/sysom_clear.sh +++ b/script/sysom_clear.sh @@ -68,9 +68,12 @@ sub_help(){ echo "" } +sub_ALL() { + sub_ms ALL +} sub_all() { - echo "all" + sub_ALL } sub_env() { diff --git a/script/sysom_init.sh b/script/sysom_init.sh index d47ee72a..13bc7655 100755 --- a/script/sysom_init.sh +++ b/script/sysom_init.sh @@ -157,10 +157,8 @@ ensure_supervisor_active() { if [[ $result =~ "active" ]]; then systemctl enable supervisord systemctl start supervisord - fi - result=`systemctl is-active supervisord` - if [[ $result =~ "active" ]]; then - red "supervisord not work" + else + red "supervisord not work" exit 1 fi } @@ -185,9 +183,15 @@ sub_help(){ echo "" } +sub_ALL() { + sub_deps ALL + sub_env ALL + sub_ms ALL + sub_web +} sub_all() { - echo "all" + sub_ALL } sub_env() { diff --git a/script/sysom_install.sh b/script/sysom_install.sh index 2c4b02f8..43185b8d 100755 --- a/script/sysom_install.sh +++ b/script/sysom_install.sh @@ -64,7 +64,7 @@ ensure_config_exist() { fi } -do_deploy_microservices() { +do_install_microservices() { mkdir -p ${MICROSERVICE_HOME} mkdir -p ${MICROSERVICE_SCRIPT_HOME} local_microservice_dir=${LocalAppHome}/sysom_server @@ -76,15 +76,15 @@ do_deploy_microservices() { for service in $(ls ${local_microservice_dir} | grep ${target}) do local_service_dir=${local_microservice_dir}/${service} - local_service_deploy_script_dir=${local_script_dir}/${service} + local_service_install_script_dir=${local_script_dir}/${service} service_dir=${MICROSERVICE_HOME}/${service} - service_deploy_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} + service_install_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} if [ "$LocalAppHome" != "$APP_HOME" ] then rm -rf ${service_dir} cp -r ${local_service_dir} ${service_dir} - rm -rf ${service_deploy_script_dir} - cp -r ${local_service_deploy_script_dir} ${service_deploy_script_dir} + rm -rf ${service_install_script_dir} + cp -r ${local_service_install_script_dir} ${service_install_script_dir} fi ################################################################################################ @@ -109,22 +109,22 @@ do_deploy_microservices() { # Perform service initial script ################################################################################################ - pushd ${service_deploy_script_dir} + pushd ${service_install_script_dir} # update *.ini replace /var/log/sysom to ${LOG_HOME} for ini_conf in $(ls sysom-*.ini) do - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_deploy_script_dir}/${ini_conf} + sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_install_script_dir}/${ini_conf} done - green "$service_deploy_script_dir deploying..................................." - if [ ! -f "${service_deploy_script_dir}/deploy.sh" ];then + green "$service_install_script_dir installing..................................." + if [ ! -f "${service_install_script_dir}/install.sh" ];then popd continue fi - bash -x ${service_deploy_script_dir}/deploy.sh + bash -x ${service_install_script_dir}/install.sh if [ $? -ne 0 ];then - red "$service_deploy_script_dir deploy fail, please check..................................." - red "sysom deploy failed, exit 1" + red "$service_install_script_dir install fail, please check..................................." + red "sysom install failed, exit 1" exit 1 fi popd @@ -132,7 +132,7 @@ do_deploy_microservices() { done } -do_deploy_environment() { +do_install_environment() { mkdir -p ${ENVIRONMENT_HOME} local_environment_dir=${LocalAppHome}/environment target=$1 @@ -149,11 +149,11 @@ do_deploy_environment() { target_dir=${ENVIRONMENT_HOME}/${env} pushd ${target_dir} - green "$target_dir deploying..................................." - bash -x ${target_dir}/deploy.sh + green "$target_dir installing..................................." + bash -x ${target_dir}/install.sh if [ $? -ne 0 ];then - red "$target_dir deploy fail, please check..................................." - red "sysom deploy failed, exit 1" + red "$target_dir install fail, please check..................................." + red "sysom install failed, exit 1" exit 1 fi popd @@ -162,7 +162,7 @@ do_deploy_environment() { done } -do_deploy_deps() { +do_install_deps() { mkdir -p ${DEPS_HOME} local_deps_dir=${LocalAppHome}/deps target=$1 @@ -179,11 +179,11 @@ do_deploy_deps() { target_dir=${DEPS_HOME}/${dep} pushd ${target_dir} - green "$target_dir deploying..................................." - bash -x ${target_dir}/deploy.sh + green "$target_dir installing..................................." + bash -x ${target_dir}/install.sh if [ $? -ne 0 ];then - red "$target_dir deploy fail, please check..................................." - red "sysom deploy failed, exit 1" + red "$target_dir install fail, please check..................................." + red "sysom install failed, exit 1" exit 1 fi popd @@ -212,9 +212,15 @@ sub_help(){ echo "" } +sub_ALL() { + sub_deps ALL + sub_env ALL + sub_ms ALL + sub_web +} sub_all() { - echo "all" + sub_ALL } sub_env() { @@ -239,11 +245,11 @@ sub_environment() { if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ];then continue fi - do_deploy_environment $env + do_install_environment $env done else # Deploy specific microservices - do_deploy_environment $target + do_install_environment $target fi } @@ -260,27 +266,27 @@ sub_microservice() { target=$1 if [ "$target" == "ALL" ] then - local_microservice_deploy_dir=${LocalAppHome}/sysom_server - if [ ! -d ${local_microservice_deploy_dir} ]; then - local_microservice_deploy_dir=${LocalAppHome}/server + local_microservice_install_dir=${LocalAppHome}/sysom_server + if [ ! -d ${local_microservice_install_dir} ]; then + local_microservice_install_dir=${LocalAppHome}/server fi # Load deploy_excludes deploy_excludes="" - if [ -f "${local_microservice_deploy_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_microservice_deploy_dir}/deploy_exclude` + if [ -f "${local_microservice_install_dir}/deploy_exclude" ];then + deploy_excludes=`cat ${local_microservice_install_dir}/deploy_exclude` fi # Deploy all microservices - for microservice in $(ls $local_microservice_deploy_dir) + for microservice in $(ls $local_microservice_install_dir) do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_deploy_dir}/${microservice}" ];then + if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_install_dir}/${microservice}" ];then continue fi - do_deploy_microservices ${microservice} + do_install_microservices ${microservice} done else # Deploy specific microservices - do_deploy_microservices $target + do_install_microservices $target fi } @@ -288,16 +294,16 @@ sub_deps() { target=$1 if [ "$target" == "ALL" ] then - local_deps_deploy_dir=${LocalAppHome}/deps + local_deps_install_dir=${LocalAppHome}/deps # Deploy all microservices - for dep in $(ls $local_deps_deploy_dir) + for dep in $(ls $local_deps_install_dir) do - do_deploy_deps ${dep} + do_install_deps ${dep} done else # Deploy specific microservices - do_deploy_deps $target + do_install_deps $target fi } diff --git a/script/sysom_start.sh b/script/sysom_start.sh index 21ec80d3..f266487c 100755 --- a/script/sysom_start.sh +++ b/script/sysom_start.sh @@ -218,9 +218,15 @@ sub_help(){ echo "" } +sub_ALL() { + sub_deps ALL + sub_env ALL + sub_ms ALL + sub_web +} sub_all() { - echo "all" + sub_ALL } sub_env() { @@ -307,25 +313,9 @@ sub_deps() { fi } -# Deploy web sub_web() { - local_web_home=${local_app_home}/sysom_web - if [ ! -d ${local_web_home} ]; then - local_web_home=${local_app_home}/web - fi - rm -rf ${WEB_HOME} - pushd ${local_web_home} - if [ -f "index.html" ] - then - # dist - cp -r ${local_web_home} ${WEB_HOME} - else - # source - yarn - yarn build - cp -r dist ${WEB_HOME} - fi - popd + # do nothing + echo "" } diff --git a/script/sysom_stop.sh b/script/sysom_stop.sh index c1e52f89..3a48b8f7 100755 --- a/script/sysom_stop.sh +++ b/script/sysom_stop.sh @@ -219,9 +219,15 @@ sub_help(){ echo "" } +sub_ALL() { + sub_web + sub_ms ALL + sub_env ALL + sub_deps ALL +} sub_all() { - echo "all" + sub_ALL } sub_env() { @@ -308,25 +314,9 @@ sub_deps() { fi } -# Deploy web sub_web() { - local_web_home=${local_app_home}/sysom_web - if [ ! -d ${local_web_home} ]; then - local_web_home=${local_app_home}/web - fi - rm -rf ${WEB_HOME} - pushd ${local_web_home} - if [ -f "index.html" ] - then - # dist - cp -r ${local_web_home} ${WEB_HOME} - else - # source - yarn - yarn build - cp -r dist ${WEB_HOME} - fi - popd + # Do nothing + echo "" } diff --git a/script/sysom_uninstall.sh b/script/sysom_uninstall.sh index 2380b4f8..10b92361 100755 --- a/script/sysom_uninstall.sh +++ b/script/sysom_uninstall.sh @@ -13,7 +13,7 @@ green() { } -do_clear_environment() { +do_uninstall_environment() { target=$1 targets=(${target//,/ }) for target in ${targets[*]} @@ -27,15 +27,15 @@ do_clear_environment() { pushd ${target_dir} - if [ ! -f "${target_dir}/clear.sh" ]; then + if [ ! -f "${target_dir}/uninstall.sh" ]; then popd continue fi - green "$target_dir clear..................................." - bash -x ${target_dir}/clear.sh + green "$target_dir uninstall..................................." + bash -x ${target_dir}/uninstall.sh if [ $? -ne 0 ];then - red "$target_dir clear fail, please check..................................." + red "$target_dir uninstall fail, please check..................................." exit 1 fi popd @@ -52,7 +52,7 @@ do_clear_environment() { fi } -do_clear_microservice() { +do_uninstall_microservice() { target=$1 targets=(${target//,/ }) for target in ${targets[*]} @@ -67,17 +67,17 @@ do_clear_microservice() { fi pushd ${target_script_dir} - if [ ! -f ${target_script_dir}/clear.sh ] + if [ ! -f "${target_script_dir}/uninstall.sh" ] then continue fi - green "$target_script_dir clear..................................." - bash -x ${target_script_dir}/clear.sh + green "$target_script_dir uninstall..................................." + bash -x ${target_script_dir}/uninstall.sh if [ $? -ne 0 ];then - red "$target_script_dir clear fail, please check..................................." - # red "sysom init failed, exit 1" - # exit 1 + red "$target_script_dir uninstall fail, please check..................................." + red "sysom uninstall failed, exit 1" + exit 1 fi popd @@ -101,7 +101,7 @@ do_clear_microservice() { } -do_clear_deps() { +do_uninstall_deps() { target=$1 targets=(${target//,/ }) for target in ${targets[*]} @@ -120,11 +120,11 @@ do_clear_deps() { continue fi - green "$target_dir clear..................................." - bash -x ${target_dir}/clear.sh + green "$target_dir uninstall..................................." + bash -x ${target_dir}/uninstall.sh if [ $? -ne 0 ];then - red "$target_dir clear fail, please check..................................." - red "sysom init failed, exit 1" + red "$target_dir uninstall fail, please check..................................." + red "sysom uninstall failed, exit 1" exit 1 fi popd @@ -160,9 +160,15 @@ sub_help(){ echo "" } +sub_ALL() { + sub_web + sub_ms ALL + sub_env ALL + sub_deps ALL +} sub_all() { - echo "all" + sub_ALL } sub_env() { @@ -185,11 +191,11 @@ sub_environment() { if [[ $clear_excludes =~ $infrastructure ]] || [ ! -d "${ENVIRONMENT_HOME}/${infrastructure}" ];then continue fi - do_clear_environment $infrastructure + do_uninstall_environment $infrastructure done else # Clear specific microservices - do_clear_environment $target + do_uninstall_environment $target fi } @@ -213,11 +219,11 @@ sub_microservice() { if [[ $clear_excludes =~ $microservice ]] || [ ! -d "${MICROSERVICE_HOME}/${microservice}" ];then continue fi - do_clear_microservice ${microservice} + do_uninstall_microservice ${microservice} done else # Deploy specific microservices - do_clear_microservice $target + do_uninstall_microservice $target fi } @@ -228,11 +234,11 @@ sub_deps() { # Deploy all deps for dep in $(ls -r $DEPS_HOME) do - do_clear_deps ${dep} + do_uninstall_deps ${dep} done else # Deploy specific dep - do_clear_deps $target + do_uninstall_deps $target fi } -- Gitee From 91052cdfc7de9e28127a1b886c80d972fc68f235 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 8 Jun 2023 14:31:57 +0800 Subject: [PATCH 092/196] fix(script): Fix scripts deploy error --- script/deploy/clear.sh | 19 +- script/deploy/deploy.sh | 4 +- script/server/clear.sh | 19 +- script/server/init.sh | 24 +- script/sysom.sh | 21 +- script/sysom_clear.sh | 167 ++++++--- script/sysom_init.sh | 142 ++++--- script/sysom_install.sh | 182 +++++---- script/sysom_run.sh | 351 ------------------ script/sysom_start.sh | 251 ++++--------- script/sysom_stop.sh | 248 ++++--------- script/sysom_uninstall.sh | 189 +++++----- .../sysom_hotfix/template/template.py | 2 +- 13 files changed, 536 insertions(+), 1083 deletions(-) delete mode 100755 script/sysom_run.sh diff --git a/script/deploy/clear.sh b/script/deploy/clear.sh index a2190696..32f4ab3f 100755 --- a/script/deploy/clear.sh +++ b/script/deploy/clear.sh @@ -18,7 +18,7 @@ export SERVER_HOME=${APP_HOME}/server del_app_home() { # Whether delete APP_HOME - if [ "`ls -A ${APP_HOME}`" = "script" ]; then + if [ -d "${APP_HOME}" ] && [ "`ls -A ${APP_HOME}`" = "script" ]; then rm -r ${APP_HOME} fi } @@ -40,20 +40,9 @@ del_conf() { BaseDir=$(dirname $(readlink -f "$0")) clear() { - if [ -d "${APP_HOME}" ];then - if [ -d "script" ]; then - # release dir - pushd "script" - else - # source code dir - pushd `dirname ${BaseDir}` - fi - bash +x sysom.sh clear ALL - bash +x sysom.sh uninstall ALL - popd - del_conf - del_app_home - fi + bash -x ${APP_HOME}/init_scripts/server/clear.sh + del_conf + del_app_home } clear diff --git a/script/deploy/deploy.sh b/script/deploy/deploy.sh index a507e3a0..d216c7c4 100755 --- a/script/deploy/deploy.sh +++ b/script/deploy/deploy.sh @@ -28,6 +28,7 @@ fi export APP_NAME=${APP_NAME} export APP_HOME=${APP_HOME} +export SCRIPT_HOME=${APP_HOME}/init_scripts export SERVER_LOCAL_IP=${SERVER_LOCAL_IP} export SERVER_PUBLIC_IP=${SERVER_PUBLIC_IP} export SERVER_PORT=${SERVER_PORT} @@ -48,8 +49,7 @@ deploy() { # source code dir pushd `dirname ${BaseDir}` fi - bash +x sysom.sh install ALL - bash +x sysom.sh init ALL + bash +x server/init.sh popd } diff --git a/script/server/clear.sh b/script/server/clear.sh index 7d9abf94..6d1bf585 100644 --- a/script/server/clear.sh +++ b/script/server/clear.sh @@ -1,18 +1,5 @@ #!/bin/bash -x BaseDir=$(dirname $(readlink -f "$0")) -pushd `dirname $BaseDir` -bash +x sysom.sh clear web -bash +x sysom.sh clear ms ALL -bash +x sysom.sh clear env ALL -bash +x sysom.sh clear deps ALL -popd - -# Whether delete SERVER_HOME -if [ "`ls -A ${SERVER_HOME}`" = "" ];then - rm -r ${SERVER_HOME} -fi - -# Whether delete APP_HOME -if [ "`ls -A ${APP_HOME}`" = "" ]; then - rm -r ${APP_HOME} -fi \ No newline at end of file +script_home=`dirname $BaseDir` +bash +x ${script_home}/sysom.sh clear ALL +bash +x ${script_home}/sysom.sh uninstall ALL \ No newline at end of file diff --git a/script/server/init.sh b/script/server/init.sh index eb78dc05..e1f1af52 100644 --- a/script/server/init.sh +++ b/script/server/init.sh @@ -1,8 +1,20 @@ #!/bin/bash -x BaseDir=$(dirname $(readlink -f "$0")) -pushd `dirname $BaseDir` -bash +x sysom.sh init web -bash +x sysom.sh init env ALL -bash +x sysom.sh init deps ALL -bash +x sysom.sh init ms ALL -popd \ No newline at end of file +FIRST_INIT_DONE=0 + +cur_script_home=$(dirname $BaseDir) +pushd ${cur_script_home} +if [ $FIRST_INIT_DONE == 0 ]; then + bash +x sysom.sh install ALL + bash +x sysom.sh init ALL + + local_app_home=$(dirname ${cur_script_home}) + if [ "${local_app_home}" == "${APP_HOME}" ]; then + sed -i 's/^FIRST_INIT_DONE=0/FIRST_INIT_DONE=1/g' $0 + else + sed -i 's/^FIRST_INIT_DONE=0/FIRST_INIT_DONE=1/g' ${SCRIPT_HOME}/server/init.sh + fi +else + bash +x sysom.sh start ALL +fi +popd diff --git a/script/sysom.sh b/script/sysom.sh index c3d53bdf..bd7e66e2 100755 --- a/script/sysom.sh +++ b/script/sysom.sh @@ -12,14 +12,23 @@ if [ "$APP_HOME" == "" ] then export APP_HOME=/usr/local/sysom fi -export SERVER_HOME=${APP_HOME}/server -export NODE_HOME=${APP_HOME}/node -export SCRIPT_HOME=${APP_HOME}/init_scripts -export WEB_HOME=${APP_HOME}/web -export DEPS_HOME=${APP_HOME}/deps + +# For rpm build phase only +# 1. Normally, this environment variable is the empty string +# 2. In rpm build stage, this environment variable is "%{build_root}" +if [ "$INSTALL_PREFIX" == "" ] +then + export INSTALL_PREFIX="" +fi + +export SERVER_HOME=${INSTALL_PREFIX}${APP_HOME}/server +export NODE_HOME=${INSTALL_PREFIX}${APP_HOME}/node +export SCRIPT_HOME=${INSTALL_PREFIX}${APP_HOME}/init_scripts +export WEB_HOME=${INSTALL_PREFIX}${APP_HOME}/web +export DEPS_HOME=${INSTALL_PREFIX}${APP_HOME}/deps +export ENVIRONMENT_HOME=${INSTALL_PREFIX}${APP_HOME}/environment export MICROSERVICE_HOME=${SERVER_HOME} export MICROSERVICE_SCRIPT_HOME=${SCRIPT_HOME}/server -export ENVIRONMENT_HOME=${APP_HOME}/environment export GLOBAL_VIRTUALENV_HOME=${ENVIRONMENT_HOME}/virtualenv if [ "$SERVER_LOCAL_IP" == "" ] diff --git a/script/sysom_clear.sh b/script/sysom_clear.sh index 48b34e9f..4c6a29f3 100755 --- a/script/sysom_clear.sh +++ b/script/sysom_clear.sh @@ -1,4 +1,8 @@ #!/bin/bash -x +set -x +ProgName=$(basename $0) +BaseDir=$(dirname $(readlink -f "$0")) +SYSOM_CLEAR_LOG=$LOG_HOME/sysom_clear.log #################################################################################################################### # Helper functions @@ -20,48 +24,93 @@ green() { do_clear_microservice() { target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for service in $(ls ${MICROSERVICE_SCRIPT_HOME} | grep ${target}) - do + for target in ${targets[*]}; do + for service in $(ls ${MICROSERVICE_HOME} | grep ${target}); do target_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} - target_service_dir=${MICROSERVICE_HOME}/${service} - if [ ! -d "$target_script_dir" ] - then + if [ ! -d "$target_script_dir" ] || [ ! -f "${target_script_dir}/clear.sh" ]; then continue fi + pushd ${target_script_dir} - - if [ ! -f ${target_script_dir}/clear.sh ] - then - continue - fi green "$target_script_dir clear..................................." bash -x ${target_script_dir}/clear.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_script_dir clear fail, please check..................................." - # red "sysom init failed, exit 1" - # exit 1 + red "sysom clear failed, exit 1" + exit 1 + fi + popd + done + done +} + +do_clear_environment() { + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]}; do + for env in $(ls ${ENVIRONMENT_HOME} | grep ${target}); do + target_dir=${ENVIRONMENT_HOME}/${env} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/clear.sh" ]; then + continue + fi + + pushd ${target_dir} + + green "$target_dir Clear..................................." + bash -x ${target_dir}/clear.sh + if [ $? -ne 0 ]; then + red "$target_dir clear fail, please check..................................." + red "sysom clear failed, exit 1" + exit 1 + fi + + popd + done + done +} + +do_clear_deps() { + target=$1 + targets=(${target//,/ }) + for target in ${targets[*]}; do + for dep in $(ls ${DEPS_HOME} | grep ${target}); do + target_dir=${DEPS_HOME}/${dep} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/clear.sh" ]; then + continue + fi + + pushd ${target_dir} + + green "$target_dir Clear..................................." + bash -x ${target_dir}/clear.sh + if [ $? -ne 0 ]; then + red "$target_dir clear fail, please check..................................." + red "sysom clear failed, exit 1" + exit 1 fi popd done + done } #################################################################################################################### # Subcommands #################################################################################################################### - -sub_help(){ + +sub_help() { echo "Usage: $ProgName [options]" echo "Subcommands:" echo " all Clear all modules" - echo " infrastructure [ALL | ] Clear all infrastructure or specific infrastructure" - echo " Example: $ProgName infrastructure env" - echo " Example: $ProgName infrastructure local_services" - echo " microservice [ALL | ] Clear all microservices or specific microservice" - echo " Example: $ProgName microservice sysom_diagnosis" + echo " env [ALL | ] Clear all enviroment or specific enviroment" + echo " Example: $ProgName env env" + echo " Example: $ProgName env sdk" + echo " deps [ALL | ] Clear all deps or specific dep" + echo " Example: $ProgName dep mysql" + echo " Example: $ProgName dep grafana" + echo " ms [ALL | ] Clear all microservices or specific microservice" + echo " Example: $ProgName ms sysom_diagnosis" echo "" echo "For help with each subcommand run:" echo "$ProgName -h|--help" @@ -70,6 +119,8 @@ sub_help(){ sub_ALL() { sub_ms ALL + sub_env ALL + sub_deps ALL } sub_all() { @@ -80,9 +131,21 @@ sub_env() { sub_environment $@ } -# -> env + local_services +# -> env + sdk sub_environment() { - yellow "all sysom servers depend on the ${subcommand}, it can't be clear." + target=$1 + if [ "$target" == "ALL" ]; then + # Clear all microservices + for env in $(ls -r $ENVIRONMENT_HOME); do + if [ ! -d "${ENVIRONMENT_HOME}/${env}" ]; then + continue + fi + do_clear_environment $env + done + else + # Clear specific microservices + do_clear_environment $target + fi } sub_ms() { @@ -92,29 +155,34 @@ sub_ms() { # All microservices sub_microservice() { target=$1 - if [ "$target" == "ALL" ] - then - # Load clear_excludes - clear_excludes="" - if [ -f "${MICROSERVICE_HOME}/clear_exclude" ];then - clear_excludes=`cat ${MICROSERVICE_HOME}/clear_exclude` - fi - # Deploy all microservices - for microservice in $(ls -r $MICROSERVICE_HOME) - do - if [[ $clear_excludes =~ $microservice ]] || [ ! -d "${MICROSERVICE_HOME}/${microservice}" ];then + if [ "$target" == "ALL" ]; then + # Clear all microservices + for microservice in $(ls -r $MICROSERVICE_HOME); do + if [ ! -d "${MICROSERVICE_HOME}/${microservice}" ]; then continue fi do_clear_microservice ${microservice} done else - # Deploy specific microservices + # CLear specific microservices do_clear_microservice $target fi } sub_deps() { - yellow "all sysom servers depend on the ${subcommand}, it can't be clear." + target=$1 + if [ "$target" == "ALL" ]; then + # Clear all deps + for dep in $(ls -r $DEPS_HOME); do + if [ ! -d "${DEPS_HOME}/${dep}" ]; then + continue + fi + do_clear_deps ${dep} + done + else + # Clear specific dep + do_clear_deps $target + fi } # Clear web @@ -122,19 +190,18 @@ sub_web() { yellow "all sysom servers depend on the ${subcommand}, it can't be clear." } - subcommand=$1 case $subcommand in - "" | "-h" | "--help") - sub_help - ;; - *) - shift - sub_${subcommand} $@ - if [ $? = 127 ]; then - echo "Error: '$subcommand' is not a known subcommand." >&2 - echo " Run '$ProgName --help' for a list of known subcommands." >&2 - exit 1 - fi - ;; +"" | "-h" | "--help") + sub_help + ;; +*) + shift + sub_${subcommand} $@ | tee ${SYSOM_CLEAR_LOG} + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; esac diff --git a/script/sysom_init.sh b/script/sysom_init.sh index 13bc7655..949df4f3 100755 --- a/script/sysom_init.sh +++ b/script/sysom_init.sh @@ -2,7 +2,7 @@ set -x ProgName=$(basename $0) BaseDir=$(dirname $(readlink -f "$0")) -LocalAppHome=`dirname $BaseDir` +LocalAppHome=$(dirname $BaseDir) SYSOM_DEPLOY_LOG=$LOG_HOME/sysom_deploy.log #################################################################################################################### @@ -24,19 +24,19 @@ do_init_microservices() { mkdir -p ${MICROSERVICE_HOME} mkdir -p ${MICROSERVICE_SCRIPT_HOME} local_microservice_dir=${LocalAppHome}/sysom_server + if [ ! -d ${local_microservice_dir} ]; then + local_microservice_dir=${LocalAppHome}/server + fi local_script_dir=$BaseDir/server target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for service in $(ls ${local_microservice_dir} | grep ${target}) - do + for target in ${targets[*]}; do + for service in $(ls ${local_microservice_dir} | grep ${target}); do local_service_dir=${local_microservice_dir}/${service} local_service_deploy_script_dir=${local_script_dir}/${service} service_dir=${MICROSERVICE_HOME}/${service} service_deploy_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} - if [ "$LocalAppHome" != "$APP_HOME" ] - then + if [ "$LocalAppHome" != "$APP_HOME" ]; then rm -rf ${service_dir} cp -r ${local_service_dir} ${service_dir} rm -rf ${service_deploy_script_dir} @@ -67,18 +67,17 @@ do_init_microservices() { pushd ${service_deploy_script_dir} # update *.ini replace /var/log/sysom to ${LOG_HOME} - for ini_conf in $(ls sysom-*.ini) - do + for ini_conf in $(ls sysom-*.ini); do sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_deploy_script_dir}/${ini_conf} done green "$service_deploy_script_dir initialing..................................." - if [ ! -f "${service_deploy_script_dir}/init.sh" ];then + if [ ! -f "${service_deploy_script_dir}/init.sh" ]; then popd continue fi bash -x ${service_deploy_script_dir}/init.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$service_deploy_script_dir initial fail, please check..................................." red "sysom init failed, exit 1" exit 1 @@ -93,28 +92,30 @@ do_init_environment() { local_environment_dir=${LocalAppHome}/environment target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for env in $(ls ${local_environment_dir} | grep ${target}) - do - if [ "$LocalAppHome" != "$APP_HOME" ] - then + for target in ${targets[*]}; do + for env in $(ls ${local_environment_dir} | grep ${target}); do + if [ "$LocalAppHome" != "$APP_HOME" ]; then # Copy env dir to deploy path if need cp -r $local_environment_dir/${env} $ENVIRONMENT_HOME/${env} fi + target_dir=${ENVIRONMENT_HOME}/${env} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/init.sh" ]; then + continue + fi + pushd ${target_dir} green "$target_dir initialing..................................." bash -x ${target_dir}/init.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir initial fail, please check..................................." red "sysom init failed, exit 1" exit 1 fi popd done - + done } @@ -123,28 +124,30 @@ do_init_deps() { local_deps_dir=${LocalAppHome}/deps target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for dep in $(ls ${local_deps_dir} | grep ${target}) - do - if [ "$LocalAppHome" != "$APP_HOME" ] - then + for target in ${targets[*]}; do + for dep in $(ls ${local_deps_dir} | grep ${target}); do + if [ "$LocalAppHome" != "$APP_HOME" ]; then # Copy dep dir to deploy path if need cp -r $local_deps_dir/${dep} $DEPS_HOME/${dep} fi + target_dir=${DEPS_HOME}/${dep} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/init.sh" ]; then + continue + fi + pushd ${target_dir} green "$target_dir initialing..................................." bash -x ${target_dir}/init.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir initial fail, please check..................................." red "sysom init failed, exit 1" exit 1 fi popd done - + done } @@ -153,30 +156,32 @@ ensure_supervisor_active() { rpm -q --quiet supervisor || yum install -y supervisor # 2. Then ensure supervisord is active - result=`systemctl is-active supervisord` - if [[ $result =~ "active" ]]; then + result=$(systemctl is-active supervisord) + if [ "$result" != "active" ]; then systemctl enable supervisord systemctl start supervisord else - red "supervisord not work" + red "supervisord not work" exit 1 fi } - #################################################################################################################### # Subcommands #################################################################################################################### -sub_help(){ +sub_help() { echo "Usage: $ProgName [options]" echo "Subcommands:" - echo " all Deploy all modules" - echo " infrastructure [ALL | ] Deploy infrastructure components" - echo " Example: $ProgName infrastructure env" - echo " Example: $ProgName infrastructure local_services" - echo " microservice [ALL | ] Deploy all microservices or specific microservice" - echo " Example: $ProgName microservice sysom_diagnosis" + echo " all Init all modules" + echo " env [ALL | ] Init all enviroment or specific enviroment" + echo " Example: $ProgName env env" + echo " Example: $ProgName env sdk" + echo " deps [ALL | ] Init all deps or specific dep" + echo " Example: $ProgName dep mysql" + echo " Example: $ProgName dep grafana" + echo " ms [ALL | ] Init all microservices or specific microservice" + echo " Example: $ProgName ms sysom_diagnosis" echo "" echo "For help with each subcommand run:" echo "$ProgName -h|--help" @@ -201,19 +206,17 @@ sub_env() { # -> env + local_services sub_environment() { target=$1 - if [ "$target" == "ALL" ] - then + if [ "$target" == "ALL" ]; then local_environment_dir=${LocalAppHome}/environment # Load deploy_excludes deploy_excludes="" - if [ -f "${local_environment_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_environment_dir}/deploy_exclude` + if [ -f "${local_environment_dir}/deploy_exclude" ]; then + deploy_excludes=$(cat ${local_environment_dir}/deploy_exclude) fi # Deploy all microservices - for env in $(ls $local_environment_dir) - do - if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ];then + for env in $(ls $local_environment_dir); do + if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ]; then continue fi do_init_environment $env @@ -235,8 +238,7 @@ sub_server() { # All microservices sub_microservice() { target=$1 - if [ "$target" == "ALL" ] - then + if [ "$target" == "ALL" ]; then local_microservice_deploy_dir=${LocalAppHome}/sysom_server if [ ! -d ${local_microservice_deploy_dir} ]; then local_microservice_deploy_dir=${LocalAppHome}/server @@ -244,13 +246,12 @@ sub_microservice() { # Load deploy_excludes deploy_excludes="" - if [ -f "${local_microservice_deploy_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_microservice_deploy_dir}/deploy_exclude` + if [ -f "${local_microservice_deploy_dir}/deploy_exclude" ]; then + deploy_excludes=$(cat ${local_microservice_deploy_dir}/deploy_exclude) fi # Deploy all microservices - for microservice in $(ls $local_microservice_deploy_dir) - do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_deploy_dir}/${microservice}" ];then + for microservice in $(ls $local_microservice_deploy_dir); do + if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_deploy_dir}/${microservice}" ]; then continue fi do_init_microservices ${microservice} @@ -263,13 +264,11 @@ sub_microservice() { sub_deps() { target=$1 - if [ "$target" == "ALL" ] - then + if [ "$target" == "ALL" ]; then local_deps_deploy_dir=${LocalAppHome}/deps # Deploy all microservices - for dep in $(ls $local_deps_deploy_dir) - do + for dep in $(ls $local_deps_deploy_dir); do do_init_deps ${dep} done else @@ -284,8 +283,7 @@ sub_web() { } check_config_exist() { - if [ ! -f "$SYSOM_CONF" ] - then + if [ ! -f "$SYSOM_CONF" ]; then red "old $SYSOM_CONF not exist, please install sysom first .............." exit 1 fi @@ -293,18 +291,18 @@ check_config_exist() { subcommand=$1 case $subcommand in - "" | "-h" | "--help") - sub_help - ;; - *) - ensure_supervisor_active - check_config_exist - shift - sub_${subcommand} $@ - if [ $? = 127 ]; then - echo "Error: '$subcommand' is not a known subcommand." >&2 - echo " Run '$ProgName --help' for a list of known subcommands." >&2 - exit 1 - fi - ;; +"" | "-h" | "--help") + sub_help + ;; +*) + ensure_supervisor_active + check_config_exist + shift + sub_${subcommand} $@ + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; esac diff --git a/script/sysom_install.sh b/script/sysom_install.sh index 43185b8d..b1e1ed82 100755 --- a/script/sysom_install.sh +++ b/script/sysom_install.sh @@ -2,8 +2,8 @@ set -x ProgName=$(basename $0) BaseDir=$(dirname $(readlink -f "$0")) -LocalAppHome=`dirname $BaseDir` -SYSOM_DEPLOY_LOG=$LOG_HOME/sysom_deploy.log +LocalAppHome=$(dirname $BaseDir) +SYSOM_INSTALL_LOG=$LOG_HOME/sysom_install.log #################################################################################################################### # Helper functions @@ -19,8 +19,10 @@ green() { install_new_scripts() { green "now install new scripts ..................." - rm -rf $SCRIPT_HOME - cp -r ${LocalAppHome}/script $SCRIPT_HOME + if [ "${LocalAppHome}" != "${APP_HOME}" ]; then + rm -rf $SCRIPT_HOME + cp -r ${LocalAppHome}/script $SCRIPT_HOME + fi } update_global_config() { @@ -43,10 +45,9 @@ update_global_config() { sed "/redis:/,/password/s/password:/password: $REDIS_PASSWORD/g" -i ${SYSOM_CONF} ###update local timezone### - local_timezone=`timedatectl status | grep "Time zone" | awk '{print $3}'` + local_timezone=$(timedatectl status | grep "Time zone" | awk '{print $3}') green "current timezone is : $local_timezone" - if [ "$local_timezone" == "" ] - then + if [ "$local_timezone" == "" ]; then echo "get time zone fail, use default time zone Asia/Shanghai" else sed "s;Asia/Shanghai;$local_timezone;g" -i ${SYSOM_CONF} @@ -54,8 +55,7 @@ update_global_config() { } ensure_config_exist() { - if [ ! -f "$SYSOM_CONF" ] - then + if [ ! -f "$SYSOM_CONF" ]; then grean "old $SYSOM_CONF not exist, now init and set local configure .............." cp ${LocalAppHome}/conf/config.yml $SYSOM_CONF update_global_config @@ -68,19 +68,19 @@ do_install_microservices() { mkdir -p ${MICROSERVICE_HOME} mkdir -p ${MICROSERVICE_SCRIPT_HOME} local_microservice_dir=${LocalAppHome}/sysom_server + if [ ! -d ${local_microservice_dir} ]; then + local_microservice_dir=${LocalAppHome}/server + fi local_script_dir=$BaseDir/server target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for service in $(ls ${local_microservice_dir} | grep ${target}) - do + for target in ${targets[*]}; do + for service in $(ls ${local_microservice_dir} | grep ${target}); do local_service_dir=${local_microservice_dir}/${service} local_service_install_script_dir=${local_script_dir}/${service} service_dir=${MICROSERVICE_HOME}/${service} service_install_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} - if [ "$LocalAppHome" != "$APP_HOME" ] - then + if [ "$LocalAppHome" != "$APP_HOME" ]; then rm -rf ${service_dir} cp -r ${local_service_dir} ${service_dir} rm -rf ${service_install_script_dir} @@ -111,18 +111,17 @@ do_install_microservices() { pushd ${service_install_script_dir} # update *.ini replace /var/log/sysom to ${LOG_HOME} - for ini_conf in $(ls sysom-*.ini) - do + for ini_conf in $(ls sysom-*.ini); do sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_install_script_dir}/${ini_conf} done green "$service_install_script_dir installing..................................." - if [ ! -f "${service_install_script_dir}/install.sh" ];then + if [ ! -f "${service_install_script_dir}/install.sh" ]; then popd continue fi bash -x ${service_install_script_dir}/install.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$service_install_script_dir install fail, please check..................................." red "sysom install failed, exit 1" exit 1 @@ -137,28 +136,29 @@ do_install_environment() { local_environment_dir=${LocalAppHome}/environment target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for env in $(ls ${local_environment_dir} | grep ${target}) - do - if [ "$LocalAppHome" != "$APP_HOME" ] - then + for target in ${targets[*]}; do + for env in $(ls ${local_environment_dir} | grep ${target}); do + if [ "$LocalAppHome" != "$APP_HOME" ]; then # Copy env dir to deploy path if need cp -r $local_environment_dir/${env} $ENVIRONMENT_HOME/${env} fi + target_dir=${ENVIRONMENT_HOME}/${env} - pushd ${target_dir} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/install.sh" ]; then + continue + fi + pushd ${target_dir} green "$target_dir installing..................................." bash -x ${target_dir}/install.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir install fail, please check..................................." red "sysom install failed, exit 1" exit 1 fi popd done - + done } @@ -167,45 +167,48 @@ do_install_deps() { local_deps_dir=${LocalAppHome}/deps target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for dep in $(ls ${local_deps_dir} | grep ${target}) - do - if [ "$LocalAppHome" != "$APP_HOME" ] - then + for target in ${targets[*]}; do + for dep in $(ls ${local_deps_dir} | grep ${target}); do + if [ "$LocalAppHome" != "$APP_HOME" ]; then # Copy dep dir to deploy path if need cp -r $local_deps_dir/${dep} $DEPS_HOME/${dep} fi + target_dir=${DEPS_HOME}/${dep} - pushd ${target_dir} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/install.sh" ]; then + continue + fi + pushd ${target_dir} green "$target_dir installing..................................." bash -x ${target_dir}/install.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir install fail, please check..................................." red "sysom install failed, exit 1" exit 1 fi popd done - + done } - #################################################################################################################### # Subcommands #################################################################################################################### -sub_help(){ +sub_help() { echo "Usage: $ProgName [options]" echo "Subcommands:" - echo " all Deploy all modules" - echo " infrastructure [ALL | ] Deploy infrastructure components" - echo " Example: $ProgName infrastructure env" - echo " Example: $ProgName infrastructure local_services" - echo " microservice [ALL | ] Deploy all microservices or specific microservice" - echo " Example: $ProgName microservice sysom_diagnosis" + echo " all Install all modules" + echo " env [ALL | ] Install all enviroment or specific enviroment" + echo " Example: $ProgName env env" + echo " Example: $ProgName env sdk" + echo " deps [ALL | ] Install all deps or specific dep" + echo " Example: $ProgName dep mysql" + echo " Example: $ProgName dep grafana" + echo " ms [ALL | ] Install all microservices or specific microservice" + echo " Example: $ProgName ms sysom_diagnosis" echo "" echo "For help with each subcommand run:" echo "$ProgName -h|--help" @@ -230,25 +233,18 @@ sub_env() { # -> env + local_services sub_environment() { target=$1 - if [ "$target" == "ALL" ] - then + if [ "$target" == "ALL" ]; then local_environment_dir=${LocalAppHome}/environment - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_environment_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_environment_dir}/deploy_exclude` - fi - # Deploy all microservices - for env in $(ls $local_environment_dir) - do - if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ];then + # Install all microservices + for env in $(ls $local_environment_dir); do + if [ ! -d "${local_environment_dir}/${env}" ]; then continue fi do_install_environment $env done else - # Deploy specific microservices + # Install specific microservices do_install_environment $target fi } @@ -264,8 +260,7 @@ sub_server() { # All microservices sub_microservice() { target=$1 - if [ "$target" == "ALL" ] - then + if [ "$target" == "ALL" ]; then local_microservice_install_dir=${LocalAppHome}/sysom_server if [ ! -d ${local_microservice_install_dir} ]; then local_microservice_install_dir=${LocalAppHome}/server @@ -273,50 +268,52 @@ sub_microservice() { # Load deploy_excludes deploy_excludes="" - if [ -f "${local_microservice_install_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_microservice_install_dir}/deploy_exclude` + if [ -f "${local_microservice_install_dir}/deploy_exclude" ]; then + deploy_excludes=$(cat ${local_microservice_install_dir}/deploy_exclude) fi - # Deploy all microservices - for microservice in $(ls $local_microservice_install_dir) - do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_install_dir}/${microservice}" ];then + # Install all microservices + for microservice in $(ls $local_microservice_install_dir); do + if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_install_dir}/${microservice}" ]; then continue fi do_install_microservices ${microservice} done else - # Deploy specific microservices + # Install specific microservices do_install_microservices $target fi } sub_deps() { target=$1 - if [ "$target" == "ALL" ] - then + if [ "$target" == "ALL" ]; then local_deps_install_dir=${LocalAppHome}/deps - # Deploy all microservices - for dep in $(ls $local_deps_install_dir) - do + # Install all microservices + for dep in $(ls $local_deps_install_dir); do + if [ ! -d "${local_deps_install_dir}/${dep}" ]; then + continue + fi do_install_deps ${dep} done else - # Deploy specific microservices + # Install specific microservices do_install_deps $target fi } -# Deploy web +# Install web sub_web() { local_web_home=${LocalAppHome}/sysom_web if [ ! -d ${local_web_home} ]; then local_web_home=${LocalAppHome}/web + else + rm -rf ${WEB_HOME} fi - rm -rf ${WEB_HOME} - pushd ${local_web_home} - if [ -f "index.html" ] - then + + if [ "${local_web_home}" != "${WEB_HOME}" ]; then + pushd ${local_web_home} + if [ -f "index.html" ]; then # dist cp -r ${local_web_home} ${WEB_HOME} else @@ -325,24 +322,23 @@ sub_web() { yarn build cp -r dist ${WEB_HOME} fi - popd + popd + fi } - subcommand=$1 case $subcommand in - "" | "-h" | "--help") - sub_help - ;; - *) - ensure_config_exists - install_new_scripts - shift - sub_${subcommand} $@ - if [ $? = 127 ]; then - echo "Error: '$subcommand' is not a known subcommand." >&2 - echo " Run '$ProgName --help' for a list of known subcommands." >&2 - exit 1 - fi - ;; +"" | "-h" | "--help") + sub_help + ;; +*) + install_new_scripts + shift + sub_${subcommand} $@ | tee ${SYSOM_INSTALL_LOG} + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; esac diff --git a/script/sysom_run.sh b/script/sysom_run.sh deleted file mode 100755 index c1e52f89..00000000 --- a/script/sysom_run.sh +++ /dev/null @@ -1,351 +0,0 @@ -#!/bin/bash -x -set -x -ProgName=$(basename $0) -BaseDir=$(dirname $(readlink -f "$0")) -local_app_home=`dirname $BaseDir` -SYSOM_DEPLOY_LOG=$LOG_HOME/sysom_deploy.log - -#################################################################################################################### -# Helper functions -#################################################################################################################### - -red() { - printf '\33[1;31m%b\n\33[0m' "$1" -} - -green() { - printf '\33[1;32m%b\n\33[0m' "$1" -} - -ensure_sysom_conf_exists() { - if [ ! -f "$SYSOM_CONF" ] - then - cp `dirname $BaseDir`/conf/config.yml $SYSOM_CONF - fi -} - -encure_scripts_exists() { - rm -rf $SCRIPT_HOME - cp -r $BaseDir $SCRIPT_HOME -} - -update_global_config() { - sed "s/SERVER_LOCAL_IP: 127.0.0.1/SERVER_LOCAL_IP: $SERVER_LOCAL_IP/g" -i ${SYSOM_CONF} - sed "s/SERVER_PUBLIC_IP: 127.0.0.1/SERVER_PUBLIC_IP: $SERVER_PUBLIC_IP/g" -i ${SYSOM_CONF} - sed "s/SERVER_PORT: 80/SERVER_PORT: $SERVER_PORT/g" -i ${SYSOM_CONF} - sed "s/global_root_path \/usr\/local\/sysom/global_root_path $APP_HOME/g" -i ${SYSOM_CONF} - - # MySQL - sed "/mysql/,/host/s/host: localhost/host: $DB_MYSQL_HOST/g" -i ${SYSOM_CONF} - sed "/mysql/,/port/s/port: 3306/port: $DB_MYSQL_PORT/g" -i ${SYSOM_CONF} - sed "/mysql/,/user/s/user: sysom/user: $DB_MYSQL_USERNAME/g" -i ${SYSOM_CONF} - sed "/mysql/,/password/s/password: sysom_admin/password: $DB_MYSQL_PASSWORD/g" -i ${SYSOM_CONF} - sed "/mysql/,/database/s/database: sysom/database: $DB_MYSQL_DATABASE/g" -i ${SYSOM_CONF} - - # Redis - sed "/redis:/,/host/s/host: localhost/host: $REDIS_HOST/g" -i ${SYSOM_CONF} - sed "/redis:/,/port/s/port: 6379/port: $REDIS_PORT/g" -i ${SYSOM_CONF} - sed "/redis:/,/username/s/username:/username: $REDIS_USERNAME/g" -i ${SYSOM_CONF} - sed "/redis:/,/password/s/password:/password: $REDIS_PASSWORD/g" -i ${SYSOM_CONF} - - ###update local timezone### - local_timezone=`timedatectl status | grep "Time zone" | awk '{print $3}'` - green "current timezone is : $local_timezone" - if [ "$local_timezone" == "" ] - then - echo "get time zone fail, use default time zone Asia/Shanghai" - else - sed "s;Asia/Shanghai;$local_timezone;g" -i ${SYSOM_CONF} - fi -} - -do_stop_microservices() { - mkdir -p ${MICROSERVICE_HOME} - mkdir -p ${MICROSERVICE_SCRIPT_HOME} - local_microservice_dir=`dirname $BaseDir`/sysom_server - local_script_dir=$BaseDir/server - target=$1 - targets=(${target//,/ }) - for target in ${targets[*]} - do - for service in $(ls ${local_microservice_dir} | grep ${target}) - do - local_service_dir=${local_microservice_dir}/${service} - local_service_deploy_script_dir=${local_script_dir}/${service} - service_dir=${MICROSERVICE_HOME}/${service} - service_deploy_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} - if [ "$local_app_home" != "$APP_HOME" ] - then - rm -rf ${service_dir} - cp -r ${local_service_dir} ${service_dir} - rm -rf ${service_deploy_script_dir} - cp -r ${local_service_deploy_script_dir} ${service_deploy_script_dir} - fi - - ################################################################################################ - # Service conf initial - ################################################################################################ - pushd ${service_dir} - # update microservice common.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} - if [ -f "${service_dir}/conf/common.py" ]; then - sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${service_dir}/conf/common.py - fi - # update fastapi based microservice alembic/env.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} - if [ -f "${service_dir}/alembic/env.py" ]; then - sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${service_dir}/alembic/env.py - fi - # update microservice gunicorn.py replace /var/log/sysom to ${LOG_HOME} - if [ -f "${targetservice_dir_dir}/conf/gunicorn.py" ]; then - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_dir}/conf/gunicorn.py - fi - popd - - ################################################################################################ - # Perform service initial script - ################################################################################################ - - pushd ${service_deploy_script_dir} - # update *.ini replace /var/log/sysom to ${LOG_HOME} - for ini_conf in $(ls sysom-*.ini) - do - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_deploy_script_dir}/${ini_conf} - done - - green "$service_deploy_script_dir Stopping..................................." - if [ ! -f "${service_deploy_script_dir}/stop.sh" ];then - popd - continue - fi - bash -x ${service_deploy_script_dir}/stop.sh - if [ $? -ne 0 ];then - red "$service_deploy_script_dir stop fail, please check..................................." - red "sysom stop failed, exit 1" - exit 1 - fi - popd - done - done -} - -do_stop_environment() { - mkdir -p ${ENVIRONMENT_HOME} - local_environment_dir=`dirname $BaseDir`/environment - target=$1 - targets=(${target//,/ }) - for target in ${targets[*]} - do - for env in $(ls ${local_environment_dir} | grep ${target}) - do - if [ "$local_app_home" != "$APP_HOME" ] - then - # Copy env dir to deploy path if need - cp -r $local_environment_dir/${env} $ENVIRONMENT_HOME/${env} - fi - target_dir=${ENVIRONMENT_HOME}/${env} - pushd ${target_dir} - - green "$target_dir Stopping..................................." - bash -x ${target_dir}/stop.sh - if [ $? -ne 0 ];then - red "$target_dir stop fail, please check..................................." - red "sysom stop failed, exit 1" - exit 1 - fi - popd - done - - done -} - -do_stop_deps() { - mkdir -p ${DEPS_HOME} - local_deps_dir=`dirname $BaseDir`/deps - target=$1 - targets=(${target//,/ }) - for target in ${targets[*]} - do - for dep in $(ls ${local_deps_dir} | grep ${target}) - do - if [ "$local_app_home" != "$APP_HOME" ] - then - # Copy dep dir to deploy path if need - cp -r $local_deps_dir/${dep} $DEPS_HOME/${dep} - fi - target_dir=${DEPS_HOME}/${dep} - pushd ${target_dir} - - green "$target_dir Stopping..................................." - bash -x ${target_dir}/stop.sh - if [ $? -ne 0 ];then - red "$target_dir stop fail, please check..................................." - red "sysom stop failed, exit 1" - exit 1 - fi - popd - done - - done -} - -ensure_supervisor_active() { - # 1. First ensure supervisor installed - rpm -q --quiet supervisor || yum install -y supervisor - - # 2. Then ensure supervisord is active - result=`systemctl is-active supervisord` - if [[ $result =~ "active" ]]; then - systemctl enable supervisord - systemctl start supervisord - fi -} - - -#################################################################################################################### -# Subcommands -#################################################################################################################### - -sub_help(){ - echo "Usage: $ProgName [options]" - echo "Subcommands:" - echo " all Deploy all modules" - echo " infrastructure [ALL | ] Deploy infrastructure components" - echo " Example: $ProgName infrastructure env" - echo " Example: $ProgName infrastructure local_services" - echo " microservice [ALL | ] Deploy all microservices or specific microservice" - echo " Example: $ProgName microservice sysom_diagnosis" - echo "" - echo "For help with each subcommand run:" - echo "$ProgName -h|--help" - echo "" -} - - -sub_all() { - echo "all" -} - -sub_env() { - sub_environment $@ -} - -# -> env + local_services -sub_environment() { - target=$1 - if [ "$target" == "ALL" ] - then - local_environment_dir=`dirname $BaseDir`/environment - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_environment_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_environment_dir}/deploy_exclude` - fi - # Deploy all microservices - for env in $(ls $local_environment_dir) - do - if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ];then - continue - fi - do_stop_environment $env - done - else - # Deploy specific microservices - do_stop_environment $target - fi -} - -sub_ms() { - sub_microservice $@ -} - -sub_server() { - sub_microservice $@ -} - -# All microservices -sub_microservice() { - target=$1 - if [ "$target" == "ALL" ] - then - local_microservice_deploy_dir=${local_app_home}/sysom_server - if [ ! -d ${local_microservice_deploy_dir} ]; then - local_microservice_deploy_dir=${local_app_home}/server - fi - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_microservice_deploy_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_microservice_deploy_dir}/deploy_exclude` - fi - # Deploy all microservices - for microservice in $(ls $local_microservice_deploy_dir) - do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_deploy_dir}/${microservice}" ];then - continue - fi - do_stop_microservices ${microservice} - done - else - # Deploy specific microservices - do_stop_microservices $target - fi -} - -sub_deps() { - target=$1 - if [ "$target" == "ALL" ] - then - local_deps_deploy_dir=${local_app_home}/deps - - # Deploy all microservices - for dep in $(ls $local_deps_deploy_dir) - do - do_stop_deps ${dep} - done - else - # Deploy specific microservices - do_stop_deps $target - fi -} - -# Deploy web -sub_web() { - local_web_home=${local_app_home}/sysom_web - if [ ! -d ${local_web_home} ]; then - local_web_home=${local_app_home}/web - fi - rm -rf ${WEB_HOME} - pushd ${local_web_home} - if [ -f "index.html" ] - then - # dist - cp -r ${local_web_home} ${WEB_HOME} - else - # source - yarn - yarn build - cp -r dist ${WEB_HOME} - fi - popd -} - - -subcommand=$1 -case $subcommand in - "" | "-h" | "--help") - sub_help - ;; - *) - ensure_sysom_conf_exists - encure_scripts_exists - update_global_config - ensure_supervisor_active - shift - sub_${subcommand} $@ - if [ $? = 127 ]; then - echo "Error: '$subcommand' is not a known subcommand." >&2 - echo " Run '$ProgName --help' for a list of known subcommands." >&2 - exit 1 - fi - ;; -esac \ No newline at end of file diff --git a/script/sysom_start.sh b/script/sysom_start.sh index f266487c..9687decd 100755 --- a/script/sysom_start.sh +++ b/script/sysom_start.sh @@ -2,8 +2,7 @@ set -x ProgName=$(basename $0) BaseDir=$(dirname $(readlink -f "$0")) -local_app_home=`dirname $BaseDir` -SYSOM_DEPLOY_LOG=$LOG_HOME/sysom_deploy.log +SYSOM_START_LOG=$LOG_HOME/sysom_start.log #################################################################################################################### # Helper functions @@ -17,107 +16,24 @@ green() { printf '\33[1;32m%b\n\33[0m' "$1" } -ensure_sysom_conf_exists() { - if [ ! -f "$SYSOM_CONF" ] - then - cp `dirname $BaseDir`/conf/config.yml $SYSOM_CONF - fi -} - -encure_scripts_exists() { - rm -rf $SCRIPT_HOME - cp -r $BaseDir $SCRIPT_HOME -} - -update_global_config() { - sed "s/SERVER_LOCAL_IP: 127.0.0.1/SERVER_LOCAL_IP: $SERVER_LOCAL_IP/g" -i ${SYSOM_CONF} - sed "s/SERVER_PUBLIC_IP: 127.0.0.1/SERVER_PUBLIC_IP: $SERVER_PUBLIC_IP/g" -i ${SYSOM_CONF} - sed "s/SERVER_PORT: 80/SERVER_PORT: $SERVER_PORT/g" -i ${SYSOM_CONF} - sed "s/global_root_path \/usr\/local\/sysom/global_root_path $APP_HOME/g" -i ${SYSOM_CONF} - - # MySQL - sed "/mysql/,/host/s/host: localhost/host: $DB_MYSQL_HOST/g" -i ${SYSOM_CONF} - sed "/mysql/,/port/s/port: 3306/port: $DB_MYSQL_PORT/g" -i ${SYSOM_CONF} - sed "/mysql/,/user/s/user: sysom/user: $DB_MYSQL_USERNAME/g" -i ${SYSOM_CONF} - sed "/mysql/,/password/s/password: sysom_admin/password: $DB_MYSQL_PASSWORD/g" -i ${SYSOM_CONF} - sed "/mysql/,/database/s/database: sysom/database: $DB_MYSQL_DATABASE/g" -i ${SYSOM_CONF} - - # Redis - sed "/redis:/,/host/s/host: localhost/host: $REDIS_HOST/g" -i ${SYSOM_CONF} - sed "/redis:/,/port/s/port: 6379/port: $REDIS_PORT/g" -i ${SYSOM_CONF} - sed "/redis:/,/username/s/username:/username: $REDIS_USERNAME/g" -i ${SYSOM_CONF} - sed "/redis:/,/password/s/password:/password: $REDIS_PASSWORD/g" -i ${SYSOM_CONF} - - ###update local timezone### - local_timezone=`timedatectl status | grep "Time zone" | awk '{print $3}'` - green "current timezone is : $local_timezone" - if [ "$local_timezone" == "" ] - then - echo "get time zone fail, use default time zone Asia/Shanghai" - else - sed "s;Asia/Shanghai;$local_timezone;g" -i ${SYSOM_CONF} - fi -} - do_start_microservices() { - mkdir -p ${MICROSERVICE_HOME} - mkdir -p ${MICROSERVICE_SCRIPT_HOME} - local_microservice_dir=`dirname $BaseDir`/sysom_server - local_script_dir=$BaseDir/server target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for service in $(ls ${local_microservice_dir} | grep ${target}) - do - local_service_dir=${local_microservice_dir}/${service} - local_service_deploy_script_dir=${local_script_dir}/${service} - service_dir=${MICROSERVICE_HOME}/${service} + for target in ${targets[*]}; do + for service in $(ls ${MICROSERVICE_HOME} | grep ${target}); do service_deploy_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} - if [ "$local_app_home" != "$APP_HOME" ] - then - rm -rf ${service_dir} - cp -r ${local_service_dir} ${service_dir} - rm -rf ${service_deploy_script_dir} - cp -r ${local_service_deploy_script_dir} ${service_deploy_script_dir} - fi - - ################################################################################################ - # Service conf initial - ################################################################################################ - pushd ${service_dir} - # update microservice common.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} - if [ -f "${service_dir}/conf/common.py" ]; then - sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${service_dir}/conf/common.py - fi - # update fastapi based microservice alembic/env.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} - if [ -f "${service_dir}/alembic/env.py" ]; then - sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${service_dir}/alembic/env.py - fi - # update microservice gunicorn.py replace /var/log/sysom to ${LOG_HOME} - if [ -f "${targetservice_dir_dir}/conf/gunicorn.py" ]; then - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_dir}/conf/gunicorn.py - fi - popd ################################################################################################ # Perform service initial script ################################################################################################ + if [ ! -f "${service_deploy_script_dir}/start.sh" ]; then + continue + fi pushd ${service_deploy_script_dir} - # update *.ini replace /var/log/sysom to ${LOG_HOME} - for ini_conf in $(ls sysom-*.ini) - do - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_deploy_script_dir}/${ini_conf} - done - green "$service_deploy_script_dir Starting..................................." - if [ ! -f "${service_deploy_script_dir}/start.sh" ];then - popd - continue - fi bash -x ${service_deploy_script_dir}/start.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$service_deploy_script_dir start fail, please check..................................." red "sysom start failed, exit 1" exit 1 @@ -128,62 +44,52 @@ do_start_microservices() { } do_start_environment() { - mkdir -p ${ENVIRONMENT_HOME} - local_environment_dir=`dirname $BaseDir`/environment target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for env in $(ls ${local_environment_dir} | grep ${target}) - do - if [ "$local_app_home" != "$APP_HOME" ] - then - # Copy env dir to deploy path if need - cp -r $local_environment_dir/${env} $ENVIRONMENT_HOME/${env} - fi + for target in ${targets[*]}; do + for env in $(ls ${ENVIRONMENT_HOME} | grep ${target}); do target_dir=${ENVIRONMENT_HOME}/${env} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/start.sh" ]; then + continue + fi + pushd ${target_dir} green "$target_dir Starting..................................." bash -x ${target_dir}/start.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir start fail, please check..................................." red "sysom start failed, exit 1" exit 1 fi + popd done - done } do_start_deps() { - mkdir -p ${DEPS_HOME} - local_deps_dir=`dirname $BaseDir`/deps target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for dep in $(ls ${local_deps_dir} | grep ${target}) - do - if [ "$local_app_home" != "$APP_HOME" ] - then - # Copy dep dir to deploy path if need - cp -r $local_deps_dir/${dep} $DEPS_HOME/${dep} - fi + for target in ${targets[*]}; do + for dep in $(ls ${DEPS_HOME} | grep ${target}); do target_dir=${DEPS_HOME}/${dep} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/start.sh" ]; then + continue + fi + pushd ${target_dir} green "$target_dir Starting..................................." bash -x ${target_dir}/start.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir start fail, please check..................................." red "sysom start failed, exit 1" exit 1 fi popd done - + done } @@ -192,8 +98,8 @@ ensure_supervisor_active() { rpm -q --quiet supervisor || yum install -y supervisor # 2. Then ensure supervisord is active - result=`systemctl is-active supervisord` - if [[ $result =~ "active" ]]; then + result=$(systemctl is-active supervisord) + if [ "$result" != "active" ]; then systemctl enable supervisord systemctl start supervisord fi @@ -203,15 +109,18 @@ ensure_supervisor_active() { # Subcommands #################################################################################################################### -sub_help(){ +sub_help() { echo "Usage: $ProgName [options]" echo "Subcommands:" - echo " all Deploy all modules" - echo " infrastructure [ALL | ] Deploy infrastructure components" - echo " Example: $ProgName infrastructure env" - echo " Example: $ProgName infrastructure local_services" - echo " microservice [ALL | ] Deploy all microservices or specific microservice" - echo " Example: $ProgName microservice sysom_diagnosis" + echo " all Start all modules" + echo " env [ALL | ] Start all enviroment or specific enviroment" + echo " Example: $ProgName env env" + echo " Example: $ProgName env sdk" + echo " deps [ALL | ] Start all deps or specific dep" + echo " Example: $ProgName dep mysql" + echo " Example: $ProgName dep grafana" + echo " ms [ALL | ] Start all microservices or specific microservice" + echo " Example: $ProgName ms sysom_diagnosis" echo "" echo "For help with each subcommand run:" echo "$ProgName -h|--help" @@ -233,28 +142,19 @@ sub_env() { sub_environment $@ } -# -> env + local_services +# -> env + sdk sub_environment() { target=$1 - if [ "$target" == "ALL" ] - then - local_environment_dir=`dirname $BaseDir`/environment - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_environment_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_environment_dir}/deploy_exclude` - fi - # Deploy all microservices - for env in $(ls $local_environment_dir) - do - if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ];then + if [ "$target" == "ALL" ]; then + # Start all enviroment + for env in $(ls $ENVIRONMENT_HOME); do + if [ ! -d "${ENVIRONMENT_HOME}/${env}" ]; then continue fi do_start_environment $env done else - # Deploy specific microservices + # Start specific enviroment do_start_environment $target fi } @@ -270,45 +170,32 @@ sub_server() { # All microservices sub_microservice() { target=$1 - if [ "$target" == "ALL" ] - then - local_microservice_deploy_dir=${local_app_home}/sysom_server - if [ ! -d ${local_microservice_deploy_dir} ]; then - local_microservice_deploy_dir=${local_app_home}/server - fi - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_microservice_deploy_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_microservice_deploy_dir}/deploy_exclude` - fi - # Deploy all microservices - for microservice in $(ls $local_microservice_deploy_dir) - do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_deploy_dir}/${microservice}" ];then + if [ "$target" == "ALL" ]; then + # start all microservices + for microservice in $(ls $MICROSERVICE_HOME); do + if [ ! -d "${MICROSERVICE_HOME}/${microservice}" ]; then continue fi do_start_microservices ${microservice} done else - # Deploy specific microservices + # start specific microservices do_start_microservices $target fi } sub_deps() { target=$1 - if [ "$target" == "ALL" ] - then - local_deps_deploy_dir=${local_app_home}/deps - - # Deploy all microservices - for dep in $(ls $local_deps_deploy_dir) - do + if [ "$target" == "ALL" ]; then + # start all deps + for dep in $(ls $DEPS_HOME); do + if [ ! -d "${DEPS_HOME}/${dep}" ]; then + continue + fi do_start_deps ${dep} done else - # Deploy specific microservices + # start specific deps do_start_deps $target fi } @@ -318,23 +205,19 @@ sub_web() { echo "" } - subcommand=$1 case $subcommand in - "" | "-h" | "--help") - sub_help - ;; - *) - ensure_sysom_conf_exists - encure_scripts_exists - update_global_config - ensure_supervisor_active - shift - sub_${subcommand} $@ - if [ $? = 127 ]; then - echo "Error: '$subcommand' is not a known subcommand." >&2 - echo " Run '$ProgName --help' for a list of known subcommands." >&2 - exit 1 - fi - ;; -esac \ No newline at end of file +"" | "-h" | "--help") + sub_help + ;; +*) + ensure_supervisor_active | tee ${SYSOM_START_LOG} + shift + sub_${subcommand} $@ | tee ${SYSOM_START_LOG} + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac diff --git a/script/sysom_stop.sh b/script/sysom_stop.sh index 3a48b8f7..56e785ec 100755 --- a/script/sysom_stop.sh +++ b/script/sysom_stop.sh @@ -2,8 +2,7 @@ set -x ProgName=$(basename $0) BaseDir=$(dirname $(readlink -f "$0")) -local_app_home=`dirname $BaseDir` -SYSOM_DEPLOY_LOG=$LOG_HOME/sysom_deploy.log +SYSOM_STOP_LOG=$LOG_HOME/sysom_stop.log #################################################################################################################### # Helper functions @@ -17,107 +16,26 @@ green() { printf '\33[1;32m%b\n\33[0m' "$1" } -ensure_sysom_conf_exists() { - if [ ! -f "$SYSOM_CONF" ] - then - cp `dirname $BaseDir`/conf/config.yml $SYSOM_CONF - fi -} - -encure_scripts_exists() { - rm -rf $SCRIPT_HOME - cp -r $BaseDir $SCRIPT_HOME -} - -update_global_config() { - sed "s/SERVER_LOCAL_IP: 127.0.0.1/SERVER_LOCAL_IP: $SERVER_LOCAL_IP/g" -i ${SYSOM_CONF} - sed "s/SERVER_PUBLIC_IP: 127.0.0.1/SERVER_PUBLIC_IP: $SERVER_PUBLIC_IP/g" -i ${SYSOM_CONF} - sed "s/SERVER_PORT: 80/SERVER_PORT: $SERVER_PORT/g" -i ${SYSOM_CONF} - sed "s/global_root_path \/usr\/local\/sysom/global_root_path $APP_HOME/g" -i ${SYSOM_CONF} - - # MySQL - sed "/mysql/,/host/s/host: localhost/host: $DB_MYSQL_HOST/g" -i ${SYSOM_CONF} - sed "/mysql/,/port/s/port: 3306/port: $DB_MYSQL_PORT/g" -i ${SYSOM_CONF} - sed "/mysql/,/user/s/user: sysom/user: $DB_MYSQL_USERNAME/g" -i ${SYSOM_CONF} - sed "/mysql/,/password/s/password: sysom_admin/password: $DB_MYSQL_PASSWORD/g" -i ${SYSOM_CONF} - sed "/mysql/,/database/s/database: sysom/database: $DB_MYSQL_DATABASE/g" -i ${SYSOM_CONF} - - # Redis - sed "/redis:/,/host/s/host: localhost/host: $REDIS_HOST/g" -i ${SYSOM_CONF} - sed "/redis:/,/port/s/port: 6379/port: $REDIS_PORT/g" -i ${SYSOM_CONF} - sed "/redis:/,/username/s/username:/username: $REDIS_USERNAME/g" -i ${SYSOM_CONF} - sed "/redis:/,/password/s/password:/password: $REDIS_PASSWORD/g" -i ${SYSOM_CONF} - - ###update local timezone### - local_timezone=`timedatectl status | grep "Time zone" | awk '{print $3}'` - green "current timezone is : $local_timezone" - if [ "$local_timezone" == "" ] - then - echo "get time zone fail, use default time zone Asia/Shanghai" - else - sed "s;Asia/Shanghai;$local_timezone;g" -i ${SYSOM_CONF} - fi -} - do_stop_microservices() { - mkdir -p ${MICROSERVICE_HOME} - mkdir -p ${MICROSERVICE_SCRIPT_HOME} - local_microservice_dir=`dirname $BaseDir`/sysom_server - local_script_dir=$BaseDir/server target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for service in $(ls ${local_microservice_dir} | grep ${target}) - do - local_service_dir=${local_microservice_dir}/${service} - local_service_deploy_script_dir=${local_script_dir}/${service} - service_dir=${MICROSERVICE_HOME}/${service} + for target in ${targets[*]}; do + for service in $(ls ${MICROSERVICE_HOME} | grep ${target}); do service_deploy_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} - if [ "$local_app_home" != "$APP_HOME" ] - then - rm -rf ${service_dir} - cp -r ${local_service_dir} ${service_dir} - rm -rf ${service_deploy_script_dir} - cp -r ${local_service_deploy_script_dir} ${service_deploy_script_dir} - fi - - ################################################################################################ - # Service conf initial - ################################################################################################ - pushd ${service_dir} - # update microservice common.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} - if [ -f "${service_dir}/conf/common.py" ]; then - sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${service_dir}/conf/common.py - fi - # update fastapi based microservice alembic/env.py, replace {BASE_DIR.parent.parent}/conf/config.yml to ${SYSOM_CONF} - if [ -f "${service_dir}/alembic/env.py" ]; then - sed -i "s;{BASE_DIR.parent.parent}/conf/config.yml;/etc/sysom/config.yml;g" ${service_dir}/alembic/env.py - fi - # update microservice gunicorn.py replace /var/log/sysom to ${LOG_HOME} - if [ -f "${targetservice_dir_dir}/conf/gunicorn.py" ]; then - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_dir}/conf/gunicorn.py - fi - popd ################################################################################################ # Perform service initial script ################################################################################################ pushd ${service_deploy_script_dir} - # update *.ini replace /var/log/sysom to ${LOG_HOME} - for ini_conf in $(ls sysom-*.ini) - do - sed -i "s;/var/log/sysom;${LOG_HOME};g" ${service_deploy_script_dir}/${ini_conf} - done green "$service_deploy_script_dir Stopping..................................." - if [ ! -f "${service_deploy_script_dir}/stop.sh" ];then + if [ ! -f "${service_deploy_script_dir}/stop.sh" ]; then popd continue fi bash -x ${service_deploy_script_dir}/stop.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$service_deploy_script_dir stop fail, please check..................................." red "sysom stop failed, exit 1" exit 1 @@ -128,62 +46,52 @@ do_stop_microservices() { } do_stop_environment() { - mkdir -p ${ENVIRONMENT_HOME} - local_environment_dir=`dirname $BaseDir`/environment target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for env in $(ls ${local_environment_dir} | grep ${target}) - do - if [ "$local_app_home" != "$APP_HOME" ] - then - # Copy env dir to deploy path if need - cp -r $local_environment_dir/${env} $ENVIRONMENT_HOME/${env} - fi + for target in ${targets[*]}; do + for env in $(ls ${ENVIRONMENT_HOME} | grep ${target}); do target_dir=${ENVIRONMENT_HOME}/${env} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/stop.sh" ]; then + continue + fi + pushd ${target_dir} green "$target_dir Stopping..................................." bash -x ${target_dir}/stop.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir stop fail, please check..................................." red "sysom stop failed, exit 1" exit 1 fi + popd done - done } do_stop_deps() { - mkdir -p ${DEPS_HOME} - local_deps_dir=`dirname $BaseDir`/deps target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for dep in $(ls ${local_deps_dir} | grep ${target}) - do - if [ "$local_app_home" != "$APP_HOME" ] - then - # Copy dep dir to deploy path if need - cp -r $local_deps_dir/${dep} $DEPS_HOME/${dep} - fi + for target in ${targets[*]}; do + for dep in $(ls ${DEPS_HOME} | grep ${target}); do target_dir=${DEPS_HOME}/${dep} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/stop.sh" ]; then + continue + fi + pushd ${target_dir} green "$target_dir Stopping..................................." bash -x ${target_dir}/stop.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir stop fail, please check..................................." red "sysom stop failed, exit 1" exit 1 fi popd done - + done } @@ -192,27 +100,29 @@ ensure_supervisor_active() { rpm -q --quiet supervisor || yum install -y supervisor # 2. Then ensure supervisord is active - result=`systemctl is-active supervisord` - if [[ $result =~ "active" ]]; then + result=$(systemctl is-active supervisord) + if [ "$result" != "active" ]; then systemctl enable supervisord systemctl start supervisord fi } - #################################################################################################################### # Subcommands #################################################################################################################### -sub_help(){ +sub_help() { echo "Usage: $ProgName [options]" echo "Subcommands:" - echo " all Deploy all modules" - echo " infrastructure [ALL | ] Deploy infrastructure components" - echo " Example: $ProgName infrastructure env" - echo " Example: $ProgName infrastructure local_services" - echo " microservice [ALL | ] Deploy all microservices or specific microservice" - echo " Example: $ProgName microservice sysom_diagnosis" + echo " all Stop all modules" + echo " env [ALL | ] Stop all enviroment or specific enviroment" + echo " Example: $ProgName env env" + echo " Example: $ProgName env sdk" + echo " deps [ALL | ] Stop all deps or specific dep" + echo " Example: $ProgName dep mysql" + echo " Example: $ProgName dep grafana" + echo " ms [ALL | ] Stop all microservices or specific microservice" + echo " Example: $ProgName ms sysom_diagnosis" echo "" echo "For help with each subcommand run:" echo "$ProgName -h|--help" @@ -234,28 +144,19 @@ sub_env() { sub_environment $@ } -# -> env + local_services +# -> env + sdk sub_environment() { target=$1 - if [ "$target" == "ALL" ] - then - local_environment_dir=`dirname $BaseDir`/environment - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_environment_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_environment_dir}/deploy_exclude` - fi - # Deploy all microservices - for env in $(ls $local_environment_dir) - do - if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ];then + if [ "$target" == "ALL" ]; then + # Stop all enviroment + for env in $(ls $ENVIRONMENT_HOME); do + if [ ! -d "${ENVIRONMENT_HOME}/${env}" ]; then continue fi do_stop_environment $env done else - # Deploy specific microservices + # Stop specific enviroment do_stop_environment $target fi } @@ -271,71 +172,54 @@ sub_server() { # All microservices sub_microservice() { target=$1 - if [ "$target" == "ALL" ] - then - local_microservice_deploy_dir=${local_app_home}/sysom_server - if [ ! -d ${local_microservice_deploy_dir} ]; then - local_microservice_deploy_dir=${local_app_home}/server - fi - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_microservice_deploy_dir}/deploy_exclude" ];then - deploy_excludes=`cat ${local_microservice_deploy_dir}/deploy_exclude` - fi - # Deploy all microservices - for microservice in $(ls $local_microservice_deploy_dir) - do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_deploy_dir}/${microservice}" ];then + if [ "$target" == "ALL" ]; then + # stop all microservices + for microservice in $(ls $MICROSERVICE_HOME); do + if [ ! -d "${MICROSERVICE_HOME}/${microservice}" ]; then continue fi do_stop_microservices ${microservice} done else - # Deploy specific microservices + # stop specific microservices do_stop_microservices $target fi } sub_deps() { target=$1 - if [ "$target" == "ALL" ] - then - local_deps_deploy_dir=${local_app_home}/deps - - # Deploy all microservices - for dep in $(ls $local_deps_deploy_dir) - do + if [ "$target" == "ALL" ]; then + # stop all deps + for dep in $(ls $DEPS_HOME); do + if [ ! -d "${DEPS_HOME}/${dep}" ]; then + continue + fi do_stop_deps ${dep} done else - # Deploy specific microservices + # stop specific deps do_stop_deps $target fi } sub_web() { - # Do nothing + # do nothing echo "" } - subcommand=$1 case $subcommand in - "" | "-h" | "--help") - sub_help - ;; - *) - ensure_sysom_conf_exists - encure_scripts_exists - update_global_config - ensure_supervisor_active - shift - sub_${subcommand} $@ - if [ $? = 127 ]; then - echo "Error: '$subcommand' is not a known subcommand." >&2 - echo " Run '$ProgName --help' for a list of known subcommands." >&2 - exit 1 - fi - ;; -esac \ No newline at end of file +"" | "-h" | "--help") + sub_help + ;; +*) + ensure_supervisor_active | tee ${SYSOM_STOP_LOG} + shift + sub_${subcommand} $@ | tee ${SYSOM_STOP_LOG} + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac diff --git a/script/sysom_uninstall.sh b/script/sysom_uninstall.sh index 10b92361..8086839c 100755 --- a/script/sysom_uninstall.sh +++ b/script/sysom_uninstall.sh @@ -1,4 +1,8 @@ #!/bin/bash -x +set -x +ProgName=$(basename $0) +BaseDir=$(dirname $(readlink -f "$0")) +SYSOM_UNINSTALL_LOG=$LOG_HOME/sysom_uninstall.log #################################################################################################################### # Helper functions @@ -12,29 +16,20 @@ green() { printf '\33[1;32m%b\n\33[0m' "$1" } - do_uninstall_environment() { target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for service_dir in $(ls ${ENVIRONMENT_HOME} | grep ${target}) - do - target_dir=${ENVIRONMENT_HOME}/${service_dir} - if [ ! -d "${target_dir}" ]; then + for target in ${targets[*]}; do + for env in $(ls ${ENVIRONMENT_HOME} | grep ${target}); do + target_dir=${ENVIRONMENT_HOME}/${env} + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/uninstall.sh" ]; then continue fi pushd ${target_dir} - - if [ ! -f "${target_dir}/uninstall.sh" ]; then - popd - continue - fi - green "$target_dir uninstall..................................." bash -x ${target_dir}/uninstall.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir uninstall fail, please check..................................." exit 1 fi @@ -45,9 +40,8 @@ do_uninstall_environment() { done done - # rm infrastructure if the folder is empty - if [ "`ls ${ENVIRONMENT_HOME} | wc -l`" == "0" ] - then + # rm env if the folder is empty + if [ "$(ls ${ENVIRONMENT_HOME} | wc -l)" == "0" ]; then rm -rf ${ENVIRONMENT_HOME} fi } @@ -55,26 +49,18 @@ do_uninstall_environment() { do_uninstall_microservice() { target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for service in $(ls ${MICROSERVICE_SCRIPT_HOME} | grep ${target}) - do + for target in ${targets[*]}; do + for service in $(ls ${MICROSERVICE_HOME} | grep ${target}); do target_script_dir=${MICROSERVICE_SCRIPT_HOME}/${service} target_service_dir=${MICROSERVICE_HOME}/${service} - if [ ! -d "$target_script_dir" ] - then + if [ ! -d "$target_script_dir" ] || [ ! -f "${target_script_dir}/uninstall.sh" ]; then continue fi pushd ${target_script_dir} - - if [ ! -f "${target_script_dir}/uninstall.sh" ] - then - continue - fi green "$target_script_dir uninstall..................................." bash -x ${target_script_dir}/uninstall.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_script_dir uninstall fail, please check..................................." red "sysom uninstall failed, exit 1" exit 1 @@ -87,42 +73,31 @@ do_uninstall_microservice() { done done - dircount=`ls -l ${MICROSERVICE_HOME}| grep "^d" | wc -l` - # rm infrastructure if the folder is empty - if [ "$dircount" == "0" ] - then + dircount=$(ls -l ${MICROSERVICE_HOME} | grep "^d" | wc -l) + # rm env if the folder is empty + if [ "$dircount" == "0" ]; then rm -rf ${MICROSERVICE_HOME} else - if [ "$dircount" == "1" ] && [ -d "${MICROSERVICE_HOME}/conf" ] - then + if [ "$dircount" == "1" ] && [ -d "${MICROSERVICE_HOME}/conf" ]; then rm -rf ${MICROSERVICE_HOME} fi fi } - do_uninstall_deps() { target=$1 targets=(${target//,/ }) - for target in ${targets[*]} - do - for dep in $(ls ${DEPS_HOME} | grep ${target}) - do + for target in ${targets[*]}; do + for dep in $(ls ${DEPS_HOME} | grep ${target}); do target_dir=${DEPS_HOME}/${dep} - if [ ! -d "${target_dir}" ]; then + if [ ! -d "${target_dir}" ] || [ ! -f "${target_dir}/uninstall.sh" ]; then continue fi pushd ${target_dir} - - if [ ! -f "${target_dir}/clear.sh" ]; then - popd - continue - fi - green "$target_dir uninstall..................................." bash -x ${target_dir}/uninstall.sh - if [ $? -ne 0 ];then + if [ $? -ne 0 ]; then red "$target_dir uninstall fail, please check..................................." red "sysom uninstall failed, exit 1" exit 1 @@ -134,26 +109,42 @@ do_uninstall_deps() { done done - # rm infrastructure if the folder is empty - if [ "`ls ${DEPS_HOME} | wc -l`" == "0" ] - then + # rm env if the folder is empty + if [ "$(ls ${DEPS_HOME} | wc -l)" == "0" ]; then rm -rf ${DEPS_HOME} fi } +check_and_rm_server_home() { + # Whether delete SERVER_HOME + if [ -d "${SERVER_HOME}" ] && [ "`ls -A ${SERVER_HOME}`" = "" ];then + rm -r ${SERVER_HOME} + fi +} + +check_and_rm_app_home() { + # Whether delete APP_HOME + if [ -d "${APP_HOME}" ] && [ "`ls -A ${APP_HOME}`" = "init_scripts" ]; then + rm -r ${APP_HOME} + fi +} + #################################################################################################################### # Subcommands #################################################################################################################### - -sub_help(){ + +sub_help() { echo "Usage: $ProgName [options]" echo "Subcommands:" - echo " all Clear all modules" - echo " infrastructure [ALL | ] Clear all infrastructure or specific infrastructure" - echo " Example: $ProgName infrastructure env" - echo " Example: $ProgName infrastructure local_services" - echo " microservice [ALL | ] Clear all microservices or specific microservice" - echo " Example: $ProgName microservice sysom_diagnosis" + echo " all Uninstall all modules" + echo " env [ALL | ] Uninstall all enviroment or specific enviroment" + echo " Example: $ProgName env env" + echo " Example: $ProgName env sdk" + echo " deps [ALL | ] Uninstall all deps or specific dep" + echo " Example: $ProgName dep mysql" + echo " Example: $ProgName dep grafana" + echo " ms [ALL | ] Uninstall all microservices or specific microservice" + echo " Example: $ProgName ms sysom_diagnosis" echo "" echo "For help with each subcommand run:" echo "$ProgName -h|--help" @@ -175,26 +166,19 @@ sub_env() { sub_environment $@ } -# -> env + local_services +# -> env + sdk sub_environment() { target=$1 - if [ "$target" == "ALL" ] - then - # Load clear_excludes - clear_excludes="" - if [ -f "${ENVIRONMENT_HOME}/clear_exclude" ];then - clear_excludes=`cat ${ENVIRONMENT_HOME}/clear_exclude` - fi - # Clear all microservices - for infrastructure in $(ls -r $ENVIRONMENT_HOME) - do - if [[ $clear_excludes =~ $infrastructure ]] || [ ! -d "${ENVIRONMENT_HOME}/${infrastructure}" ];then + if [ "$target" == "ALL" ]; then + # Uninstall all microservices + for env in $(ls -r $ENVIRONMENT_HOME); do + if [ ! -d "${ENVIRONMENT_HOME}/${env}" ]; then continue fi - do_uninstall_environment $infrastructure + do_uninstall_environment $env done else - # Clear specific microservices + # Uninstall specific microservices do_uninstall_environment $target fi } @@ -206,60 +190,55 @@ sub_ms() { # All microservices sub_microservice() { target=$1 - if [ "$target" == "ALL" ] - then - # Load clear_excludes - clear_excludes="" - if [ -f "${MICROSERVICE_HOME}/clear_exclude" ];then - clear_excludes=`cat ${MICROSERVICE_HOME}/clear_exclude` - fi - # Deploy all microservices - for microservice in $(ls -r $MICROSERVICE_HOME) - do - if [[ $clear_excludes =~ $microservice ]] || [ ! -d "${MICROSERVICE_HOME}/${microservice}" ];then + if [ "$target" == "ALL" ]; then + # Uninstall all microservices + for microservice in $(ls -r $MICROSERVICE_HOME); do + if [ ! -d "${MICROSERVICE_HOME}/${microservice}" ]; then continue fi do_uninstall_microservice ${microservice} done else - # Deploy specific microservices + # Uninstall specific microservices do_uninstall_microservice $target fi } sub_deps() { target=$1 - if [ "$target" == "ALL" ] - then - # Deploy all deps - for dep in $(ls -r $DEPS_HOME) - do + if [ "$target" == "ALL" ]; then + # Uninstall all deps + for dep in $(ls -r $DEPS_HOME); do + if [ ! -d "${DEPS_HOME}/${dep}" ]; then + continue + fi do_uninstall_deps ${dep} done else - # Deploy specific dep + # Uninstall specific dep do_uninstall_deps $target fi } -# Clear web +# Uninstall web sub_web() { rm -rf ${WEB_HOME} } - subcommand=$1 case $subcommand in - "" | "-h" | "--help") - sub_help - ;; - *) - shift - sub_${subcommand} $@ - if [ $? = 127 ]; then - echo "Error: '$subcommand' is not a known subcommand." >&2 - echo " Run '$ProgName --help' for a list of known subcommands." >&2 - exit 1 - fi - ;; +"" | "-h" | "--help") + sub_help + ;; +*) + shift + sub_${subcommand} $@ | tee ${SYSOM_UNINSTALL_LOG} + check_and_rm_server_home | tee ${SYSOM_UNINSTALL_LOG} + check_and_rm_app_home | tee ${SYSOM_UNINSTALL_LOG} + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; esac diff --git a/sysom_server/sysom_hotfix/template/template.py b/sysom_server/sysom_hotfix/template/template.py index bb89767a..bcee3825 100755 --- a/sysom_server/sysom_hotfix/template/template.py +++ b/sysom_server/sysom_hotfix/template/template.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python3 """ # Please follow the input of this script # input : kernel_version : eg. 4.19.91-26.an7.x86_64 -- Gitee From 434985f4a65bf322ff9f94c1f82f9f0dd3f3a05c Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 8 Jun 2023 15:03:59 +0800 Subject: [PATCH 093/196] fix(rpm): Compatible with new version deploy scripts --- tools/deploy/sysom.spec | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/tools/deploy/sysom.spec b/tools/deploy/sysom.spec index f16f87cd..05136209 100755 --- a/tools/deploy/sysom.spec +++ b/tools/deploy/sysom.spec @@ -147,12 +147,19 @@ echo "build sysom end........" %pre %install -mkdir -p %{buildroot}/usr/local/sysom/server/target -#mkdir -p %{buildroot}/usr/local/sysom/server/redis -cp -a sysom_server %{buildroot}/usr/local/sysom/server/target -cp -a conf %{buildroot}/usr/local/sysom/server/target -cp -a sysom_web/dist %{buildroot}/usr/local/sysom/server/target/sysom_web +mkdir -p %{buildroot}/usr/local/sysom +export INSTALL_PREFIX=%{buildroot} +cp -a sysom_server %{buildroot}/usr/local/sysom/server cp -a script %{buildroot}/usr/local/sysom/init_scripts +cp -a sysom_web/dist %{buildroot}/usr/local/sysom/web +cp -a deps %{buildroot}/usr/local/sysom/deps +cp -a environment %{buildroot}/usr/local/sysom/environment + +#mkdir -p %{buildroot}/usr/local/sysom/server/redis +# cp -a sysom_server %{buildroot}/usr/local/sysom/server/target +# cp -a conf %{buildroot}/usr/local/sysom/server/target +# cp -a sysom_web/dist %{buildroot}/usr/local/sysom/server/target/sysom_web +# cp -a script %{buildroot}/usr/local/sysom/init_scripts #cp -a monitor %{buildroot}/usr/local/sysom/server/ #cp -a /usr/local/sysom/server/virtualenv.tar.gz %{buildroot}/usr/local/sysom/server/ @@ -160,13 +167,23 @@ cp -a script %{buildroot}/usr/local/sysom/init_scripts /usr/local/sysom/ %post +# bash +x /usr/local/sysom/init_scripts/sysom.sh install ALL +# bash +x /usr/local/sysom/init_scripts/sysom.sh init ALL +# pushd /usr/local/sysom +# bash +x script/sysom.sh install ALL +# bash +x script/sysom.sh init ALL +# popd #pushd /usr/local/sysom/server/ #tar -xvf virtualenv.tar.gz #rm -rf virtualenv.tar.gz #popd %preun -bash -x /usr/local/sysom/init_scripts/server/clear.sh +bash +x /usr/local/sysom/init_scripts/sysom.sh clear ALL +bash +x /usr/local/sysom/init_scripts/sysom.sh uninstall ALL +# bash +x /usr/local/sysom/init_scripts/sysom.sh clear ALL +# bash +x /usr/local/sysom/init_scripts/sysom.sh uninstall ALL +# bash -x /usr/local/sysom/init_scripts/server/clear.sh %postun rm -rf /usr/local/sysom -- Gitee From 68cc2cf2b70d697385e9afc994ada7467279a32a Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Thu, 8 Jun 2023 15:40:55 +0800 Subject: [PATCH 094/196] fix(script): Can not execute init script while supervisord is activate --- script/sysom_init.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/script/sysom_init.sh b/script/sysom_init.sh index 949df4f3..50a7ec32 100755 --- a/script/sysom_init.sh +++ b/script/sysom_init.sh @@ -160,9 +160,6 @@ ensure_supervisor_active() { if [ "$result" != "active" ]; then systemctl enable supervisord systemctl start supervisord - else - red "supervisord not work" - exit 1 fi } -- Gitee From e2426ccdd6734800e44c6b1083d5b54ae474db95 Mon Sep 17 00:00:00 2001 From: wb-msm261421 Date: Thu, 8 Jun 2023 15:49:03 +0800 Subject: [PATCH 095/196] Fix [sysom-account]: model migration file --- .../0003_alter_handlerlog_request_option.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 sysom_server/sysom_api/apps/accounts/migrations/0003_alter_handlerlog_request_option.py diff --git a/sysom_server/sysom_api/apps/accounts/migrations/0003_alter_handlerlog_request_option.py b/sysom_server/sysom_api/apps/accounts/migrations/0003_alter_handlerlog_request_option.py new file mode 100644 index 00000000..cd10c626 --- /dev/null +++ b/sysom_server/sysom_api/apps/accounts/migrations/0003_alter_handlerlog_request_option.py @@ -0,0 +1,18 @@ +# Generated by Django 3.2.16 on 2023-06-08 07:45 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('accounts', '0002_user_allow_login'), + ] + + operations = [ + migrations.AlterField( + model_name='handlerlog', + name='request_option', + field=models.IntegerField(choices=[(0, '登录'), (1, '操作'), (2, '注销')], default=1, verbose_name='请求动作'), + ), + ] -- Gitee From 91a9cbc8b5a4254f7866f9aa0f30e113914b0833 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 9 Jun 2023 10:58:03 +0800 Subject: [PATCH 096/196] fix(script): Hotfix builder init.sh missing BaseDir define --- script/server/sysom_hotfix_builder/init.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/script/server/sysom_hotfix_builder/init.sh b/script/server/sysom_hotfix_builder/init.sh index 09b9b030..a06a00e9 100644 --- a/script/server/sysom_hotfix_builder/init.sh +++ b/script/server/sysom_hotfix_builder/init.sh @@ -3,6 +3,7 @@ SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME +BaseDir=$(dirname $(readlink -f "$0")) SERVICE_NAME=sysom-hotfix-builder NFS_SERVER_IP=${SERVER_LOCAL_IP} -- Gitee From 9988d72e60b66d93766be2f358ef0e4e09ffff66 Mon Sep 17 00:00:00 2001 From: ydzhang Date: Fri, 9 Jun 2023 14:33:00 +0800 Subject: [PATCH 097/196] adjust the new process of sysom ms clear revert the operation in init.sh for hotfix builder and hotfix server --- script/server/sysom_hotfix/clear.sh | 8 ++++++++ script/server/sysom_hotfix_builder/clear.sh | 17 +++++++---------- script/server/sysom_hotfix_builder/stop.sh | 12 +++++++++++- 3 files changed, 26 insertions(+), 11 deletions(-) diff --git a/script/server/sysom_hotfix/clear.sh b/script/server/sysom_hotfix/clear.sh index 1a68c312..58b9a6dc 100644 --- a/script/server/sysom_hotfix/clear.sh +++ b/script/server/sysom_hotfix/clear.sh @@ -3,7 +3,15 @@ BaseDir=$(dirname $(readlink -f "$0")) SERVICE_NAME=sysom-hotfix clear_app() { + + sed -i '/hotfix/d' /etc/exports + exportfs -rv + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + # remove the hotfix_builder nfs share base directory + rm -rf ${SERVER_HOME}/hotfix_builder + + systemctl stop nfs-server ###use supervisorctl update to stop and clear services### supervisorctl update } diff --git a/script/server/sysom_hotfix_builder/clear.sh b/script/server/sysom_hotfix_builder/clear.sh index eb5dded3..3dd3743b 100644 --- a/script/server/sysom_hotfix_builder/clear.sh +++ b/script/server/sysom_hotfix_builder/clear.sh @@ -6,17 +6,14 @@ clear_app() { LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix sed -i '/hotfix_builder/d' /etc/exports - # kill all kpatch-build process - echo "find : `ps -eo comm,pid | grep "kpatch-build"`" - for each_line in `ps -eo comm,pid | grep "kpatch-build"`; do - echo $each_line - if [[ $each_line =~ "kpatch-build" ]]; then - echo "find kpatch-build" - else - kill -9 $each_line - fi - done + result=$(systemctl is-active nfs-server) + if [[ $result =~ "inactive" ]]; then + systemctl restart nfs-server + fi + umount -f $LOCAL_NFS_HOME + rm -rf ${SERVER_HOME}/builder rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + systemctl stop nfs-server ###use supervisorctl update to stop and clear services### supervisorctl update } diff --git a/script/server/sysom_hotfix_builder/stop.sh b/script/server/sysom_hotfix_builder/stop.sh index f1ece2eb..d6535439 100644 --- a/script/server/sysom_hotfix_builder/stop.sh +++ b/script/server/sysom_hotfix_builder/stop.sh @@ -2,7 +2,17 @@ SERVICE_NAME=sysom-hotfix-builder stop_app() { - umount $LOCAL_NFS_HOME + # kill all kpatch-build process + echo "find : `ps -eo comm,pid | grep "kpatch-build"`" + for each_line in `ps -eo comm,pid | grep "kpatch-build"`; do + echo $each_line + if [[ $each_line =~ "kpatch-build" ]]; then + echo "find kpatch-build" + else + kill -9 $each_line + fi + done + supervisorctl stop $SERVICE_NAME } -- Gitee From a6091fa87566f64308cf5579be4b33b3b50612de Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 9 Jun 2023 17:10:25 +0800 Subject: [PATCH 098/196] fix(script): sysom_install.sh missing invoke ensure_conf_exists --- environment/0_env/install.sh | 17 ++++++++--------- script/sysom_install.sh | 1 + 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/environment/0_env/install.sh b/environment/0_env/install.sh index b4eff2d3..b644bed7 100644 --- a/environment/0_env/install.sh +++ b/environment/0_env/install.sh @@ -8,15 +8,12 @@ if [ "$UID" -ne 0 ]; then exit 1 fi -check_selinux_status() -{ +check_selinux_status() { ###check selinux rpm### rpm -qa | grep selinux-policy - if [ $? -eq 0 ] - then - cat /etc/selinux/config | grep "SELINUX=disabled" - if [ $? -eq 0 ] - then + if [ $? -eq 0 ]; then + cat /etc/selinux/config | grep "SELINUX=disabled" + if [ $? -eq 0 ]; then echo "selinux disable..." else echo "selinux enable, please set selinux disable" @@ -51,7 +48,7 @@ check_requirements() { ### atomic-0.7.3 need cffi, we show install cffi first### pip install --upgrade pip pip install cffi - pip install -r requirements.txt -i "${ALIYUN_MIRROR}" |tee -a "${requirements_log}" || exit 1 + pip install -r requirements.txt -i "${ALIYUN_MIRROR}" | tee -a "${requirements_log}" || exit 1 local pip_res=$? if [ $pip_res -ne 0 ]; then echo "ERROR: requirements not satisfied and auto install failed, please check ${requirements_log}" @@ -69,6 +66,9 @@ touch_virtualenv() { cp pip.conf ~/.pip/ cp .pydistutils.cfg ~/.pydistutils.cfg python3 -m venv ${VIRTUALENV_HOME} + if [ "$?" != 0 ]; then + python3 -m virtualenv ${VIRTUALENV_HOME} + fi if [ "$?" = 0 ]; then echo "INFO: create virtualenv success" else @@ -80,7 +80,6 @@ touch_virtualenv() { fi } - install_app() { check_selinux_status touch_env_rpms diff --git a/script/sysom_install.sh b/script/sysom_install.sh index b1e1ed82..7aa40250 100755 --- a/script/sysom_install.sh +++ b/script/sysom_install.sh @@ -333,6 +333,7 @@ case $subcommand in ;; *) install_new_scripts + ensure_config_exist shift sub_${subcommand} $@ | tee ${SYSOM_INSTALL_LOG} if [ $? = 127 ]; then -- Gitee From d8df5cdefdba539c0ceb6c363444c2f441d3ed2a Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 12 Jun 2023 16:02:04 +0800 Subject: [PATCH 099/196] fix(script): Maridb is in-active while execut init.sh --- deps/0_mysql/init.sh | 13 +++++++++++++ deps/2_nginx/init.sh | 3 +++ 2 files changed, 16 insertions(+) diff --git a/deps/0_mysql/init.sh b/deps/0_mysql/init.sh index 31dc59a3..d1815ed5 100644 --- a/deps/0_mysql/init.sh +++ b/deps/0_mysql/init.sh @@ -1,6 +1,18 @@ #!/bin/bash BaseDir=$(dirname $(readlink -f "$0")) +ensure_mysql_active() { + # 1. First ensure mysql installed + rpm -q --quiet mariadb-server || yum install -y mariadb-server + + # 2. Then ensure mariadb.service is active + result=$(systemctl is-active mariadb.service) + if [ "$result" != "active" ]; then + systemctl enable mariadb.service + systemctl start mariadb.service + fi +} + setup_database() { mysql -uroot -e "create user if not exists 'sysom'@'%' identified by 'sysom_admin';" mysql -uroot -e "grant usage on *.* to 'sysom'@'localhost' identified by 'sysom_admin'" @@ -12,6 +24,7 @@ setup_database() { } init_app() { + ensure_mysql_active setup_database } diff --git a/deps/2_nginx/init.sh b/deps/2_nginx/init.sh index 2346eba5..be415b8c 100644 --- a/deps/2_nginx/init.sh +++ b/deps/2_nginx/init.sh @@ -8,6 +8,9 @@ init_app() { ###change the install dir base on param $1### sed -i "s;SERVER_PORT;${SERVER_PORT};g" /etc/nginx/conf.d/sysom.conf sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/nginx/conf.d/sysom.conf + + # Restart nginx services + systemctl restart nginx } init_app -- Gitee From 6e632692c5a82aa3bc0383055dd1873c2dff5c65 Mon Sep 17 00:00:00 2001 From: wb-672209 Date: Tue, 13 Jun 2023 17:13:28 +0800 Subject: [PATCH 100/196] =?UTF-8?q?hotfix=E6=A8=A1=E5=9D=97=20=E8=87=AA?= =?UTF-8?q?=E5=AE=9A=E4=B9=89=E5=86=85=E6=A0=B8=E9=85=8D=E7=BD=AE=E7=89=88?= =?UTF-8?q?=E6=9C=AC=20=E6=93=8D=E4=BD=9C=E7=B3=BB=E7=BB=9F=E9=85=8D?= =?UTF-8?q?=E7=BD=AE=20=E5=8E=BB=E6=8E=89=E6=AD=A3=E5=88=99=E8=A7=84?= =?UTF-8?q?=E5=88=99=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../Version/VersionConfig/KernelVersion.jsx | 34 +------------------ .../hotfix/components/KernelConfigForm.jsx | 9 ----- sysom_web/src/pages/hotfix/service.js | 19 ----------- 3 files changed, 1 insertion(+), 61 deletions(-) diff --git a/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx index b313bb6f..c8e33aa3 100644 --- a/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx +++ b/sysom_web/src/pages/hotfix/Version/VersionConfig/KernelVersion.jsx @@ -3,7 +3,7 @@ import { useIntl, FormattedMessage } from 'umi'; import { PageContainer } from '@ant-design/pro-layout'; import ProTable from '@ant-design/pro-table'; import { Popconfirm, message, Upload, Button, Select, Form, Switch, Tag} from 'antd'; -import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeOsType, PostWhetherSyncRequest, downloadHotfixFile, downloadHotfixFileregular } from '../../service'; +import { delOSType, delKernelVersion, getOSTypeList, getKernelVersionList, submitOSType, submitKernelVersion, postChangeOsType, PostWhetherSyncRequest } from '../../service'; import { ProForm, ModalForm, ProFormText, ProFormTextArea, ProFormSelect, ProFormSwitch } from '@ant-design/pro-form'; import { async } from '@antv/x6/lib/registry/marker/async'; import { PropertySafetyFilled, DownloadOutlined } from '@ant-design/icons'; @@ -33,22 +33,6 @@ const handleDelOSType = async (record) => { } } -const downloadHotfix = async (record) => { - const res = await downloadHotfixFileregular(record.git_rule); - if (res) { - const url = window.URL.createObjectURL(res.data); - const link = document.createElement('a'); //创建a标签 - link.style.display = 'none'; - link.href = url; // 设置a标签路径 - link.download = res.response.headers.get('content-disposition').split("attachment;filename=")[1]; //设置文件名 - document.body.appendChild(link); - link.click(); - URL.revokeObjectURL(link.href); // 释放 URL对象 - document.body.removeChild(link); - console.log(res.response.headers.get('content-disposition').split("attachment;filename=")[1]) - } -} - const OSTypeConfigList = React.forwardRef((props, ref) => { const oslistRef = useRef(); const intl = useIntl(); @@ -72,15 +56,6 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { valueType: 'input', tooltip: '该操作系统类型的源码仓库地址', }, - { - title: , - dataIndex: 'git_rule', - valueType: 'input', - hideInSearch: true, - render: (_, record) => [ - downloadHotfix(record)}>{ record.git_rule } - ] - }, { title: , dataIndex: 'image', @@ -166,7 +141,6 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { os_type_name: values.os_type_name, git_repo_link: values.git_repo_link, image: values.building_image, - git_rule: values.git_rule, source_devel: values.source_devel, source_debuginfo: values.source_debuginfo } @@ -201,12 +175,6 @@ const OSTypeConfigList = React.forwardRef((props, ref) => { placeholder="请输入git仓库地址" initialValue={record.source_repo} /> - } diff --git a/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx b/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx index 79397879..5a825034 100644 --- a/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx +++ b/sysom_web/src/pages/hotfix/components/KernelConfigForm.jsx @@ -71,15 +71,6 @@ export default (props, ref) => { tooltip="该操作系统类型的git仓库地址" label="源码git仓库地址" /> - } Date: Tue, 13 Jun 2023 19:26:32 +0800 Subject: [PATCH 101/196] feat(template): Update server template Usage: ./sysom.sh create server --- script/sysom_create.sh | 141 ++++++++++++++++++++++++++++ template/fastapi/scripts/clear.sh | 8 +- template/fastapi/scripts/init.sh | 47 ++-------- template/fastapi/scripts/install.sh | 29 ++++++ template/fastapi/scripts/start.sh | 22 ++++- template/fastapi/scripts/stop.sh | 1 + template/fastapi/service/main.py | 1 - 7 files changed, 208 insertions(+), 41 deletions(-) create mode 100644 script/sysom_create.sh create mode 100644 template/fastapi/scripts/install.sh diff --git a/script/sysom_create.sh b/script/sysom_create.sh new file mode 100644 index 00000000..d3b40edc --- /dev/null +++ b/script/sysom_create.sh @@ -0,0 +1,141 @@ +#!/bin/bash -x +set -x +ProgName=$(basename $0) +BaseDir=$(dirname $(readlink -f "$0")) + +#################################################################################################################### +# Helper functions +#################################################################################################################### + +red() { + printf '\33[1;31m%b\n\33[0m' "$1" +} + +green() { + printf '\33[1;32m%b\n\33[0m' "$1" +} + +check_port_conflict() { + port=$1 + + # scan all service + local_microservice_dir=`dirname $BaseDir`/sysom_server + for service in $(ls ${local_microservice_dir}) + do + service_config_dir=${local_microservice_dir}/${service}/config.yml + if [ ! -f "${service_config_dir}" ]; then + continue + fi + _service_port=`cat $service_config_dir | grep 'sysom_service:' -A 6 | awk '/port/ {print $2}'` + if [ "${port}" == "${_service_port}" ]; then + red "Error: ${port} already used by ${service}" + exit 1 + fi + done +} + +check_service_name_conflict() { + service_name=$1 + local_microservice_dir=`dirname $BaseDir`/sysom_server + for service in $(ls ${local_microservice_dir}) + do + if [ "${service_name}" == "${service}" ];then + red "Error: ${service} already exists" + exit 1 + fi + done +} + +do_fill_template() { + service_name_shorter=$3 + service_name=sysom_$3 + service_middle=sysom-$3 + + for file in `ls $1` + do + if [ -d $1"/"$file ] + then + do_fill_template $1"/"$file $2 $3 + else + target_file=$1"/"$file + sed "s/\$TEMPLATE_SERVICE_NAME_SHORTER/$service_name_shorter/g" -i $target_file + sed "s/\$TEMPLATE_SERVICE_NAME_MIDDLE/$service_middle/g" -i $target_file + sed "s/\$TEMPLATE_SERVICE_NAME/$service_name/g" -i $target_file + sed "s/\$TEMPLATE_SERVICE_PORT/$2/g" -i $target_file + fi + done +} + + +#################################################################################################################### +# Subcommands +#################################################################################################################### + +sub_help(){ + echo "Usage: $ProgName [options]" + echo "Subcommands:" + echo " server Create one new server(microservice)" + echo " Example: $ProgName server demo" + echo "" + echo "For help with each subcommand run:" + echo "$ProgName -h|--help" + echo "" +} + +sub_server() { + sub_microservice $@ +} + +sub_service() { + sub_microservice $@ +} + +sub_microservice() { + name=$1 + service_port=$2 + service_name_shorter=$name + service_name=sysom_$name + service_middle=sysom-$name + + green $service_port + + # Check port conflict + check_port_conflict $service_port + + # Check service_name conflict + check_service_name_conflict $service_name + + # Copy template file to sysom_service and script/server + target_script_dir=`dirname $BaseDir`/script/server/${service_name} + target_service_dir=`dirname $BaseDir`/sysom_server/${service_name} + pushd `dirname $BaseDir`/template/fastapi + cp -r scripts $target_script_dir + cp -r service $target_service_dir + popd + + # Fill service and port info to template + do_fill_template $target_script_dir $service_port $name + do_fill_template $target_service_dir $service_port $name + + # rename supervisor conf + pushd $target_script_dir + mv template.ini $service_middle.ini + popd +} + + +subcommand=$1 +case $subcommand in + "" | "-h" | "--help") + sub_help + ;; + *) + shift + sub_${subcommand} $@ + if [ $? = 127 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; +esac \ No newline at end of file diff --git a/template/fastapi/scripts/clear.sh b/template/fastapi/scripts/clear.sh index 71e088b3..ed82f81e 100644 --- a/template/fastapi/scripts/clear.sh +++ b/template/fastapi/scripts/clear.sh @@ -1,8 +1,14 @@ #!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE + clear_app() { - rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini ###use supervisorctl update to stop and clear services### supervisorctl update } + +# Stop first +bash -x $BaseDir/stop.sh + clear_app diff --git a/template/fastapi/scripts/init.sh b/template/fastapi/scripts/init.sh index a93249c3..e18e972e 100644 --- a/template/fastapi/scripts/init.sh +++ b/template/fastapi/scripts/init.sh @@ -1,54 +1,25 @@ #!/bin/bash SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} -SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} -VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME +BaseDir=$(dirname $(readlink -f "$0")) SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE -if [ "$UID" -ne 0 ]; then - echo "Please run as root" - exit 1 -fi - -install_requirement() { - pushd ${SERVICE_SCRIPT_HOME} - pip install -r requirements.txt - popd -} - -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - init_conf() { - pushd ${SERVICE_HOME} - alembic upgrade head - popd - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ ###change the install dir base on param $1### sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=$(cat /proc/cpuinfo | grep processor | wc -l) + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/gunicorn.py } -start_app() { +init_app() { + init_conf + bash -x $BaseDir/db_migrate.sh ###if supervisor service started, we need use "supervisorctl update" to start new conf#### supervisorctl update - supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then - echo "supervisorctl start ${SERVICE_NAME} success..." - return 0 - fi - echo "${SERVICE_NAME} service start fail, please check log" - exit 1 } -deploy() { - source_virtualenv - install_requirement - init_conf - start_app -} +init_app -deploy +# Start +bash -x $BaseDir/start.sh diff --git a/template/fastapi/scripts/install.sh b/template/fastapi/scripts/install.sh new file mode 100644 index 00000000..d481b1f7 --- /dev/null +++ b/template/fastapi/scripts/install.sh @@ -0,0 +1,29 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME +SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +install_requirement() { + pushd ${SERVICE_SCRIPT_HOME} + pip install -r requirements.txt + popd +} + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_app() { + source_virtualenv + install_requirement +} + +install_app diff --git a/template/fastapi/scripts/start.sh b/template/fastapi/scripts/start.sh index e6c8d1fa..927049a0 100644 --- a/template/fastapi/scripts/start.sh +++ b/template/fastapi/scripts/start.sh @@ -1,7 +1,27 @@ #!/bin/bash SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE + +is_start() { + supervisorctl status ${SERVICE_NAME} + if [ $? -eq 0 ]; then + return 1 + else + return 0 + fi +} + start_app() { - supervisorctl start $SERVICE_NAME + is_start + if [[ $? == 0 ]]; then + supervisorctl start $SERVICE_NAME + is_start + if [[ $? == 0 ]]; then + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 + else + echo "supervisorctl start ${SERVICE_NAME} success..." + fi + fi } start_app diff --git a/template/fastapi/scripts/stop.sh b/template/fastapi/scripts/stop.sh index 4b514ad1..63d16cec 100644 --- a/template/fastapi/scripts/stop.sh +++ b/template/fastapi/scripts/stop.sh @@ -1,5 +1,6 @@ #!/bin/bash SERVICE_NAME=$TEMPLATE_SERVICE_NAME_MIDDLE + stop_app() { supervisorctl stop $SERVICE_NAME } diff --git a/template/fastapi/service/main.py b/template/fastapi/service/main.py index 19041020..4faec919 100644 --- a/template/fastapi/service/main.py +++ b/template/fastapi/service/main.py @@ -23,7 +23,6 @@ app.include_router(health.router, prefix="/api/v1/$TEMPLATE_SERVICE_NAME_SHORTER ############################################################################# - def init_framwork(): SysomFramework\ .init(YAML_CONFIG) \ -- Gitee From 083810673854f58054b344b245b66d30080f06a6 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 19 Jun 2023 10:19:28 +0800 Subject: [PATCH 102/196] fix(rpm): Missing copy config file --- tools/deploy/sysom.spec | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/deploy/sysom.spec b/tools/deploy/sysom.spec index 0867bcd8..d38d5dbe 100755 --- a/tools/deploy/sysom.spec +++ b/tools/deploy/sysom.spec @@ -154,6 +154,7 @@ cp -a script %{buildroot}/usr/local/sysom/init_scripts cp -a sysom_web/dist %{buildroot}/usr/local/sysom/web cp -a deps %{buildroot}/usr/local/sysom/deps cp -a environment %{buildroot}/usr/local/sysom/environment +cp -a conf/config.yml %{buildroot}/etc/sysom/config.yml #mkdir -p %{buildroot}/usr/local/sysom/server/redis # cp -a sysom_server %{buildroot}/usr/local/sysom/server/target -- Gitee From ffd721b637a617a5efa7d31bdb0f029675205e6f Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 19 Jun 2023 17:05:00 +0800 Subject: [PATCH 103/196] fix(rpm): Missing add /etc/sysom to %files --- tools/deploy/sysom.spec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/deploy/sysom.spec b/tools/deploy/sysom.spec index d38d5dbe..64273c81 100755 --- a/tools/deploy/sysom.spec +++ b/tools/deploy/sysom.spec @@ -154,7 +154,8 @@ cp -a script %{buildroot}/usr/local/sysom/init_scripts cp -a sysom_web/dist %{buildroot}/usr/local/sysom/web cp -a deps %{buildroot}/usr/local/sysom/deps cp -a environment %{buildroot}/usr/local/sysom/environment -cp -a conf/config.yml %{buildroot}/etc/sysom/config.yml +mkdir -p %{buildroot}/etc +cp -a conf %{buildroot}/etc/sysom #mkdir -p %{buildroot}/usr/local/sysom/server/redis # cp -a sysom_server %{buildroot}/usr/local/sysom/server/target @@ -166,6 +167,7 @@ cp -a conf/config.yml %{buildroot}/etc/sysom/config.yml %files /usr/local/sysom/ +/etc/sysom %post # bash +x /usr/local/sysom/init_scripts/sysom.sh install ALL -- Gitee From 4876b42570f01173982d37efbdb33edcba098b80 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 20 Jun 2023 16:45:27 +0800 Subject: [PATCH 104/196] fix(rpm): rpm deployment error --- deps/3_prometheus/install.sh | 1 + deps/4_grafana/install.sh | 1 + environment/1_sdk/sysom_utils/config_parser.py | 2 ++ script/server/sysom_migration/init.sh | 2 +- script/server/sysom_monitor_server/db_migrate.sh | 4 +++- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/deps/3_prometheus/install.sh b/deps/3_prometheus/install.sh index 76886c07..06321546 100644 --- a/deps/3_prometheus/install.sh +++ b/deps/3_prometheus/install.sh @@ -24,6 +24,7 @@ install_prometheus() ls | grep ${PROMETHEUS_TAR} 1>/dev/null 2>/dev/null if [ $? -ne 0 ] then + rpm -q --quiet wget || yum install -y wget wget ${OSS_URL}/${PROMETHEUS_TAR} || wget ${PROMETHEUS_DL_URL}/${PROMETHEUS_TAR} ls fi diff --git a/deps/4_grafana/install.sh b/deps/4_grafana/install.sh index 2b24b55f..53a89b8b 100644 --- a/deps/4_grafana/install.sh +++ b/deps/4_grafana/install.sh @@ -14,6 +14,7 @@ install_grafana() ls | grep ${GRAFANA_PKG} 1>/dev/null 2>/dev/null if [ $? -ne 0 ] then + rpm -q --quiet wget || yum install -y wget wget ${OSS_URL}/${GRAFANA_PKG} || wget ${GRAFANA_DL_URL}/${GRAFANA_PKG} ls fi diff --git a/environment/1_sdk/sysom_utils/config_parser.py b/environment/1_sdk/sysom_utils/config_parser.py index 030dd984..69845937 100644 --- a/environment/1_sdk/sysom_utils/config_parser.py +++ b/environment/1_sdk/sysom_utils/config_parser.py @@ -25,6 +25,8 @@ def dict_merge(dct: dict, merge_dct: dict): Return: None """ + if dct is None or merge_dct is None: + return for k, v in merge_dct.items(): if (k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], dict)): # noqa dict_merge(dct[k], merge_dct[k]) diff --git a/script/server/sysom_migration/init.sh b/script/server/sysom_migration/init.sh index d9157ed1..1d9796e0 100644 --- a/script/server/sysom_migration/init.sh +++ b/script/server/sysom_migration/init.sh @@ -1,7 +1,7 @@ #!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME -SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) BaseDir=$(dirname $(readlink -f "$0")) SERVICE_NAME=sysom-migration diff --git a/script/server/sysom_monitor_server/db_migrate.sh b/script/server/sysom_monitor_server/db_migrate.sh index f530e1f8..ac7881c3 100644 --- a/script/server/sysom_monitor_server/db_migrate.sh +++ b/script/server/sysom_monitor_server/db_migrate.sh @@ -10,7 +10,9 @@ source_virtualenv() { db_migrate() { pushd ${SERVICE_HOME} - alembic upgrade head + if [ -d "alembic" ]; then + alembic upgrade head + fi popd } -- Gitee From 59dd9c2bd81c4a5b197661c69bfe2af5dc44d21d Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 21 Jun 2023 11:11:41 +0800 Subject: [PATCH 105/196] fix(script): Change FIRST_INIT_DONE mark failed due to missing envs --- script/server/init.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/server/init.sh b/script/server/init.sh index e1f1af52..31adb98b 100644 --- a/script/server/init.sh +++ b/script/server/init.sh @@ -9,7 +9,7 @@ if [ $FIRST_INIT_DONE == 0 ]; then bash +x sysom.sh init ALL local_app_home=$(dirname ${cur_script_home}) - if [ "${local_app_home}" == "${APP_HOME}" ]; then + if [ "${local_app_home}" == "${APP_HOME}" ] || [ -z "$APP_HOME" ] || [ -z "$SCRIPT_HOME" ]; then sed -i 's/^FIRST_INIT_DONE=0/FIRST_INIT_DONE=1/g' $0 else sed -i 's/^FIRST_INIT_DONE=0/FIRST_INIT_DONE=1/g' ${SCRIPT_HOME}/server/init.sh -- Gitee From c335ad08e3b8ef8b0c4037edc47bfce63de1d926 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 21 Jun 2023 11:25:14 +0800 Subject: [PATCH 106/196] fix(sysom_install): Misspelled green as grean --- script/sysom_install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/sysom_install.sh b/script/sysom_install.sh index 7aa40250..505a0ea7 100755 --- a/script/sysom_install.sh +++ b/script/sysom_install.sh @@ -56,11 +56,11 @@ update_global_config() { ensure_config_exist() { if [ ! -f "$SYSOM_CONF" ]; then - grean "old $SYSOM_CONF not exist, now init and set local configure .............." + green "old $SYSOM_CONF not exist, now init and set local configure .............." cp ${LocalAppHome}/conf/config.yml $SYSOM_CONF update_global_config else - grean "old $SYSOM_CONF exist, not update configure .............." + green "old $SYSOM_CONF exist, not update configure .............." fi } -- Gitee From 6459bb2c9c87ccb55ebe6a696b5b892edfebc664 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Wed, 21 Jun 2023 15:26:58 +0800 Subject: [PATCH 107/196] fix(diagnosis): Node-side scripts are not multi-architecture compatible --- sysom_server/sysom_diagnosis/config.yml | 4 +++- .../sysom_diagnosis/scripts/node_init.sh | 21 ++++++++++--------- .../sysom_diagnosis/scripts/node_update.sh | 12 +++++++---- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/sysom_server/sysom_diagnosis/config.yml b/sysom_server/sysom_diagnosis/config.yml index 33504e0f..1e256eb3 100644 --- a/sysom_server/sysom_diagnosis/config.yml +++ b/sysom_server/sysom_diagnosis/config.yml @@ -1,6 +1,6 @@ vars: SYSAK_DOWNLOAD_URL: &SYSAK_DOWNLOAD_URL https://mirrors.openanolis.cn/sysak/packages/ - SYSAK_VERSION: &SYSAK_VERSION 1.3.0-2 + SYSAK_VERSION: &SYSAK_VERSION 2.1.0-2 SERVICE_NAME: &SERVICE_NAME sysom_diagnosis SERVICE_CONSUMER_GROUP: !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] @@ -41,6 +41,8 @@ sysom_service: # 节点测配置 sysom_node: version: 2.1 + env: + SYSAK_VERSION: *SYSAK_VERSION # 节点分发配置 delivery: from_dir: scripts diff --git a/sysom_server/sysom_diagnosis/scripts/node_init.sh b/sysom_server/sysom_diagnosis/scripts/node_init.sh index f25253ba..4c76ea5b 100755 --- a/sysom_server/sysom_diagnosis/scripts/node_init.sh +++ b/sysom_server/sysom_diagnosis/scripts/node_init.sh @@ -1,25 +1,26 @@ #!/bin/bash -x - RESOURCE_DIR=${NODE_HOME}/${SERVICE_NAME} -SYSAK_VER=1.3.0-2 -SYSAK_ARCH=x86_64 -SYSAK_PKG=sysak-${SYSAK_VER}.${SYSAK_ARCH}.rpm -main() -{ +if [ "$SYSAK_VERTION" == "" ]; then + export SYSAK_VERTION=2.1.0-2 +fi +if [ "$ARCH" == "" ]; then + export ARCH=x86_64 +fi + +main() { rpm -qa | grep sysak - if [ $? -eq 0 ] - then + if [ $? -eq 0 ]; then exit 0 fi ###install sysak from local yum repo first### yum install -y sysak - if [ $? -eq 0 ] - then + if [ $? -eq 0 ]; then exit 0 fi + SYSAK_PKG=sysak-${SYSAK_VERTION}.${ARCH}.rpm yum install -y ${SYSAK_PKG} exit $? } diff --git a/sysom_server/sysom_diagnosis/scripts/node_update.sh b/sysom_server/sysom_diagnosis/scripts/node_update.sh index d895e97b..c000871b 100755 --- a/sysom_server/sysom_diagnosis/scripts/node_update.sh +++ b/sysom_server/sysom_diagnosis/scripts/node_update.sh @@ -1,9 +1,12 @@ #!/bin/bash -x - RESOURCE_DIR=${NODE_HOME}/${SERVICE_NAME} -SYSAK_VER=1.3.0-2 -SYSAK_ARCH=x86_64 -SYSAK_PKG=sysak-${SYSAK_VER}.${SYSAK_ARCH}.rpm + +if [ "$SYSAK_VERTION" == "" ]; then + export SYSAK_VERTION=2.1.0-2 +fi +if [ "$ARCH" == "" ]; then + export ARCH=x86_64 +fi main() { @@ -14,6 +17,7 @@ main() yum erase -y `rpm -qa | grep sysak` fi + SYSAK_PKG=sysak-${SYSAK_VERTION}.${ARCH}.rpm yum install -y ${SYSAK_PKG} exit $? } -- Gitee From efdb9a0ebc23ceba5a82c061643da0ab5c138fa1 Mon Sep 17 00:00:00 2001 From: ydzhang Date: Tue, 13 Jun 2023 22:01:58 +0800 Subject: [PATCH 108/196] adjust hotfix_builder to ms change builder info micro service unite the hotfix job topic of builder and server to hotfix_msg add the sys env into ini and renew the usage page of check_env change building state update method This commit contains: 1. Delete all the API communication with server. Use cec communication instead. 2. Add cec consumer crash recovery. Under the situation that redis is crash, the consumer process will exit. This commit add max_retry_time to sleep for a while, and reload the cec consumer again. 3. Config the work process from 2 to 1, in gunicorn.py remove comment change hotfix builder service name --- script/server/deploy_exclude | 1 - script/server/sysom_hotfix_builder/clear.sh | 7 +- .../server/sysom_hotfix_builder/db_migrate.sh | 20 ++ script/server/sysom_hotfix_builder/init.sh | 23 +- script/server/sysom_hotfix_builder/install.sh | 12 +- .../sysom_hotfix_builder/requirements.txt | 15 + script/server/sysom_hotfix_builder/start.sh | 9 +- script/server/sysom_hotfix_builder/stop.sh | 2 +- .../sysom-hotfix-builder.ini | 9 - .../sysom-hotfix_builder.ini | 9 + sysom_server/deploy_exclude | 1 - sysom_server/sysom_hotfix/apps/hotfix/apps.py | 7 + .../sysom_hotfix/apps/hotfix/views.py | 11 +- sysom_server/sysom_hotfix/config.yml | 3 + .../{apps/hotfix => lib}/function.py | 67 ++-- sysom_server/sysom_hotfix_builder/alembic.ini | 102 ++++++ .../sysom_hotfix_builder/alembic/README | 1 + .../sysom_hotfix_builder/alembic/env.py | 110 ++++++ .../alembic/script.py.mako | 24 ++ .../sysom_hotfix_builder/app/__init__.py | 8 + .../sysom_hotfix_builder/{ => app}/builder.py | 312 ++++++------------ sysom_server/sysom_hotfix_builder/app/crud.py | 30 ++ .../sysom_hotfix_builder/app/database.py | 20 ++ .../sysom_hotfix_builder/app/models.py | 22 ++ .../app/routers/health.py | 21 ++ .../sysom_hotfix_builder/app/schemas.py | 22 ++ sysom_server/sysom_hotfix_builder/builder.ini | 19 -- .../sysom_hotfix_builder/conf/common.py | 33 ++ .../sysom_hotfix_builder/conf/develop.py | 15 + .../sysom_hotfix_builder/conf/gunicorn.py | 23 ++ .../{ => conf}/img_list.json | 0 .../sysom_hotfix_builder/conf/product.py | 15 + .../sysom_hotfix_builder/conf/settings.py | 19 ++ .../sysom_hotfix_builder/conf/testing.py | 14 + sysom_server/sysom_hotfix_builder/config.yml | 51 +++ .../sysom_hotfix_builder/lib/README.md | 1 + sysom_server/sysom_hotfix_builder/main.py | 47 +++ .../{ => script}/build_hotfix.sh | 0 .../{ => script}/build_rpm.sh | 0 .../{ => script}/check_env.sh | 10 +- sysom_web/yarn.lock | 10 +- 41 files changed, 832 insertions(+), 293 deletions(-) create mode 100644 script/server/sysom_hotfix_builder/db_migrate.sh delete mode 100644 script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini create mode 100644 script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini rename sysom_server/sysom_hotfix/{apps/hotfix => lib}/function.py (91%) create mode 100644 sysom_server/sysom_hotfix_builder/alembic.ini create mode 100644 sysom_server/sysom_hotfix_builder/alembic/README create mode 100644 sysom_server/sysom_hotfix_builder/alembic/env.py create mode 100644 sysom_server/sysom_hotfix_builder/alembic/script.py.mako create mode 100644 sysom_server/sysom_hotfix_builder/app/__init__.py rename sysom_server/sysom_hotfix_builder/{ => app}/builder.py (69%) create mode 100644 sysom_server/sysom_hotfix_builder/app/crud.py create mode 100644 sysom_server/sysom_hotfix_builder/app/database.py create mode 100644 sysom_server/sysom_hotfix_builder/app/models.py create mode 100644 sysom_server/sysom_hotfix_builder/app/routers/health.py create mode 100644 sysom_server/sysom_hotfix_builder/app/schemas.py delete mode 100644 sysom_server/sysom_hotfix_builder/builder.ini create mode 100644 sysom_server/sysom_hotfix_builder/conf/common.py create mode 100644 sysom_server/sysom_hotfix_builder/conf/develop.py create mode 100644 sysom_server/sysom_hotfix_builder/conf/gunicorn.py rename sysom_server/sysom_hotfix_builder/{ => conf}/img_list.json (100%) create mode 100644 sysom_server/sysom_hotfix_builder/conf/product.py create mode 100644 sysom_server/sysom_hotfix_builder/conf/settings.py create mode 100644 sysom_server/sysom_hotfix_builder/conf/testing.py create mode 100644 sysom_server/sysom_hotfix_builder/config.yml create mode 100644 sysom_server/sysom_hotfix_builder/lib/README.md create mode 100644 sysom_server/sysom_hotfix_builder/main.py rename sysom_server/sysom_hotfix_builder/{ => script}/build_hotfix.sh (100%) rename sysom_server/sysom_hotfix_builder/{ => script}/build_rpm.sh (100%) rename sysom_server/sysom_hotfix_builder/{ => script}/check_env.sh (91%) diff --git a/script/server/deploy_exclude b/script/server/deploy_exclude index 5e69798f..e69de29b 100644 --- a/script/server/deploy_exclude +++ b/script/server/deploy_exclude @@ -1 +0,0 @@ -sysom_hotfix_builder \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/clear.sh b/script/server/sysom_hotfix_builder/clear.sh index 3dd3743b..30ffeb37 100644 --- a/script/server/sysom_hotfix_builder/clear.sh +++ b/script/server/sysom_hotfix_builder/clear.sh @@ -1,17 +1,18 @@ #!/bin/bash BaseDir=$(dirname $(readlink -f "$0")) -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder_service clear_app() { LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix sed -i '/hotfix_builder/d' /etc/exports - + result=$(systemctl is-active nfs-server) if [[ $result =~ "inactive" ]]; then - systemctl restart nfs-server + systemctl start nfs-server fi umount -f $LOCAL_NFS_HOME rm -rf ${SERVER_HOME}/builder + systemctl stop nfs-server rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini systemctl stop nfs-server ###use supervisorctl update to stop and clear services### diff --git a/script/server/sysom_hotfix_builder/db_migrate.sh b/script/server/sysom_hotfix_builder/db_migrate.sh new file mode 100644 index 00000000..f7007941 --- /dev/null +++ b/script/server/sysom_hotfix_builder/db_migrate.sh @@ -0,0 +1,20 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + if [ -d "alembic/versions" ]; then + alembic upgrade head + fi + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/init.sh b/script/server/sysom_hotfix_builder/init.sh index a06a00e9..679dd04c 100644 --- a/script/server/sysom_hotfix_builder/init.sh +++ b/script/server/sysom_hotfix_builder/init.sh @@ -1,16 +1,10 @@ -#! /bin/bash +#!/bin/bash SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} -SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} -VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME BaseDir=$(dirname $(readlink -f "$0")) -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder_service NFS_SERVER_IP=${SERVER_LOCAL_IP} -init_conf() { - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ -} - mount_nfs() { HOTFIX_NFS_HOME=${SERVER_HOME}/hotfix_builder/hotfix-nfs @@ -18,10 +12,23 @@ mount_nfs() mkdir -p ${LOCAL_NFS_HOME} + result=$(systemctl is-active nfs-server) + if [[ $result =~ "inactive" ]]; then + systemctl start nfs-server + fi + sudo umount $LOCAL_NFS_HOME # Remove the mounted directory in case it mounted before sudo mount -t nfs ${NFS_SERVER_IP}:${HOTFIX_NFS_HOME} ${LOCAL_NFS_HOME} || exit 1 } +init_conf() { + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=$(cat /proc/cpuinfo | grep processor | wc -l) + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/gunicorn.py +} + init_app() { init_conf mount_nfs diff --git a/script/server/sysom_hotfix_builder/install.sh b/script/server/sysom_hotfix_builder/install.sh index 050d16fd..47d17d6f 100644 --- a/script/server/sysom_hotfix_builder/install.sh +++ b/script/server/sysom_hotfix_builder/install.sh @@ -3,7 +3,7 @@ SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder_service if [ "$UID" -ne 0 ]; then echo "Please run as root" @@ -16,17 +16,17 @@ install_requirement() { popd } -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - install_package() { rpm -q --quiet make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || yum install -y make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || exit 1 rpm -q --quiet docker git || yum install -y docker git || echo "Warngin : Docker is not installed in this machine!" } +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + install_app() { rpm -q --quiet gcc || yum install -y gcc rpm -q --quiet make || yum install -y make diff --git a/script/server/sysom_hotfix_builder/requirements.txt b/script/server/sysom_hotfix_builder/requirements.txt index d3d398e7..ee975fdb 100644 --- a/script/server/sysom_hotfix_builder/requirements.txt +++ b/script/server/sysom_hotfix_builder/requirements.txt @@ -2,4 +2,19 @@ clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 +sysom_utils>=0.0.1 +alembic==1.7.7 +anyio==3.6.2 +asyncer==0.0.2 +asyncssh==2.12.0 +fastapi==0.83.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +uvicorn==0.16.0 +gunicorn==20.1.0 +python-multipart==0.0.5 +###################################################################### +# Add your custom python requirements here +###################################################################### requests==2.27.1 \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/start.sh b/script/server/sysom_hotfix_builder/start.sh index b6464a14..c9afe102 100644 --- a/script/server/sysom_hotfix_builder/start.sh +++ b/script/server/sysom_hotfix_builder/start.sh @@ -1,10 +1,9 @@ #!/bin/bash -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder_service is_start() { supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then + if [ $? -eq 0 ]; then return 1 else return 0 @@ -13,11 +12,11 @@ is_start() { start_app() { is_start - if [[ $? == 0 ]];then + if [[ $? == 0 ]]; then supervisorctl start $SERVICE_NAME is_start if [[ $? == 0 ]]; then - echo "${SERVICE_NAME} service start fail, please check log" + echo "${SERVICE_NAME} service start fail, please check log" exit 1 else echo "supervisorctl start ${SERVICE_NAME} success..." diff --git a/script/server/sysom_hotfix_builder/stop.sh b/script/server/sysom_hotfix_builder/stop.sh index d6535439..6c63f186 100644 --- a/script/server/sysom_hotfix_builder/stop.sh +++ b/script/server/sysom_hotfix_builder/stop.sh @@ -1,5 +1,5 @@ #!/bin/bash -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder_service stop_app() { # kill all kpatch-build process diff --git a/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini b/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini deleted file mode 100644 index 3359d39b..00000000 --- a/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-hotfix-builder] -directory = /usr/local/sysom/server/sysom_hotfix_builder -command=/usr/local/sysom/environment/virtualenv/bin/python3 builder.py -startsecs=3 -autostart=true -autorestart=true -environment=PATH=/usr/local/sysom/environment/virtualenv/bin:%(ENV_PATH)s -stderr_logfile=/var/log/sysom/sysom-hotfix-builder-error.log -stdout_logfile=/var/log/sysom/sysom-hotfix-builder.log diff --git a/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini b/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini new file mode 100644 index 00000000..cbdf497d --- /dev/null +++ b/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini @@ -0,0 +1,9 @@ +[program:sysom-hotfix_builder_service] +directory=/usr/local/sysom/server/sysom-hotfix_builder_service +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +startsecs=3 +autostart=true +autorestart=true +stderr_logfile=/var/log/sysom/sysom-hotfix_builder_service.log +stdout_logfile=/var/log/sysom/sysom-hotfix_builder_service.log +environment=PATH=/usr/local/sysom/virtualenv/bin:%(ENV_PATH)s diff --git a/sysom_server/deploy_exclude b/sysom_server/deploy_exclude index 5e69798f..e69de29b 100644 --- a/sysom_server/deploy_exclude +++ b/sysom_server/deploy_exclude @@ -1 +0,0 @@ -sysom_hotfix_builder \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/apps.py b/sysom_server/sysom_hotfix/apps/hotfix/apps.py index 56f2514b..1542c435 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/apps.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/apps.py @@ -26,6 +26,13 @@ class HotfixConfig(AppConfig): .load_plugin_cls(CmgPlugin) \ .enable_channel_job() \ .start() + + # create CECListerner object for cec message listening + from lib.function import CECListener + ceclistener = CECListener(settings.YAML_CONFIG.get_service_config(), settings.SYSOM_CEC_URL, settings.SYSOM_CEC_HOTFIX_SERVER_MSG_TOPIC) + if ceclistener is None: + logger.error("INIT CECListener Failed...") + ceclistener.run() else: # nothing to do when execute database migrations pass diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index b4f89c03..5d490ad2 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -33,7 +33,7 @@ from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer from cec_base.event import Event from django.http import HttpResponse, FileResponse -from apps.hotfix.function import FunctionClass, RedisCache +from lib.function import FunctionClass, RedisCache class SaveUploadFile(APIView): authentication_classes = [] @@ -79,8 +79,7 @@ class SaveUploadFile(APIView): patch_file_repo = os.path.join(settings.HOTFIX_FILE_BRANCH_RULE) if not os.path.exists(patch_file_repo): os.makedirs(patch_file_repo) - # patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + str(time.time()).split(".")[0] + "." + file_upload_type - patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + datetime.now().strftime('%Y%m%d%H%M%S-%f') + "." + file_upload_type + patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + datetime.now().strftime('%Y%m%d%H%M%S') + "." + file_upload_type file_path = os.path.join(patch_file_repo, patch_file_name) try: with open(file_path, 'wb') as f: @@ -123,7 +122,7 @@ class HotfixAPIView(GenericViewSet, log_file = "{}-{}.log".format(request.data["hotfix_name"], time.strftime("%Y%m%d%H%M%S")) hotfix_name = request.data["hotfix_name"] hotfix_name.replace(" ","-") - kernel_version = request.data['kernel_version'] + kernel_version = request.data['kernel_version'].replace(" ","") patch_file_name = request.data['patch_file_name'] # check if this kernel_version is customize @@ -191,6 +190,10 @@ class HotfixAPIView(GenericViewSet, if hotfix_object.building_status != self.build_failed: return other_response(message="This Hotfix Job is Not Failed", code=401) + # clear the original build log + hotfix_object.log="" + hotfix_object.save() + try: customize_version_object = KernelVersionModel.objects.all().filter(kernel_version=kernel_version).first() except Exception as e: diff --git a/sysom_server/sysom_hotfix/config.yml b/sysom_server/sysom_hotfix/config.yml index 08d3e700..54a1e154 100644 --- a/sysom_server/sysom_hotfix/config.yml +++ b/sysom_server/sysom_hotfix/config.yml @@ -36,3 +36,6 @@ sysom_service: deregister: 25 header: tls_skip_verify: false + cec: + max_retry_time: 10 # max retry the recovery of listener + sleep_time: 5 # second \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/function.py b/sysom_server/sysom_hotfix/lib/function.py similarity index 91% rename from sysom_server/sysom_hotfix/apps/hotfix/function.py rename to sysom_server/sysom_hotfix/lib/function.py index 8619d416..95f27546 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -2,16 +2,22 @@ import os import re import urllib import subprocess +import sys from clogger import logger from django.conf import settings import threading import redis import requests +import json +import time from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel from cec_base.producer import dispatch_producer, Producer +from cec_base.consumer import Consumer, dispatch_consumer +from cec_base.admin import dispatch_admin from bs4 import BeautifulSoup from lib.utils import human_datetime +from sysom_utils import SysomFramework """ Function class @@ -20,7 +26,7 @@ This class contains the support/tool function class FunctionClass(): def __init__(self): - self.cec = dispatch_producer(settings.SYSOM_CEC_URL) + self.producer = SysomFramework.cec_producer() def delete_document(self, doc_path, doc_name): document = os.path.join(doc_path, doc_name) @@ -133,7 +139,7 @@ class FunctionClass(): try: if not customize: if re.search('anolis', os_type): - self.cec.produce(cec_topic, { + self.producer.produce(cec_topic, { "hotfix_id" : hotfix_id, "kernel_version" : kernel_version, "patch_name" : patch_file, @@ -145,7 +151,6 @@ class FunctionClass(): "git_repo": "git@gitee.com:anolis/cloud-kernel.git", "customize": 0 }) - return True else: # this is customize kernel source_repo = kwargs['source_repo'] @@ -154,7 +159,7 @@ class FunctionClass(): debuginfo_link = kwargs['debuginfo_link'] image = kwargs['image'] is_src_package = kwargs["is_src_package"] - self.cec.produce(cec_topic, { + self.producer.produce(cec_topic, { "hotfix_id" : hotfix_id, "kernel_version" : kernel_version, "hotfix_name" : hotfix_name, @@ -171,6 +176,7 @@ class FunctionClass(): "image": image, "is_src_package": is_src_package }) + self.producer.flush() return True except Exception as e: logger.exception(e) @@ -446,17 +452,17 @@ class FunctionClass(): logger.error(e) - class CECListener(): - def __init__(self, cec_url, listen_topic) -> None: + def __init__(self, con, cec_url, listen_topic) -> None: try: logger.info("Server CECListener init ...") - print(">>>>>>>>>>>>>>>>init CECListener....") + self.parameters = con self.cec_url = cec_url self.listen_topic = listen_topic self.sync_key = "sync" # this key is to tag this message for sync job self.rpm_key = "rpm_name" # this key is to tag this message for sync rpm name + self.log_key = "log" # this key is to tag this message for sync log self.thread_runner = threading.Thread(target=self.listener, name="hotfix_server_listener") except Exception as e: return None @@ -477,21 +483,28 @@ class CECListener(): group_id="server_listener") logger.info("Server CECListener init finished...") - for event in self.consumer: - try: - parameters = event.value - hotfix_id = parameters["hotfix_id"] - if self.sync_key in parameters: - if self.rpm_key in parameters: - - self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) - else: - self.update_hotfix_job_status(hotfix_id, parameters["status"]) - except Exception as e: - logger.error(str(e)) - finally: - logger.info("ack one msg from builder...") - self.consumer.ack(event=event) + + retry_time = 0 + while retry_time < self.parameters.cec.max_retry_time: + for event in self.consumer: + logger.info("processing one msg...") + try: + parameters = event.value + hotfix_id = parameters["hotfix_id"] + if self.sync_key in parameters: + if self.rpm_key in parameters: + self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) + if self.log_key in parameters: + self.sync_hotfix_log(hotfix_id) + else: + self.update_hotfix_job_status(hotfix_id, parameters["status"]) + except Exception as e: + logger.error(str(e)) + finally: + logger.info("ack one msg from builder...") + self.consumer.ack(event=event) + time.sleep(self.parameters.cec.sleep_time) + retry_time += 1 def update_hotfix_job_status(self, hotfix_id, status): hotfix = HotfixModel.objects.filter(id=hotfix_id).first() @@ -523,6 +536,16 @@ class CECListener(): logger.error("%s : Exception raised..." % sys._getframe().f_code.co_name) return None + def sync_hotfix_log(self, hotfix_id): + hotfix = HotfixModel.objects.filter(id=hotfix_id).first() + try: + log = "" + for line in open(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): + log = log + str(line) + hotfix.log = log + hotfix.save() + except Exception as e: + logger.error(str(e)) class RedisCache(): def __init__(self): diff --git a/sysom_server/sysom_hotfix_builder/alembic.ini b/sysom_server/sysom_hotfix_builder/alembic.ini new file mode 100644 index 00000000..f6ab9feb --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic.ini @@ -0,0 +1,102 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = "" + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/sysom_server/sysom_hotfix_builder/alembic/README b/sysom_server/sysom_hotfix_builder/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/alembic/env.py b/sysom_server/sysom_hotfix_builder/alembic/env.py new file mode 100644 index 00000000..8fcb7206 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/env.py @@ -0,0 +1,110 @@ +import inspect +import app.models as models +from logging.config import fileConfig +from sqlalchemy import engine_from_config +from sqlalchemy import pool +from app.models import Base +from alembic import context +from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL + +################################################################## +# Load yaml config first +################################################################## +mysql_config = YAML_CONFIG.get_server_config().db.mysql + +################################################################## +# Scan models +################################################################## +service_tables = [] +for name, data in inspect.getmembers(models, inspect.isclass): + if data.__module__ != "app.models": + continue + if "__tablename__" in data.__dict__: + service_tables.append(data.__dict__["__tablename__"]) + elif "__table__" in data.__dict__: + service_tables.append(data.__dict__["__table__"]) + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Update mysql config according config.yml +config.set_main_option( + "sqlalchemy.url", + SQLALCHEMY_DATABASE_URL +) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def include_object(object, name, type_, reflected, compare_to): + if type_ == "table" and name not in service_tables: + return False + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + include_object=include_object, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, + include_object=include_object + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/sysom_server/sysom_hotfix_builder/alembic/script.py.mako b/sysom_server/sysom_hotfix_builder/alembic/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/sysom_server/sysom_hotfix_builder/app/__init__.py b/sysom_server/sysom_hotfix_builder/app/__init__.py new file mode 100644 index 00000000..3fb0576a --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py +Description: +""" \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/builder.py b/sysom_server/sysom_hotfix_builder/app/builder.py similarity index 69% rename from sysom_server/sysom_hotfix_builder/builder.py rename to sysom_server/sysom_hotfix_builder/app/builder.py index 1083e8c0..45e77a26 100644 --- a/sysom_server/sysom_hotfix_builder/builder.py +++ b/sysom_server/sysom_hotfix_builder/app/builder.py @@ -19,85 +19,20 @@ import sys import subprocess import configparser import redis +import time from cec_base.consumer import Consumer, dispatch_consumer from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer, Producer +from conf.settings import YAML_CONFIG +from sysom_utils import CecTarget, SysomFramework class ServerConnector(): - def __init__(self, server_ip, username, password, cec_url=None, cec_topic=None): - self.server_ip = server_ip - self.username = username - self.password = password - self.token = None - self.cec_url = cec - self.cec = dispatch_producer(cec_url) + def __init__(self, cec_topic=None): + self.cec = dispatch_producer(YAML_CONFIG.get_cec_url(CecTarget.PRODUCER)) + #self.cec = SysomFramework.cec_consumer(cec_topic) self.cec_produce_topic = cec_topic - def get_token(self): - url = self.server_ip + "/api/v1/auth/" - headers = {"Content-Type":"application/json"} - cond = {"username":self.username, "password":self.password} - resp = requests.post(url, data=json.dumps(cond), headers = headers) - data = resp.json().get('data') ## it is a dict - token = data.get("token") - self.token = token - return token - - def insert_log_to_server(self, hotfix_id, logs): - url = self.server_ip + "/api/v1/hotfix/insert_building_log/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "log" : logs} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'content-type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("insert_log_to_server : can not connect to the server correctly..") - - def change_building_status(self, hotfix_id, status): - url = self.server_ip + "/api/v1/hotfix/update_building_status/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "status" : status} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("change_building_status : can not connect to the server correctly..") - - def sync_building_log(self, hotfix_id): - url = self.server_ip + "/api/v1/hotfix/sync_building_log/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("sync_building_log : can not connect to the server correctly..") - return resp - - def sync_rpm_name(self, hotfix_id, rpm_name): - url = self.server_ip + "/api/v1/hotfix/update_hotfix_name/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "rpm":rpm_name} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("sync_rpm_name : can not connect to the server correctly..") - return resp - # Here, we design to use some status more than building status # in building status , we include : waiting\building\success\failed # but more, we can send : sync to server for log and hotfix name sync @@ -108,58 +43,62 @@ class ServerConnector(): }) - def sync_rpm_name(self, hotfix_id, rpm_name): + def sync_rpm_name_cec(self, hotfix_id, rpm_name): logger.info("produce rpm_name ") self.cec.produce(self.cec_produce_topic, { "hotfix_id" : hotfix_id, + "type": "rpm", "sync" : True, "rpm_name" : rpm_name }) + def sync_building_log_cec(self, hotfix_id): + logger.info("sync hotfix build log") + self.cec.produce(self.cec_produce_topic, { + "hotfix_id": hotfix_id, + "type": "log", + "sync": True + }) + """ Object : Cache Function : initalize the Redis Cache Object """ class Cache(): - def __init__(self, con : configparser.ConfigParser): - redis_conn = dict(con.items('redis')) - host = redis_conn['host'] - port = redis_conn['port'] + def __init__(self, con : dict): + host = con.redis.host + port = con.redis.port pool = redis.ConnectionPool(host=host, port=port) self.conn = redis.Redis(connection_pool=pool) def get(self,key): return self.conn.get(key) + + def all_keys(self): + return self.conn.keys() - -class HotfixBuilder(): - - def __init__(self, con : configparser.ConfigParser): - # read init parameters from builder.ini - # cec - cec = dict(con.items('cec')) - cec_url = cec['cec_url'] - cec_hotfix_topic = cec['cec_hotfix_topic'] - hotfix_building_status_topic = cec['job_status_topic'] class HotfixBuilder(): - def __init__(self, nfs_dir_home, hotfix_base, cec_url, cec_topic, server_ip, username, password, packages_repo): + def __init__(self, con: dict): cache = Cache(con) self.cache = cache - self.nfs_dir_home = nfs_dir_home - self.hotfix_base = hotfix_base - self.cec_url = cec_url - self.builder_hotfix_package_repo = packages_repo - self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder") - self.cec_hotfix_topic = "hotfix_job" + ####################### + # Server config + ####################### + self.nfs_dir_home = con.builder.nfs_dir_home + self.max_retry_time = con.cec.max_retry + self.sleep_time = con.cec.retry_sleep_time + self.hotfix_base = con.builder.hotfix_base + self.builder_hotfix_package_repo = con.builder.package_repo + self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder", daemon=True) + self.cec_hotfix_topic = con.cec.hotfix_topic self.local_arch = os.uname().release.split(".")[-1] - self.connector = ServerConnector(server_ip, username, password, cec_url=cec_url, cec_topic=cec_topic) - self.tmpdir="/hotfix_tmpdir" - self.src_rpm_tmpdir="/tmp/hotfix_src" + self.connector = ServerConnector(cec_topic=con.cec.job_status_topic) + self.tmpdir=con.builder.tmpdir + self.src_rpm_tmpdir=con.builder.src_rpm_tmpdir self.hotfix_id = None self.fd = None - self.token = self.connector.get_token() self.prepare_env() ################################################################## @@ -174,20 +113,16 @@ class HotfixBuilder(): self.thread_runner.start() def change_building_status(self, status): - self.connector.change_building_status(self.hotfix_id, status) + self.connector.update_hotfix_building_status(self.hotfix_id, status) def die(self, msg): self.fd.write("%s \n" % msg) self.change_building_status("failed") def prepare_env(self): - # prepare kernel src and kaptch-build - # cmd = "chmod +x check_env.sh && ./check_env.sh -b %s -n %s" % (self.hotfix_base, self.nfs_dir_home) - # with os.popen(cmd) as process: - # output = process.read() # get the img_list image information and pull them based on machine's kernel arch - image_config_file = open(os.path.join(os.getcwd(), "img_list.json")) + image_config_file = open(os.path.join(os.getcwd(), "conf" , "img_list.json")) config_data = json.load(image_config_file) machine_kernel = platform.uname().release arch = machine_kernel.split(".")[-1] @@ -226,13 +161,13 @@ class HotfixBuilder(): os.makedirs(os.path.join(self.builder_hotfix_package_repo, "src_pack")) # copy build_hotfix.sh to BASE - if os.path.exists("./build_hotfix.sh"): - shutil.copy("./build_hotfix.sh", self.hotfix_base) + if os.path.exists(os.path.join(os.getcwd(), "script", "build_hotfix.sh")): + shutil.copy(os.path.join(os.getcwd(), "script", "build_hotfix.sh"), self.hotfix_base) else: logger.error("ERROR: cannot find build_hotfix.sh") - if os.path.exists("./build_rpm.sh"): - shutil.copy("./build_rpm.sh", self.hotfix_base) + if os.path.exists(os.path.join(os.getcwd(), "script", "build_rpm.sh")): + shutil.copy(os.path.join(os.getcwd(), "script", "build_rpm.sh"), self.hotfix_base) else: logger.error("ERROR: cannot find build_rpm.sh") @@ -240,8 +175,9 @@ class HotfixBuilder(): os.system("rm -rf {} && mkdir {} ".format(self.tmpdir, self.tmpdir)) os.system("rm -rf {}/*patch".format(self.hotfix_base)) - def find_build_rpm(self): - directory = os.path.join(self.tmpdir, "rpmbuild") + def find_build_rpm(self, kernel_version): + arch = kernel_version.split(".")[-1] + directory = os.path.join(self.tmpdir, "rpmbuild", "RPMS", arch) rpms = [] for root, dirs, files in os.walk(directory): for eachfile in files: @@ -334,7 +270,7 @@ class HotfixBuilder(): def get_building_image(self, kernel_version): arch = kernel_version.split(".")[-1] - image_list_file = open('./img_list.json') + image_list_file = open(os.path.join(os.getcwd(), "conf" , "img_list.json")) images = json.load(image_list_file) return images[arch]['anolis'] @@ -435,7 +371,6 @@ class HotfixBuilder(): except Exception as e: self.fd.write(str(e)) logger.error(str(e)) - # self.connector.change_building_status(hotfix_id, "failed") description = self.extract_description_from_patch(local_patch) # run the build hotfix script @@ -453,23 +388,20 @@ class HotfixBuilder(): return_code=p.wait() logger.info("The return code is %d" % return_code) - rpm_names = self.find_build_rpm() - - # when finished building, sync the build log - #self.connector.sync_building_log(hotfix_id) + rpm_names = self.find_build_rpm(kernel_version) # if rpm is more than one, upload it one by one for each_rpm in rpm_names: - resp = self.connector.sync_rpm_name(hotfix_id, each_rpm) - if resp.status_code != 200: - self.connector.insert_log_to_server(hotfix_id, "cannot sync rpm package name %s" % each_rpm) + self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) + + self.connector.sync_building_log_cec(self.hotfix_id) # check the last output if return_code == 0: - self.connector.change_building_status(hotfix_id, "success") + self.change_building_status("success") else: os.system("echo \"BUILD FAILED\" >> %s" % log_file_path) - self.connector.change_building_status(hotfix_id, "failed") + self.change_building_status("failed") """ build the customize kernel from user defined @@ -586,23 +518,22 @@ class HotfixBuilder(): return_code=p.wait() logger.info("The return code is %d" % return_code) - rpm_names = self.find_build_rpm() + rpm_names = self.find_build_rpm(kernel_version) # when finished building, sync the build log #self.connector.sync_building_log(hotfix_id) # if rpm is more than one, upload it one by one for each_rpm in rpm_names: - resp = self.connector.sync_rpm_name(hotfix_id, each_rpm) - if resp.status_code != 200: - self.connector.insert_log_to_server(hotfix_id, "cannot sync rpm package name %s" % each_rpm) + self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) + self.connector.sync_building_log_cec(self.hotfix_id) # check the last output if return_code == 0: - self.connector.change_building_status(hotfix_id, "success") + self.change_building_status("success") else: os.system("echo \"BUILD FAILED\" >> %s" % log_file_path) - self.connector.change_building_status(hotfix_id, "failed") + self.change_building_status("failed") ''' Each event is an object, the parameter is inside event.value @@ -614,91 +545,68 @@ class HotfixBuilder(): ''' def build(self): - with dispatch_admin(self.cec_url) as admin: - if not admin.is_topic_exist(self.cec_hotfix_topic): - admin.create_topic(self.cec_hotfix_topic) consumer_id = Consumer.generate_consumer_id() + if self.local_arch == "x86_64": - consumer = dispatch_consumer(self.cec_url, self.cec_hotfix_topic, + consumer = SysomFramework.cec_consumer(self.cec_hotfix_topic, consumer_id=consumer_id, group_id="hotfix_job_group_x86") else: - consumer = dispatch_consumer(self.cec_url, self.cec_hotfix_topic, + consumer = SysomFramework.cec_consumer(self.cec_hotfix_topic, consumer_id=consumer_id, group_id="hotfix_job_group_arm") - - for event in consumer: - try: - parameters = event.value - log_file = parameters['log_file'] - self.hotfix_id = parameters['hotfix_id'] - log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) - is_deleted = self.cache.get(str(self.hotfix_id)) - # if this hotfix_id is not exist in the deleted pool - if not is_deleted: - if parameters['arch'] == self.local_arch: - # this operation aims to clear the previous log if rebuild - with open(log_file_path, "w") as f: - f.write("=========================================================\n") - f.write("==========*******Sysom Hotfix Building System*******==============\n") - f.write("=========================================================\n") - - self.change_building_status("building") - - # for each run, update the repo - cmd = "chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) - with os.popen(cmd) as process: - output = process.read() - - customize = parameters['customize'] - # before build one job, clear the tmpdir - self.clear_tmpdir() - if not customize: - self.build_supported_kernel(parameters) - else: - self.build_customize_kernel(parameters) - logger.info(log_file) + retry_time = 0 + while retry_time < self.max_retry_time: + for event in consumer: + try: + parameters = event.value + log_file = parameters['log_file'] + self.hotfix_id = parameters['hotfix_id'] + log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) + is_deleted = self.cache.get(str(self.hotfix_id)) + # if this hotfix_id is not exist in the deleted pool + if not is_deleted: + if parameters['arch'] == self.local_arch: + # this operation aims to clear the previous log if rebuild + with open(log_file_path, "w") as f: + f.write("=========================================================\n") + f.write("=================Sysom Hotfix Building System===================\n") + f.write("=========================================================\n") + + self.change_building_status("building") + + # for each run, update the repo + cmd = "echo \"hello\" && chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) + with os.popen(cmd) as process: + output = process.read() + + customize = parameters['customize'] + # before build one job, clear the tmpdir + self.clear_tmpdir() + if not customize: + self.build_supported_kernel(parameters) + else: + self.build_customize_kernel(parameters) + logger.info(log_file) + self.fd.close() + else: + logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id) + except Exception as e: + logger.error(str(e)) + self.change_building_status("failed") + finally: + if self.fd is not None: self.fd.close() - else: - logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id) - except Exception as e: - logger.error(str(e)) - self.connector.change_building_status(self.hotfix_id, "failed") - finally: - if self.fd is not None: - self.fd.close() - consumer.ack(event) + consumer.ack(event) + time.sleep(self.sleep_time) + retry_time += 1 - -if __name__ == "__main__": - config_file = "builder.ini" - - con = configparser.ConfigParser() - con.read(config_file, encoding='utf-8') +class HotfixBuilderMain(): - # cec - cec = dict(con.items('cec')) - cec_url = cec['cec_url'] - hotfix_building_status_topic = cec['job_status_topic'] - - # server config - sysom_server = dict(con.items('sysom_server')) - server_ip = sysom_server['server_ip'] - server_login_account = sysom_server['account'] - server_login_password = sysom_server['password'] - - # builder config - builder = dict(con.items('builder')) - hotfix_base = builder['hotfix_base'] - nfs_dir_home = builder['nfs_dir_home'] - builder_hotfix_package_repo = builder['package_repo'] + def __init__(self, config: YAML_CONFIG) -> None: + self.parameters = config.get_service_config() - - # if this builder run in local, we should use the local repo directory instead of the nfs directory - if re.search("127.0.0.1", server_ip): - nfs_dir_home="/usr/local/sysom/server/builder/hotfix" - - hotfix_builder = HotfixBuilder(nfs_dir_home, hotfix_base, cec_url, hotfix_building_status_topic, - server_ip, server_login_account, server_login_password, builder_hotfix_package_repo) - hotfix_builder.run() \ No newline at end of file + def start(self): + hotfix_builder = HotfixBuilder(self.parameters) + hotfix_builder.run() diff --git a/sysom_server/sysom_hotfix_builder/app/crud.py b/sysom_server/sysom_hotfix_builder/app/crud.py new file mode 100644 index 00000000..6e0e26c3 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/crud.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File crud.py +Description: +""" +from typing import Optional +from sqlalchemy.orm import Session +from app import models, schemas + +################################################################################################ +# Define database crud here +################################################################################################ + +# def get_person_by_name(db: Session, name: str) -> Optional[models.Person]: +# return db.query(models.Person).filter(models.Person.name == name).first() + +# def create_person(db: Session, person: schemas.Person) -> models.Person: +# person = models.Person(**person.dict()) +# db.add(person) +# db.commit() +# db.refresh(person) +# return person + +# def del_person_by_id(db: Session, person_id: int): +# person = db.get(models.Person, person_id) +# db.delete(person) +# db.commit() diff --git a/sysom_server/sysom_hotfix_builder/app/database.py b/sysom_server/sysom_hotfix_builder/app/database.py new file mode 100644 index 00000000..d1500828 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/database.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File database.py +Description: +""" +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from conf.settings import SQLALCHEMY_DATABASE_URL + +engine = create_engine( + SQLALCHEMY_DATABASE_URL, connect_args={} +) + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/app/models.py b/sysom_server/sysom_hotfix_builder/app/models.py new file mode 100644 index 00000000..60fa74d4 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/models.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File models.py +Description: +""" +from sqlalchemy import Column, Integer, String +from app.database import Base + + +########################################################################### +# Define databse model here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/sql-databases/ +# class Person(Base): +# __tablename__ = "sys_person" +# id = Column(Integer, primary_key=True) +# name = Column(String(254), unique=True) +# age = Column(Integer) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/app/routers/health.py b/sysom_server/sysom_hotfix_builder/app/routers/health.py new file mode 100644 index 00000000..9b6f43fc --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/routers/health.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/04/17 19:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get("/check") +async def get_channel_config(): + return { + "code": 0, + "err_msg": "", + "data": "" + } diff --git a/sysom_server/sysom_hotfix_builder/app/schemas.py b/sysom_server/sysom_hotfix_builder/app/schemas.py new file mode 100644 index 00000000..aa413a9d --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/schemas.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File schemas.py +Description: +""" +from pydantic import BaseModel + +########################################################################### +# Define schemas here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/response-model/ +# class Person(BaseModel): +# id: int +# name: str +# age: int + +# class Config: +# orm_mode = True \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/builder.ini b/sysom_server/sysom_hotfix_builder/builder.ini deleted file mode 100644 index 02aeff21..00000000 --- a/sysom_server/sysom_hotfix_builder/builder.ini +++ /dev/null @@ -1,19 +0,0 @@ -[sysom_server] -server_ip = http://127.0.0.1 -account = account -password = password - -[cec] -cec_url = redis://127.0.0.1:6379 -job_status_topic = hotfix_building_status - -[builder] -hotfix_base = /hotfix_build/hotfix -nfs_dir_home = /usr/local/sysom/server/builder/hotfix -package_repo = /hotfix/packages -tmpdir = /hotfix_tmpdir -src_rpm_tmpdir = /tmp/hotfix_src - -[redis] -host = localhost -port = 6379 diff --git a/sysom_server/sysom_hotfix_builder/conf/common.py b/sysom_server/sysom_hotfix_builder/conf/common.py new file mode 100644 index 00000000..1b38077e --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/common.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" +from pathlib import Path +from sysom_utils import ConfigParser, SysomFramework + +BASE_DIR = Path(__file__).resolve().parent.parent + +################################################################## +# Load yaml config first +################################################################## +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" + +YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) + +mysql_config = YAML_CONFIG.get_server_config().db.mysql +service_config = YAML_CONFIG.get_service_config() + +SysomFramework.init(YAML_CONFIG) + +################################################################## +# fastapi config +################################################################## +SQLALCHEMY_DATABASE_URL = ( + f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" + f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" +) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/develop.py b/sysom_server/sysom_hotfix_builder/conf/develop.py new file mode 100644 index 00000000..bb64bf49 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/develop.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File develoop.py +Description: +""" +from .common import * + +''' +开发环境配置项 +''' + +DEBUG = True \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/gunicorn.py b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py new file mode 100644 index 00000000..08c69325 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py @@ -0,0 +1,23 @@ +''' +Channel Service Gunicorn Settings +''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + +workers = 1 # 指定工作进程数, builder应该只开一个工作进程 + +threads = 3 + +bind = f'{bind}:{port}' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/sysom-hotfix_builder_service.log' + +loglevel = 'error' + +proc_name = 'channel_service' diff --git a/sysom_server/sysom_hotfix_builder/img_list.json b/sysom_server/sysom_hotfix_builder/conf/img_list.json similarity index 100% rename from sysom_server/sysom_hotfix_builder/img_list.json rename to sysom_server/sysom_hotfix_builder/conf/img_list.json diff --git a/sysom_server/sysom_hotfix_builder/conf/product.py b/sysom_server/sysom_hotfix_builder/conf/product.py new file mode 100644 index 00000000..c748de1d --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/product.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File product.py +Description: +""" +from .common import * + +''' +生产环境配置项 +''' + +DEBUG = False diff --git a/sysom_server/sysom_hotfix_builder/conf/settings.py b/sysom_server/sysom_hotfix_builder/conf/settings.py new file mode 100644 index 00000000..b9156df0 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/settings.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File settings.py +Description: +""" +import os + +env = os.environ.get("env", "product") + + +if env == "develop": + from .develop import * +elif env == "testing": + from .testing import * +elif env == "product": + from .product import * \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/testing.py b/sysom_server/sysom_hotfix_builder/conf/testing.py new file mode 100644 index 00000000..864f53a2 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/testing.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File testing.py +Description: +""" +from .common import * + +''' +测试环境配置项 +''' +DEBUG = True diff --git a/sysom_server/sysom_hotfix_builder/config.yml b/sysom_server/sysom_hotfix_builder/config.yml new file mode 100644 index 00000000..03efc403 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/config.yml @@ -0,0 +1,51 @@ +vars: + SERVICE_NAME: &SERVICE_NAME sysom_hotfix_builder + SERVICE_CONSUMER_GROUP: + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] + +sysom_server: + cec: + consumer_group: SYSOM_CEC_CHANNEL_CONSUMER_GROUP + +sysom_service: + service_name: *SERVICE_NAME + service_dir: *SERVICE_NAME + protocol: http + host: 127.0.0.1 + bind: 127.0.0.1 + port: 7011 + framework: + gcache: + protocol: redis + node_dispatch: + cmg: + tags: + - hotfix_builder + - FastApi + # Metadata of service + metadata: + check: + type: http + url: "/api/v1/hotfix_builder/health/check" + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false + cec: + job_status_topic: "hotfix_msg" + hotfix_topic: "hotfix_job" + max_retry: 10 # retry consumer listen time + retry_sleep_time: 5 # second + + builder: + hotfix_base: "/hotfix_build/hotfix" + nfs_dir_home: "/usr/local/sysom/server/builder/hotfix" + package_repo: "/hotfix/packages" + tmpdir: "/hotfix_tmpdir" + src_rpm_tmpdir: "/tmp/hotfix_src" + + redis: + host: "127.0.0.1" + port: 6379 + diff --git a/sysom_server/sysom_hotfix_builder/lib/README.md b/sysom_server/sysom_hotfix_builder/lib/README.md new file mode 100644 index 00000000..3ec74424 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/lib/README.md @@ -0,0 +1 @@ +The current directory holds the public libraries or utils needed for microservices \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/main.py b/sysom_server/sysom_hotfix_builder/main.py new file mode 100644 index 00000000..62dcb404 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/main.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File ssh.py +Description: +""" +from clogger import logger +from fastapi import FastAPI +import app.routers.health as health +from app.builder import HotfixBuilderMain +from conf.settings import YAML_CONFIG +from sysom_utils import CmgPlugin, SysomFramework + + +app = FastAPI() + +app.include_router(health.router, prefix="/api/v1/hotfix_builder/health") + + +############################################################################# +# Write your API interface here, or add to app/routes +############################################################################# + + +def init_framwork(): + SysomFramework\ + .init(YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() + logger.info("SysomFramework init finished!") + + +@app.on_event("startup") +async def on_start(): + init_framwork() + ############################################################################# + # Perform some microservice initialization operations over here + ############################################################################# + hotfix_builder_obj = HotfixBuilderMain(YAML_CONFIG) + hotfix_builder_obj.start() + + +@app.on_event("shutdown") +async def on_shutdown(): + pass \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/build_hotfix.sh b/sysom_server/sysom_hotfix_builder/script/build_hotfix.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/build_hotfix.sh rename to sysom_server/sysom_hotfix_builder/script/build_hotfix.sh diff --git a/sysom_server/sysom_hotfix_builder/build_rpm.sh b/sysom_server/sysom_hotfix_builder/script/build_rpm.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/build_rpm.sh rename to sysom_server/sysom_hotfix_builder/script/build_rpm.sh diff --git a/sysom_server/sysom_hotfix_builder/check_env.sh b/sysom_server/sysom_hotfix_builder/script/check_env.sh similarity index 91% rename from sysom_server/sysom_hotfix_builder/check_env.sh rename to sysom_server/sysom_hotfix_builder/script/check_env.sh index 6441ca0f..d7177638 100755 --- a/sysom_server/sysom_hotfix_builder/check_env.sh +++ b/sysom_server/sysom_hotfix_builder/script/check_env.sh @@ -5,15 +5,19 @@ usage() { echo "./check_env.sh [OPTIONS]"; echo ""; echo "Options:"; - echo " -p|--patch"; - echo " The patch (must not null)"; + echo " -b|--hotfix_base"; + echo " The hotfix running base directory"; + echo " -n|--nfs"; + echo " Select the NFS directory home"; + echo " -l|--log"; + echo " Select the log file path"; echo " -k|--kernelversion"; echo " The complete kernel version include architecture (must not null)"; echo " -h|--help"; echo " For help"; echo ""; echo "For example:"; - echo "./kpatch-packager.sh -p \${patch} -k \${kernel} -d \"\${description}\" -v --prefix=\${prefix_patch}"; + echo "./check_env.sh -k {kernel_version} -b {hotfix_base_directory} -n {nfs_directory_home} -l {log_file_path}"; } # this is to output the error msg diff --git a/sysom_web/yarn.lock b/sysom_web/yarn.lock index e7d2ee39..5bae1478 100644 --- a/sysom_web/yarn.lock +++ b/sysom_web/yarn.lock @@ -13961,7 +13961,7 @@ urix@^0.1.0: resolved "https://registry.npmmirror.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg== -url-parse@^1.5.10: +url-parse@^1.5.10, url-parse@^1.5.3: version "1.5.10" resolved "https://mirrors.cloud.tencent.com/npm/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== @@ -13969,14 +13969,6 @@ url-parse@^1.5.10: querystringify "^2.1.1" requires-port "^1.0.0" -url-parse@^1.5.3: - version "1.5.10" - resolved "https://registry.npmmirror.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" - integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== - dependencies: - querystringify "^2.1.1" - requires-port "^1.0.0" - url@^0.11.0: version "0.11.0" resolved "https://mirrors.cloud.tencent.com/npm/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" -- Gitee From ad58d012d91c51fc0c6bd042c36c20812c99896d Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 26 Jun 2023 15:47:02 +0800 Subject: [PATCH 109/196] fix(template): Replace proc_name with SERVICE_NAME --- template/fastapi/service/conf/gunicorn.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/template/fastapi/service/conf/gunicorn.py b/template/fastapi/service/conf/gunicorn.py index edca8d44..2635abf0 100644 --- a/template/fastapi/service/conf/gunicorn.py +++ b/template/fastapi/service/conf/gunicorn.py @@ -20,4 +20,4 @@ accesslog = '/var/log/sysom/$TEMPLATE_SERVICE_NAME_MIDDLE-access.log' loglevel = 'error' -proc_name = 'channel_service' +proc_name = '$TEMPLATE_SERVICE_NAME_service' -- Gitee From d4ba32fc13ce9f4f3547ea7111b42575e59c6c51 Mon Sep 17 00:00:00 2001 From: "fuqiu.ffq" Date: Tue, 27 Jun 2023 00:26:07 +0800 Subject: [PATCH 110/196] new channel design --- .../sysom_diagnosis/apps/task/executor.py | 2 +- .../sysom_diagnosis/apps/task/helper.py | 525 ++++++++++++------ .../sysom_diagnosis/service_scripts/base.py | 149 +++++ .../service_scripts/command_post.py | 22 + .../service_scripts/command_pre.py | 30 + 5 files changed, 567 insertions(+), 161 deletions(-) create mode 100644 sysom_server/sysom_diagnosis/service_scripts/base.py create mode 100644 sysom_server/sysom_diagnosis/service_scripts/command_post.py create mode 100644 sysom_server/sysom_diagnosis/service_scripts/command_pre.py diff --git a/sysom_server/sysom_diagnosis/apps/task/executor.py b/sysom_server/sysom_diagnosis/apps/task/executor.py index 66e6e558..aa4e9eef 100644 --- a/sysom_server/sysom_diagnosis/apps/task/executor.py +++ b/sysom_server/sysom_diagnosis/apps/task/executor.py @@ -65,7 +65,7 @@ class DiagnosisTaskExecutor(AsyncEventExecutor): # 2. Execute and Postprocess if res: - job_result = await DiagnosisHelper.execute(instance) + job_result = await DiagnosisHelper.execute(instance, res) await DiagnosisHelper.postprocess(instance, job_result) # 3. TODO: produce task execute result to cec diff --git a/sysom_server/sysom_diagnosis/apps/task/helper.py b/sysom_server/sysom_diagnosis/apps/task/helper.py index a912a218..ccdf633d 100644 --- a/sysom_server/sysom_diagnosis/apps/task/helper.py +++ b/sysom_server/sysom_diagnosis/apps/task/helper.py @@ -1,15 +1,26 @@ import json import os import subprocess +import asyncio from clogger import logger import tempfile import ast -from typing import List +from typing import List, Type, Optional from apps.task.models import JobModel +from importlib import import_module from django.conf import settings from lib.utils import uuid_8 from asgiref.sync import sync_to_async from channel_job.job import default_channel_job_executor, JobResult +from service_scripts.base import ( + DiagnosisJob, + DiagnosisJobResult, + DiagnosisTask, + DiagnosisTaskResult, + DiagnosisPreProcessor, + PostProcessResult, + DiagnosisPostProcessor, +) class DiagnosisHelper: @@ -20,7 +31,7 @@ class DiagnosisHelper: """Update JobModel object""" try: if isinstance(kwargs.get("result", ""), dict): - kwargs['result'] = json.dumps(kwargs.get('result', "")) + kwargs["result"] = json.dumps(kwargs.get("result", "")) instance.__dict__.update(**kwargs) await sync_to_async(instance.save)() except Exception as e: @@ -40,13 +51,17 @@ class DiagnosisHelper: # 1. Determines if there is a task with the same parameters # and a status of Running. - if JobModel.objects.filter( - status__in=["Ready", "Running"], service_name=service_name, - params=json.dumps(params) - ).first() \ - is not None: + if ( + JobModel.objects.filter( + status__in=["Ready", "Running"], + service_name=service_name, + params=json.dumps(params), + ).first() + is not None + ): raise Exception( - f"node:{data.get('instance', '')}, There are tasks in progress, {service_name}") + f"node:{data.get('instance', '')}, There are tasks in progress, {service_name}" + ) # 2. Create a task with Ready status task_params = { @@ -55,7 +70,7 @@ class DiagnosisHelper: "created_by": user_id, "params": json.dumps(params), "service_name": service_name, - "status": "Ready" + "status": "Ready", } return JobModel.objects.create(**task_params) @@ -71,34 +86,22 @@ class DiagnosisHelper: "created_by": user_id, "params": json.dumps(data), "service_name": service_name, - "status": "Ready" + "status": "Ready", } return JobModel.objects.create(**task_params) @staticmethod async def run_subprocess(cmd: List[str]) -> dict: - resp = subprocess.run(cmd, stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + resp = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) return { "stdout": resp.stdout.decode("utf-8"), "stderr": resp.stderr.decode("utf-8"), - "returncode": resp.returncode + "returncode": resp.returncode, } - # proc = await asyncio.create_subprocess_shell( - # cmd, - # stdout=asyncio.subprocess.PIPE, - # stderr=asyncio.subprocess.PIPE - # ) - # stdout, stderr = await proc.communicate() - # return { - # "stdout": stdout.decode("utf-8"), - # "stderr": stderr.decode("utf-8"), - # "returncode": proc.returncode - # } @staticmethod - async def preprocess(instance: JobModel) -> bool: - """"Perform diagnosis preprocessing + async def preprocess_v1(instance: JobModel) -> DiagnosisTask: + """ "Perform diagnosis preprocessing { "commands":[ @@ -110,106 +113,327 @@ class DiagnosisHelper: } } ] - } + } """ - success = False + service_name = instance.service_name + params = instance.params + if isinstance(params, str): + try: + params = json.loads(params) + except Exception as exc: + logger.exception(f"Task params loads error: {str(exc)}") + + # 2. Invoke preprocessing script（preprocessing script） + SCRIPTS_DIR = settings.SCRIPTS_DIR + service_path = os.path.join(SCRIPTS_DIR, service_name) + if not os.path.exists(service_path): + raise Exception("Can not find script file, please check service name") try: - service_name = instance.service_name - params = instance.params - if isinstance(params, str): - try: - params = json.loads(params) - except Exception as exc: - logger.exception( - f"Task params loads error: {str(exc)}") + resp = await DiagnosisHelper.run_subprocess( + [service_path, json.dumps(params)] + ) + except Exception as exc: + raise Exception(f"Execute preprocess script error: {str(exc)}") from exc - # 2. Invoke preprocessing script（preprocessing script） - SCRIPTS_DIR = settings.SCRIPTS_DIR - service_path = os.path.join(SCRIPTS_DIR, service_name) - if not os.path.exists(service_path): - raise Exception( - "Can not find script file, please check service name") + # 3. If the preprocessing script executes with an error + if resp["returncode"] != 0: + raise (Exception(f"Execute preprocess script error: {resp['stderr']}")) + + # 4. If the preprocessing script executes successfully, + # take out the processing result + stdout = resp["stdout"] + resp = ast.literal_eval(stdout) + resp_scripts = resp.get("commands") + + # 5. If the preprocessing result not contains 'commands', it's a not expect bug + if not resp_scripts: + raise ( + Exception( + f"Not find commands, please check the preprocess script return" + ) + ) + + diagnosis_task = DiagnosisTask( + jobs=[DiagnosisJob.from_dict(item) for item in resp_scripts], in_order=True + ) + + return diagnosis_task + + @staticmethod + async def preprocess_v2(instance: JobModel) -> Optional[DiagnosisTask]: + """Pre-processing V2 + + Args: + instance (JobModel): JobModel + + Returns: + Optional[DiagnosisTask]: Diagnosis task + """ + + def _get_pre_processor(service_name: str) -> Type[DiagnosisPreProcessor]: + """ + 根据要执行的命令，动态引入一个 PreProcessor 的实现用于执行前处理 + """ + try: + return import_module(f"service_scripts.{service_name}_pre").PreProcessor + except Exception as e: + raise Exception(f"No Pre-processor available => {str(e)}") + + # 1. Get params + service_name = instance.service_name + params = {} + if isinstance(instance.params, str): try: - resp = await DiagnosisHelper.run_subprocess([service_path, json.dumps(params)]) + params = json.loads(instance.params) except Exception as exc: + raise Exception(f"Task params loads error: {str(exc)}") + + instances = params.pop("instances", []) + if "instance" in params and len(instances) == 0: + instances.append(params.pop("instance", "")) + + if len(instances) == 0: + raise Exception("Missing params, instances or instance") + + # 2. Use PreProcessor to check if the version of the tool meets the requirements + try: + pre_processor = _get_pre_processor(service_name)(service_name, **params) + except Exception as e: + return None + for instance in instances: + job_params = { + "channel_type": params.pop("channel", "ssh"), + "channel_opt": "cmd", + "params": { + **params, + "instance": instance, + "command": pre_processor.get_version_cmd(), + }, + "timeout": 1000 * 60, # 1 minutes + } + job_result = await default_channel_job_executor.dispatch_job( + **job_params + ).execute_async() + if job_result.code != 0: raise Exception( - f"Execute preprocess script error: {str(exc)}" - ) from exc + f"Check tool version for '{instance}' failed: {job_result.err_msg}" + ) + if not pre_processor.version_check(job_result.result.strip()): + raise Exception( + f"Tool version less than {pre_processor.get_min_version_support()}" + ) - # 3. If the preprocessing script executes with an error - if resp["returncode"] != 0: - raise (Exception( - f"Execute preprocess script error: {resp['stderr']}" - )) - - # 4. If the preprocessing script executes successfully, - # take out the processing result - stdout = resp["stdout"] - resp = ast.literal_eval(stdout) - # - resp_scripts = resp.get("commands") - - # 5. If the preprocessing result not contains 'commands', it's a not expect bug - if not resp_scripts: - raise (Exception( - f"Not find commands, please check the preprocess script return" - )) + # 3. Use PreProcessor to convert params to diagnosis jobs + diagnosis_task = pre_processor.get_diagnosis_cmds(instances, params) + if diagnosis_task is None or len(diagnosis_task.jobs) == 0: + raise Exception(f"Pre-processor not return any diagnosis job") + + return diagnosis_task + + @staticmethod + async def preprocess(instance: JobModel) -> Optional[DiagnosisTask]: + """ "Perform diagnosis preprocessing + { + "commands":[ + { + "instance":"xxx", + "cmd":"xxx", + "params":{ => overide initial param + "region":"target_region" + } + } + ] + } + """ + diagnosis_task: Optional[DiagnosisTask] = None + try: + diagnosis_task = await DiagnosisHelper.preprocess_v2(instance) + if diagnosis_task is None: + diagnosis_task = await DiagnosisHelper.preprocess_v1(instance) - # 6. If the preprocessing script executes successfully, the parameters are compliant - # and the Job instance is updated + # If the pre-processor executes successfully, the parameters are compliant + # and the Job instance is updated await DiagnosisHelper._update_job( - instance, command=json.dumps(resp_scripts), status="Running" + instance, command=json.dumps(diagnosis_task.to_dict()), status="Running" ) - success = True except Exception as exc: - logger.exception( - f"Diagnosis preprocess error: {str(exc)}") + logger.exception(f"Diagnosis preprocess error: {str(exc)}") await DiagnosisHelper._update_job( - instance, result="Diagnosis preprocess error", status="Fail", - code=1, err_msg=f"Diagnosis preprocess error: {str(exc)}") - return success + instance, + result="Diagnosis preprocess error", + status="Fail", + code=1, + err_msg=f"Diagnosis preprocess error: {str(exc)}", + ) + return diagnosis_task @staticmethod - async def execute(instance: JobModel) -> JobResult: + async def execute( + instance: JobModel, diagnosis_task: DiagnosisTask + ) -> DiagnosisTaskResult: """Execute diagnosis task""" - job_result = JobResult(code=1, err_msg="Diagnosis execute task error") + diagnosis_task_result = DiagnosisTaskResult( + code=1, + job_results=[], + err_msg="Diagnosis execute task error", + in_order=diagnosis_task.in_order, + ) try: - resp_scripts = json.loads(instance.command) params = instance.params if isinstance(params, str): try: params = json.loads(params) except Exception as exc: - raise Exception( - f"Task params loads error: {str(exc)}") from exc - for idx, script in enumerate(resp_scripts): - job_params = { - "channel_type": params.pop("channel", "ssh"), - "channel_opt": "cmd", - "params": { - **params, - "instance": script.get("instance", "Unknown"), - "command": script.get("cmd", "Unknown"), - }, - "echo": { - "task_id": instance.task_id - }, - "timeout": 1000 * 60 * 10, # 10 minutes - } - job_result = await default_channel_job_executor.dispatch_job(**job_params).execute_async() - if job_result.code != 0: - raise Exception( - f"Task execute failed: {job_result.err_msg}") + raise Exception(f"Task params loads error: {str(exc)}") from exc + + if diagnosis_task.in_order: + # Execute diagnosis jobs in order + for job in diagnosis_task.jobs: + job_params = { + "channel_type": params.pop("channel", "ssh"), + "channel_opt": "cmd", + "params": { + **params, + "instance": job.instance, + "command": job.cmd, + }, + "echo": {"task_id": instance.task_id}, + "timeout": 1000 * 60 * 10, # 10 minutes + } + job_result = await default_channel_job_executor.dispatch_job( + **job_params + ).execute_async() + + diagnosis_task_result.job_results.append( + DiagnosisJobResult( + code=job_result.code, + err_msg=job_result.err_msg, + job=job, + stdout=job_result.result, + ) + ) + + if job_result.code != 0: + raise Exception(f"Task execute failed: {job_result.err_msg}") + else: + # Execute diagnosis jobs in parallel + tasks = [] + for job in diagnosis_task.jobs: + job_params = { + "channel_type": params.pop("channel", "ssh"), + "channel_opt": "cmd", + "params": { + **params, + "instance": job.instance, + "command": job.cmd, + }, + "echo": {"task_id": instance.task_id}, + "timeout": 1000 * 60 * 10, # 10 minutes + } + tasks.append( + default_channel_job_executor.dispatch_job( + **job_params + ).execute_async() + ) + job_results = asyncio.gather(*tasks) + diagnosis_task_result.job_results = [ + DiagnosisJobResult( + code=job_result.code, + err_msg=job_result.err_msg, + job=diagnosis_task.jobs[idx], + stdout=job_result.result, + ) + for idx, job_result in enumerate(job_results) + ] + for job_result in job_results: + if job_result.code != 0: + raise Exception(f"Task execut failed: {job_result.err_msg}") + diagnosis_task_result.code = 0 + diagnosis_task_result.err_msg = "" except Exception as exc: - job_result.err_msg = f"Diagnosis execute task error: {str(exc)}" - logger.exception(job_result.err_msg) + diagnosis_task_result.err_msg = f"Diagnosis execute task error: {str(exc)}" + logger.exception(diagnosis_task_result.err_msg) await DiagnosisHelper._update_job( - instance, result="Diagnosis execute task error", status="Fail", - code=1, err_msg=job_result.err_msg) - return job_result + instance, + result="Diagnosis execute task error", + status="Fail", + code=1, + err_msg=diagnosis_task_result.err_msg, + ) + return diagnosis_task_result + + @staticmethod + async def postprocess_v1( + instance: JobModel, diagnosis_task_result: DiagnosisTaskResult + ) -> PostProcessResult: + service_name = instance.service_name + # 执行后处理脚本，将结果整理成前端可识别的规范结构 + SCRIPTS_DIR = settings.SCRIPTS_DIR + service_post_name = service_name + "_post" + service_post_path = os.path.join(SCRIPTS_DIR, service_post_name) + if not os.path.exists(service_post_path): + raise Exception( + f"No matching post-processing script found: {service_post_path}" + ) + + # 创建一个临时文件，用于暂存中间结果 + with tempfile.NamedTemporaryFile(mode="w") as tmp_file: + try: + # 将要传递的中间结果写入到临时文件当中 + tmp_file.write(diagnosis_task_result.job_results[0].stdout) + tmp_file.flush() + resp = await DiagnosisHelper.run_subprocess( + [service_post_path, tmp_file.name, instance.task_id] + ) + except Exception as exc: + raise Exception(f"Execute postprocess script error: {str(exc)}") + + if resp["returncode"] != 0: + raise ( + Exception( + f"Execute postprocess script error: {str(resp['stderr'])}" + ) + ) + result = json.loads(resp["stdout"].strip()) + return PostProcessResult.from_dict(result) @staticmethod - async def postprocess(instance: JobModel, job_result: JobResult): + async def postprocess_v2( + instance: JobModel, diagnosis_task_result: DiagnosisTaskResult + ) -> Optional[PostProcessResult]: + """Post-processing V2 + + Args: + instance (JobModel): JobModel + diagnosis_task_result(DiagnosisTaskResult): Diagnosis task result, contain all job results + + Returns: + Optional[DiagnosisTask]: Diagnosis task + """ + + def _get_post_processor(service_name: str) -> Type[DiagnosisPostProcessor]: + """ + 根据要执行的命令，动态引入一个 PreProcessor 的实现用于执行前处理 + """ + try: + return import_module( + f"service_scripts.{service_name}_post" + ).PostProcessor + except Exception as e: + raise Exception(f"No Pre-processor available => {str(e)}") + + service_name = instance.service_name + # Use PostProcessor to convert diagnosis result to fron-end format data + try: + post_processor = _get_post_processor(service_name)(service_name) + except Exception as e: + return None + return post_processor.parse_diagnosis_result(diagnosis_task_result.job_results) + + @staticmethod + async def postprocess( + instance: JobModel, diagnosis_task_result: DiagnosisTaskResult + ): """Perform diagnosis postprocessing JobResult -> { "code": 0, => 0 表示成功，1表示失败 @@ -227,66 +451,47 @@ class DiagnosisHelper: "result": {} => 后处理脚本处理的结果，应该是一个 JSON Object } """ + + code = diagnosis_task_result.code + err_msg = diagnosis_task_result.err_msg + if code != 0: + # Diagnosis task execute failed + await DiagnosisHelper._update_job( + instance, + status="Fail", + code=code, + result=diagnosis_task_result.job_results[0].stdout, + err_msg=err_msg, + ) + return + post_process_result: Optional[PostProcessResult] = None try: - code = job_result.code - err_msg = job_result.err_msg - if code != 0: + # 1. 优先尝试使用 V2 方案 + post_process_result = await DiagnosisHelper.postprocess_v2( + instance, diagnosis_task_result + ) + if post_process_result is None: + # 2. 没有为目标诊断匹配到 v2 方案，回退尝试 v1 方案 + post_process_result = await DiagnosisHelper.postprocess_v1( + instance, diagnosis_task_result + ) + + if post_process_result.code != 0: + # 后处理脚本认为诊断出错 await DiagnosisHelper._update_job( - instance, status="Fail", code=code, - result=job_result.result, err_msg=err_msg) - return - # 如果任务执行成功，则执行后处理脚本 - params = instance.params - if isinstance(params, str): - try: - params = json.loads(params) - except Exception as exc: - raise Exception( - f"Task params loads error: {str(exc)}") from exc - - service_name = instance.service_name - # 执行后处理脚本，将结果整理成前端可识别的规范结构 - SCRIPTS_DIR = settings.SCRIPTS_DIR - service_post_name = service_name + '_post' - service_post_path = os.path.join( - SCRIPTS_DIR, service_post_name) - if not os.path.exists(service_post_path): - raise Exception( - f"No matching post-processing script found: {service_post_path}") - - # 创建一个临时文件，用于暂存中间结果 - with tempfile.NamedTemporaryFile(mode="w") as tmp_file: - try: - # 将要传递的中间结果写入到临时文件当中 - tmp_file.write(job_result.result) - tmp_file.flush() - resp = await DiagnosisHelper.run_subprocess([service_post_path, tmp_file.name, instance.task_id]) - except Exception as exc: - raise Exception( - f"Execute postprocess script error: {str(exc)}" - ) - - if resp["returncode"] != 0: - raise (Exception( - f"Execute postprocess script error: {str(resp['stderr'])}" - )) - result = json.loads(resp["stdout"].strip()) - code = result.get("code", 1) - if code != 0: - err_msg = result.get("err_msg", "Postprocess error") - # 后处理脚本认为诊断出错 - await DiagnosisHelper._update_job( - instance, err_msg=err_msg, status="Fail") - else: - rel_result = result.get("result", {}) - # 后处理脚本执行结束，更新任务状态 - await DiagnosisHelper._update_job( - instance, result=rel_result, status="Success" - ) - pass + instance, err_msg=post_process_result.err_msg, status="Fail" + ) + else: + # 后处理脚本执行成功，更新任务状态 + await DiagnosisHelper._update_job( + instance, result=post_process_result.result, status="Success" + ) except Exception as exc: - logger.exception( - f"Diagnosis postprocess error: {str(exc)}") + logger.exception(f"Diagnosis postprocess error: {str(exc)}") await DiagnosisHelper._update_job( - instance, result="Diagnosis postprocess error", status="Fail", - code=1, err_msg=f"Diagnosis postprocess error: {str(exc)}") + instance, + result="Diagnosis postprocess error", + status="Fail", + code=1, + err_msg=f"Diagnosis postprocess error: {str(exc)}", + ) diff --git a/sysom_server/sysom_diagnosis/service_scripts/base.py b/sysom_server/sysom_diagnosis/service_scripts/base.py new file mode 100644 index 00000000..d3f8360b --- /dev/null +++ b/sysom_server/sysom_diagnosis/service_scripts/base.py @@ -0,0 +1,149 @@ +""" +Time 2023/06/19 11:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File base.py +Description: +""" +from abc import ABC, abstractmethod +from typing import List, Dict + + +class DiagnosisJob: + def __init__(self, instance: str, cmd: str) -> None: + self.instance = instance + self.cmd = cmd + + @classmethod + def from_dict(cls, data: dict) -> "DiagnosisJob": + return DiagnosisJob(instance=data.get("instance", ""), cmd=data.get("cmd", "")) + + def to_dict(self): + return {"instance": self.instance, "cmd": self.cmd} + + +class DiagnosisTask: + """Diagnosis Task + + Args: + jobs([DiagnosisJob]): Diagnosis jobs + in_order(bool): Whether to execute all jobs in order + False => Concurrent execution of all Jobs + True => Execute each job in turn + """ + + def __init__(self, jobs: List[DiagnosisJob] = [], in_order: bool = False) -> None: + self.jobs = jobs + self.in_order = in_order + + def to_dict(self): + return { + "in_order": self.in_order, + "jobs": [item.to_dict() for item in self.jobs], + } + + +class DiagnosisJobResult: + def __init__( + self, code: int, job: DiagnosisJob, err_msg: str = "", stdout: str = "" + ) -> None: + self.job = job + self.code = code + self.err_msg = err_msg + self.stdout = stdout + + +class DiagnosisTaskResult: + def __init__( + self, + code: int, + job_results: List[DiagnosisJobResult], + err_msg: str = "", + in_order: bool = False, + ) -> None: + self.code = code + self.err_msg = err_msg + self.job_results = job_results + self.in_order = in_order + + +class PostProcessResult: + def __init__(self, code: int, result: str, err_msg: str = "") -> None: + self.code = code + self.result = result + self.err_msg = err_msg + + def to_dict(self): + return {"code": self.code, "result": self.result, "err_msg": self.err_msg} + + @classmethod + def from_dict(cls, data: dict) -> 'PostProcessResult': + return PostProcessResult( + code=data.get("code", 1), + result=data.get("result", ""), + err_msg=data.get("err_msg", ""), + ) + + +class DiagnosisProcessorBase(ABC): + def version_check(self, version: str) -> bool: + """Override this method to check tool version""" + return version >= self.get_min_version_support() + + def get_min_version_support(self) -> str: + return "1.3.0-2" + + +class DiagnosisPreProcessor(DiagnosisProcessorBase): + """Pre-processor used to perform: -> """ + + def __init__(self, service_name: str, **kwargs): + self.service_name = service_name + + def get_version_cmd(self) -> str: + """Get tool version + + The default is to use the sysak command for diagnostics, so + the default implement is to query the sysak version number + + Returns: + str: tool version + """ + return 'rpm -q sysak --queryformat "%{VERSION}-%{RELEASE}"' + + @abstractmethod + def get_diagnosis_cmds(self, instances: List[str], params: dict) -> DiagnosisTask: + """Convert params to diagnosis cmds + + params => { "service_name": "xxx", "time": 15 } + + cmds => [ + DiagnosisJob(instance="127.0.0.1", command="sysak memleak"), + DiagnosisJob(instance="192.168.0.1", command="sysak memleak") + ] + + Args: + params (dict): Diagnosis parameters + + Returns: + DiagnosisTask: Diagnosis task + """ + + +class DiagnosisPostProcessor(DiagnosisProcessorBase): + """Post-processor used to perform: -> + + Args: + DiagnosisProcessorBase (_type_): _description_ + """ + + def __init__(self, service_name: str, **kwargs): + self.service_name = service_name + + @abstractmethod + def parse_diagnosis_result(self, results: List[DiagnosisJobResult]) -> PostProcessResult: + """Parse diagnosis results to front-end formmated data + + Args: + results (List[DiagnosisResult]): Diagnosis results + """ diff --git a/sysom_server/sysom_diagnosis/service_scripts/command_post.py b/sysom_server/sysom_diagnosis/service_scripts/command_post.py new file mode 100644 index 00000000..ccc25cf8 --- /dev/null +++ b/sysom_server/sysom_diagnosis/service_scripts/command_post.py @@ -0,0 +1,22 @@ +""" +Time 2023/06/19 17:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File command.py +Description: +""" +from typing import List +from .base import DiagnosisJobResult, DiagnosisPostProcessor + + +class CommandPostProcessor(DiagnosisPostProcessor): + def parse_diagnosis_result(self, results: List[DiagnosisJobResult]) -> dict: + postprocess_result = { + "code": 0, + "err_msg": "", + "result": {}, + } + postprocess_result["result"] = { + "CommandResult": {"data": [{"key": "", "value": results[0].stdout}]} + } + return postprocess_result diff --git a/sysom_server/sysom_diagnosis/service_scripts/command_pre.py b/sysom_server/sysom_diagnosis/service_scripts/command_pre.py new file mode 100644 index 00000000..2a388978 --- /dev/null +++ b/sysom_server/sysom_diagnosis/service_scripts/command_pre.py @@ -0,0 +1,30 @@ +""" +Time 2023/06/19 17:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File command.py +Description: +""" +from typing import List + +from sysom_server.sysom_diagnosis.service_scripts.base import DiagnosisTask +from .base import DiagnosisJob, DiagnosisPreProcessor + + +class PreProcessor(DiagnosisPreProcessor): + """Command diagnosis + + Just invoke command in target instance and get stdout result + + Args: + DiagnosisPreProcessor (_type_): _description_ + """ + + def get_diagnosis_cmds(self, instances: List[str], params: dict) -> DiagnosisTask: + command = params.get("command", "") + return DiagnosisTask( + jobs=[ + DiagnosisJob(instance=instance, cmd=command) for instance in instances + ], + in_order=False, + ) -- Gitee From df87cd15551053b89fcacc7789963bb7bb79c905 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 27 Jun 2023 14:03:31 +0800 Subject: [PATCH 111/196] feat(deploy): Support used enviroment variable to control deploy DEPLOY_ENV_LIST: Used to control the list of environments that need to be installed DEPLOY_ENV_EXCLUDE: Used to control the list of environments that don't need to be installed DEPLOY_DEPS_LIST: Used to control the list of deps that need to be installed DEPLOY_DEPS_EXCLUDE: Used to control the list of deps that don't need to be installed DEPLOY_SERVER_LIST: Used to control the list of service that need to be installed DEPLOY_SERVER_EXCLUDE: Used to control the list of service that don't need to be installed example 1: (The following command will install all envs,deps and serivce except hotfix service): DEPLOY_SERVER_EXCLUDE=sysom_hotfix,sysom_hotfix_builder ./deploy.sh example 2: (The following command will install all envs,deps, then only install sysom_api service): DEPLOY_SERVER_LIST=sysom_api ./deploy.sh --- environment/clear_exclude | 1 - environment/deploy_exclude | 0 script/server/clear_exclude | 0 script/server/deploy_exclude | 0 script/sysom.sh | 132 ++++++++++++++++++++++------------- script/sysom_init.sh | 40 +---------- script/sysom_install.sh | 39 ++--------- script/sysom_stop.sh | 6 +- sysom_server/clear_exclude | 0 sysom_server/deploy_exclude | 0 10 files changed, 93 insertions(+), 125 deletions(-) delete mode 100644 environment/clear_exclude delete mode 100644 environment/deploy_exclude delete mode 100644 script/server/clear_exclude delete mode 100644 script/server/deploy_exclude delete mode 100644 sysom_server/clear_exclude delete mode 100644 sysom_server/deploy_exclude diff --git a/environment/clear_exclude b/environment/clear_exclude deleted file mode 100644 index e1994367..00000000 --- a/environment/clear_exclude +++ /dev/null @@ -1 +0,0 @@ -0_env \ No newline at end of file diff --git a/environment/deploy_exclude b/environment/deploy_exclude deleted file mode 100644 index e69de29b..00000000 diff --git a/script/server/clear_exclude b/script/server/clear_exclude deleted file mode 100644 index e69de29b..00000000 diff --git a/script/server/deploy_exclude b/script/server/deploy_exclude deleted file mode 100644 index e69de29b..00000000 diff --git a/script/sysom.sh b/script/sysom.sh index bd7e66e2..c33908b2 100755 --- a/script/sysom.sh +++ b/script/sysom.sh @@ -1,23 +1,21 @@ #!/bin/bash -x - +BaseDir=$(dirname $(readlink -f "$0")) +LocalAppHome=$(dirname $BaseDir) #################################################################################################################### # Initialize environment variables #################################################################################################################### -if [ "$APP_NAME" == "" ] -then +if [ "$APP_NAME" == "" ]; then export APP_NAME="sysom" fi -if [ "$APP_HOME" == "" ] -then +if [ "$APP_HOME" == "" ]; then export APP_HOME=/usr/local/sysom fi # For rpm build phase only # 1. Normally, this environment variable is the empty string # 2. In rpm build stage, this environment variable is "%{build_root}" -if [ "$INSTALL_PREFIX" == "" ] -then +if [ "$INSTALL_PREFIX" == "" ]; then export INSTALL_PREFIX="" fi @@ -31,80 +29,116 @@ export MICROSERVICE_HOME=${SERVER_HOME} export MICROSERVICE_SCRIPT_HOME=${SCRIPT_HOME}/server export GLOBAL_VIRTUALENV_HOME=${ENVIRONMENT_HOME}/virtualenv -if [ "$SERVER_LOCAL_IP" == "" ] -then - local_ip=`ip -4 route | grep "link src" | awk -F"link src " '{print $2}' | awk '{print $1}' | head -n 1` +if [ "$SERVER_LOCAL_IP" == "" ]; then + local_ip=$(ip -4 route | grep "link src" | awk -F"link src " '{print $2}' | awk '{print $1}' | head -n 1) export SERVER_LOCAL_IP=$local_ip fi -if [ "$SERVER_PUBLIC_IP" == "" ] -then +if [ "$SERVER_PUBLIC_IP" == "" ]; then export SERVER_PUBLIC_IP=$SERVER_LOCAL_IP fi -if [ "$SERVER_PORT" == "" ] -then +if [ "$SERVER_PORT" == "" ]; then export SERVER_PORT=80 fi -if [ "$LOG_HOME" == "" ] -then +if [ "$LOG_HOME" == "" ]; then export LOG_HOME=/var/log/sysom fi mkdir -p $LOG_HOME -if [ "$CONF_HOME" == "" ] -then +if [ "$CONF_HOME" == "" ]; then export CONF_HOME=/etc/sysom export SYSOM_CONF=${CONF_HOME}/config.yml fi mkdir -p $CONF_HOME -if [ "$DB_MYSQL_HOST" == "" ] -then +if [ "$DB_MYSQL_HOST" == "" ]; then export DB_MYSQL_HOST=localhost fi -if [ "$DB_MYSQL_PORT" == "" ] -then +if [ "$DB_MYSQL_PORT" == "" ]; then export DB_MYSQL_PORT=3306 fi -if [ "$DB_MYSQL_USERNAME" == "" ] -then +if [ "$DB_MYSQL_USERNAME" == "" ]; then export DB_MYSQL_USERNAME=sysom fi -if [ "$DB_MYSQL_PASSWORD" == "" ] -then +if [ "$DB_MYSQL_PASSWORD" == "" ]; then export DB_MYSQL_PASSWORD=sysom_admin fi -if [ "$DB_MYSQL_DATABASE" == "" ] -then +if [ "$DB_MYSQL_DATABASE" == "" ]; then export DB_MYSQL_DATABASE=sysom fi -if [ "$REDIS_HOST" == "" ] -then +if [ "$REDIS_HOST" == "" ]; then export REDIS_HOST=localhost fi -if [ "$REDIS_PORT" == "" ] -then +if [ "$REDIS_PORT" == "" ]; then export REDIS_PORT=6379 fi -if [ "$REDIS_USERNAME" == "" ] -then +if [ "$REDIS_USERNAME" == "" ]; then export REDIS_USERNAME="" fi -if [ "$REDIS_PASSWORD" == "" ] -then +if [ "$REDIS_PASSWORD" == "" ]; then export REDIS_PASSWORD="" fi +# Deploy env list +if [ "$DEPLOY_ENV_LIST" == "" ]; then + local_environment_dir=${LocalAppHome}/environment + export DEPLOY_ENV_LIST=$(echo $(ls $local_environment_dir) | sed 's/ /,/g') +fi +if [ "$DEPLOY_ENV_EXCLUDE" != "" ]; then + DEPLOY_ENV_EXCLUDE=$(echo $DEPLOY_ENV_EXCLUDE | sed 's/,/\n/g' | awk '{print length(), $0 | "sort -n -r" }' | awk '{print $2}' | tr '\n' ',' | sed 's/.$//') + targets=(${DEPLOY_ENV_EXCLUDE//,/ }) + env_list=$DEPLOY_ENV_LIST + for target in ${targets[*]}; do + env_list=$(echo $env_list | sed "s/${target}//g" | sed 's/,,/,/g') + done + export DEPLOY_ENV_LIST=$env_list +fi +export DEPLOY_ENV_LIST_REVERT=$(echo $DEPLOY_ENV_LIST | sed 's/,/\n/g' | tac | tr '\n' ',' | sed 's/.$//') + +# Deploy deps list +if [ "$DEPLOY_DEPS_LIST" == "" ]; then + local_deps_install_dir=${LocalAppHome}/deps + export DEPLOY_DEPS_LIST=$(echo $(ls $local_deps_install_dir) | sed 's/ /,/g') +fi +if [ "$DEPLOY_DEPS_EXCLUDE" != "" ]; then + DEPLOY_DEPS_EXCLUDE=$(echo $DEPLOY_DEPS_EXCLUDE | sed 's/,/\n/g' | awk '{print length(), $0 | "sort -n -r" }' | awk '{print $2}' | tr '\n' ',' | sed 's/.$//') + targets=(${DEPLOY_DEPS_EXCLUDE//,/ }) + deps_list=$DEPLOY_DEPS_LIST + for target in ${targets[*]}; do + deps_list=$(echo $deps_list | sed "s/${target}//g" | sed 's/,,/,/g') + done + export DEPLOY_DEPS_LIST=$deps_list +fi +export DEPLOY_DEPS_LIST_REVERT=$(echo $DEPLOY_DEPS_LIST | sed 's/,/\n/g' | tac | tr '\n' ',' | sed 's/.$//') + +# Deploy server list +if [ "$DEPLOY_SERVER_LIST" == "" ]; then + local_microservice_install_dir=${LocalAppHome}/sysom_server + if [ ! -d ${local_microservice_install_dir} ]; then + local_microservice_install_dir=${LocalAppHome}/server + fi + export DEPLOY_SERVER_LIST=$(echo $(ls $local_microservice_install_dir) | sed 's/ /,/g') +fi +if [ "$DEPLOY_SERVER_EXCLUDE" != "" ]; then + DEPLOY_SERVER_EXCLUDE=$(echo $DEPLOY_SERVER_EXCLUDE | sed 's/,/\n/g' | awk '{print length(), $0 | "sort -n -r" }' | awk '{print $2}' | tr '\n' ',' | sed 's/.$//') + targets=(${DEPLOY_SERVER_EXCLUDE//,/ }) + service_list=$DEPLOY_SERVER_LIST + for target in ${targets[*]}; do + service_list=$(echo $service_list | sed "s/${target}//g" | sed 's/,,/,/g') + done + export DEPLOY_SERVER_LIST=$service_list +fi +export DEPLOY_SERVER_LIST_REVERT=$(echo $DEPLOY_SERVER_LIST | sed 's/,/\n/g' | tac | tr '\n' ',' | sed 's/.$//') #################################################################################################################### # Subcommands @@ -114,7 +148,7 @@ ProgName=$(basename $0) BaseDir=$(dirname $(readlink -f "$0")) subcommand=$1 -sub_help(){ +sub_help() { echo "Usage: $ProgName [options]" echo "Subcommands:" echo " install Install SysOM module" @@ -136,16 +170,16 @@ sub_func() { } case $subcommand in - "" | "-h" | "--help") - sub_help - ;; - *) - shift - sub_func $@ - if [ $? -ne 0 ]; then - echo "Error: '$subcommand' is not a known subcommand." >&2 - echo " Run '$ProgName --help' for a list of known subcommands." >&2 - exit 1 - fi - ;; +"" | "-h" | "--help") + sub_help + ;; +*) + shift + sub_func $@ + if [ $? -ne 0 ]; then + echo "Error: '$subcommand' is not a known subcommand." >&2 + echo " Run '$ProgName --help' for a list of known subcommands." >&2 + exit 1 + fi + ;; esac diff --git a/script/sysom_init.sh b/script/sysom_init.sh index 50a7ec32..311169c5 100755 --- a/script/sysom_init.sh +++ b/script/sysom_init.sh @@ -204,20 +204,7 @@ sub_env() { sub_environment() { target=$1 if [ "$target" == "ALL" ]; then - local_environment_dir=${LocalAppHome}/environment - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_environment_dir}/deploy_exclude" ]; then - deploy_excludes=$(cat ${local_environment_dir}/deploy_exclude) - fi - # Deploy all microservices - for env in $(ls $local_environment_dir); do - if [[ $deploy_excludes =~ $env ]] || [ ! -d "${local_environment_dir}/${env}" ]; then - continue - fi - do_init_environment $env - done + do_init_environment $DEPLOY_ENV_LIST else # Deploy specific microservices do_init_environment $target @@ -236,23 +223,7 @@ sub_server() { sub_microservice() { target=$1 if [ "$target" == "ALL" ]; then - local_microservice_deploy_dir=${LocalAppHome}/sysom_server - if [ ! -d ${local_microservice_deploy_dir} ]; then - local_microservice_deploy_dir=${LocalAppHome}/server - fi - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_microservice_deploy_dir}/deploy_exclude" ]; then - deploy_excludes=$(cat ${local_microservice_deploy_dir}/deploy_exclude) - fi - # Deploy all microservices - for microservice in $(ls $local_microservice_deploy_dir); do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_deploy_dir}/${microservice}" ]; then - continue - fi - do_init_microservices ${microservice} - done + do_init_microservices $DEPLOY_SERVER_LIST else # Deploy specific microservices do_init_microservices $target @@ -262,12 +233,7 @@ sub_microservice() { sub_deps() { target=$1 if [ "$target" == "ALL" ]; then - local_deps_deploy_dir=${LocalAppHome}/deps - - # Deploy all microservices - for dep in $(ls $local_deps_deploy_dir); do - do_init_deps ${dep} - done + do_init_deps $DEPLOY_DEPS_LIST else # Deploy specific microservices do_init_deps $target diff --git a/script/sysom_install.sh b/script/sysom_install.sh index 505a0ea7..b6e16b7f 100755 --- a/script/sysom_install.sh +++ b/script/sysom_install.sh @@ -234,15 +234,7 @@ sub_env() { sub_environment() { target=$1 if [ "$target" == "ALL" ]; then - local_environment_dir=${LocalAppHome}/environment - - # Install all microservices - for env in $(ls $local_environment_dir); do - if [ ! -d "${local_environment_dir}/${env}" ]; then - continue - fi - do_install_environment $env - done + do_install_environment $DEPLOY_ENV_LIST else # Install specific microservices do_install_environment $target @@ -261,23 +253,8 @@ sub_server() { sub_microservice() { target=$1 if [ "$target" == "ALL" ]; then - local_microservice_install_dir=${LocalAppHome}/sysom_server - if [ ! -d ${local_microservice_install_dir} ]; then - local_microservice_install_dir=${LocalAppHome}/server - fi - - # Load deploy_excludes - deploy_excludes="" - if [ -f "${local_microservice_install_dir}/deploy_exclude" ]; then - deploy_excludes=$(cat ${local_microservice_install_dir}/deploy_exclude) - fi - # Install all microservices - for microservice in $(ls $local_microservice_install_dir); do - if [[ $deploy_excludes =~ $microservice ]] || [ ! -d "${local_microservice_install_dir}/${microservice}" ]; then - continue - fi - do_install_microservices ${microservice} - done + echo !!!!$DEPLOY_SERVER_LIST + do_install_microservices $DEPLOY_SERVER_LIST else # Install specific microservices do_install_microservices $target @@ -287,15 +264,7 @@ sub_microservice() { sub_deps() { target=$1 if [ "$target" == "ALL" ]; then - local_deps_install_dir=${LocalAppHome}/deps - - # Install all microservices - for dep in $(ls $local_deps_install_dir); do - if [ ! -d "${local_deps_install_dir}/${dep}" ]; then - continue - fi - do_install_deps ${dep} - done + do_install_deps $DEPLOY_DEPS_LIST else # Install specific microservices do_install_deps $target diff --git a/script/sysom_stop.sh b/script/sysom_stop.sh index 56e785ec..ad66d6a1 100755 --- a/script/sysom_stop.sh +++ b/script/sysom_stop.sh @@ -149,7 +149,7 @@ sub_environment() { target=$1 if [ "$target" == "ALL" ]; then # Stop all enviroment - for env in $(ls $ENVIRONMENT_HOME); do + for env in $(ls -r $ENVIRONMENT_HOME); do if [ ! -d "${ENVIRONMENT_HOME}/${env}" ]; then continue fi @@ -174,7 +174,7 @@ sub_microservice() { target=$1 if [ "$target" == "ALL" ]; then # stop all microservices - for microservice in $(ls $MICROSERVICE_HOME); do + for microservice in $(ls -r $MICROSERVICE_HOME); do if [ ! -d "${MICROSERVICE_HOME}/${microservice}" ]; then continue fi @@ -190,7 +190,7 @@ sub_deps() { target=$1 if [ "$target" == "ALL" ]; then # stop all deps - for dep in $(ls $DEPS_HOME); do + for dep in $(ls -r $DEPS_HOME); do if [ ! -d "${DEPS_HOME}/${dep}" ]; then continue fi diff --git a/sysom_server/clear_exclude b/sysom_server/clear_exclude deleted file mode 100644 index e69de29b..00000000 diff --git a/sysom_server/deploy_exclude b/sysom_server/deploy_exclude deleted file mode 100644 index e69de29b..00000000 -- Gitee From 5c83edf440f18c39d0dac165775cb26d72fcddf8 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 27 Jun 2023 17:57:13 +0800 Subject: [PATCH 112/196] fix(diagnosis): Sysak download url error --- sysom_server/sysom_diagnosis/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sysom_server/sysom_diagnosis/config.yml b/sysom_server/sysom_diagnosis/config.yml index 1e256eb3..738e5e03 100644 --- a/sysom_server/sysom_diagnosis/config.yml +++ b/sysom_server/sysom_diagnosis/config.yml @@ -1,5 +1,5 @@ vars: - SYSAK_DOWNLOAD_URL: &SYSAK_DOWNLOAD_URL https://mirrors.openanolis.cn/sysak/packages/ + SYSAK_DOWNLOAD_URL: &SYSAK_DOWNLOAD_URL https://mirrors.openanolis.cn/sysak/packages/release-v2.1.0/ SYSAK_VERSION: &SYSAK_VERSION 2.1.0-2 SERVICE_NAME: &SERVICE_NAME sysom_diagnosis SERVICE_CONSUMER_GROUP: !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] -- Gitee From 9a9980115f38bf9aabb3ff95e14ea53c166b2017 Mon Sep 17 00:00:00 2001 From: ydzhang Date: Mon, 3 Jul 2023 16:20:18 +0800 Subject: [PATCH 113/196] fix bug of database sync problem of cec communication This commit fix the following problems: 1. Fix the wrong service name of builder. 2. Fix the bug of hotfix job data update from cec communication 3. Fix the bug of cec message explosion because producer and consumer use same topic --- script/server/sysom_hotfix_builder/clear.sh | 2 +- script/server/sysom_hotfix_builder/init.sh | 2 +- script/server/sysom_hotfix_builder/install.sh | 2 +- script/server/sysom_hotfix_builder/start.sh | 2 +- script/server/sysom_hotfix_builder/stop.sh | 2 +- ...x_builder.ini => sysom-hotfix-builder.ini} | 8 +-- .../sysom_hotfix/apps/hotfix/views.py | 41 +++++++----- sysom_server/sysom_hotfix/lib/function.py | 66 +++++++++++-------- .../sysom_hotfix_builder/app/builder.py | 26 ++++---- .../sysom_hotfix_builder/conf/gunicorn.py | 4 +- sysom_server/sysom_hotfix_builder/config.yml | 8 +-- .../script/build_hotfix.sh | 6 +- .../sysom_hotfix_builder/script/build_rpm.sh | 11 ++-- 13 files changed, 98 insertions(+), 82 deletions(-) rename script/server/sysom_hotfix_builder/{sysom-hotfix_builder.ini => sysom-hotfix-builder.ini} (46%) diff --git a/script/server/sysom_hotfix_builder/clear.sh b/script/server/sysom_hotfix_builder/clear.sh index 30ffeb37..cfae33ca 100644 --- a/script/server/sysom_hotfix_builder/clear.sh +++ b/script/server/sysom_hotfix_builder/clear.sh @@ -1,6 +1,6 @@ #!/bin/bash BaseDir=$(dirname $(readlink -f "$0")) -SERVICE_NAME=sysom-hotfix_builder_service +SERVICE_NAME=sysom-hotfix-builder clear_app() { LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix diff --git a/script/server/sysom_hotfix_builder/init.sh b/script/server/sysom_hotfix_builder/init.sh index 679dd04c..b361b540 100644 --- a/script/server/sysom_hotfix_builder/init.sh +++ b/script/server/sysom_hotfix_builder/init.sh @@ -2,7 +2,7 @@ SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} BaseDir=$(dirname $(readlink -f "$0")) -SERVICE_NAME=sysom-hotfix_builder_service +SERVICE_NAME=sysom-hotfix-builder NFS_SERVER_IP=${SERVER_LOCAL_IP} mount_nfs() diff --git a/script/server/sysom_hotfix_builder/install.sh b/script/server/sysom_hotfix_builder/install.sh index 47d17d6f..1110d561 100644 --- a/script/server/sysom_hotfix_builder/install.sh +++ b/script/server/sysom_hotfix_builder/install.sh @@ -3,7 +3,7 @@ SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME -SERVICE_NAME=sysom-hotfix_builder_service +SERVICE_NAME=sysom-hotfix-builder if [ "$UID" -ne 0 ]; then echo "Please run as root" diff --git a/script/server/sysom_hotfix_builder/start.sh b/script/server/sysom_hotfix_builder/start.sh index c9afe102..bb764f41 100644 --- a/script/server/sysom_hotfix_builder/start.sh +++ b/script/server/sysom_hotfix_builder/start.sh @@ -1,5 +1,5 @@ #!/bin/bash -SERVICE_NAME=sysom-hotfix_builder_service +SERVICE_NAME=sysom-hotfix-builder is_start() { supervisorctl status ${SERVICE_NAME} diff --git a/script/server/sysom_hotfix_builder/stop.sh b/script/server/sysom_hotfix_builder/stop.sh index 6c63f186..d6535439 100644 --- a/script/server/sysom_hotfix_builder/stop.sh +++ b/script/server/sysom_hotfix_builder/stop.sh @@ -1,5 +1,5 @@ #!/bin/bash -SERVICE_NAME=sysom-hotfix_builder_service +SERVICE_NAME=sysom-hotfix-builder stop_app() { # kill all kpatch-build process diff --git a/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini b/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini similarity index 46% rename from script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini rename to script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini index cbdf497d..b83c7c70 100644 --- a/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini +++ b/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini @@ -1,9 +1,9 @@ -[program:sysom-hotfix_builder_service] -directory=/usr/local/sysom/server/sysom-hotfix_builder_service +[program:sysom-hotfix-builder] +directory=/usr/local/sysom/server/sysom_hotfix_builder command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app startsecs=3 autostart=true autorestart=true -stderr_logfile=/var/log/sysom/sysom-hotfix_builder_service.log -stdout_logfile=/var/log/sysom/sysom-hotfix_builder_service.log +stderr_logfile=/var/log/sysom/sysom-hotfix-builder.log +stdout_logfile=/var/log/sysom/sysom-hotfix-builder-err.log environment=PATH=/usr/local/sysom/virtualenv/bin:%(ENV_PATH)s diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 5d490ad2..e2f2852d 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -300,28 +300,32 @@ class HotfixAPIView(GenericViewSet, hotfix_id = request.GET.get('id') hotfix = HotfixModel.objects.filter(id=hotfix_id).first() if hotfix: - if hotfix.building_status == 2 or hotfix.building_status == 3: - # this hotfix task is finished - if os.path.exists(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): + try: + if hotfix.building_status == 2 or hotfix.building_status == 3: + # this hotfix task is finished + if os.path.exists(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): + msg = "" + for line in open(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): + msg += line + hotfix.log = msg + hotfix.save() + return success(result=msg, message="hotfix build log return") + else: + msg = hotfix.log + + if len(msg) > 0: + return success(result=msg, message="hotfix build log return") + else: + return other_response(message="No build log found", result={"msg":"No build log found"}, code=400) + else: + # this job is not finished.. read from the log file msg = "" for line in open(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): msg += line - hotfix.log = msg - hotfix.save() - return success(result=msg, message="hotfix build log return") - else: - msg = hotfix.log - - if len(msg) > 0: return success(result=msg, message="hotfix build log return") - else: - return other_response(message="No build log found", result={"msg":"No build log found"}, code=400) - else: - # this job is not finished.. read from the log file - msg = "" - for line in open(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): - msg += line - return success(result=msg, message="hotfix build log return") + except Exception as e: + logger.error(str(e)) + return other_response(message="No such log", result={"msg":"Log not found"}, code=400) else: return other_response(message="No such record", result={"msg":"Hotfix not found"}, code=400) @@ -357,6 +361,7 @@ class HotfixAPIView(GenericViewSet, rpm_package = request.data["rpm"] hotfix.rpm_package += rpm_package hotfix.save() + logger.info("invoking update hotfix name, this function is duplicated...") except Exception as e: return other_response(message=str(e), code=400) return success(result={"msg":"update hotfix name success"}, message="updated hotfix name") diff --git a/sysom_server/sysom_hotfix/lib/function.py b/sysom_server/sysom_hotfix/lib/function.py index 95f27546..db00af87 100644 --- a/sysom_server/sysom_hotfix/lib/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -451,7 +451,9 @@ class FunctionClass(): except Exception as e: logger.error(e) - +""" +CECListener listen topic of hotfix_msg, which is send by builder +""" class CECListener(): def __init__(self, con, cec_url, listen_topic) -> None: @@ -485,29 +487,38 @@ class CECListener(): logger.info("Server CECListener init finished...") retry_time = 0 - while retry_time < self.parameters.cec.max_retry_time: - for event in self.consumer: - logger.info("processing one msg...") - try: - parameters = event.value - hotfix_id = parameters["hotfix_id"] - if self.sync_key in parameters: - if self.rpm_key in parameters: - self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) - if self.log_key in parameters: - self.sync_hotfix_log(hotfix_id) - else: - self.update_hotfix_job_status(hotfix_id, parameters["status"]) - except Exception as e: - logger.error(str(e)) - finally: - logger.info("ack one msg from builder...") - self.consumer.ack(event=event) - time.sleep(self.parameters.cec.sleep_time) - retry_time += 1 + try: + while retry_time < self.parameters.cec.max_retry_time: + """ + for each event in hotfix_msg topic, use the key inside to decide the message type + """ + for event in self.consumer: + time.sleep(1) + logger.info("processing one msg...") + try: + cec_values = event.value + hotfix_id = cec_values["hotfix_id"] + if self.sync_key in cec_values: + if self.rpm_key in cec_values: + self.sync_hotfix_job_rpm_name(hotfix_id, cec_values[self.rpm_key]) + if self.log_key in cec_values: + self.sync_hotfix_log(hotfix_id) + else: + self.update_hotfix_job_status(hotfix_id, cec_values["status"]) + except Exception as e: + logger.error(str(e)) + finally: + logger.info("ack one msg from builder...") + self.consumer.ack(event=event) + time.sleep(0.5) + time.sleep(self.parameters.cec.sleep_time) + retry_time += 1 + except Exception as e: + logger.error(str(e)) + logger.error("Hotfix Server CEC Listener exit ...") def update_hotfix_job_status(self, hotfix_id, status): - hotfix = HotfixModel.objects.filter(id=hotfix_id).first() + hotfix = HotfixModel.objects.get(id=hotfix_id) logger.info("ack one job status of : %s from builder of hotfix id : %s " % (status, str(hotfix_id))) if hotfix is None: logger.error("%s : hotfix job is not exist filtered by hotfix id : %s" % (sys._getframe().f_code.co_name, str(hotfix_id))) @@ -527,17 +538,16 @@ class CECListener(): def sync_hotfix_job_rpm_name(self, hotfix_id, rpm_name): try: - logger.info("get rpm_name of %s from builfer..." % rpm_name) - hotfix = HotfixModel.objects.filter(id=hotfix_id).first() - rpm_package = rpm_name - hotfix.rpm_package = rpm_package + logger.info("get rpm_name of %s from builder..." % rpm_name) + hotfix = HotfixModel.objects.get(id=hotfix_id) + hotfix.rpm_package = rpm_name hotfix.save() except Exception as e: logger.error("%s : Exception raised..." % sys._getframe().f_code.co_name) - return None + return hotfix_id def sync_hotfix_log(self, hotfix_id): - hotfix = HotfixModel.objects.filter(id=hotfix_id).first() + hotfix = HotfixModel.objects.get(id=hotfix_id) try: log = "" for line in open(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): diff --git a/sysom_server/sysom_hotfix_builder/app/builder.py b/sysom_server/sysom_hotfix_builder/app/builder.py index 45e77a26..39452b94 100644 --- a/sysom_server/sysom_hotfix_builder/app/builder.py +++ b/sysom_server/sysom_hotfix_builder/app/builder.py @@ -28,16 +28,17 @@ from sysom_utils import CecTarget, SysomFramework class ServerConnector(): - def __init__(self, cec_topic=None): + def __init__(self, hotfix_msg_topic=None, hotfix_job_received_topic=None): self.cec = dispatch_producer(YAML_CONFIG.get_cec_url(CecTarget.PRODUCER)) #self.cec = SysomFramework.cec_consumer(cec_topic) - self.cec_produce_topic = cec_topic + self.cec_msg_topic = hotfix_msg_topic + self.cec_hotfix_job_received_topic = hotfix_job_received_topic # Here, we design to use some status more than building status # in building status , we include : waiting\building\success\failed # but more, we can send : sync to server for log and hotfix name sync def update_hotfix_building_status(self, id, status): - self.cec.produce(self.cec_produce_topic, { + self.cec.produce(self.cec_msg_topic, { "hotfix_id" : id, "status" : status }) @@ -45,7 +46,7 @@ class ServerConnector(): def sync_rpm_name_cec(self, hotfix_id, rpm_name): logger.info("produce rpm_name ") - self.cec.produce(self.cec_produce_topic, { + self.cec.produce(self.cec_msg_topic, { "hotfix_id" : hotfix_id, "type": "rpm", "sync" : True, @@ -54,7 +55,7 @@ class ServerConnector(): def sync_building_log_cec(self, hotfix_id): logger.info("sync hotfix build log") - self.cec.produce(self.cec_produce_topic, { + self.cec.produce(self.cec_msg_topic, { "hotfix_id": hotfix_id, "type": "log", "sync": True @@ -92,9 +93,10 @@ class HotfixBuilder(): self.hotfix_base = con.builder.hotfix_base self.builder_hotfix_package_repo = con.builder.package_repo self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder", daemon=True) - self.cec_hotfix_topic = con.cec.hotfix_topic + self.cec_hotfix_topic = con.cec.hotfix_topic # send + self.cec_hotfix_job_topic = con.cec.job_topic # listen self.local_arch = os.uname().release.split(".")[-1] - self.connector = ServerConnector(cec_topic=con.cec.job_status_topic) + self.connector = ServerConnector(hotfix_msg_topic=con.cec.hotfix_topic) self.tmpdir=con.builder.tmpdir self.src_rpm_tmpdir=con.builder.src_rpm_tmpdir self.hotfix_id = None @@ -389,9 +391,10 @@ class HotfixBuilder(): logger.info("The return code is %d" % return_code) rpm_names = self.find_build_rpm(kernel_version) - + logger.info(rpm_names) # if rpm is more than one, upload it one by one for each_rpm in rpm_names: + logger.info("==> sync rpm : %s" % each_rpm) self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) self.connector.sync_building_log_cec(self.hotfix_id) @@ -520,9 +523,6 @@ class HotfixBuilder(): rpm_names = self.find_build_rpm(kernel_version) - # when finished building, sync the build log - #self.connector.sync_building_log(hotfix_id) - # if rpm is more than one, upload it one by one for each_rpm in rpm_names: self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) @@ -548,11 +548,11 @@ class HotfixBuilder(): consumer_id = Consumer.generate_consumer_id() if self.local_arch == "x86_64": - consumer = SysomFramework.cec_consumer(self.cec_hotfix_topic, + consumer = SysomFramework.cec_consumer(self.cec_hotfix_job_topic, consumer_id=consumer_id, group_id="hotfix_job_group_x86") else: - consumer = SysomFramework.cec_consumer(self.cec_hotfix_topic, + consumer = SysomFramework.cec_consumer(self.cec_hotfix_job_topic, consumer_id=consumer_id, group_id="hotfix_job_group_arm") retry_time = 0 diff --git a/sysom_server/sysom_hotfix_builder/conf/gunicorn.py b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py index 08c69325..43052f03 100644 --- a/sysom_server/sysom_hotfix_builder/conf/gunicorn.py +++ b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py @@ -16,8 +16,8 @@ worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为s max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) -accesslog = '/var/log/sysom/sysom-hotfix_builder_service.log' +accesslog = '/var/log/sysom/sysom-hotfix-builder.log' loglevel = 'error' -proc_name = 'channel_service' +proc_name = 'hotfix_builder_service' diff --git a/sysom_server/sysom_hotfix_builder/config.yml b/sysom_server/sysom_hotfix_builder/config.yml index 03efc403..7efd5dd1 100644 --- a/sysom_server/sysom_hotfix_builder/config.yml +++ b/sysom_server/sysom_hotfix_builder/config.yml @@ -33,10 +33,10 @@ sysom_service: header: tls_skip_verify: false cec: - job_status_topic: "hotfix_msg" - hotfix_topic: "hotfix_job" - max_retry: 10 # retry consumer listen time - retry_sleep_time: 5 # second + job_topic: "hotfix_job" # this topic is used to update the building status + hotfix_topic: "hotfix_msg" # this topic is used to communicate the hotfix message + max_retry: 10 # retry consumer listen time + retry_sleep_time: 5 # second builder: hotfix_base: "/hotfix_build/hotfix" diff --git a/sysom_server/sysom_hotfix_builder/script/build_hotfix.sh b/sysom_server/sysom_hotfix_builder/script/build_hotfix.sh index d284b246..772b1401 100644 --- a/sysom_server/sysom_hotfix_builder/script/build_hotfix.sh +++ b/sysom_server/sysom_hotfix_builder/script/build_hotfix.sh @@ -449,9 +449,9 @@ function do_rpmbuild(){ kofile=${kpatch_id//./-} if [[ $USE_KPATCH_MODULE -eq 1 ]]; then - ${BASE}/../build_rpm.sh --kpatch-module -m "${tmpdir}"/"${kofile}".ko -d ${dist} -e "${description}" -r "${tmpdir}"/rpmbuild -k "${kernel_version}" -c "${CHANGEINFOFILE}" -l "${release}" 2>&1 >> ${LOGFILE} + ${BASE}/../build_rpm.sh --kpatch-module -n ${hotfix_name} -m "${tmpdir}"/"${kofile}".ko -d ${dist} -e "${description}" -r "${tmpdir}"/rpmbuild -k "${kernel_version}" -c "${CHANGEINFOFILE}" -l "${release}" 2>&1 >> ${LOGFILE} else - ${BASE}/../build_rpm.sh -m "${tmpdir}"/"${kofile}".ko -d ${dist} -e "${description}" -r "${tmpdir}"/rpmbuild -k "${kernel_version}" -c "${CHANGEINFOFILE}" -l "${release}" 2>&1 >> ${LOGFILE} + ${BASE}/../build_rpm.sh -n ${hotfix_name} -m "${tmpdir}"/"${kofile}".ko -d ${dist} -e "${description}" -r "${tmpdir}"/rpmbuild -k "${kernel_version}" -c "${CHANGEINFOFILE}" -l "${release}" 2>&1 >> ${LOGFILE} fi if [[ $? -ne 0 ]]; then @@ -459,7 +459,7 @@ function do_rpmbuild(){ fi cp ${tmpdir}/rpmbuild/RPMS/${arch}/*.rpm ${NFS_RPM_DIR} || die "FAILED"; - echo "The rpm is : `ls ${pwddir}/*.rpm`"; + echo "The rpm is : `ls ${tmpdir}/rpmbuild/RPMS/${arch}/`"; cd $pwddir [ -d "./${kernel_version}" ] && rm -rf "./${kernel_version}" diff --git a/sysom_server/sysom_hotfix_builder/script/build_rpm.sh b/sysom_server/sysom_hotfix_builder/script/build_rpm.sh index d3ef9895..b442862f 100644 --- a/sysom_server/sysom_hotfix_builder/script/build_rpm.sh +++ b/sysom_server/sysom_hotfix_builder/script/build_rpm.sh @@ -23,7 +23,7 @@ function usage(){ } function parse_args(){ - ARGS=`getopt -l module:,distro:,rpmbuild:,kernel:,release:,description:,changeinfo:,help,kpatch-module -o hm:d:r:k:c:e:l: -- "$@" 2>/dev/null` || { usage; exit 1; } + ARGS=`getopt -l module:,distro:,rpmbuild:,kernel:,release:,description:,changeinfo:,name:,help,kpatch-module -o hm:d:r:k:c:e:l:n: -- "$@" 2>/dev/null` || { usage; exit 1; } eval set -- "${ARGS}" while [ -n "$1" ] do @@ -56,6 +56,10 @@ function parse_args(){ release="$2" shift ;; + -n|--name) + hotfix_name="$2" + shift + ;; -h|--help) usage ;; @@ -127,7 +131,7 @@ function prepare_spec(){ %define _ks_prefix /usr/local Summary: Hotfix for Kernel -Name: kernel-hotfix-${hotfix_base} +Name: kernel-hotfix-${hotfix_name} version: 1.0 Release: ${RELEASE:-1}.${distro} License: GPL @@ -145,9 +149,6 @@ fi hotfix_spec=${hotfix_spec}" -Requires: kpatch >= 0.8.0-1.5 - - %description hotfix rpm build : ${description} -- Gitee From ea51f679a3aa2773024273afddaed984ca7b5e2f Mon Sep 17 00:00:00 2001 From: ydzhang Date: Tue, 4 Jul 2023 15:54:13 +0800 Subject: [PATCH 114/196] fix bug of missing function of oneclick to deploy add hotfix file exist check --- sysom_server/sysom_hotfix/apps/hotfix/urls.py | 1 + .../sysom_hotfix/apps/hotfix/views.py | 22 +++++++++++++++++++ sysom_server/sysom_hotfix/lib/function.py | 6 ++++- 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/urls.py b/sysom_server/sysom_hotfix/apps/hotfix/urls.py index ebbe4e5f..0762e342 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/urls.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/urls.py @@ -32,6 +32,7 @@ urlpatterns = [ path('api/v1/hotfix/update_kernel_relation/', views.HotfixAPIView.as_view({'post': 'update_kernel_version'})), path('api/v1/hotfix/update_ostype/', views.HotfixAPIView.as_view({'post': 'update_ostype'})), path('api/v1/hotfix/rebuild_hotfix/',views.HotfixAPIView.as_view({'post': 'rebuild_hotfix'})), + path('api/v1/hotfix/oneclick_deploy/', views.HotfixAPIView.as_view({'post': 'oneclick_deploy'})), path('api/v1/hotfix/health_check/', views.HealthViewset.as_view({'get': 'health_check'})), path('api/v1/', include(router.urls)), ] diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index e2f2852d..51d522b6 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -582,6 +582,28 @@ class HotfixAPIView(GenericViewSet, thread_runner=threading.Thread(target=self.function.sync_kernel, name="sync_kernel",args=(id,)) thread_runner.start() return success(result={"msg":"success trigger sync kernel"}) + + def oneclick_deploy(self, request): + kernel_version = request.data.get("kernel_version") + rpm_package = request.data.get("rpm_package") + + server_hotfix_path = os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "rpm", rpm_package) + + if not os.path.exists(server_hotfix_path): + logger.error("hotfix file %s not exist..." % server_hotfix_path) + return other_response(msg="hotfix file %s not found." % server_hotfix_path, code=400) + + host_list = self.function.get_host_list(kernel_version=kernel_version) + host_num = len(host_list) + success_num = self.function.deploy_hotfix_to_machine(kernel_version=kernel_version, hotfix_path=server_hotfix_path) + + if success_num == -1: + return other_response(msg="No host matches kernel version : %s" % kernel_version, code=400) + + if success_num is not None: + return success(result={"msg": "deploy hotfix to machine, %s of %s success" % (success_num, host_num)}) + else: + return other_response(msg="deploy to hotfix failed", code=400) class HealthViewset(CommonModelViewSet): def health_check(self, request, *args, **kwargs): diff --git a/sysom_server/sysom_hotfix/lib/function.py b/sysom_server/sysom_hotfix/lib/function.py index db00af87..4b9e4371 100644 --- a/sysom_server/sysom_hotfix/lib/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -18,6 +18,7 @@ from cec_base.admin import dispatch_admin from bs4 import BeautifulSoup from lib.utils import human_datetime from sysom_utils import SysomFramework +from channel_job import default_channel_job_executor """ Function class @@ -239,7 +240,10 @@ class FunctionClass(): "remote_path": "/root/%s" % hotfix_name, "instances": hotfix_list }).execute().result - # the res is string + + if hotfix_list is None or len(hotfix_list) == 0: + return -1 + res = json.loads(res) for each_instance in res: if not each_instance["success"]: -- Gitee From b8560855129cb0cc0654b85725cdc9f2d0d47802 Mon Sep 17 00:00:00 2001 From: ydzhang Date: Mon, 12 Jun 2023 12:52:26 +0800 Subject: [PATCH 115/196] optimize the function of sync kernel --- .../sysom_hotfix/apps/hotfix/views.py | 4 +- sysom_server/sysom_hotfix/lib/function.py | 146 +++++++----------- 2 files changed, 54 insertions(+), 96 deletions(-) diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 51d522b6..847a7de3 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -428,7 +428,7 @@ class HotfixAPIView(GenericViewSet, return other_response(msg="can not find the OS type record", code=400) if os_type_object.sync_status == 0: return other_response(msg="Synchronizing, please wait for synchronization to complete", code=400) - thread_runner = threading.Thread(target=self.sync_kernel, name="sync_kernel",args=(os_type_object.id,)) + thread_runner = threading.Thread(target=self.function.sync_kernel, name="sync_kernel",args=(os_type_object.id,)) thread_runner.start() else: return other_response(message="same OS Type found in record..", code=400) @@ -566,7 +566,7 @@ class HotfixAPIView(GenericViewSet, os.remove(file_path) os_type_object.git_rule = request.data['git_rule'] os_type_object.save() - thread_runner = threading.Thread(target=self.sync_kernel, name="sync_kernel",args=(os_type_object.id,)) + thread_runner = threading.Thread(target=self.function.sync_kernel, name="sync_kernel",args=(os_type_object.id,)) thread_runner.start() except Exception as e: return other_response(msg=str(e), code=400) diff --git a/sysom_server/sysom_hotfix/lib/function.py b/sysom_server/sysom_hotfix/lib/function.py index 4b9e4371..eab810b3 100644 --- a/sysom_server/sysom_hotfix/lib/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -274,23 +274,24 @@ class FunctionClass(): def sync_git(self, id, os_type, source_devel, source_debuginfo, git_rule): try: + devel_lists = self.get_rpm_list(source_devel, "devel") + debuginfo_lists = self.get_rpm_list(source_debuginfo, "debuginfo") self.update_ostype_sync_status(id=id, status=0) - source_devel_list = source_devel.split(",") - x86_devel_source = list(filter(lambda x: "x86_64" in x, source_devel_list))[0] - arm_devel_source = list(filter(lambda x: "aarch64" in x, source_devel_list))[0] - x86_source_devel_lists = self.get_git_devel_list(x86_devel_source) - arm_source_devel_lists = self.get_git_devel_list(arm_devel_source) - source_debuginfo_list = source_debuginfo.split(",") - x86_debuginfo_source = list(filter(lambda x: "x86_64" in x, source_debuginfo_list))[0] - arm_debuginfo_source = list(filter(lambda x: "aarch64" in x, source_debuginfo_list))[0] - self.insert_kernel_version_relation_git(os_type, git_rule, x86_devel_source, x86_source_devel_lists, x86_debuginfo_source) - self.insert_kernel_version_relation_git(os_type, git_rule, arm_devel_source, arm_source_devel_lists, arm_debuginfo_source) + for each_rpm in debuginfo_lists: + version = each_rpm.replace("kernel-debuginfo-", '').replace(".rpm",'') + debuginfo_url = source_debuginfo + each_rpm + devel_rpm = list(filter(lambda x: version in x, devel_lists))[0] + devel_url = source_devel + devel_rpm + self.insert_kernel_version_relation_internal(kernel_version=version,os_type=os_type, + source="", devel_link=devel_url,debuginfo_link=debuginfo_url) self.update_ostype_sync_status(id=id, status=2) - logger.info("successfully sync kernel") except Exception as e: logger.error(e) self.update_ostype_sync_status(id=id, status=1) + """ + deprecated + """ def insert_kernel_version_relation_git(self, os_type, git_rule, devel_source, source_devel_list, debuginfo_source): for kernel_devel_rpm in source_devel_list: version = ".".join(kernel_devel_rpm.replace("kernel-devel-", '').split(".")[:-1]) @@ -306,7 +307,9 @@ class FunctionClass(): develinfo_link = devel_source + kernel_devel_rpm self.insert_kernel_version_relation_internal(kernel_version=version,os_type=os_type, source=source, devel_link=develinfo_link,debuginfo_link=debuginfo_link) - + """ + deprecated + """ def git_branch_by_git_rule(self, git_rule, version): branch = None rule_file_dir = os.path.join(settings.HOTFIX_FILE_BRANCH_RULE) @@ -330,30 +333,21 @@ class FunctionClass(): def sync_source(self, id, os_type, source_repo, source_devel, source_debuginfo): try: self.update_ostype_sync_status(id=id, status=0) - source_lists = self.get_source_list(source_repo) - for kernel_rpm in source_lists: - version = ".".join(kernel_rpm.replace("kernel-", '').split(".")[:-1]).replace(".src", "") - kernel_url = source_repo + kernel_rpm - source_devel_list = source_devel.split(",") - x86_devel = list(filter(lambda x: "x86_64" in x, source_devel_list))[0] - source_debuginfo_list = source_debuginfo.split(",") - x86_debuginfo = list(filter(lambda x: "x86_64" in x, source_debuginfo_list))[0] - aarch64_devel = list(filter(lambda x: "aarch64" in x, source_devel_list))[0] - aarch64_debuginfo = list(filter(lambda x: "aarch64" in x, source_debuginfo_list))[0] - x86_devel_link = self.get_devel_rpm(x86_devel, version) - x86_devel_link = x86_devel+ x86_devel_link - x86_debuginfo_link = self.get_debuginfo_rpm(x86_debuginfo, version) - x86_debuginfo_link = x86_debuginfo + x86_debuginfo_link - aarch64devel_link = self.get_devel_rpm(aarch64_devel, version) - aarch64devel_link = aarch64_devel + aarch64devel_link - aarch64debuginfo_link = self.get_debuginfo_rpm(aarch64_debuginfo, version) - aarch64debuginfo_link = aarch64_debuginfo + aarch64debuginfo_link - self.insert_kernel_version_relation_internal(kernel_version=version+".x86_64",os_type=os_type, - source=kernel_url, devel_link=aarch64devel_link,debuginfo_link=aarch64debuginfo_link) - self.insert_kernel_version_relation_internal(kernel_version=version+".aarch64",os_type=os_type, - source=kernel_url, devel_link=x86_devel_link,debuginfo_link=x86_debuginfo_link) + source_lists = self.get_rpm_list(source_repo, "source") + devel_lists = self.get_rpm_list(source_devel, "devel") + debuginfo_lists = self.get_rpm_list(source_debuginfo, "debuginfo") + for each_rpm in debuginfo_lists: # find each kernel debuginfo package + version = each_rpm.replace("kernel-debuginfo-", '').replace(".rpm",'') # kernel-debuginfo-5.10.60-9.an8.x86_64.rpm -> 5.10.60-9.an8.x86_64 + version_no_arch = version.replace(version.split(".")[-1], '') + kernel_package = list(filter(lambda x: version_no_arch in x, source_lists))[0] # find the specific kernel version rpm from kernel package + kernel_url = source_repo + kernel_package + debuginfo_rpm = list(filter(lambda x: version in x, debuginfo_lists))[0] + debuginfo_url = source_debuginfo + debuginfo_rpm + devel_rpm = list(filter(lambda x: version in x, devel_lists))[0] + devel_url = source_devel + devel_rpm + self.insert_kernel_version_relation_internal(kernel_version=version,os_type=os_type, + source=kernel_url, devel_link=devel_url,debuginfo_link=debuginfo_url) self.update_ostype_sync_status(id=id, status=2) - logger.info("msg:successfully sync kernel") except Exception as e: logger.error(e) self.update_ostype_sync_status(id=id, status=1) @@ -379,70 +373,34 @@ class FunctionClass(): kernel_object.save() logger.info("update kernel version in record...") - def get_source_list(self, url): + def get_rpm_list(self, package_url, packge_type): try: - response = urllib.request.urlopen(url) + response = urllib.request.urlopen(package_url) html = response.read() soup = BeautifulSoup(html, "html.parser") - linklist = soup.tbody.select('a') - linkrpmlists = [] - for kernel_rpm in linklist: - kernel_rpm = kernel_rpm.get_text().strip() - if re.findall('^kernel-\d', kernel_rpm): - linkrpmlists.append(kernel_rpm) - logger.info(linkrpmlists) - return linkrpmlists - except Exception as e: - logger.error(e) - return [] - - def get_git_devel_list(self, url): - try: - logger.info(url) - response = urllib.request.urlopen(url) - html = response.read() - soup = BeautifulSoup(html, "html.parser") - linklist = soup.tbody.select('a') - linkrpmlists = [] - for kernel_rpm in linklist: - kernel_rpm = kernel_rpm.get_text().strip() - if re.findall('^kernel-devel-\d', kernel_rpm): - linkrpmlists.append(kernel_rpm) - logger.info(linkrpmlists) - return linkrpmlists - except Exception as e: - logger.error(e) - return [] - - def get_devel_rpm(self, source_devel, version): - try: - response = urllib.request.urlopen(source_devel) - html = response.read() - soup = BeautifulSoup(html, "html.parser") - linklist = soup.tbody.select('a') - for kernel_rpm in linklist: - kernel_rpm = kernel_rpm.get_text().strip() - if re.findall('^kernel-devel-\d', kernel_rpm) and version in kernel_rpm: - return kernel_rpm - return None - except Exception as e: - logger.error(e) - return None - - def get_debuginfo_rpm(self, source_debuginfo, version): - try: - response = urllib.request.urlopen(source_debuginfo) - html = response.read() - soup = BeautifulSoup(html, "html.parser") - linklist = soup.tbody.select('a') - for kernel_rpm in linklist: - kernel_rpm = kernel_rpm.get_text().strip() - if re.findall('^kernel-debuginfo-\d', kernel_rpm) and version in kernel_rpm: - return kernel_rpm - return None + all_rpm_lists = soup.tbody.select('a') + rpm_list = [] + if packge_type == "source": + for each_rpm in all_rpm_lists: + rpm_name = each_rpm.get_text().strip() + if re.findall('^kernel-\d', rpm_name): + rpm_list.append(rpm_name) + elif packge_type == "devel": + for each_rpm in all_rpm_lists: + rpm_name = each_rpm.get_text().strip() + if re.findall('^kernel-devel-\d', rpm_name): + rpm_list.append(rpm_name) + elif packge_type == "debuginfo": + for each_rpm in all_rpm_lists: + rpm_name = each_rpm.get_text().strip() + if re.findall('^kernel-debuginfo-\d', rpm_name): + rpm_list.append(rpm_name) + else: + logger.error("This package type is not supported") + return None + return rpm_list except Exception as e: logger.error(e) - return None def update_ostype_sync_status(self, id, status): try: -- Gitee From ac41b875092da8ca7030c24656607d744abc0c04 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Fri, 7 Jul 2023 11:30:08 +0800 Subject: [PATCH 116/196] fix(diagnosis): Diagnosis v2 call failed --- .../sysom_diagnosis/apps/task/helper.py | 28 +++--- .../sysom_diagnosis/apps/task/views.py | 96 ++++++++++--------- .../sysom_diagnosis/service_scripts/base.py | 8 +- .../service_scripts/command_post.py | 18 ++-- .../service_scripts/command_pre.py | 10 +- 5 files changed, 78 insertions(+), 82 deletions(-) diff --git a/sysom_server/sysom_diagnosis/apps/task/helper.py b/sysom_server/sysom_diagnosis/apps/task/helper.py index ccdf633d..f81549bb 100644 --- a/sysom_server/sysom_diagnosis/apps/task/helper.py +++ b/sysom_server/sysom_diagnosis/apps/task/helper.py @@ -188,25 +188,26 @@ class DiagnosisHelper: except Exception as exc: raise Exception(f"Task params loads error: {str(exc)}") - instances = params.pop("instances", []) - if "instance" in params and len(instances) == 0: - instances.append(params.pop("instance", "")) - - if len(instances) == 0: - raise Exception("Missing params, instances or instance") - # 2. Use PreProcessor to check if the version of the tool meets the requirements try: + params.pop("service_name", "") pre_processor = _get_pre_processor(service_name)(service_name, **params) except Exception as e: return None - for instance in instances: + + # 3. Use PreProcessor to convert params to diagnosis jobs + diagnosis_task = pre_processor.get_diagnosis_cmds(params) + if diagnosis_task is None or len(diagnosis_task.jobs) == 0: + raise Exception(f"Pre-processor not return any diagnosis job") + + # 4. Check whether tool version satisfied current job + for job in diagnosis_task.jobs: job_params = { "channel_type": params.pop("channel", "ssh"), "channel_opt": "cmd", "params": { **params, - "instance": instance, + "instance": job.instance, "command": pre_processor.get_version_cmd(), }, "timeout": 1000 * 60, # 1 minutes @@ -223,11 +224,6 @@ class DiagnosisHelper: f"Tool version less than {pre_processor.get_min_version_support()}" ) - # 3. Use PreProcessor to convert params to diagnosis jobs - diagnosis_task = pre_processor.get_diagnosis_cmds(instances, params) - if diagnosis_task is None or len(diagnosis_task.jobs) == 0: - raise Exception(f"Pre-processor not return any diagnosis job") - return diagnosis_task @staticmethod @@ -335,7 +331,7 @@ class DiagnosisHelper: **job_params ).execute_async() ) - job_results = asyncio.gather(*tasks) + job_results = await asyncio.gather(*tasks) diagnosis_task_result.job_results = [ DiagnosisJobResult( code=job_result.code, @@ -475,7 +471,7 @@ class DiagnosisHelper: post_process_result = await DiagnosisHelper.postprocess_v1( instance, diagnosis_task_result ) - + if post_process_result.code != 0: # 后处理脚本认为诊断出错 await DiagnosisHelper._update_job( diff --git a/sysom_server/sysom_diagnosis/apps/task/views.py b/sysom_server/sysom_diagnosis/apps/task/views.py index 6f59f519..313f4cab 100644 --- a/sysom_server/sysom_diagnosis/apps/task/views.py +++ b/sysom_server/sysom_diagnosis/apps/task/views.py @@ -1,4 +1,3 @@ -import json import requests from clogger import logger from django.http.response import FileResponse @@ -17,23 +16,27 @@ from lib.authentications import TokenAuthentication from channel_job.job import JobResult from asgiref.sync import async_to_sync from .helper import DiagnosisHelper - - -class TaskAPIView(CommonModelViewSet, - mixins.ListModelMixin, - mixins.RetrieveModelMixin, - mixins.DestroyModelMixin, - mixins.CreateModelMixin - ): - queryset = JobModel.objects.all().order_by('-created_at') +from service_scripts.base import ( + DiagnosisJobResult, + DiagnosisTaskResult, +) + + +class TaskAPIView( + CommonModelViewSet, + mixins.ListModelMixin, + mixins.RetrieveModelMixin, + mixins.DestroyModelMixin, + mixins.CreateModelMixin, +): + queryset = JobModel.objects.all().order_by("-created_at") serializer_class = seriaizer.JobListSerializer filter_backends = (DjangoFilterBackend, SearchFilter, OrderingFilter) - search_fields = ('id', 'task_id', 'created_by__id', - 'status', 'params') # 模糊查询 + search_fields = ("id", "task_id", "created_by__id", "status", "params") # 模糊查询 filterset_class = TaskFilter # 精确查询 authentication_classes = [TokenAuthentication] - create_requird_fields = ['service_name'] - lookup_field = 'task_id' + create_requird_fields = ["service_name"] + lookup_field = "task_id" def __init__(self, **kwargs) -> None: super().__init__(**kwargs) @@ -41,7 +44,9 @@ class TaskAPIView(CommonModelViewSet, def get_authenticators(self): # 判断请求是否是单查task task_id = self.kwargs.get(self.lookup_field, None) - if self.request.path.endswith("health_check/") or (task_id is not None and self.request.method == 'GET'): + if self.request.path.endswith("health_check/") or ( + task_id is not None and self.request.method == "GET" + ): return [] else: return [auth() for auth in self.authentication_classes] @@ -53,14 +58,14 @@ class TaskAPIView(CommonModelViewSet, try: instance = self.get_object() except Http404: - return other_response(result={}, message='task不存在', success=False, code=400) + return other_response(result={}, message="task不存在", success=False, code=400) response = seriaizer.JobRetrieveSerializer(instance) res = response.data - result = res['result'] - if 'state' in result: - res['result'] = result['result'] - res['url'] = "/".join(["", "diagnose", "detail", instance.task_id]) + result = res["result"] + if "state" in result: + res["result"] = result["result"] + res["url"] = "/".join(["", "diagnose", "detail", instance.task_id]) return success(result=res) def list(self, request, *args, **kwargs): @@ -77,13 +82,13 @@ class TaskAPIView(CommonModelViewSet, return success(message="删除成功", code=200, result={}) def get_task_svg(self, request, task_id: str, etx: str, *args, **kwargs): - if etx != 'svg': + if etx != "svg": return not_found(message="请输入正确参数: SVG") instance = get_object_or_404(JobModel, task_id=task_id) - if instance.status == 'Success': + if instance.status == "Success": result = instance.result - svg_context = result.get('flamegraph', None) + svg_context = result.get("flamegraph", None) if svg_context is None: return success(success=False, message='Result 未包含 "flamegraph"字段') return FileResponse(svg_context) @@ -93,23 +98,18 @@ class TaskAPIView(CommonModelViewSet, def create_task_v2(self, request, *args, **kwargs): try: # 1. Check required params - res = self.require_param_validate( - request, ['service_name']) - if not res['success']: - return ErrorResponse(msg=res.get('message', 'Missing parameters')) + res = self.require_param_validate(request, ["service_name"]) + if not res["success"]: + return ErrorResponse(msg=res.get("message", "Missing parameters")) data = request.data # 3. Create Task - instance = DiagnosisHelper.init(data, getattr(request, 'user')) + instance = DiagnosisHelper.init(data, getattr(request, "user")) self.produce_event_to_cec( settings.SYSOM_CEC_DIAGNOSIS_TASK_DISPATCH_TOPIC, - { - "task_id": instance.task_id - } + {"task_id": instance.task_id}, ) - return success({ - "task_id": instance.task_id - }) + return success({"task_id": instance.task_id}) except Exception as e: logger.exception(e) return ErrorResponse(msg=str(e)) @@ -119,34 +119,36 @@ class TaskAPIView(CommonModelViewSet, try: # 1. Check required params res = self.require_param_validate( - request, ['instance', 'offline_log', 'service_name']) - if not res['success']: - return ErrorResponse(message=res.get('message', 'Missing parameters')) + request, ["instance", "offline_log", "service_name"] + ) + if not res["success"]: + return ErrorResponse(message=res.get("message", "Missing parameters")) data = request.data # 2. Offline import offline_log = data.pop("offline_log", "") - instance = DiagnosisHelper.offline_import( - data, getattr(request, 'user')) + instance = DiagnosisHelper.offline_import(data, getattr(request, "user")) # 3. postprocess + async_to_sync(DiagnosisHelper.postprocess)( - instance, job_result=JobResult(code=0, result=offline_log)) - return success({ - "task_id": instance.task_id - }) + instance, + diagnosis_task_result=DiagnosisTaskResult( + 0, job_results=[DiagnosisJobResult(0, stdout=offline_log)] + ), + ) + return success({"task_id": instance.task_id}) except Exception as e: logger.exception(e) return ErrorResponse(msg=str(e)) def health_check(self, request, *args, **kwargs): return success(result={}) - + def get_host(self, request): - host_url = f'{settings.SYSOM_API_URL}/api/v1/host/' + host_url = f"{settings.SYSOM_API_URL}/api/v1/host/" res = requests.get(host_url) if res.status_code == 200: - return success(result=res.json().get('data', [])) + return success(result=res.json().get("data", [])) else: return ErrorResponse(msg=f"Get host failed, status_code={res.status_code}") - diff --git a/sysom_server/sysom_diagnosis/service_scripts/base.py b/sysom_server/sysom_diagnosis/service_scripts/base.py index d3f8360b..441fae0c 100644 --- a/sysom_server/sysom_diagnosis/service_scripts/base.py +++ b/sysom_server/sysom_diagnosis/service_scripts/base.py @@ -6,7 +6,7 @@ File base.py Description: """ from abc import ABC, abstractmethod -from typing import List, Dict +from typing import List, Optional class DiagnosisJob: @@ -45,7 +45,7 @@ class DiagnosisTask: class DiagnosisJobResult: def __init__( - self, code: int, job: DiagnosisJob, err_msg: str = "", stdout: str = "" + self, code: int, job: Optional[DiagnosisJob] = None, err_msg: str = "", stdout: str = "" ) -> None: self.job = job self.code = code @@ -112,10 +112,10 @@ class DiagnosisPreProcessor(DiagnosisProcessorBase): return 'rpm -q sysak --queryformat "%{VERSION}-%{RELEASE}"' @abstractmethod - def get_diagnosis_cmds(self, instances: List[str], params: dict) -> DiagnosisTask: + def get_diagnosis_cmds(self, params: dict) -> DiagnosisTask: """Convert params to diagnosis cmds - params => { "service_name": "xxx", "time": 15 } + params => { "instance": "127.0.0.1", "service_name": "xxx", "time": 15 } cmds => [ DiagnosisJob(instance="127.0.0.1", command="sysak memleak"), diff --git a/sysom_server/sysom_diagnosis/service_scripts/command_post.py b/sysom_server/sysom_diagnosis/service_scripts/command_post.py index ccc25cf8..5332b696 100644 --- a/sysom_server/sysom_diagnosis/service_scripts/command_post.py +++ b/sysom_server/sysom_diagnosis/service_scripts/command_post.py @@ -6,17 +6,17 @@ File command.py Description: """ from typing import List -from .base import DiagnosisJobResult, DiagnosisPostProcessor +from .base import DiagnosisJobResult, DiagnosisPostProcessor, PostProcessResult -class CommandPostProcessor(DiagnosisPostProcessor): - def parse_diagnosis_result(self, results: List[DiagnosisJobResult]) -> dict: - postprocess_result = { - "code": 0, - "err_msg": "", - "result": {}, - } - postprocess_result["result"] = { +class PostProcessor(DiagnosisPostProcessor): + def parse_diagnosis_result(self, results: List[DiagnosisJobResult]) -> PostProcessResult: + postprocess_result = PostProcessResult( + code=0, + err_msg="", + result={} + ) + postprocess_result.result = { "CommandResult": {"data": [{"key": "", "value": results[0].stdout}]} } return postprocess_result diff --git a/sysom_server/sysom_diagnosis/service_scripts/command_pre.py b/sysom_server/sysom_diagnosis/service_scripts/command_pre.py index 2a388978..e2665635 100644 --- a/sysom_server/sysom_diagnosis/service_scripts/command_pre.py +++ b/sysom_server/sysom_diagnosis/service_scripts/command_pre.py @@ -5,10 +5,7 @@ Email mfeng@linux.alibaba.com File command.py Description: """ -from typing import List - -from sysom_server.sysom_diagnosis.service_scripts.base import DiagnosisTask -from .base import DiagnosisJob, DiagnosisPreProcessor +from .base import DiagnosisJob, DiagnosisPreProcessor, DiagnosisTask class PreProcessor(DiagnosisPreProcessor): @@ -20,11 +17,12 @@ class PreProcessor(DiagnosisPreProcessor): DiagnosisPreProcessor (_type_): _description_ """ - def get_diagnosis_cmds(self, instances: List[str], params: dict) -> DiagnosisTask: + def get_diagnosis_cmds(self, params: dict) -> DiagnosisTask: command = params.get("command", "") + instance = params.get("instance", "") return DiagnosisTask( jobs=[ - DiagnosisJob(instance=instance, cmd=command) for instance in instances + DiagnosisJob(instance=instance, cmd=command) ], in_order=False, ) -- Gitee From 9647fadf5cd583cd5db3a71607de6dca885e6b6c Mon Sep 17 00:00:00 2001 From: wb-672209 Date: Fri, 7 Jul 2023 15:51:33 +0800 Subject: [PATCH 117/196] =?UTF-8?q?fix(hotfix):=E8=A1=A5=E4=B8=81=E4=B8=AD?= =?UTF-8?q?=E5=BF=83=E6=A8=A1=E5=9D=97=20=E7=83=AD=E8=A1=A5=E4=B8=81?= =?UTF-8?q?=E5=88=97=E8=A1=A8=E9=A1=B5=E9=9D=A2=20=E4=B8=80=E9=94=AE?= =?UTF-8?q?=E9=83=A8=E7=BD=B2=E5=8A=9F=E8=83=BD=E6=8A=A5=E9=94=99=E4=BF=AE?= =?UTF-8?q?=E6=94=B9=20=E6=B7=BB=E5=8A=A0=E6=88=90=E5=8A=9F=E5=A4=B1?= =?UTF-8?q?=E8=B4=A5=E7=9A=84=E6=8F=90=E7=A4=BA=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pages/hotfix/FormalHotfixList/index.jsx | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/sysom_web/src/pages/hotfix/FormalHotfixList/index.jsx b/sysom_web/src/pages/hotfix/FormalHotfixList/index.jsx index 492c2ccf..175e9413 100644 --- a/sysom_web/src/pages/hotfix/FormalHotfixList/index.jsx +++ b/sysom_web/src/pages/hotfix/FormalHotfixList/index.jsx @@ -46,7 +46,6 @@ const FormalHotfixList = () => { const intl = useIntl(); const [loadings, setLoadings] = useState([]); const [disabled, setDisabled] = useState([]); - const [messageApi, contextHolder] = message.useMessage(); const enterLoading = async (record) => { setLoadings((prevLoadings) => { @@ -59,8 +58,29 @@ const FormalHotfixList = () => { newdisabled[record.id] = true; return newdisabled; }); - const res = await postOneclickDeployment(record); - if (res) { + const res = await postOneclickDeployment(record).then(res=>{ + if (res) { + setLoadings((prevLoadings) => { + const newLoadings = [...prevLoadings]; + newLoadings[record.id] = false; + return newLoadings; + }); + setDisabled((prevLoadings) => { + const newdisabled = [...prevLoadings]; + newdisabled[record.id] = false; + return newdisabled; + }); + if(res.code === 200){ + message.success('部署成功！'); + message.success(res.data.msg); + }else{ + message.error('部署失败！'); + } + } else{ + message.error('部署有问题！'); + } + }).catch((res)=>{ + message.error('部署有问题，请查找原因后再部署！'); setLoadings((prevLoadings) => { const newLoadings = [...prevLoadings]; newLoadings[record.id] = false; @@ -71,18 +91,8 @@ const FormalHotfixList = () => { newdisabled[record.id] = false; return newdisabled; }); - if(res.code === 200){ - messageApi.open({ - type: 'success', - content: res.data.msg, - }); - }else{ - messageApi.open({ - type: 'error', - content: res.data.msg, - }); - } - } + }); + }; const columns = [ -- Gitee From e38838687a0568a4c4d69d70b57478281b6bc937 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 10 Jul 2023 14:15:09 +0800 Subject: [PATCH 118/196] feat(monitor): Add sysak-based monitor dashboard --- deps/4_grafana/grafana_api_set.sh | 9 + deps/4_grafana/sysom-migration-dashboard.json | 2 +- .../4_grafana/sysom-sysak-base-dashboard.json | 14273 ++++++++++++++++ .../sysom_diagnosis/scripts/node_clear.sh | 5 +- .../sysom_diagnosis/scripts/node_init.sh | 10 +- .../app/routeres/prometheus.py | 2 +- sysom_server/sysom_monitor_server/config.yml | 13 + .../scripts/node_clear.sh | 15 +- .../sysom_monitor_server/scripts/node_init.sh | 51 +- 9 files changed, 14360 insertions(+), 20 deletions(-) create mode 100644 deps/4_grafana/sysom-sysak-base-dashboard.json diff --git a/deps/4_grafana/grafana_api_set.sh b/deps/4_grafana/grafana_api_set.sh index 4adb9c57..2a7fb716 100755 --- a/deps/4_grafana/grafana_api_set.sh +++ b/deps/4_grafana/grafana_api_set.sh @@ -111,6 +111,15 @@ then exit 1 fi +curl -c cookie -b cookie --location --request POST 'http://127.0.0.1:3000/api/dashboards/db' \ +--header 'Content-Type: application/json' \ +-d @"sysom-sysak-base-dashboard.json" +if [ $? -ne 0 ] +then + echo "grafana configure sysom-cec-status-dashboard error" + exit 1 +fi + #curl -c cookie -b cookie --location --request POST 'http://127.0.0.1:3000/api/dashboards/db' \ diff --git a/deps/4_grafana/sysom-migration-dashboard.json b/deps/4_grafana/sysom-migration-dashboard.json index c861207b..5c124da4 100644 --- a/deps/4_grafana/sysom-migration-dashboard.json +++ b/deps/4_grafana/sysom-migration-dashboard.json @@ -19,7 +19,7 @@ } ] }, - "editable": true, + "editable": false, "fiscalYearStartMonth": 0, "graphTooltip": 0, "id": null, diff --git a/deps/4_grafana/sysom-sysak-base-dashboard.json b/deps/4_grafana/sysom-sysak-base-dashboard.json new file mode 100644 index 00000000..672e440c --- /dev/null +++ b/deps/4_grafana/sysom-sysak-base-dashboard.json @@ -0,0 +1,14273 @@ +{ + "dashboard": { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "sysom-prometheus", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "gnetId": 1860, + "graphTooltip": 0, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": true, + "datasource": "sysom-prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 261, + "panels": [ + { + "datasource": "sysom-prometheus", + "description": "Total number of CPU cores", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 14, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 30 + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "count(sysak_proc_cpus{mode=\"nice\",instance=\"$node\"})", + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Cores", + "type": "stat" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 75, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 30 + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{value=\"MemTotal\",instance=\"$node\"} * 1024", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Mem Total", + "type": "stat" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 321, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 30 + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_fs_stat{mount=\"/\",counter=\"f_blocks\",instance=\"$node\"} * 4096", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Rootfs Total", + "type": "stat" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 390, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 30 + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "count(sysak_proc_networks{instance=\"$node\",counter=\"if_ibytes\"})", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Eth Total", + "type": "stat" + }, + { + "datasource": "sysom-prometheus", + "description": "System date", + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "dateTimeAsSystem" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 16, + "y": 1 + }, + "hideTimeOverride": true, + "id": 15, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": {}, + "textMode": "auto" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_uptime{instance=\"$node\",value=\"stamp\"} * 1000", + "format": "table", + "instant": true, + "interval": "", + "intervalFactor": 1, + "range": false, + "refId": "A", + "step": 240 + } + ], + "title": "System Date", + "type": "stat" + }, + { + "datasource": "sysom-prometheus", + "description": "System uptime", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 1, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 4, + "x": 20, + "y": 1 + }, + "hideTimeOverride": true, + "id": 325, + "links": [], + "maxDataPoints": 100, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 30 + }, + "textMode": "auto" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_uptime{instance=\"$node\",value=\"uptime\"}", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": "sysom-prometheus", + "description": "Busy state of all CPU cores together", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "orange", + "value": 85 + }, + { + "color": "red", + "value": 95 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 0, + "y": 3 + }, + "id": 20, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "text": {} + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "100 - avg(sysak_proc_cpus{mode='idle',instance=\"$node\"})", + "hide": false, + "intervalFactor": 1, + "legendFormat": "", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "CPU Busy", + "type": "gauge" + }, + { + "datasource": "sysom-prometheus", + "description": "Non available memory", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 3 + }, + "hideTimeOverride": false, + "id": 16, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "100 - sysak_proc_meminfo{value=\"MemAvailable\",instance=\"$node\"} / on(instance) sysak_proc_meminfo{value=\"MemTotal\",instance=\"$node\"} * 100", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Mem Used", + "type": "gauge" + }, + { + "datasource": "sysom-prometheus", + "description": "Used Root FS", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 2, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "max": 100, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "rgba(50, 172, 45, 0.97)" + }, + { + "color": "yellow", + "value": 50 + }, + { + "color": "orange", + "value": 80 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 6, + "y": 3 + }, + "id": 154, + "links": [], + "options": { + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "100 - sysak_fs_stat{mount=\"/\",counter=\"f_bavail\",instance=\"$node\"} / on(instance)(sysak_fs_stat{mount=\"/\",counter=\"f_blocks\",instance=\"$node\"} - on(instance)sysak_fs_stat{mount=\"/\",counter=\"f_bfree\",instance=\"$node\"} + on(instance)sysak_fs_stat{mount=\"/\",counter=\"f_bavail\",instance=\"$node\"}) * 100", + "format": "time_series", + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Rootfs Used", + "type": "gauge" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 9, + "y": 3 + }, + "id": 391, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 1 + }, + "textMode": "value_and_name" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sum(sysak_proc_networks{instance=\"$node\",counter=\"if_ipackets\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "rx", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sum(sysak_proc_networks{instance=\"$node\",counter=\"if_opackets\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "tx", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network pps", + "type": "stat" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 14, + "y": 3 + }, + "id": 392, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "text": { + "valueSize": 1 + }, + "textMode": "value_and_name" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sum(sysak_proc_networks{instance=\"$node\",counter=\"if_ibytes\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "rx", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sum(sysak_proc_networks{instance=\"$node\",counter=\"if_obytes\"})", + "format": "time_series", + "hide": false, + "instant": false, + "intervalFactor": 1, + "legendFormat": "tx", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network bps", + "type": "stat" + }, + { + "datasource": "sysom-prometheus", + "description": "Busy state of all CPU cores together (1 min average)", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 5, + "x": 19, + "y": 3 + }, + "id": 155, + "links": [], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "value" + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_loadavg{instance=\"$node\",value=\"load1\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Sys Load", + "type": "stat" + } + ], + "targets": [ + { + "datasource": "sysom-prometheus", + "refId": "A" + } + ], + "title": "Summary", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 347, + "panels": [ + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "links": [], + "mappings": [], + "unit": "%" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 2 + }, + "id": 345, + "options": { + "displayLabels": [ + "percent", + "name" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"idle\"}", + "legendFormat": "idle", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"user\"} + on(instance)sysak_proc_cpu_total{instance=\"$node\",mode=\"nice\"}", + "hide": false, + "legendFormat": "user", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"sys\"} + on(instance)sysak_proc_cpu_total{instance=\"$node\",mode=\"iowait\"} + on(instance)sysak_proc_cpu_total{instance=\"$node\",mode=\"hardirq\"} + on(instance)sysak_proc_cpu_total{instance=\"$node\",mode=\"softirq\"}", + "hide": false, + "legendFormat": "kernel", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"steal\"}", + "hide": false, + "legendFormat": "steal", + "range": true, + "refId": "D" + } + ], + "title": "CPU Graph", + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "%" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 2 + }, + "id": 348, + "options": { + "displayLabels": [ + "name", + "value" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"sys\"}", + "legendFormat": "sys", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"iowait\"}", + "hide": false, + "legendFormat": "iowait", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"hardirq\"}", + "hide": false, + "legendFormat": "hardirq", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"softirq\"}", + "hide": false, + "legendFormat": "softirq", + "range": true, + "refId": "E" + } + ], + "title": "Kernel Used CPU", + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 2 + }, + "id": 356, + "options": { + "displayLabels": [ + "value", + "percent" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_cgroups{instance=\"$node\",type=\"num_cgroups\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{value}}", + "range": false, + "refId": "A" + } + ], + "title": "Cgroup Numbers", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "__name__": true, + "exported_instance": true, + "instance": true, + "job": true, + "type": true + }, + "indexByName": {}, + "renameByName": { + "Value": "Count", + "value": "Type" + } + } + } + ], + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 4, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "%" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 352, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"user\"}", + "hide": false, + "legendFormat": "user", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"sys\"}", + "hide": false, + "legendFormat": "sys", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"nice\"}", + "hide": false, + "legendFormat": "nice", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"idle\"}", + "legendFormat": "idle", + "range": true, + "refId": "D" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"iowait\"}", + "hide": false, + "legendFormat": "iowait", + "range": true, + "refId": "E" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"hardirq\"}", + "hide": false, + "legendFormat": "hardirq", + "range": true, + "refId": "F" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"softirq\"}", + "hide": false, + "legendFormat": "softirq", + "range": true, + "refId": "G" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_cpu_total{instance=\"$node\",mode=\"steal\"}", + "hide": false, + "legendFormat": "steal", + "range": true, + "refId": "H" + } + ], + "title": "CPU Used", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 4, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "%" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 398, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "100 - sysak_proc_cpus{instance=\"$node\",mode=\"idle\"}", + "hide": false, + "instant": false, + "legendFormat": "{{cpu_name}} busy", + "range": true, + "refId": "A" + } + ], + "title": "CPU Used", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sys" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "hardirq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 351, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_loadavg{instance=\"$node\",value=\"load1\"}", + "legendFormat": "load 1m", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_loadavg{instance=\"$node\",value=\"load5\"}", + "hide": false, + "legendFormat": "load 5m", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_loadavg{instance=\"$node\",value=\"load15\"}", + "hide": false, + "legendFormat": "load 15m", + "range": true, + "refId": "C" + } + ], + "title": "System Load", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 4, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 372, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sum by(value)(rate(sysak_interrupts{instance=\"$node\"}[$__rate_interval]))", + "hide": false, + "legendFormat": "{{value}}", + "range": true, + "refId": "A" + } + ], + "title": "Hardirqs Rate", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 4, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 350, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_sirq{instance=\"$node\"}[$__rate_interval])", + "hide": false, + "legendFormat": "{{type}}", + "range": true, + "refId": "A" + } + ], + "title": "Softirqs Rate", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "log": 10, + "type": "log" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "sys" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "hardirq" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 353, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_stat_counters{instance=\"$node\",counter=\"ctxt\"} / 20", + "legendFormat": "Context Switches", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_stat_counters{instance=\"$node\",counter=\"processes_forks\"} / 20", + "hide": false, + "legendFormat": "Forks", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_loadavg{instance=\"$node\",value=\"runq\"}", + "hide": false, + "legendFormat": "Running Threads", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_loadavg{instance=\"$node\",value=\"plit\"}", + "hide": false, + "legendFormat": "Total Threads", + "range": true, + "refId": "D" + } + ], + "title": "Context Switches / Forks", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "cgroup子系统周期内增长速率", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 4, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 373, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "delta(sysak_cgroups{type=\"num_cgroups\",instance=\"$node\"}[$__range])", + "hide": false, + "legendFormat": "{{value}}", + "range": true, + "refId": "A" + } + ], + "title": "Cgroup Numbers Increase Rate", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "关中断过长统计", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 4, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 396, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sched_moni_jitter{instance=\"$node\",value=\"gt50ms\",mod=\"irqoff\"}", + "legendFormat": "DelayTime in (50ms,100ms)", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sched_moni_jitter{instance=\"$node\",value=\"gt100ms\",mod=\"irqoff\"}", + "hide": false, + "legendFormat": "DelayTime in [100ms, 500ms)", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sched_moni_jitter{instance=\"$node\",value=\"gt500ms\",mod=\"irqoff\"}", + "hide": false, + "legendFormat": "DelayTime in [500ms, 1s)", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sched_moni_jitter{instance=\"$node\",value=\"gt1s\",mod=\"irqoff\"}", + "hide": false, + "legendFormat": "DelayTime >= 1s", + "range": true, + "refId": "D" + } + ], + "title": "IrqOff Count", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "发生长时间不调度的次数", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 4, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 354, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sched_moni_jitter{instance=\"$node\",value=\"gt50ms\",mod=\"noschd\"}", + "legendFormat": "DelayTime in (50ms,100ms)", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sched_moni_jitter{instance=\"$node\",value=\"gt100ms\",mod=\"noschd\"}", + "hide": false, + "legendFormat": "DelayTime in [100ms, 500ms)", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sched_moni_jitter{instance=\"$node\",value=\"gt500ms\",mod=\"noschd\"}", + "hide": false, + "legendFormat": "DelayTime in [500ms, 1s)", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sched_moni_jitter{instance=\"$node\",value=\"gt1s\",mod=\"noschd\"}", + "hide": false, + "legendFormat": "DelayTime >= 1s", + "range": true, + "refId": "D" + } + ], + "title": "NoSched Count", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "在就绪队列中等待的时间", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 4, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ns" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 11, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 355, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_schedstat{instance=\"$node\",value=\"delay\"}[$__rate_interval])", + "legendFormat": "{{cpu}} WakeUp2Sched Delay", + "range": true, + "refId": "A" + } + ], + "title": "WaitOnRunq Delay", + "type": "timeseries" + } + ], + "title": "System CPU and Schedule", + "type": "row" + }, + { + "collapsed": true, + "datasource": "sysom-prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 2 + }, + "id": 266, + "panels": [ + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 2, + "mappings": [], + "min": -4, + "unit": "kbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel reserved" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 3 + }, + "id": 333, + "options": { + "displayLabels": [ + "percent", + "value" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"kernel_reserved\"}", + "legendFormat": "system reserved", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"MemFree\"}", + "hide": false, + "legendFormat": "free", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_used\"}", + "hide": false, + "legendFormat": "app use", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"kernel_used\"} - on(instance)sysak_proc_meminfo{value=\"kernel_reserved\"}", + "hide": false, + "legendFormat": "kernel use", + "range": true, + "refId": "D" + } + ], + "title": "Memory Graph", + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "kbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 3 + }, + "id": 334, + "options": { + "displayLabels": [ + "percent", + "value" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"VmallocUsed\"}", + "hide": false, + "legendFormat": "VmallocUsed", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"SReclaimable\"}", + "hide": false, + "legendFormat": "SReclaimable", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"SUnreclaim\"}", + "hide": false, + "legendFormat": "SUnreclaim", + "range": true, + "refId": "D" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"PageTables\"}", + "hide": false, + "legendFormat": "PageTables", + "range": true, + "refId": "E" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"alloc_page\"}", + "hide": false, + "legendFormat": "alloc_page", + "range": true, + "refId": "F" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"KernelStack\"}", + "hide": false, + "legendFormat": "KernelStack", + "range": true, + "refId": "G" + } + ], + "title": "Kernel Used Memory", + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "kbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 3 + }, + "id": 335, + "options": { + "displayLabels": [ + "percent", + "value" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_filecache\"}", + "legendFormat": "filecache", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_anon\"}", + "hide": false, + "legendFormat": "anon", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_mlock\"}", + "hide": false, + "legendFormat": "mlock", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_buffers\"}", + "hide": false, + "legendFormat": "buffers", + "range": true, + "refId": "D" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_shmem\"}", + "hide": false, + "legendFormat": "shmem", + "range": true, + "refId": "E" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_huge_1G\"}", + "hide": false, + "legendFormat": "huge_1G", + "range": true, + "refId": "F" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_huge_2M\"}", + "hide": false, + "legendFormat": "huge_2M", + "range": true, + "refId": "G" + } + ], + "title": "User Used Memory", + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "description": "内存使用率", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "kbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 11 + }, + "id": 331, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "#node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}\nsysak_proc_meminfo{value=\"total\",instance=\"$node\"}\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Total", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "#node_memory_Active_bytes{instance=\"$node\",job=\"$job\"}\nsysak_proc_meminfo{value=\"total\",instance=\"$node\"} - on(instance)sysak_proc_meminfo{value=\"MemFree\",instance=\"$node\"} - on(instance)sysak_proc_meminfo{value=\"Cached\",instance=\"$node\"} - on(instance)sysak_proc_meminfo{value=\"Buffers\",instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{value=\"MemFree\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Free", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{value=\"Cached\",instance=\"$node\"} + on(instance)sysak_proc_meminfo{value=\"Buffers\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Buffers+Cached", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory Usage", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "内存伙伴系统情况", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*CommitLimit - *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 11 + }, + "id": 135, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo0\",instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "4K", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo1\",instance=\"$node\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "8K", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo2\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "16K", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo3\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "32K", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo4\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "64K", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo5\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "128K", + "range": true, + "refId": "G", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo6\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "256K", + "range": true, + "refId": "H", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo7\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "512K", + "range": true, + "refId": "I", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo8\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "1M", + "range": true, + "refId": "J", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo9\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "2M", + "range": true, + "refId": "K", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_buddyinfo{value=\"buddyinfo10\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "4M", + "range": true, + "refId": "L", + "step": 240 + } + ], + "title": "Memory BuddyInfo", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "kbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 336, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"kernel_reserved\"}", + "hide": false, + "legendFormat": "reserved", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"VmallocUsed\"}", + "hide": false, + "legendFormat": "VmallocUsed", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"SReclaimable\"}", + "hide": false, + "legendFormat": "SReclaimable", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"SUnreclaim\"}", + "hide": false, + "legendFormat": "SUnreclaim", + "range": true, + "refId": "D" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"PageTables\"}", + "hide": false, + "legendFormat": "PageTables", + "range": true, + "refId": "E" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"alloc_page\"}", + "hide": false, + "legendFormat": "alloc_page", + "range": true, + "refId": "F" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"KernelStack\"}", + "hide": false, + "legendFormat": "KernelStack", + "range": true, + "refId": "G" + } + ], + "title": "Kernel Used Memory", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "kbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "app" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "kernel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 337, + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_filecache\"}", + "legendFormat": "filecache", + "range": true, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_anon\"}", + "hide": false, + "legendFormat": "anon", + "range": true, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_mlock\"}", + "hide": false, + "legendFormat": "mlock", + "range": true, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_buffers\"}", + "hide": false, + "legendFormat": "buffers", + "range": true, + "refId": "D" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_shmem\"}", + "hide": false, + "legendFormat": "shmem", + "range": true, + "refId": "E" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_huge_1G\"}", + "hide": false, + "legendFormat": "huge_1G", + "range": true, + "refId": "F" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{instance=\"$node\",value=\"user_huge_2M\"}", + "hide": false, + "legendFormat": "huge_2M", + "range": true, + "refId": "G" + } + ], + "title": "User Used Memory", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "THP申请频率", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 31 + }, + "id": 191, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"thp_fault_alloc\",instance=\"$node\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "THP Fault Alloc Times", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"thp_fault_fallback\",instance=\"$node\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "THP Fault Fallback", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"thp_collapse_alloc\",instance=\"$node\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "THP Collapse Alloc", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"thp_collapse_alloc_failed\",instance=\"$node\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "THP Collapse Alloc Fail", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_meminfo{value=\"AnonHugePages\",instance=\"$node\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "THP Page Count", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "THP Alloc Rate", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "可回收和不可回收Slab内存大小", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "left", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "kbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 31 + }, + "id": 136, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "#node_memory_Inactive_bytes{instance=\"$node\",job=\"$job\"}\nsysak_proc_meminfo{value=\"SReclaimable\",instance=\"$node\"}\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Reclaimable", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_meminfo{value=\"SUnreclaim\",instance=\"$node\"}", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Unreclaim", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Memory Slab", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "Page换进换出频率", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Apps" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#629E51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A437C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Hardware Corrupted - Amount of RAM that the kernel identified as corrupted / not working" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#CFFAFF", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "RAM_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#806EB7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#2F575E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Unused" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "ShmemHugePages - Memory used by shared memory (shmem) and tmpfs allocated with huge pages" + }, + "properties": [ + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 138, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 350 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"pgpgin\",instance=\"$node\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Page In", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"pgpgout\",instance=\"$node\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Page Out", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"pswpin\",instance=\"$node\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "pswpin", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"pswpout\",instance=\"$node\"}[1m])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "pswpout", + "range": true, + "refId": "D", + "step": 240 + } + ], + "title": "Page In/Out", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "PageFault频率", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 131, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"pgfault\",instance=\"$node\"}[1m])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "PageFault", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"pgmajfault\",instance=\"$node\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Pgmajfault", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{value=\"pgfault\",instance=\"$node\"}[1m]) - on(instance)rate(sysak_proc_vmstat{value=\"pgmajfault\",instance=\"$node\"}[1m])", + "format": "time_series", + "hide": false, + "intervalFactor": 1, + "legendFormat": "Pgminfault", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "PageFault Rate", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 51 + }, + "id": 307, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"compact_stall\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "compact_stall", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"compact_success\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "compact_success", + "range": true, + "refId": "F", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"compact_fail\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "compact_fail", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Memory Compact ", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Active" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#99440A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Buffers" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#58140C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6D1F62", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Cached" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Committed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#508642", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Dirty" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#B7DBAB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Inactive" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Mapped" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "PageTables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Page_Tables" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Slab_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Swap_Cache" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#C15C17", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#511749", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total RAM + Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#052B51", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Swap" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "VmallocUsed" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 51 + }, + "id": 339, + "links": [], + "options": { + "legend": { + "calcs": [ + "min", + "mean", + "max", + "lastNotNull" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"pgscan_kswapd\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pgscan_kswapd", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"pgscan_direct\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pgscan_direct", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"oom_kill\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "oom_kill", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"pgsteal_kswapd\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pgsteal_kswapd", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"pgsteal_direct\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pgsteal_direct", + "range": true, + "refId": "E", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_proc_vmstat{instance=\"$node\",value=\"pgscan_direct_throttle\"}[1m])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "pgscan_direct_throttle", + "range": true, + "refId": "F", + "step": 240 + } + ], + "title": "Memory Others", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": "sysom-prometheus", + "refId": "A" + } + ], + "title": "System Memory", + "type": "row" + }, + { + "collapsed": true, + "datasource": "sysom-prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 3 + }, + "id": 270, + "panels": [ + { + "datasource": "sysom-prometheus", + "description": "磁盘空间使用情况，类似df -h的结果", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false, + "minWidth": 50 + }, + "decimals": 2, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "kbytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "exported_instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "Used" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mount 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Size" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mount" + }, + "properties": [ + { + "id": "displayName", + "value": "Mounted on" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "counter 1" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "counter 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Time 3" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fs 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "Available" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Use%" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total Size(ignore reserve)" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fs 1" + }, + "properties": [ + { + "id": "displayName", + "value": "Filesystem" + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 4 + }, + "id": 383, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "Use%" + } + ] + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_fs_stat{instance=\"$node\",counter=\"f_blocks\"} * 4", + "format": "table", + "instant": true, + "legendFormat": "{{fs}}", + "range": false, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "(sysak_fs_stat{instance=\"$node\",counter=\"f_blocks\"} - on(mount)sysak_fs_stat{instance=\"$node\",counter=\"f_bfree\"}) * 4", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "{{fs}}", + "range": false, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_fs_stat{instance=\"$node\",counter=\"f_bavail\"} * 4", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "{{fs}}", + "range": false, + "refId": "C" + } + ], + "title": "df -h info", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "mount", + "mode": "outer" + } + }, + { + "id": "calculateField", + "options": { + "alias": "Total Size(ignore reserve)", + "binary": { + "left": "Value #B", + "operator": "+", + "reducer": "sum", + "right": "Value #C" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + }, + { + "id": "calculateField", + "options": { + "alias": "Use%", + "binary": { + "left": "Value #B", + "operator": "/", + "reducer": "sum", + "right": "Total Size(ignore reserve)" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "table" + }, + { + "datasource": "sysom-prometheus", + "description": "类似df -i的结果，查看文件系统inode使用情况", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "displayMode": "auto", + "inspect": false, + "minWidth": 50 + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Time" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "__name__" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "exported_instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #B" + }, + "properties": [ + { + "id": "displayName", + "value": "IUsed" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mount 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #A" + }, + "properties": [ + { + "id": "displayName", + "value": "Inodes" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "mount" + }, + "properties": [ + { + "id": "displayName", + "value": "Mounted on" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fs 1" + }, + "properties": [ + { + "id": "displayName", + "value": "Filesystem" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "counter" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Value #C" + }, + "properties": [ + { + "id": "displayName", + "value": "IFree" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "IUse%" + }, + "properties": [ + { + "id": "unit", + "value": "percentunit" + }, + { + "id": "decimals", + "value": 2 + }, + { + "id": "custom.displayMode", + "value": "gradient-gauge" + }, + { + "id": "max", + "value": 1 + }, + { + "id": "color", + "value": { + "mode": "continuous-GrYlRd" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "job" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "instance" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "fs 2" + }, + "properties": [ + { + "id": "custom.hidden", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 4 + }, + "id": 385, + "options": { + "footer": { + "enablePagination": true, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "IUse%" + } + ] + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_fs_stat{instance=\"$node\",counter=\"f_files\"}", + "format": "table", + "instant": true, + "legendFormat": "{{fs}}", + "range": false, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_fs_stat{instance=\"$node\",counter=\"f_files\"} - on(mount)sysak_fs_stat{instance=\"$node\",counter=\"f_ffree\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "{{fs}}", + "range": false, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_fs_stat{instance=\"$node\",counter=\"f_ffree\"}", + "format": "table", + "hide": false, + "instant": true, + "legendFormat": "{{fs}}", + "range": false, + "refId": "C" + } + ], + "title": "df -i info", + "transformations": [ + { + "id": "joinByField", + "options": { + "byField": "mount", + "mode": "outer" + } + }, + { + "id": "calculateField", + "options": { + "alias": "IUse%", + "binary": { + "left": "Value #B", + "operator": "/", + "reducer": "sum", + "right": "Value #A" + }, + "mode": "binary", + "reduce": { + "reducer": "sum" + } + } + } + ], + "type": "table" + }, + { + "datasource": "sysom-prometheus", + "description": "文件系统空间使用量", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "KBs" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 0, + "y": 16 + }, + "id": 387, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "- delta(sysak_fs_stat{instance=\"$node\",counter=\"f_bfree\"}[1m]) * 4 / 60", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{mount}}({{fs}})", + "range": true, + "refId": "B" + } + ], + "title": "Filesystem used increase rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "Filesystem Inode 使用情况", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 12, + "w": 12, + "x": 12, + "y": 16 + }, + "id": 386, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "-delta(sysak_fs_stat{instance=\"$node\",counter=\"f_ffree\"}[1m])", + "format": "time_series", + "hide": false, + "instant": false, + "legendFormat": "{{mount}}({{fs}})", + "range": true, + "refId": "B" + } + ], + "title": "Filesystem Inode used increase rate", + "transformations": [], + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "磁盘发生io hang的次数", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 301, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_IOMonIndForDisksIO{instance=\"$node\",value=\"iohangCnt\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{devname}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk IO iohangCnt", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "磁盘发生io delay的次数", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 401, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_IOMonIndForDisksIO{instance=\"$node\",value=\"iodelayCnt\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{devname}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk IO iodelayCnt", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "磁盘发生io burst的次数", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 402, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_IOMonIndForDisksIO{instance=\"$node\",value=\"ioburstCnt\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{devname}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk IO ioburstCnt", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "系统发生io wait 高的次数", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 403, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_IOMonIndForSystemIO{instance=\"$node\",value=\"iowaithighCnt\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{devname}} iowait high count", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "System IO iowaithighCnt", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IOs", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 48 + }, + "id": 400, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"busy\",disk_name!~\"[a-z]*[0-9]$\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{disk_name}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Disk IO Utils", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "The number (after merges) of I/O requests completed per second for the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "IO read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 48 + }, + "id": 9, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"reads\",disk_name!~\"[a-z]*[0-9]$\"}", + "intervalFactor": 4, + "legendFormat": "{{disk_name}} - Reads completed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"writes\",disk_name!~\"[a-z]*[0-9]$\"}", + "intervalFactor": 1, + "legendFormat": "{{disk_name}} - Writes completed", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk IOps Completed", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "The average time for requests issued to the device to be served. This includes the time spent by the requests in queue and the time spent servicing them.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "time. read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 30, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 58 + }, + "id": 37, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"rmsec\",disk_name!~\"[a-z]*[0-9]$\"}", + "hide": false, + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{disk_name}} - Read wait time avg", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"wmsec\",disk_name!~\"[a-z]*[0-9]$\"}", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{disk_name}} - Write wait time avg", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk Average Wait Time", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "The number of bytes read from or written to the device per second", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "bytes read (-) / write (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "KBs" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 58 + }, + "id": 33, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"rkb\",disk_name!~\"[a-z]*[0-9]$\"}", + "format": "time_series", + "intervalFactor": 4, + "legendFormat": "{{disk_name}} - Read bytes", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"wkb\",disk_name!~\"[a-z]*[0-9]$\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{disk_name}} - Written bytes", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Data", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "The average queue length of the requests that were issued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "aqu-sz", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 2, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 68 + }, + "id": 35, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"backlog\",disk_name!~\"[a-z]*[0-9]$\"} / 1000", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{disk_name}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Average Queue Size", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "The number of read and write requests merged per second that were queued to the device", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "I/Os", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "iops" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Read.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 68 + }, + "id": 133, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"rmerge\",disk_name!~\"[a-z]*[0-9]$\"}", + "intervalFactor": 1, + "legendFormat": "{{disk_name}} - Read merged", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"wmerge\",disk_name!~\"[a-z]*[0-9]$\"}", + "intervalFactor": 1, + "legendFormat": "{{disk_name}} - Write merged", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Disk R/W Merged", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "The number of outstanding requests at the instant the sample was taken. Incremented as requests are given to appropriate struct request_queue and decremented as they finish.", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "Outstanding req", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#584477", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda2_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BA43A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sda3_.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F4D598", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#0A50A1", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#BF1B00", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdb3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0752D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#962D82", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#614D93", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdc3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#9AC48A", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#65C5DB", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9934E", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EA6460", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde1.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E0F9D7", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sdd2.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FCEACA", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*sde3.*/" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#F9E2D2", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 78 + }, + "id": 34, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_disks{instance=\"$node\",counter=\"inflight\",disk_name!~\"[a-z]*[0-9]$\"}", + "interval": "", + "intervalFactor": 4, + "legendFormat": "{{disk_name}} - IO now", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Inflight IO/s", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": "sysom-prometheus", + "refId": "A" + } + ], + "title": "Storage Filesystem and IO", + "type": "row" + }, + { + "collapsed": true, + "datasource": "sysom-prometheus", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 272, + "panels": [ + { + "datasource": "sysom-prometheus", + "description": "统计各个网口的收发包情况", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 0, + "mappings": [], + "min": -2, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 5 + }, + "id": 393, + "options": { + "displayLabels": [ + "percent", + "value" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "asc" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"if_ipackets\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "{{network_name}} rx", + "range": false, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"if_opackets\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "{{network_name}} tx", + "range": false, + "refId": "B" + } + ], + "title": "Network Traffic RTX", + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 1, + "mappings": [], + "min": -4, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 5 + }, + "id": 394, + "options": { + "displayLabels": [ + "percent", + "value" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_net_tcp_count{instance=\"$node\",value=\"InSegs\"}", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "tcp in", + "range": false, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_net_tcp_count{instance=\"$node\",value=\"OutSegs\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "tcp out", + "range": false, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_net_udp_count{instance=\"$node\",value=\"InDatagrams\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "udp in", + "range": false, + "refId": "C" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_net_udp_count{instance=\"$node\",value=\"OutDatagrams\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "udp out", + "range": false, + "refId": "D" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_net_ip_count{instance=\"$node\",value=\"InReceives\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "ip in", + "range": false, + "refId": "E" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_net_ip_count{instance=\"$node\",value=\"OutRequests\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "ip out", + "range": false, + "refId": "F" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_net_icmp_count{instance=\"$node\",value=\"InMsgs\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "icmp in", + "range": false, + "refId": "G" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sysak_net_icmp_count{instance=\"$node\",value=\"OutMsgs\"}", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "icmp out", + "range": false, + "refId": "H" + } + ], + "title": "Netstat RTX Graph", + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "decimals": 1, + "mappings": [], + "min": -4, + "unit": "pps" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 5 + }, + "id": 397, + "options": { + "displayLabels": [ + "percent", + "value" + ], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.2.2", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "irate(sysak_net_icmp_count{instance=\"$node\",value=\"InErrors\"}[$__rate_interval])", + "format": "time_series", + "instant": true, + "interval": "", + "legendFormat": "icmp InError", + "range": false, + "refId": "A" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sum(irate(sysak_net_udp_count{instance=\"$node\",value!=\"InDatagrams\",value!=\"OutDatagrams\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "udp errors", + "range": false, + "refId": "B" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sum (irate(sysak_net_tcp_ext_count{instance=\"$node\"}[$__rate_interval]))", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "tcp errors", + "range": false, + "refId": "G" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "exemplar": false, + "expr": "sum(sysak_net_retrans_count{instance=\"$node\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "legendFormat": "tcp retrans", + "range": false, + "refId": "C" + } + ], + "title": "Netstat Error Count", + "type": "piechart" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "receive_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "receive_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_eth0" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "transmit_packets_lo" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 60, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"if_ipackets\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{network_name}} - Receive", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"if_opackets\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{network_name}} - Transmit", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic by Packets", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 143, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"if_idrop\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{network_name}} - Receive drop", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"if_odrop\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{network_name}} - Transmit drop", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Drop", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 146, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"if_imulticast\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{network_name}} - Receive multicast", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Network Traffic Multicast", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "packets out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "pps" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Trans.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 24 + }, + "id": 142, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"if_ierrs\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{network_name}} - Receive errors", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_proc_networks{instance=\"$node\",counter=\"of_ierrs\"}", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{network_name}} - Rransmit errors", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Network Traffic Errors", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 34 + }, + "id": 63, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"tcp_alloc\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_alloc - Allocated sockets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"tcp_inuse\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_inuse - Tcp sockets currently in use", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"tcp_mem\"}", + "format": "time_series", + "hide": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_mem - Used memory for tcp", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"tcp_orphan\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_orphan - Orphan sockets", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"tcp_tw\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCP_tw - Sockets wating close", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "Sockstat TCP", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 34 + }, + "id": 124, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"udplite_inuse\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDPLITE_inuse - Udplite sockets currently in use", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"udp_inuse\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_inuse - Udp sockets currently in use", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"udp_mem\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "UDP_mem - Used memory for udp", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "Sockstat UDP", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "sockets", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 44 + }, + "id": 126, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"sockets_used\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Sockets_used - Sockets currently in use", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Sockstat Used", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "kbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 44 + }, + "id": 220, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"tcp_mem\"} * 4", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - TCP sockets in that state", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"udp_mem\"} * 4", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "mem_bytes - UDP sockets in that state", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_sock_stat{instance=\"$node\",value=\"frag_mem\"} * 4", + "interval": "", + "intervalFactor": 1, + "legendFormat": "FRAG_memory - Used memory for frag", + "range": true, + "refId": "C" + } + ], + "title": "Sockstat Memory Size", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "octects out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 221, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_ip_count{instance=\"$node\",value=\"InReceives\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InOctets - Received octets", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_ip_count{instance=\"$node\",value=\"OutRequests\"}[$__rate_interval])", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "OutOctets - Sent octets", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "Netstat IP In / Out Octets", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 81, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "width": 300 + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "rate(sysak_net_ip_count{instance=\"$node\",value=\"Forwarding\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "Forwarding - IP forwarding", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "Netstat IP Forwarding", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 64 + }, + "id": 115, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_icmp_count{instance=\"$node\",value=\"InMsgs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InMsgs - Messages which the entity received. Note that this counter includes all those counted by icmpInErrors", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_icmp_count{instance=\"$node\",value=\"OutMsgs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutMsgs - Messages which this entity attempted to send. Note that this counter includes all those counted by icmpOutErrors", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "ICMP In / Out", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "messages out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 64 + }, + "id": 50, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_icmp_count{instance=\"$node\",value=\"InErrors\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - Messages which the entity received but determined as having ICMP-specific errors (bad ICMP checksums, bad length, etc.)", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "ICMP Errors", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 74 + }, + "id": 55, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_udp_count{instance=\"$node\",value=\"InDatagrams\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InDatagrams - Datagrams received", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_udp_count{instance=\"$node\",value=\"OutDatagrams\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutDatagrams - Datagrams sent", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "UDP In / Out", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 74 + }, + "id": 109, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_udp_count{instance=\"$node\",value=\"InErrors\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "InErrors - UDP Datagrams that could not be delivered to an application", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_udp_count{instance=\"$node\",value=\"NoPorts\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "NoPorts - UDP Datagrams received on a port with no listener", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_udp_count{instance=\"$node\",value=\"RcvbufErrors\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "RcvbufErrors - UDP buffer errors received", + "range": true, + "refId": "D", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_udp_count{instance=\"$node\",value=\"SndbufErrors\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "SndbufErrors - UDP buffer errors send", + "range": true, + "refId": "E", + "step": 240 + } + ], + "title": "UDP Errors", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "datagrams out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Out.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + }, + { + "matcher": { + "id": "byRegexp", + "options": "/.*Snd.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 84 + }, + "id": 299, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_count{instance=\"$node\",value=\"InSegs\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "InSegs - Segments received, including those received in error. This count includes segments received on currently established connections", + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_count{instance=\"$node\",value=\"OutSegs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "OutSegs - Segments sent, including those on current connections but excluding those containing only retransmitted octets", + "range": true, + "refId": "B", + "step": 240 + } + ], + "title": "TCP In / Out", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 84 + }, + "id": 104, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_ext_count{instance=\"$node\",value=\"ListenOverflows\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenOverflows - Times the listen queue of a socket overflowed", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_ext_count{instance=\"$node\",value=\"ListenDrops\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "ListenDrops - SYNs to LISTEN sockets ignored", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_retrans_count{instance=\"$node\",value=\"syn_ack\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 1, + "legendFormat": "TCPSynRetrans - SYN-SYN/ACK retransmits to break down retransmissions in SYN, fast/timeout retransmits", + "range": true, + "refId": "C", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_count{instance=\"$node\",value=\"RetransSegs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "RetransSegs - Segments retransmitted - that is, the number of TCP segments transmitted containing one or more previously transmitted octets", + "range": true, + "refId": "D" + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_count{instance=\"$node\",value=\"InErrs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "InErrs - Segments received in error (e.g., bad TCP checksums)", + "range": true, + "refId": "E" + } + ], + "title": "TCP Errors", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "connections", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*MaxConn *./" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#890F02", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 94 + }, + "id": 85, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_count{instance=\"$node\",value=\"CurrEstab\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "CurrEstab - TCP connections for which the current state is either ESTABLISHED or CLOSE- WAIT", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "TCP Connections", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 94 + }, + "id": 395, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "sysak_net_retrans_count{instance=\"$node\"}", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{value}}", + "range": true, + "refId": "A", + "step": 240 + } + ], + "title": "TCP Retrans", + "type": "timeseries" + }, + { + "datasource": "sysom-prometheus", + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "counter out (-) / in (+)", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 20, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/.*Sent.*/" + }, + "properties": [ + { + "id": "custom.transform", + "value": "negative-Y" + } + ] + } + ] + }, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 91, + "links": [], + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "9.2.0", + "targets": [ + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_ext_count{instance=\"$node\",value=\"SyncookiesFailed\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesFailed - Invalid SYN cookies received", + "range": true, + "refId": "A", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_ext_count{instance=\"$node\",value=\"SyncookiesRecv\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesRecv - SYN cookies received", + "range": true, + "refId": "B", + "step": 240 + }, + { + "datasource": "sysom-prometheus", + "editorMode": "code", + "expr": "irate(sysak_net_tcp_ext_count{instance=\"$node\",value=\"SyncookiesSent\"}[$__rate_interval])", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "SyncookiesSent - SYN cookies sent", + "range": true, + "refId": "C", + "step": 240 + } + ], + "title": "TCP SynCookie", + "type": "timeseries" + } + ], + "targets": [ + { + "datasource": "sysom-prometheus", + "refId": "A" + } + ], + "title": "Network Traffic and Sockstat and Netstat", + "type": "row" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [ + "linux" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "default", + "value": "default" + }, + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "localhost:8405", + "value": "localhost:8405" + }, + "datasource": "sysom-prometheus", + "definition": "label_values(sysak_proc_meminfo, instance)", + "hide": 0, + "includeAll": false, + "label": "Host:", + "multi": false, + "name": "node", + "options": [], + "query": { + "query": "label_values(sysak_proc_meminfo, instance)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": "", + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "sysak_base", + "uid": "rYdddlPWk", + "version": 102, + "weekStart": "" + } +} \ No newline at end of file diff --git a/sysom_server/sysom_diagnosis/scripts/node_clear.sh b/sysom_server/sysom_diagnosis/scripts/node_clear.sh index 11dbf5f4..6b38f439 100755 --- a/sysom_server/sysom_diagnosis/scripts/node_clear.sh +++ b/sysom_server/sysom_diagnosis/scripts/node_clear.sh @@ -3,7 +3,10 @@ RESOURCE_DIR=${NODE_HOME}/${SERVICE_NAME} main() { - yum erase -y `rpm -qa | grep sysak` + rpm -qa | grep sysak + if [ $? -eq 0 ]; then + yum erase -y `rpm -qa | grep sysak` + fi rm -rf ${RESOURCE_DIR} exit 0 } diff --git a/sysom_server/sysom_diagnosis/scripts/node_init.sh b/sysom_server/sysom_diagnosis/scripts/node_init.sh index 4c76ea5b..f47ca8d7 100755 --- a/sysom_server/sysom_diagnosis/scripts/node_init.sh +++ b/sysom_server/sysom_diagnosis/scripts/node_init.sh @@ -14,15 +14,13 @@ main() { exit 0 fi + SYSAK_PKG=$(ls sysak-*.rpm) + yum install -y ${SYSAK_PKG} + ###install sysak from local yum repo first### - yum install -y sysak if [ $? -eq 0 ]; then - exit 0 + yum install -y sysak fi - - SYSAK_PKG=sysak-${SYSAK_VERTION}.${ARCH}.rpm - yum install -y ${SYSAK_PKG} - exit $? } main diff --git a/sysom_server/sysom_monitor_server/app/routeres/prometheus.py b/sysom_server/sysom_monitor_server/app/routeres/prometheus.py index e178c794..5278155c 100644 --- a/sysom_server/sysom_monitor_server/app/routeres/prometheus.py +++ b/sysom_server/sysom_monitor_server/app/routeres/prometheus.py @@ -11,7 +11,7 @@ async def search_grafana(response: Response): res = g_client.get("/api/v1/host/").json() return [ { - "targets": [f"{item['ip']}:9100" for item in res["data"]], + "targets": [f"{item['ip']}:9100" for item in res["data"]] + [f"{item['ip']}:8405" for item in res["data"]], "labels": { "source": "sysom_monitor_server" } diff --git a/sysom_server/sysom_monitor_server/config.yml b/sysom_server/sysom_monitor_server/config.yml index b7a62e13..864a71f5 100644 --- a/sysom_server/sysom_monitor_server/config.yml +++ b/sysom_server/sysom_monitor_server/config.yml @@ -1,6 +1,8 @@ vars: NODE_EXPORT_BASE_DOWNLOAD_URL: &NODE_EXPORT_BASE_DOWNLOAD_URL https://sysom.oss-cn-beijing.aliyuncs.com/monitor/ NODE_EXPORT_VERSION: &NODE_EXPORT_VERSION 1.5.0 + SYSAK_DOWNLOAD_URL: &SYSAK_DOWNLOAD_URL https://mirrors.openanolis.cn/sysak/packages/release-v2.1.0/ + SYSAK_VERSION: &SYSAK_VERSION 2.1.0-2 SERVICE_NAME: &SERVICE_NAME sysom_monitor_server SERVICE_CONSUMER_GROUP: !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] @@ -47,6 +49,9 @@ sysom_service: # 节点测配置 sysom_node: version: 2.1 + env: + SYSAK_VERSION: *SYSAK_VERSION + NODE_EXPORT_VERSION: *NODE_EXPORT_VERSION # 节点分发配置 delivery: from_dir: scripts @@ -69,6 +74,14 @@ sysom_node: *NODE_EXPORT_VERSION, ".linux-amd64.tar.gz", ] + - local: !concat ["sysak-", *SYSAK_VERSION, ".x86_64.rpm"] + remote: + !concat [ + *SYSAK_DOWNLOAD_URL, + "sysak-", + *SYSAK_VERSION, + ".x86_64.rpm", + ] arm64: - *code_delivery_files_comm - local: node_exporter-1.5.0.linux-arm64.tar.gz diff --git a/sysom_server/sysom_monitor_server/scripts/node_clear.sh b/sysom_server/sysom_monitor_server/scripts/node_clear.sh index 998aaf27..cd5ed450 100755 --- a/sysom_server/sysom_monitor_server/scripts/node_clear.sh +++ b/sysom_server/sysom_monitor_server/scripts/node_clear.sh @@ -2,10 +2,21 @@ RESOURCE_DIR=${NODE_HOME}/${SERVICE_NAME} -main() -{ +clear_node_exporter() { systemctl stop node_exporter rm -f /usr/lib/systemd/system/node_exporter.service +} + +clear_sysak() { + rpm -qa | grep sysak + if [ $? -eq 0 ]; then + yum erase -y `rpm -qa | grep sysak` + fi +} + +main() { + clear_node_exporter + clear_sysak rm -rf ${RESOURCE_DIR} # bash -x raptor_profiling_clear.sh exit 0 diff --git a/sysom_server/sysom_monitor_server/scripts/node_init.sh b/sysom_server/sysom_monitor_server/scripts/node_init.sh index b17dbaf5..62fac38a 100755 --- a/sysom_server/sysom_monitor_server/scripts/node_init.sh +++ b/sysom_server/sysom_monitor_server/scripts/node_init.sh @@ -1,9 +1,19 @@ #!/bin/bash -x - RESOURCE_DIR=${NODE_HOME}/${SERVICE_NAME} -##设置node_exporter开机自动启动 -cat << EOF > node_exporter.service +if [ "$SYSAK_VERTION" == "" ]; then + export SYSAK_VERTION=2.1.0-2 +fi +if [ "$NODE_EXPORT_VERSION" == "" ]; then + export NODE_EXPORT_VERSION=1.5.0 +fi +if [ "$ARCH" == "" ]; then + export ARCH=x86_64 +fi + +init_node_exporter() { + ##设置node_exporter开机自动启动 + cat <node_exporter.service [Unit] Description=SysOM Monitor Prometheus Documentation=SysOM Monitor Prometheus @@ -18,10 +28,7 @@ RestartSec=5 [Install] WantedBy=multi-user.target EOF - -main() -{ - NODE_EXPORTER_PKG=`ls node_exporter-*.tar.gz | awk -F".tar.gz" '{print $1}'` + NODE_EXPORTER_PKG=$(ls node_exporter-*.tar.gz | awk -F".tar.gz" '{print $1}') NODE_EXPORTER_TAR=${NODE_EXPORTER_PKG}.tar.gz tar -zxvf ${NODE_EXPORTER_TAR} rm -rf ${RESOURCE_DIR}/node_exporter @@ -34,11 +41,37 @@ main() systemctl enable node_exporter systemctl start node_exporter ps -elf | grep "${RESOURCE_DIR}/node_exporter/node_exporter" | grep -v grep 1>/dev/null - if [ $? -ne 0 ] - then + if [ $? -ne 0 ]; then exit 1 fi + +} + +init_sysak_monitor_service() { + rpm -qa | grep sysak + if [ $? -eq 0 ]; then + exit 0 + fi + + SYSAK_PKG=$(ls sysak-*.rpm) + yum install -y ${SYSAK_PKG} + + ###install sysak from local yum repo first### + if [ $? -eq 0 ]; then + yum install -y sysak + fi + + systemctl enable sysak + systemctl start sysak +} + +init_raptor_profiling() { bash -x raptor_profiling_deploy.sh +} + +main() { + init_node_exporter + init_sysak_monitor_service exit 0 } -- Gitee From 432ddf56de849dbc12ef2bbb18de536c794a2073 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 10 Jul 2023 14:23:39 +0800 Subject: [PATCH 119/196] feat(web): Add sysak-based dashboard page --- sysom_web/config/routes.js | 13 ++++- .../src/pages/Monitor/SysakBaseDashboard.jsx | 54 +++++++++++++++++++ .../src/pages/Monitor/SystemDashboard.jsx | 14 +---- sysom_web/src/pages/Monitor/grafana.jsx | 20 +++++-- 4 files changed, 84 insertions(+), 17 deletions(-) create mode 100644 sysom_web/src/pages/Monitor/SysakBaseDashboard.jsx diff --git a/sysom_web/config/routes.js b/sysom_web/config/routes.js index d207487a..484bd32d 100644 --- a/sysom_web/config/routes.js +++ b/sysom_web/config/routes.js @@ -22,7 +22,7 @@ export default [ path: '/welcome', name: 'welcome', layout: false, - hideInMenu:true, + hideInMenu: true, component: './Welcome', }, { @@ -109,6 +109,17 @@ export default [ path: 'migration/:host?', component: './Monitor/MigrationDashboard', }, + { + path: 'base', + name: 'base', + hideInBreadcrumb: true, + component: './Monitor/SysakBaseDashboard', + }, + { + path: 'base/:host?', + component: './Monitor/SysakBaseDashboard', + }, + { component: './404', }, diff --git a/sysom_web/src/pages/Monitor/SysakBaseDashboard.jsx b/sysom_web/src/pages/Monitor/SysakBaseDashboard.jsx new file mode 100644 index 00000000..cfc01d15 --- /dev/null +++ b/sysom_web/src/pages/Monitor/SysakBaseDashboard.jsx @@ -0,0 +1,54 @@ +import { Statistic } from 'antd'; +import { useIntl, useRequest, useParams } from 'umi'; +import { useState } from 'react' +import { getHost } from '../host/service'; +import ProCard from '@ant-design/pro-card'; +import ServerList from './components/ServerList'; +import GrafanaWrap from './grafana'; + +const Dashboard = () => { + const intl = useIntl(); + const { data, error, loading } = useRequest(getHost) + const { host } = useParams() + const [hostIP, setHostIP] = useState(host || "127.0.0.1") + const [collapsed, setCollapsed] = useState(false) + + const onCollapsed = () => { + setCollapsed(!collapsed); + } + + return ( + <> + + + + + + + + item.status == 1).length} + valueStyle={{ color: "#FF4D4F", fontSize: 30 }} /> + + + + setHostIP(ip)} onLoad={dataSource => { + if (dataSource.length > 0 && !!dataSource[0].ip) { + setHostIP(dataSource[0].ip) + } + }} /> + + + {collapsed ? + >>
展
开
实
例
面
板 + : <<
折
叠
实
例
面
板 + } + + + + + ); +}; + +export default Dashboard; diff --git a/sysom_web/src/pages/Monitor/SystemDashboard.jsx b/sysom_web/src/pages/Monitor/SystemDashboard.jsx index 2a6a829e..cf77692c 100644 --- a/sysom_web/src/pages/Monitor/SystemDashboard.jsx +++ b/sysom_web/src/pages/Monitor/SystemDashboard.jsx @@ -4,17 +4,7 @@ import { useState } from 'react' import { getHost } from '../host/service'; import ProCard from '@ant-design/pro-card'; import ServerList from './components/ServerList'; - -const GrafanaWrap = (props) => { - return ( -