diff --git a/001_mad.patch b/001_mad.patch new file mode 100644 index 0000000000000000000000000000000000000000..d70221ce5de548df282e34a4f5eef9e41d08751c --- /dev/null +++ b/001_mad.patch @@ -0,0 +1,1752 @@ +diff --git a/deps/2_nginx/sysom.conf b/deps/2_nginx/sysom.conf +index 15fc7da..9083992 100644 +--- a/deps/2_nginx/sysom.conf ++++ b/deps/2_nginx/sysom.conf +@@ -146,6 +146,13 @@ server { + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + ++ location /api/v1/metric_anomaly_detection/ { ++ proxy_pass http://127.0.0.1:7017; ++ proxy_read_timeout 180; ++ proxy_redirect off; ++ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; ++ } ++ + location /api/ { + proxy_pass http://127.0.0.1:7001; + proxy_read_timeout 180s; +diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic.ini b/sysom_server/sysom_metric_anomaly_detection/alembic.ini +new file mode 100644 +index 0000000..f6ab9fe +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/alembic.ini +@@ -0,0 +1,102 @@ ++# A generic, single database configuration. ++ ++[alembic] ++# path to migration scripts ++script_location = alembic ++ ++# template used to generate migration files ++# file_template = %%(rev)s_%%(slug)s ++ ++# sys.path path, will be prepended to sys.path if present. ++# defaults to the current working directory. ++prepend_sys_path = . ++ ++# timezone to use when rendering the date within the migration file ++# as well as the filename. ++# If specified, requires the python-dateutil library that can be ++# installed by adding `alembic[tz]` to the pip requirements ++# string value is passed to dateutil.tz.gettz() ++# leave blank for localtime ++# timezone = ++ ++# max length of characters to apply to the ++# "slug" field ++# truncate_slug_length = 40 ++ ++# set to 'true' to run the environment during ++# the 'revision' command, regardless of autogenerate ++# revision_environment = false ++ ++# set to 'true' to allow .pyc and .pyo files without ++# a source .py file to be detected as revisions in the ++# versions/ directory ++# sourceless = false ++ ++# version location specification; This defaults ++# to alembic/versions. When using multiple version ++# directories, initial revisions must be specified with --version-path. ++# The path separator used here should be the separator specified by "version_path_separator" below. ++# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions ++ ++# version path separator; As mentioned above, this is the character used to split ++# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. ++# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. ++# Valid values for version_path_separator are: ++# ++# version_path_separator = : ++# version_path_separator = ; ++# version_path_separator = space ++version_path_separator = os # Use os.pathsep. Default configuration used for new projects. ++ ++# the output encoding used when revision files ++# are written from script.py.mako ++# output_encoding = utf-8 ++ ++sqlalchemy.url = "" ++ ++ ++[post_write_hooks] ++# post_write_hooks defines scripts or Python functions that are run ++# on newly generated revision scripts. See the documentation for further ++# detail and examples ++ ++# format using "black" - use the console_scripts runner, against the "black" entrypoint ++# hooks = black ++# black.type = console_scripts ++# black.entrypoint = black ++# black.options = -l 79 REVISION_SCRIPT_FILENAME ++ ++# Logging configuration ++[loggers] ++keys = root,sqlalchemy,alembic ++ ++[handlers] ++keys = console ++ ++[formatters] ++keys = generic ++ ++[logger_root] ++level = WARN ++handlers = console ++qualname = ++ ++[logger_sqlalchemy] ++level = WARN ++handlers = ++qualname = sqlalchemy.engine ++ ++[logger_alembic] ++level = INFO ++handlers = ++qualname = alembic ++ ++[handler_console] ++class = StreamHandler ++args = (sys.stderr,) ++level = NOTSET ++formatter = generic ++ ++[formatter_generic] ++format = %(levelname)-5.5s [%(name)s] %(message)s ++datefmt = %H:%M:%S +diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic/README b/sysom_server/sysom_metric_anomaly_detection/alembic/README +new file mode 100644 +index 0000000..98e4f9c +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/alembic/README +@@ -0,0 +1 @@ ++Generic single-database configuration. +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic/env.py b/sysom_server/sysom_metric_anomaly_detection/alembic/env.py +new file mode 100644 +index 0000000..5d68d40 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/alembic/env.py +@@ -0,0 +1,115 @@ ++import inspect ++import app.models as models ++from logging.config import fileConfig ++from sqlalchemy import engine_from_config, Table ++from sqlalchemy import pool ++from app.models import Base ++from alembic import context ++from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL ++ ++################################################################## ++# Load yaml config first ++################################################################## ++mysql_config = YAML_CONFIG.get_server_config().db.mysql ++ ++################################################################## ++# Scan models ++################################################################## ++service_tables = [] ++for name, data in inspect.getmembers(models): ++ if inspect.isclass(data): ++ if data.__module__ != "app.models": ++ continue ++ if "__tablename__" in data.__dict__: ++ service_tables.append(data.__dict__["__tablename__"]) ++ elif "__table__" in data.__dict__: ++ service_tables.append(data.__dict__["__table__"]) ++ elif isinstance(data, Table): ++ service_tables.append(name) ++ ++# this is the Alembic Config object, which provides ++# access to the values within the .ini file in use. ++config = context.config ++ ++# Interpret the config file for Python logging. ++# This line sets up loggers basically. ++if config.config_file_name is not None: ++ fileConfig(config.config_file_name) ++ ++# Update mysql config according config.yml ++config.set_main_option( ++ "sqlalchemy.url", ++ SQLALCHEMY_DATABASE_URL ++) ++ ++# add your model's MetaData object here ++# for 'autogenerate' support ++# from myapp import mymodel ++# target_metadata = mymodel.Base.metadata ++target_metadata = Base.metadata ++ ++# other values from the config, defined by the needs of env.py, ++# can be acquired: ++# my_important_option = config.get_main_option("my_important_option") ++# ... etc. ++ ++def include_object(object, name, type_, reflected, compare_to): ++ if type_ == "table" and name not in service_tables: ++ return False ++ return True ++ ++ ++def run_migrations_offline(): ++ """Run migrations in 'offline' mode. ++ ++ This configures the context with just a URL ++ and not an Engine, though an Engine is acceptable ++ here as well. By skipping the Engine creation ++ we don't even need a DBAPI to be available. ++ ++ Calls to context.execute() here emit the given string to the ++ script output. ++ ++ """ ++ url = config.get_main_option("sqlalchemy.url") ++ context.configure( ++ url=url, ++ target_metadata=target_metadata, ++ literal_binds=True, ++ include_object=include_object, ++ version_table="metric_anomaly_detection_version", ++ dialect_opts={"paramstyle": "named"}, ++ ) ++ ++ with context.begin_transaction(): ++ context.run_migrations() ++ ++ ++def run_migrations_online(): ++ """Run migrations in 'online' mode. ++ ++ In this scenario we need to create an Engine ++ and associate a connection with the context. ++ ++ """ ++ connectable = engine_from_config( ++ config.get_section(config.config_ini_section), ++ prefix="sqlalchemy.", ++ poolclass=pool.NullPool, ++ ) ++ ++ with connectable.connect() as connection: ++ context.configure( ++ connection=connection, target_metadata=target_metadata, ++ include_object=include_object, ++ version_table="metric_anomaly_detection_version" ++ ) ++ ++ with context.begin_transaction(): ++ context.run_migrations() ++ ++ ++if context.is_offline_mode(): ++ run_migrations_offline() ++else: ++ run_migrations_online() +diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic/script.py.mako b/sysom_server/sysom_metric_anomaly_detection/alembic/script.py.mako +new file mode 100644 +index 0000000..2c01563 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/alembic/script.py.mako +@@ -0,0 +1,24 @@ ++"""${message} ++ ++Revision ID: ${up_revision} ++Revises: ${down_revision | comma,n} ++Create Date: ${create_date} ++ ++""" ++from alembic import op ++import sqlalchemy as sa ++${imports if imports else ""} ++ ++# revision identifiers, used by Alembic. ++revision = ${repr(up_revision)} ++down_revision = ${repr(down_revision)} ++branch_labels = ${repr(branch_labels)} ++depends_on = ${repr(depends_on)} ++ ++ ++def upgrade(): ++ ${upgrades if upgrades else "pass"} ++ ++ ++def downgrade(): ++ ${downgrades if downgrades else "pass"} +diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic/versions/.gitkeep b/sysom_server/sysom_metric_anomaly_detection/alembic/versions/.gitkeep +new file mode 100644 +index 0000000..e69de29 +diff --git a/sysom_server/sysom_metric_anomaly_detection/app/__init__.py b/sysom_server/sysom_metric_anomaly_detection/app/__init__.py +new file mode 100644 +index 0000000..311048e +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/app/__init__.py +@@ -0,0 +1,8 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File __init__.py ++Description: ++""" +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/app/crud.py b/sysom_server/sysom_metric_anomaly_detection/app/crud.py +new file mode 100644 +index 0000000..96e28c2 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/app/crud.py +@@ -0,0 +1,36 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File crud.py ++Description: ++""" ++from typing import Optional, List ++from sqlalchemy.orm import Session ++from app import models, schemas, query ++ ++################################################################################################ ++# Define database crud here ++################################################################################################ ++ ++# def get_person_by_name(db: Session, name: str) -> Optional[models.Person]: ++# return db.query(models.Person).filter(models.Person.name == name).first() ++ ++# def create_person(db: Session, person: schemas.Person) -> models.Person: ++# person = models.Person(**person.dict()) ++# db.add(person) ++# db.commit() ++# db.refresh(person) ++# return person ++ ++# def del_person_by_id(db: Session, person_id: int): ++# person = db.get(models.Person, person_id) ++# db.delete(person) ++# db.commit() ++ ++# def get_person_list(db: Session, query_params: query.PersonQueryParams) -> List[models.Person]: ++# return ( ++# query_params.get_query_exp(db) ++# .all() ++# ) +diff --git a/sysom_server/sysom_metric_anomaly_detection/app/database.py b/sysom_server/sysom_metric_anomaly_detection/app/database.py +new file mode 100644 +index 0000000..4b564ac +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/app/database.py +@@ -0,0 +1,30 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File database.py ++Description: ++""" ++from sqlalchemy import create_engine ++from sqlalchemy.ext.declarative import declarative_base ++from sqlalchemy.orm import sessionmaker ++from conf.settings import SQLALCHEMY_DATABASE_URL ++from sysom_utils import FastApiResponseHelper ++ ++engine = create_engine( ++ SQLALCHEMY_DATABASE_URL, connect_args={} ++) ++ ++SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) ++ ++def get_db(): ++ db = SessionLocal() ++ try: ++ yield db ++ finally: ++ db.close() ++ ++Base = declarative_base() ++ ++FastApiResponseHelper.bind_base_class(Base) +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/app/models.py b/sysom_server/sysom_metric_anomaly_detection/app/models.py +new file mode 100644 +index 0000000..10e1fe3 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/app/models.py +@@ -0,0 +1,24 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File models.py ++Description: ++""" ++from sqlalchemy import Column, Integer, String, DateTime ++from sqlalchemy.sql import func ++from app.database import Base ++ ++ ++########################################################################### ++# Define databse model here ++########################################################################### ++ ++# @reference https://fastapi.tiangolo.com/zh/tutorial/sql-databases/ ++# class Person(Base): ++# __tablename__ = "sys_person" ++# id = Column(Integer, primary_key=True) ++# name = Column(String(254), unique=True) ++# age = Column(Integer) ++# created_at = Column(DateTime(timezone=True), server_default=func.now()) +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/app/query.py b/sysom_server/sysom_metric_anomaly_detection/app/query.py +new file mode 100644 +index 0000000..fcbcd08 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/app/query.py +@@ -0,0 +1,28 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/19 15:41 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File schemas.py ++Description: ++""" ++from typing import Optional ++from app import models ++from sysom_utils import BaseQueryParams ++ ++ ++# class PersonQueryParams(BaseQueryParams): ++ ++# # 1. 指定要查询的模型 ++# __modelclass__ = models.Person ++ ++# # 2. 定义排序字段 ++# sort: str = "-created_at" ++ ++# # 3. 定义支持用于过滤的参数 ++# name: Optional[str] = None ++# age: Optional[str] = None ++ ++# # 4. 指定哪些字段是枚举类型，并且指明对应的枚举类 ++# __enum_fields__ = { ++# } +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/app/routers/health.py b/sysom_server/sysom_metric_anomaly_detection/app/routers/health.py +new file mode 100644 +index 0000000..092d930 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/app/routers/health.py +@@ -0,0 +1,21 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File health.py ++Description: ++""" ++from fastapi import APIRouter ++ ++ ++router = APIRouter() ++ ++ ++@router.get("/check") ++async def get_channel_config(): ++ return { ++ "code": 0, ++ "err_msg": "", ++ "data": "" ++ } +diff --git a/sysom_server/sysom_metric_anomaly_detection/app/routers/person.py b/sysom_server/sysom_metric_anomaly_detection/app/routers/person.py +new file mode 100644 +index 0000000..42b496f +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/app/routers/person.py +@@ -0,0 +1,33 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File health.py ++Description: ++""" ++from fastapi import APIRouter, Depends ++from sqlalchemy.orm import Session ++from app.query import PersonQueryParams ++from app.database import get_db ++from app.crud import get_person_list, get_person_by_name ++from app.schemas import Person ++from sysom_utils import StandardListResponse, StandardResponse ++ ++ ++router = APIRouter() ++ ++ ++@router.get("/get") ++async def get_specific_person( ++ person_name: str, db: Session = Depends(get_db) ++): ++ person = get_person_by_name(db, person_name) ++ return StandardResponse(person, Person) ++ ++@router.get("/list") ++async def get_persons( ++ query_params: PersonQueryParams = Depends(), db: Session = Depends(get_db) ++): ++ person_list = get_person_list(db, query_params) ++ return StandardListResponse(person_list, Person) +diff --git a/sysom_server/sysom_metric_anomaly_detection/app/schemas.py b/sysom_server/sysom_metric_anomaly_detection/app/schemas.py +new file mode 100644 +index 0000000..353a3cb +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/app/schemas.py +@@ -0,0 +1,24 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File schemas.py ++Description: ++""" ++from pydantic import BaseModel ++from datetime import datetime ++ ++########################################################################### ++# Define schemas here ++########################################################################### ++ ++# @reference https://fastapi.tiangolo.com/zh/tutorial/response-model/ ++# class Person(BaseModel): ++# id: int ++# name: str ++# age: int ++# created_at: datetime ++ ++# class Config: ++# orm_mode = True +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/common.py b/sysom_server/sysom_metric_anomaly_detection/conf/common.py +new file mode 100644 +index 0000000..d8c0ddb +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/conf/common.py +@@ -0,0 +1,33 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File common.py ++Description: ++""" ++from pathlib import Path ++from sysom_utils import ConfigParser, SysomFramework ++ ++BASE_DIR = Path(__file__).resolve().parent.parent ++ ++################################################################## ++# Load yaml config first ++################################################################## ++YAML_GLOBAL_CONFIG_PATH = f"/etc/sysom/config.yml" ++YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" ++ ++YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) ++ ++mysql_config = YAML_CONFIG.get_server_config().db.mysql ++service_config = YAML_CONFIG.get_service_config() ++ ++SysomFramework.init(YAML_CONFIG) ++ ++################################################################## ++# fastapi config ++################################################################## ++SQLALCHEMY_DATABASE_URL = ( ++ f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" ++ f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" ++) +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/develop.py b/sysom_server/sysom_metric_anomaly_detection/conf/develop.py +new file mode 100644 +index 0000000..e818cad +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/conf/develop.py +@@ -0,0 +1,15 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File develoop.py ++Description: ++""" ++from .common import * ++ ++''' ++开发环境配置项 ++''' ++ ++DEBUG = True +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/gunicorn.py b/sysom_server/sysom_metric_anomaly_detection/conf/gunicorn.py +new file mode 100644 +index 0000000..a1f65c7 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/conf/gunicorn.py +@@ -0,0 +1,23 @@ ++''' ++Channel Service Gunicorn Settings ++''' ++from conf.common import YAML_CONFIG ++ ++bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") ++port = YAML_CONFIG.get_service_config().get("port", "80") ++ ++workers = 2 # 指定工作进程数 ++ ++threads = 4 ++ ++bind = f'{bind}:{port}' ++ ++worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 ++ ++max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) ++ ++accesslog = '/var/log/sysom/sysom-metric_anomaly_detection-access.log' ++ ++loglevel = 'error' ++ ++proc_name = 'sysom_metric_anomaly_detection_service' +diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/product.py b/sysom_server/sysom_metric_anomaly_detection/conf/product.py +new file mode 100644 +index 0000000..84e4729 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/conf/product.py +@@ -0,0 +1,15 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File product.py ++Description: ++""" ++from .common import * ++ ++''' ++生产环境配置项 ++''' ++ ++DEBUG = False +diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/settings.py b/sysom_server/sysom_metric_anomaly_detection/conf/settings.py +new file mode 100644 +index 0000000..2aae16f +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/conf/settings.py +@@ -0,0 +1,19 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File settings.py ++Description: ++""" ++import os ++ ++env = os.environ.get("env", "product") ++ ++ ++if env == "develop": ++ from .develop import * ++elif env == "testing": ++ from .testing import * ++elif env == "product": ++ from .product import * +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/testing.py b/sysom_server/sysom_metric_anomaly_detection/conf/testing.py +new file mode 100644 +index 0000000..9dc95fa +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/conf/testing.py +@@ -0,0 +1,14 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File testing.py ++Description: ++""" ++from .common import * ++ ++''' ++测试环境配置项 ++''' ++DEBUG = True +diff --git a/sysom_server/sysom_metric_anomaly_detection/config.yml b/sysom_server/sysom_metric_anomaly_detection/config.yml +new file mode 100644 +index 0000000..1413a2b +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/config.yml +@@ -0,0 +1,35 @@ ++vars: ++ SERVICE_NAME: &SERVICE_NAME sysom_metric_anomaly_detection ++ SERVICE_CONSUMER_GROUP: ++ !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] ++ ++sysom_server: ++ cec: ++ consumer_group: SYSOM_CEC_CHANNEL_CONSUMER_GROUP ++ ++sysom_service: ++ service_name: *SERVICE_NAME ++ service_dir: *SERVICE_NAME ++ protocol: http ++ host: 127.0.0.1 ++ bind: 127.0.0.1 ++ port: 7017 ++ framework: ++ gcache: ++ protocol: redis ++ node_dispatch: ++ cmg: ++ tags: ++ - metric_anomaly_detection ++ - FastApi ++ # Metadata of service ++ metadata: ++ check: ++ type: http ++ url: "/api/v1/metric_anomaly_detection/health/check" ++ interval: 10 ++ timeout: 10 ++ deregister: 25 ++ header: ++ tls_skip_verify: false ++ +diff --git a/sysom_server/sysom_metric_anomaly_detection/lib/README.md b/sysom_server/sysom_metric_anomaly_detection/lib/README.md +new file mode 100644 +index 0000000..3ec7442 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/lib/README.md +@@ -0,0 +1 @@ ++The current directory holds the public libraries or utils needed for microservices +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/main.py b/sysom_server/sysom_metric_anomaly_detection/main.py +new file mode 100755 +index 0000000..414a192 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/main.py +@@ -0,0 +1,127 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File ssh.py ++Description: ++""" ++from clogger import logger ++from fastapi import FastAPI ++from app.routers import health ++from conf.settings import YAML_CONFIG ++from sysom_utils import CmgPlugin, SysomFramework ++import traceback ++import sys,os,json ++sys.path.append("%s/lib"%(os.path.dirname(os.path.abspath(__file__)))) ++import time ++import datetime ++from datetime import date, datetime, timedelta ++import requests ++from metric_reader import MetricReader, dispatch_metric_reader,RangeQueryTask ++ ++import pandas as pd ++import numpy as np ++import matplotlib ++from matplotlib import pyplot as plt ++import adtk ++from adtk.data import validate_series ++from adtk.visualization import plot ++from adtk.transformer import RollingAggregate ++from adtk.transformer import DoubleRollingAggregate ++from adtk.detector import PersistAD ++from adtk.detector import LevelShiftAD ++from adtk.detector import SeasonalAD ++from adtk.detector import AutoregressionAD ++ ++app = FastAPI() ++ ++app.include_router(health.router, prefix="/api/v1/metric_anomaly_detection/health") ++# app.include_router(health.router, prefix="/api/v1/metric_anomaly_detection/person") ++ ++metric_reader = dispatch_metric_reader("prometheus://localhost:9090") ++ ++def pull_monidata(start_time,end_time): ++ retdict = [] ++ try: ++ ret_items = metric_reader.get_metric_names() ++ for item in ret_items.data: ++ #print(item) ++ task = RangeQueryTask(item, start_time, end_time).append_wildcard_filter("instance", "*") ++ ret_tmp = metric_reader.range_query([task]) ++ if len(ret_tmp.data) <= 0: ++ print ("ignore: ------",item,ret_tmp.data) ++ if len(ret_tmp.data) > 0: ++ for i in range(len(ret_tmp.data)): ++ #if "labels" in ret_tmp.data[i].to_dict(): ++ #print(json.dumps(ret_tmp.data[i].to_dict(),ensure_ascii=False,indent=4)) ++ retdict.append(ret_tmp.data[i].to_dict()) ++ #print(json.dumps(retdict[0:10],ensure_ascii=False,indent=4)) ++ except: ++ traceback.print_exc() ++ pass ++ return retdict ++ ++now_time = time.time() ++retdata = pull_monidata(now_time-600, now_time) ++exp = retdata[0]['values'] ++stamps = pd.date_range('2023-08-13', periods=41, freq='15S') ++df = pd.DataFrame(columns=['values']) ++for data in exp: ++ df.loc[len(df)] = data[0] ++df.index = stamps ++#print(df) ++df = validate_series(df) ++#plot(df) ++ ++#Detect spikes using PersistAD ++spikes = PersistAD(c=3.0, side='both').fit_detect(df) ++plot(df, anomaly=spikes, ts_linewidth=1, ts_markersize=3, anomaly_color='red') ++ ++#Detect level changes using LevelShiftAD ++level_shifts = LevelShiftAD(c=6.0, side='both', window=5).fit_detect(df) ++plot(df, anomaly=level_shifts, ts_linewidth=1, ts_markersize=3, anomaly_color='red') ++ ++#Detect levels shifts using DoubleRollingAggregate ++df_doubleRolling_mean = DoubleRollingAggregate(agg='mean', window=(3,1), diff='l1').transform(df) ++plot(df_doubleRolling_mean) ++ ++#Detect seasonality using SeasonalAD ++try: ++ seasonality = SeasonalAD(c=3.0, side='both').fit_detect(df) ++ plot(seasonality) ++except Exception as e: ++ print(e) ++ ++#Detect slope shifts using AutoregressionAD and one-order differencing ++slope_shifts = AutoregressionAD(n_steps=1, step_size=1, c=3.0).fit_detect(df) ++plot(slope_shifts) ++df_diff = df.diff(periods=1) ++plot(df_diff) ++ ++ ++############################################################################# ++# Write your API interface here, or add to app/routes ++############################################################################# ++ ++ ++def init_framwork(): ++ SysomFramework\ ++ .init(YAML_CONFIG) \ ++ .load_plugin_cls(CmgPlugin) \ ++ .start() ++ logger.info("SysomFramework init finished!") ++ ++ ++@app.on_event("startup") ++async def on_start(): ++ init_framwork() ++ ++ ############################################################################# ++ # Perform some microservice initialization operations over here ++ ############################################################################# ++ ++ ++@app.on_event("shutdown") ++async def on_shutdown(): ++ pass +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic.ini b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic.ini +new file mode 100644 +index 0000000..f6ab9fe +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic.ini +@@ -0,0 +1,102 @@ ++# A generic, single database configuration. ++ ++[alembic] ++# path to migration scripts ++script_location = alembic ++ ++# template used to generate migration files ++# file_template = %%(rev)s_%%(slug)s ++ ++# sys.path path, will be prepended to sys.path if present. ++# defaults to the current working directory. ++prepend_sys_path = . ++ ++# timezone to use when rendering the date within the migration file ++# as well as the filename. ++# If specified, requires the python-dateutil library that can be ++# installed by adding `alembic[tz]` to the pip requirements ++# string value is passed to dateutil.tz.gettz() ++# leave blank for localtime ++# timezone = ++ ++# max length of characters to apply to the ++# "slug" field ++# truncate_slug_length = 40 ++ ++# set to 'true' to run the environment during ++# the 'revision' command, regardless of autogenerate ++# revision_environment = false ++ ++# set to 'true' to allow .pyc and .pyo files without ++# a source .py file to be detected as revisions in the ++# versions/ directory ++# sourceless = false ++ ++# version location specification; This defaults ++# to alembic/versions. When using multiple version ++# directories, initial revisions must be specified with --version-path. ++# The path separator used here should be the separator specified by "version_path_separator" below. ++# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions ++ ++# version path separator; As mentioned above, this is the character used to split ++# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. ++# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. ++# Valid values for version_path_separator are: ++# ++# version_path_separator = : ++# version_path_separator = ; ++# version_path_separator = space ++version_path_separator = os # Use os.pathsep. Default configuration used for new projects. ++ ++# the output encoding used when revision files ++# are written from script.py.mako ++# output_encoding = utf-8 ++ ++sqlalchemy.url = "" ++ ++ ++[post_write_hooks] ++# post_write_hooks defines scripts or Python functions that are run ++# on newly generated revision scripts. See the documentation for further ++# detail and examples ++ ++# format using "black" - use the console_scripts runner, against the "black" entrypoint ++# hooks = black ++# black.type = console_scripts ++# black.entrypoint = black ++# black.options = -l 79 REVISION_SCRIPT_FILENAME ++ ++# Logging configuration ++[loggers] ++keys = root,sqlalchemy,alembic ++ ++[handlers] ++keys = console ++ ++[formatters] ++keys = generic ++ ++[logger_root] ++level = WARN ++handlers = console ++qualname = ++ ++[logger_sqlalchemy] ++level = WARN ++handlers = ++qualname = sqlalchemy.engine ++ ++[logger_alembic] ++level = INFO ++handlers = ++qualname = alembic ++ ++[handler_console] ++class = StreamHandler ++args = (sys.stderr,) ++level = NOTSET ++formatter = generic ++ ++[formatter_generic] ++format = %(levelname)-5.5s [%(name)s] %(message)s ++datefmt = %H:%M:%S +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/README b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/README +new file mode 100644 +index 0000000..98e4f9c +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/README +@@ -0,0 +1 @@ ++Generic single-database configuration. +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/env.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/env.py +new file mode 100644 +index 0000000..5d68d40 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/env.py +@@ -0,0 +1,115 @@ ++import inspect ++import app.models as models ++from logging.config import fileConfig ++from sqlalchemy import engine_from_config, Table ++from sqlalchemy import pool ++from app.models import Base ++from alembic import context ++from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL ++ ++################################################################## ++# Load yaml config first ++################################################################## ++mysql_config = YAML_CONFIG.get_server_config().db.mysql ++ ++################################################################## ++# Scan models ++################################################################## ++service_tables = [] ++for name, data in inspect.getmembers(models): ++ if inspect.isclass(data): ++ if data.__module__ != "app.models": ++ continue ++ if "__tablename__" in data.__dict__: ++ service_tables.append(data.__dict__["__tablename__"]) ++ elif "__table__" in data.__dict__: ++ service_tables.append(data.__dict__["__table__"]) ++ elif isinstance(data, Table): ++ service_tables.append(name) ++ ++# this is the Alembic Config object, which provides ++# access to the values within the .ini file in use. ++config = context.config ++ ++# Interpret the config file for Python logging. ++# This line sets up loggers basically. ++if config.config_file_name is not None: ++ fileConfig(config.config_file_name) ++ ++# Update mysql config according config.yml ++config.set_main_option( ++ "sqlalchemy.url", ++ SQLALCHEMY_DATABASE_URL ++) ++ ++# add your model's MetaData object here ++# for 'autogenerate' support ++# from myapp import mymodel ++# target_metadata = mymodel.Base.metadata ++target_metadata = Base.metadata ++ ++# other values from the config, defined by the needs of env.py, ++# can be acquired: ++# my_important_option = config.get_main_option("my_important_option") ++# ... etc. ++ ++def include_object(object, name, type_, reflected, compare_to): ++ if type_ == "table" and name not in service_tables: ++ return False ++ return True ++ ++ ++def run_migrations_offline(): ++ """Run migrations in 'offline' mode. ++ ++ This configures the context with just a URL ++ and not an Engine, though an Engine is acceptable ++ here as well. By skipping the Engine creation ++ we don't even need a DBAPI to be available. ++ ++ Calls to context.execute() here emit the given string to the ++ script output. ++ ++ """ ++ url = config.get_main_option("sqlalchemy.url") ++ context.configure( ++ url=url, ++ target_metadata=target_metadata, ++ literal_binds=True, ++ include_object=include_object, ++ version_table="metric_anomaly_detection_version", ++ dialect_opts={"paramstyle": "named"}, ++ ) ++ ++ with context.begin_transaction(): ++ context.run_migrations() ++ ++ ++def run_migrations_online(): ++ """Run migrations in 'online' mode. ++ ++ In this scenario we need to create an Engine ++ and associate a connection with the context. ++ ++ """ ++ connectable = engine_from_config( ++ config.get_section(config.config_ini_section), ++ prefix="sqlalchemy.", ++ poolclass=pool.NullPool, ++ ) ++ ++ with connectable.connect() as connection: ++ context.configure( ++ connection=connection, target_metadata=target_metadata, ++ include_object=include_object, ++ version_table="metric_anomaly_detection_version" ++ ) ++ ++ with context.begin_transaction(): ++ context.run_migrations() ++ ++ ++if context.is_offline_mode(): ++ run_migrations_offline() ++else: ++ run_migrations_online() +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/script.py.mako b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/script.py.mako +new file mode 100644 +index 0000000..2c01563 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/script.py.mako +@@ -0,0 +1,24 @@ ++"""${message} ++ ++Revision ID: ${up_revision} ++Revises: ${down_revision | comma,n} ++Create Date: ${create_date} ++ ++""" ++from alembic import op ++import sqlalchemy as sa ++${imports if imports else ""} ++ ++# revision identifiers, used by Alembic. ++revision = ${repr(up_revision)} ++down_revision = ${repr(down_revision)} ++branch_labels = ${repr(branch_labels)} ++depends_on = ${repr(depends_on)} ++ ++ ++def upgrade(): ++ ${upgrades if upgrades else "pass"} ++ ++ ++def downgrade(): ++ ${downgrades if downgrades else "pass"} +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/versions/.gitkeep b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/alembic/versions/.gitkeep +new file mode 100644 +index 0000000..e69de29 +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/__init__.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/__init__.py +new file mode 100644 +index 0000000..311048e +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/__init__.py +@@ -0,0 +1,8 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File __init__.py ++Description: ++""" +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/crud.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/crud.py +new file mode 100644 +index 0000000..96e28c2 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/crud.py +@@ -0,0 +1,36 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File crud.py ++Description: ++""" ++from typing import Optional, List ++from sqlalchemy.orm import Session ++from app import models, schemas, query ++ ++################################################################################################ ++# Define database crud here ++################################################################################################ ++ ++# def get_person_by_name(db: Session, name: str) -> Optional[models.Person]: ++# return db.query(models.Person).filter(models.Person.name == name).first() ++ ++# def create_person(db: Session, person: schemas.Person) -> models.Person: ++# person = models.Person(**person.dict()) ++# db.add(person) ++# db.commit() ++# db.refresh(person) ++# return person ++ ++# def del_person_by_id(db: Session, person_id: int): ++# person = db.get(models.Person, person_id) ++# db.delete(person) ++# db.commit() ++ ++# def get_person_list(db: Session, query_params: query.PersonQueryParams) -> List[models.Person]: ++# return ( ++# query_params.get_query_exp(db) ++# .all() ++# ) +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/database.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/database.py +new file mode 100644 +index 0000000..4b564ac +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/database.py +@@ -0,0 +1,30 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File database.py ++Description: ++""" ++from sqlalchemy import create_engine ++from sqlalchemy.ext.declarative import declarative_base ++from sqlalchemy.orm import sessionmaker ++from conf.settings import SQLALCHEMY_DATABASE_URL ++from sysom_utils import FastApiResponseHelper ++ ++engine = create_engine( ++ SQLALCHEMY_DATABASE_URL, connect_args={} ++) ++ ++SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) ++ ++def get_db(): ++ db = SessionLocal() ++ try: ++ yield db ++ finally: ++ db.close() ++ ++Base = declarative_base() ++ ++FastApiResponseHelper.bind_base_class(Base) +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/models.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/models.py +new file mode 100644 +index 0000000..10e1fe3 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/models.py +@@ -0,0 +1,24 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File models.py ++Description: ++""" ++from sqlalchemy import Column, Integer, String, DateTime ++from sqlalchemy.sql import func ++from app.database import Base ++ ++ ++########################################################################### ++# Define databse model here ++########################################################################### ++ ++# @reference https://fastapi.tiangolo.com/zh/tutorial/sql-databases/ ++# class Person(Base): ++# __tablename__ = "sys_person" ++# id = Column(Integer, primary_key=True) ++# name = Column(String(254), unique=True) ++# age = Column(Integer) ++# created_at = Column(DateTime(timezone=True), server_default=func.now()) +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/query.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/query.py +new file mode 100644 +index 0000000..fcbcd08 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/query.py +@@ -0,0 +1,28 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/19 15:41 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File schemas.py ++Description: ++""" ++from typing import Optional ++from app import models ++from sysom_utils import BaseQueryParams ++ ++ ++# class PersonQueryParams(BaseQueryParams): ++ ++# # 1. 指定要查询的模型 ++# __modelclass__ = models.Person ++ ++# # 2. 定义排序字段 ++# sort: str = "-created_at" ++ ++# # 3. 定义支持用于过滤的参数 ++# name: Optional[str] = None ++# age: Optional[str] = None ++ ++# # 4. 指定哪些字段是枚举类型，并且指明对应的枚举类 ++# __enum_fields__ = { ++# } +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/routers/health.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/routers/health.py +new file mode 100644 +index 0000000..092d930 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/routers/health.py +@@ -0,0 +1,21 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File health.py ++Description: ++""" ++from fastapi import APIRouter ++ ++ ++router = APIRouter() ++ ++ ++@router.get("/check") ++async def get_channel_config(): ++ return { ++ "code": 0, ++ "err_msg": "", ++ "data": "" ++ } +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/routers/person.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/routers/person.py +new file mode 100644 +index 0000000..42b496f +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/routers/person.py +@@ -0,0 +1,33 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File health.py ++Description: ++""" ++from fastapi import APIRouter, Depends ++from sqlalchemy.orm import Session ++from app.query import PersonQueryParams ++from app.database import get_db ++from app.crud import get_person_list, get_person_by_name ++from app.schemas import Person ++from sysom_utils import StandardListResponse, StandardResponse ++ ++ ++router = APIRouter() ++ ++ ++@router.get("/get") ++async def get_specific_person( ++ person_name: str, db: Session = Depends(get_db) ++): ++ person = get_person_by_name(db, person_name) ++ return StandardResponse(person, Person) ++ ++@router.get("/list") ++async def get_persons( ++ query_params: PersonQueryParams = Depends(), db: Session = Depends(get_db) ++): ++ person_list = get_person_list(db, query_params) ++ return StandardListResponse(person_list, Person) +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/schemas.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/schemas.py +new file mode 100644 +index 0000000..353a3cb +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/app/schemas.py +@@ -0,0 +1,24 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File schemas.py ++Description: ++""" ++from pydantic import BaseModel ++from datetime import datetime ++ ++########################################################################### ++# Define schemas here ++########################################################################### ++ ++# @reference https://fastapi.tiangolo.com/zh/tutorial/response-model/ ++# class Person(BaseModel): ++# id: int ++# name: str ++# age: int ++# created_at: datetime ++ ++# class Config: ++# orm_mode = True +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/common.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/common.py +new file mode 100644 +index 0000000..d8c0ddb +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/common.py +@@ -0,0 +1,33 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File common.py ++Description: ++""" ++from pathlib import Path ++from sysom_utils import ConfigParser, SysomFramework ++ ++BASE_DIR = Path(__file__).resolve().parent.parent ++ ++################################################################## ++# Load yaml config first ++################################################################## ++YAML_GLOBAL_CONFIG_PATH = f"/etc/sysom/config.yml" ++YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" ++ ++YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) ++ ++mysql_config = YAML_CONFIG.get_server_config().db.mysql ++service_config = YAML_CONFIG.get_service_config() ++ ++SysomFramework.init(YAML_CONFIG) ++ ++################################################################## ++# fastapi config ++################################################################## ++SQLALCHEMY_DATABASE_URL = ( ++ f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" ++ f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" ++) +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/develop.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/develop.py +new file mode 100644 +index 0000000..e818cad +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/develop.py +@@ -0,0 +1,15 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File develoop.py ++Description: ++""" ++from .common import * ++ ++''' ++开发环境配置项 ++''' ++ ++DEBUG = True +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/gunicorn.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/gunicorn.py +new file mode 100644 +index 0000000..a1f65c7 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/gunicorn.py +@@ -0,0 +1,23 @@ ++''' ++Channel Service Gunicorn Settings ++''' ++from conf.common import YAML_CONFIG ++ ++bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") ++port = YAML_CONFIG.get_service_config().get("port", "80") ++ ++workers = 2 # 指定工作进程数 ++ ++threads = 4 ++ ++bind = f'{bind}:{port}' ++ ++worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 ++ ++max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) ++ ++accesslog = '/var/log/sysom/sysom-metric_anomaly_detection-access.log' ++ ++loglevel = 'error' ++ ++proc_name = 'sysom_metric_anomaly_detection_service' +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/product.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/product.py +new file mode 100644 +index 0000000..84e4729 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/product.py +@@ -0,0 +1,15 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File product.py ++Description: ++""" ++from .common import * ++ ++''' ++生产环境配置项 ++''' ++ ++DEBUG = False +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/settings.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/settings.py +new file mode 100644 +index 0000000..2aae16f +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/settings.py +@@ -0,0 +1,19 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File settings.py ++Description: ++""" ++import os ++ ++env = os.environ.get("env", "product") ++ ++ ++if env == "develop": ++ from .develop import * ++elif env == "testing": ++ from .testing import * ++elif env == "product": ++ from .product import * +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/testing.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/testing.py +new file mode 100644 +index 0000000..9dc95fa +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/conf/testing.py +@@ -0,0 +1,14 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File testing.py ++Description: ++""" ++from .common import * ++ ++''' ++测试环境配置项 ++''' ++DEBUG = True +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/config.yml b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/config.yml +new file mode 100644 +index 0000000..1413a2b +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/config.yml +@@ -0,0 +1,35 @@ ++vars: ++ SERVICE_NAME: &SERVICE_NAME sysom_metric_anomaly_detection ++ SERVICE_CONSUMER_GROUP: ++ !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] ++ ++sysom_server: ++ cec: ++ consumer_group: SYSOM_CEC_CHANNEL_CONSUMER_GROUP ++ ++sysom_service: ++ service_name: *SERVICE_NAME ++ service_dir: *SERVICE_NAME ++ protocol: http ++ host: 127.0.0.1 ++ bind: 127.0.0.1 ++ port: 7017 ++ framework: ++ gcache: ++ protocol: redis ++ node_dispatch: ++ cmg: ++ tags: ++ - metric_anomaly_detection ++ - FastApi ++ # Metadata of service ++ metadata: ++ check: ++ type: http ++ url: "/api/v1/metric_anomaly_detection/health/check" ++ interval: 10 ++ timeout: 10 ++ deregister: 25 ++ header: ++ tls_skip_verify: false ++ +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/lib/README.md b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/lib/README.md +new file mode 100644 +index 0000000..3ec7442 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/lib/README.md +@@ -0,0 +1 @@ ++The current directory holds the public libraries or utils needed for microservices +\ No newline at end of file +diff --git a/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/main.py b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/main.py +new file mode 100755 +index 0000000..414a192 +--- /dev/null ++++ b/sysom_server/sysom_metric_anomaly_detection/sysom_metric_anomaly_detection/main.py +@@ -0,0 +1,127 @@ ++# -*- coding: utf-8 -*- # ++""" ++Time 2023/09/20 19:03 ++Author: mingfeng (SunnyQjm) ++Email mfeng@linux.alibaba.com ++File ssh.py ++Description: ++""" ++from clogger import logger ++from fastapi import FastAPI ++from app.routers import health ++from conf.settings import YAML_CONFIG ++from sysom_utils import CmgPlugin, SysomFramework ++import traceback ++import sys,os,json ++sys.path.append("%s/lib"%(os.path.dirname(os.path.abspath(__file__)))) ++import time ++import datetime ++from datetime import date, datetime, timedelta ++import requests ++from metric_reader import MetricReader, dispatch_metric_reader,RangeQueryTask ++ ++import pandas as pd ++import numpy as np ++import matplotlib ++from matplotlib import pyplot as plt ++import adtk ++from adtk.data import validate_series ++from adtk.visualization import plot ++from adtk.transformer import RollingAggregate ++from adtk.transformer import DoubleRollingAggregate ++from adtk.detector import PersistAD ++from adtk.detector import LevelShiftAD ++from adtk.detector import SeasonalAD ++from adtk.detector import AutoregressionAD ++ ++app = FastAPI() ++ ++app.include_router(health.router, prefix="/api/v1/metric_anomaly_detection/health") ++# app.include_router(health.router, prefix="/api/v1/metric_anomaly_detection/person") ++ ++metric_reader = dispatch_metric_reader("prometheus://localhost:9090") ++ ++def pull_monidata(start_time,end_time): ++ retdict = [] ++ try: ++ ret_items = metric_reader.get_metric_names() ++ for item in ret_items.data: ++ #print(item) ++ task = RangeQueryTask(item, start_time, end_time).append_wildcard_filter("instance", "*") ++ ret_tmp = metric_reader.range_query([task]) ++ if len(ret_tmp.data) <= 0: ++ print ("ignore: ------",item,ret_tmp.data) ++ if len(ret_tmp.data) > 0: ++ for i in range(len(ret_tmp.data)): ++ #if "labels" in ret_tmp.data[i].to_dict(): ++ #print(json.dumps(ret_tmp.data[i].to_dict(),ensure_ascii=False,indent=4)) ++ retdict.append(ret_tmp.data[i].to_dict()) ++ #print(json.dumps(retdict[0:10],ensure_ascii=False,indent=4)) ++ except: ++ traceback.print_exc() ++ pass ++ return retdict ++ ++now_time = time.time() ++retdata = pull_monidata(now_time-600, now_time) ++exp = retdata[0]['values'] ++stamps = pd.date_range('2023-08-13', periods=41, freq='15S') ++df = pd.DataFrame(columns=['values']) ++for data in exp: ++ df.loc[len(df)] = data[0] ++df.index = stamps ++#print(df) ++df = validate_series(df) ++#plot(df) ++ ++#Detect spikes using PersistAD ++spikes = PersistAD(c=3.0, side='both').fit_detect(df) ++plot(df, anomaly=spikes, ts_linewidth=1, ts_markersize=3, anomaly_color='red') ++ ++#Detect level changes using LevelShiftAD ++level_shifts = LevelShiftAD(c=6.0, side='both', window=5).fit_detect(df) ++plot(df, anomaly=level_shifts, ts_linewidth=1, ts_markersize=3, anomaly_color='red') ++ ++#Detect levels shifts using DoubleRollingAggregate ++df_doubleRolling_mean = DoubleRollingAggregate(agg='mean', window=(3,1), diff='l1').transform(df) ++plot(df_doubleRolling_mean) ++ ++#Detect seasonality using SeasonalAD ++try: ++ seasonality = SeasonalAD(c=3.0, side='both').fit_detect(df) ++ plot(seasonality) ++except Exception as e: ++ print(e) ++ ++#Detect slope shifts using AutoregressionAD and one-order differencing ++slope_shifts = AutoregressionAD(n_steps=1, step_size=1, c=3.0).fit_detect(df) ++plot(slope_shifts) ++df_diff = df.diff(periods=1) ++plot(df_diff) ++ ++ ++############################################################################# ++# Write your API interface here, or add to app/routes ++############################################################################# ++ ++ ++def init_framwork(): ++ SysomFramework\ ++ .init(YAML_CONFIG) \ ++ .load_plugin_cls(CmgPlugin) \ ++ .start() ++ logger.info("SysomFramework init finished!") ++ ++ ++@app.on_event("startup") ++async def on_start(): ++ init_framwork() ++ ++ ############################################################################# ++ # Perform some microservice initialization operations over here ++ ############################################################################# ++ ++ ++@app.on_event("shutdown") ++async def on_shutdown(): ++ pass diff --git a/deps/2_nginx/sysom.conf b/deps/2_nginx/sysom.conf index 15fc7da61084c4abebf6accd3492b72541d3f251..9083992572bbac1289ae9740d5b76339477e2209 100644 --- a/deps/2_nginx/sysom.conf +++ b/deps/2_nginx/sysom.conf @@ -146,6 +146,13 @@ server { proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; } + location /api/v1/metric_anomaly_detection/ { + proxy_pass http://127.0.0.1:7017; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } + location /api/ { proxy_pass http://127.0.0.1:7001; proxy_read_timeout 180s; diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic.ini b/sysom_server/sysom_metric_anomaly_detection/alembic.ini new file mode 100644 index 0000000000000000000000000000000000000000..f6ab9febcd93add9d0ea8857f857d7f30f1fe48f --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/alembic.ini @@ -0,0 +1,102 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = "" + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic/README b/sysom_server/sysom_metric_anomaly_detection/alembic/README new file mode 100644 index 0000000000000000000000000000000000000000..98e4f9c44effe479ed38c66ba922e7bcc672916f --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic/env.py b/sysom_server/sysom_metric_anomaly_detection/alembic/env.py new file mode 100644 index 0000000000000000000000000000000000000000..5d68d40ec93f636570f8f11db58ec80fc0a940be --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/alembic/env.py @@ -0,0 +1,115 @@ +import inspect +import app.models as models +from logging.config import fileConfig +from sqlalchemy import engine_from_config, Table +from sqlalchemy import pool +from app.models import Base +from alembic import context +from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL + +################################################################## +# Load yaml config first +################################################################## +mysql_config = YAML_CONFIG.get_server_config().db.mysql + +################################################################## +# Scan models +################################################################## +service_tables = [] +for name, data in inspect.getmembers(models): + if inspect.isclass(data): + if data.__module__ != "app.models": + continue + if "__tablename__" in data.__dict__: + service_tables.append(data.__dict__["__tablename__"]) + elif "__table__" in data.__dict__: + service_tables.append(data.__dict__["__table__"]) + elif isinstance(data, Table): + service_tables.append(name) + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Update mysql config according config.yml +config.set_main_option( + "sqlalchemy.url", + SQLALCHEMY_DATABASE_URL +) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def include_object(object, name, type_, reflected, compare_to): + if type_ == "table" and name not in service_tables: + return False + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + include_object=include_object, + version_table="metric_anomaly_detection_version", + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, + include_object=include_object, + version_table="metric_anomaly_detection_version" + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic/script.py.mako b/sysom_server/sysom_metric_anomaly_detection/alembic/script.py.mako new file mode 100644 index 0000000000000000000000000000000000000000..2c0156303a8df3ffdc9de87765bf801bf6bea4a5 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/sysom_server/sysom_metric_anomaly_detection/alembic/versions/.gitkeep b/sysom_server/sysom_metric_anomaly_detection/alembic/versions/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/sysom_server/sysom_metric_anomaly_detection/app/__init__.py b/sysom_server/sysom_metric_anomaly_detection/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..311048e5a8fbabb40bf7161ac8c82a055b703f17 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/app/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py +Description: +""" \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/app/crud.py b/sysom_server/sysom_metric_anomaly_detection/app/crud.py new file mode 100644 index 0000000000000000000000000000000000000000..96e28c2b664495fc2de086c178dd3da1faa5cd94 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/app/crud.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File crud.py +Description: +""" +from typing import Optional, List +from sqlalchemy.orm import Session +from app import models, schemas, query + +################################################################################################ +# Define database crud here +################################################################################################ + +# def get_person_by_name(db: Session, name: str) -> Optional[models.Person]: +# return db.query(models.Person).filter(models.Person.name == name).first() + +# def create_person(db: Session, person: schemas.Person) -> models.Person: +# person = models.Person(**person.dict()) +# db.add(person) +# db.commit() +# db.refresh(person) +# return person + +# def del_person_by_id(db: Session, person_id: int): +# person = db.get(models.Person, person_id) +# db.delete(person) +# db.commit() + +# def get_person_list(db: Session, query_params: query.PersonQueryParams) -> List[models.Person]: +# return ( +# query_params.get_query_exp(db) +# .all() +# ) diff --git a/sysom_server/sysom_metric_anomaly_detection/app/database.py b/sysom_server/sysom_metric_anomaly_detection/app/database.py new file mode 100644 index 0000000000000000000000000000000000000000..4b564acad479e53f5d2572da9718b1993b42c9e6 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/app/database.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File database.py +Description: +""" +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from conf.settings import SQLALCHEMY_DATABASE_URL +from sysom_utils import FastApiResponseHelper + +engine = create_engine( + SQLALCHEMY_DATABASE_URL, connect_args={} +) + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + +Base = declarative_base() + +FastApiResponseHelper.bind_base_class(Base) \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/app/models.py b/sysom_server/sysom_metric_anomaly_detection/app/models.py new file mode 100644 index 0000000000000000000000000000000000000000..10e1fe3373201659f7628d57812b2ee9e23b51d0 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/app/models.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File models.py +Description: +""" +from sqlalchemy import Column, Integer, String, DateTime +from sqlalchemy.sql import func +from app.database import Base + + +########################################################################### +# Define databse model here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/sql-databases/ +# class Person(Base): +# __tablename__ = "sys_person" +# id = Column(Integer, primary_key=True) +# name = Column(String(254), unique=True) +# age = Column(Integer) +# created_at = Column(DateTime(timezone=True), server_default=func.now()) \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/app/query.py b/sysom_server/sysom_metric_anomaly_detection/app/query.py new file mode 100644 index 0000000000000000000000000000000000000000..fcbcd0898fc8d3f2be8d64ce694d7aa99ccb2332 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/app/query.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/19 15:41 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File schemas.py +Description: +""" +from typing import Optional +from app import models +from sysom_utils import BaseQueryParams + + +# class PersonQueryParams(BaseQueryParams): + +# # 1. 指定要查询的模型 +# __modelclass__ = models.Person + +# # 2. 定义排序字段 +# sort: str = "-created_at" + +# # 3. 定义支持用于过滤的参数 +# name: Optional[str] = None +# age: Optional[str] = None + +# # 4. 指定哪些字段是枚举类型，并且指明对应的枚举类 +# __enum_fields__ = { +# } \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/app/routers/health.py b/sysom_server/sysom_metric_anomaly_detection/app/routers/health.py new file mode 100644 index 0000000000000000000000000000000000000000..092d9304e6fe98d79c428d234188de9d669bf432 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/app/routers/health.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get("/check") +async def get_channel_config(): + return { + "code": 0, + "err_msg": "", + "data": "" + } diff --git a/sysom_server/sysom_metric_anomaly_detection/app/routers/person.py b/sysom_server/sysom_metric_anomaly_detection/app/routers/person.py new file mode 100644 index 0000000000000000000000000000000000000000..42b496f95c27a2effccc4047ffaf47393234387d --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/app/routers/person.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session +from app.query import PersonQueryParams +from app.database import get_db +from app.crud import get_person_list, get_person_by_name +from app.schemas import Person +from sysom_utils import StandardListResponse, StandardResponse + + +router = APIRouter() + + +@router.get("/get") +async def get_specific_person( + person_name: str, db: Session = Depends(get_db) +): + person = get_person_by_name(db, person_name) + return StandardResponse(person, Person) + +@router.get("/list") +async def get_persons( + query_params: PersonQueryParams = Depends(), db: Session = Depends(get_db) +): + person_list = get_person_list(db, query_params) + return StandardListResponse(person_list, Person) diff --git a/sysom_server/sysom_metric_anomaly_detection/app/schemas.py b/sysom_server/sysom_metric_anomaly_detection/app/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..353a3cb63b4d019bd7133da3fb19a0d3e0838775 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/app/schemas.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File schemas.py +Description: +""" +from pydantic import BaseModel +from datetime import datetime + +########################################################################### +# Define schemas here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/response-model/ +# class Person(BaseModel): +# id: int +# name: str +# age: int +# created_at: datetime + +# class Config: +# orm_mode = True \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/common.py b/sysom_server/sysom_metric_anomaly_detection/conf/common.py new file mode 100644 index 0000000000000000000000000000000000000000..d8c0ddb9d5f05a75a7dadd205ba01332dc6a9015 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/conf/common.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" +from pathlib import Path +from sysom_utils import ConfigParser, SysomFramework + +BASE_DIR = Path(__file__).resolve().parent.parent + +################################################################## +# Load yaml config first +################################################################## +YAML_GLOBAL_CONFIG_PATH = f"/etc/sysom/config.yml" +YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" + +YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) + +mysql_config = YAML_CONFIG.get_server_config().db.mysql +service_config = YAML_CONFIG.get_service_config() + +SysomFramework.init(YAML_CONFIG) + +################################################################## +# fastapi config +################################################################## +SQLALCHEMY_DATABASE_URL = ( + f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" + f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" +) \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/develop.py b/sysom_server/sysom_metric_anomaly_detection/conf/develop.py new file mode 100644 index 0000000000000000000000000000000000000000..e818cadb4636d75b48534f59dad1f469eb20ea03 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/conf/develop.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File develoop.py +Description: +""" +from .common import * + +''' +开发环境配置项 +''' + +DEBUG = True \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/gunicorn.py b/sysom_server/sysom_metric_anomaly_detection/conf/gunicorn.py new file mode 100644 index 0000000000000000000000000000000000000000..a1f65c77ba12abaa879ea580a3bd7beb5b20037e --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/conf/gunicorn.py @@ -0,0 +1,23 @@ +''' +Channel Service Gunicorn Settings +''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + +workers = 2 # 指定工作进程数 + +threads = 4 + +bind = f'{bind}:{port}' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/sysom-metric_anomaly_detection-access.log' + +loglevel = 'error' + +proc_name = 'sysom_metric_anomaly_detection_service' diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/product.py b/sysom_server/sysom_metric_anomaly_detection/conf/product.py new file mode 100644 index 0000000000000000000000000000000000000000..84e472934ce22ec3be96e9504f2b3ce79512934b --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/conf/product.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File product.py +Description: +""" +from .common import * + +''' +生产环境配置项 +''' + +DEBUG = False diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/settings.py b/sysom_server/sysom_metric_anomaly_detection/conf/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..2aae16fee8f6a6ee4e001625411b0fafa53d2714 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/conf/settings.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File settings.py +Description: +""" +import os + +env = os.environ.get("env", "product") + + +if env == "develop": + from .develop import * +elif env == "testing": + from .testing import * +elif env == "product": + from .product import * \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/conf/testing.py b/sysom_server/sysom_metric_anomaly_detection/conf/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..9dc95facab4bf875094ed319a8391e43ccb909c9 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/conf/testing.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File testing.py +Description: +""" +from .common import * + +''' +测试环境配置项 +''' +DEBUG = True diff --git a/sysom_server/sysom_metric_anomaly_detection/config.yml b/sysom_server/sysom_metric_anomaly_detection/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..1413a2b673a9df8f53f057a8f07c028548f69145 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/config.yml @@ -0,0 +1,35 @@ +vars: + SERVICE_NAME: &SERVICE_NAME sysom_metric_anomaly_detection + SERVICE_CONSUMER_GROUP: + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] + +sysom_server: + cec: + consumer_group: SYSOM_CEC_CHANNEL_CONSUMER_GROUP + +sysom_service: + service_name: *SERVICE_NAME + service_dir: *SERVICE_NAME + protocol: http + host: 127.0.0.1 + bind: 127.0.0.1 + port: 7017 + framework: + gcache: + protocol: redis + node_dispatch: + cmg: + tags: + - metric_anomaly_detection + - FastApi + # Metadata of service + metadata: + check: + type: http + url: "/api/v1/metric_anomaly_detection/health/check" + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false + diff --git a/sysom_server/sysom_metric_anomaly_detection/lib/README.md b/sysom_server/sysom_metric_anomaly_detection/lib/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ec74424ba6ecde5d035d0c113bafb0ddc6e4cfa --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/lib/README.md @@ -0,0 +1 @@ +The current directory holds the public libraries or utils needed for microservices \ No newline at end of file diff --git a/sysom_server/sysom_metric_anomaly_detection/main.py b/sysom_server/sysom_metric_anomaly_detection/main.py new file mode 100755 index 0000000000000000000000000000000000000000..414a192a56feec569aa8f695ed31e8ccde505540 --- /dev/null +++ b/sysom_server/sysom_metric_anomaly_detection/main.py @@ -0,0 +1,127 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/20 19:03 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File ssh.py +Description: +""" +from clogger import logger +from fastapi import FastAPI +from app.routers import health +from conf.settings import YAML_CONFIG +from sysom_utils import CmgPlugin, SysomFramework +import traceback +import sys,os,json +sys.path.append("%s/lib"%(os.path.dirname(os.path.abspath(__file__)))) +import time +import datetime +from datetime import date, datetime, timedelta +import requests +from metric_reader import MetricReader, dispatch_metric_reader,RangeQueryTask + +import pandas as pd +import numpy as np +import matplotlib +from matplotlib import pyplot as plt +import adtk +from adtk.data import validate_series +from adtk.visualization import plot +from adtk.transformer import RollingAggregate +from adtk.transformer import DoubleRollingAggregate +from adtk.detector import PersistAD +from adtk.detector import LevelShiftAD +from adtk.detector import SeasonalAD +from adtk.detector import AutoregressionAD + +app = FastAPI() + +app.include_router(health.router, prefix="/api/v1/metric_anomaly_detection/health") +# app.include_router(health.router, prefix="/api/v1/metric_anomaly_detection/person") + +metric_reader = dispatch_metric_reader("prometheus://localhost:9090") + +def pull_monidata(start_time,end_time): + retdict = [] + try: + ret_items = metric_reader.get_metric_names() + for item in ret_items.data: + #print(item) + task = RangeQueryTask(item, start_time, end_time).append_wildcard_filter("instance", "*") + ret_tmp = metric_reader.range_query([task]) + if len(ret_tmp.data) <= 0: + print ("ignore: ------",item,ret_tmp.data) + if len(ret_tmp.data) > 0: + for i in range(len(ret_tmp.data)): + #if "labels" in ret_tmp.data[i].to_dict(): + #print(json.dumps(ret_tmp.data[i].to_dict(),ensure_ascii=False,indent=4)) + retdict.append(ret_tmp.data[i].to_dict()) + #print(json.dumps(retdict[0:10],ensure_ascii=False,indent=4)) + except: + traceback.print_exc() + pass + return retdict + +now_time = time.time() +retdata = pull_monidata(now_time-600, now_time) +exp = retdata[0]['values'] +stamps = pd.date_range('2023-08-13', periods=41, freq='15S') +df = pd.DataFrame(columns=['values']) +for data in exp: + df.loc[len(df)] = data[0] +df.index = stamps +#print(df) +df = validate_series(df) +#plot(df) + +#Detect spikes using PersistAD +spikes = PersistAD(c=3.0, side='both').fit_detect(df) +plot(df, anomaly=spikes, ts_linewidth=1, ts_markersize=3, anomaly_color='red') + +#Detect level changes using LevelShiftAD +level_shifts = LevelShiftAD(c=6.0, side='both', window=5).fit_detect(df) +plot(df, anomaly=level_shifts, ts_linewidth=1, ts_markersize=3, anomaly_color='red') + +#Detect levels shifts using DoubleRollingAggregate +df_doubleRolling_mean = DoubleRollingAggregate(agg='mean', window=(3,1), diff='l1').transform(df) +plot(df_doubleRolling_mean) + +#Detect seasonality using SeasonalAD +try: + seasonality = SeasonalAD(c=3.0, side='both').fit_detect(df) + plot(seasonality) +except Exception as e: + print(e) + +#Detect slope shifts using AutoregressionAD and one-order differencing +slope_shifts = AutoregressionAD(n_steps=1, step_size=1, c=3.0).fit_detect(df) +plot(slope_shifts) +df_diff = df.diff(periods=1) +plot(df_diff) + + +############################################################################# +# Write your API interface here, or add to app/routes +############################################################################# + + +def init_framwork(): + SysomFramework\ + .init(YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() + logger.info("SysomFramework init finished!") + + +@app.on_event("startup") +async def on_start(): + init_framwork() + + ############################################################################# + # Perform some microservice initialization operations over here + ############################################################################# + + +@app.on_event("shutdown") +async def on_shutdown(): + pass