diff --git a/deps/2_nginx/sysom.conf b/deps/2_nginx/sysom.conf index 6dae31739cfc0e92a9fe1436e85bdb76d934ed9a..6c1cba7a57701a7302831116aba62648b468153b 100644 --- a/deps/2_nginx/sysom.conf +++ b/deps/2_nginx/sysom.conf @@ -179,6 +179,12 @@ server { # 7020 resvered for cluster_health # 7022 resvered for colocation + location /api/v1/cmg/ { + proxy_pass http://127.0.0.1:7023; + proxy_read_timeout 180; + proxy_redirect off; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } location /api/ { proxy_pass http://127.0.0.1:7001; diff --git a/script/server/sysom_cmg/clear.sh b/script/server/sysom_cmg/clear.sh new file mode 100644 index 0000000000000000000000000000000000000000..d1d29a5abc8420b660aac4da29a7b762ef89a2d1 --- /dev/null +++ b/script/server/sysom_cmg/clear.sh @@ -0,0 +1,14 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) +SERVICE_NAME=sysom-cmg + +clear_app() { + rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini + ###use supervisorctl update to stop and clear services### + supervisorctl update +} + +# Stop first +bash -x $BaseDir/stop.sh + +clear_app diff --git a/script/server/sysom_cmg/db_migrate.sh b/script/server/sysom_cmg/db_migrate.sh new file mode 100644 index 0000000000000000000000000000000000000000..f530e1f82f0e3becd05b158b68263bc37fbd0447 --- /dev/null +++ b/script/server/sysom_cmg/db_migrate.sh @@ -0,0 +1,18 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + alembic upgrade head + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/script/server/sysom_cmg/init.sh b/script/server/sysom_cmg/init.sh new file mode 100644 index 0000000000000000000000000000000000000000..1298b134f2327f2ed62631d84c0e7fc77be13861 --- /dev/null +++ b/script/server/sysom_cmg/init.sh @@ -0,0 +1,25 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +BaseDir=$(dirname $(readlink -f "$0")) +SERVICE_NAME=sysom-cmg + +init_conf() { + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=$(cat /proc/cpuinfo | grep processor | wc -l) + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/gunicorn.py +} + +init_app() { + init_conf + bash -x $BaseDir/db_migrate.sh + ###if supervisor service started, we need use "supervisorctl update" to start new conf#### + supervisorctl update +} + +init_app + +# Start +bash -x $BaseDir/start.sh diff --git a/script/server/sysom_cmg/install.sh b/script/server/sysom_cmg/install.sh new file mode 100644 index 0000000000000000000000000000000000000000..b7715e4bbebf89e2ca3689c5f2a176753b55a646 --- /dev/null +++ b/script/server/sysom_cmg/install.sh @@ -0,0 +1,29 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME +SERVICE_NAME=sysom-cmg + +if [ "$UID" -ne 0 ]; then + echo "Please run as root" + exit 1 +fi + +install_requirement() { + pushd ${SERVICE_SCRIPT_HOME} + pip install -r requirements.txt + popd +} + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +install_app() { + source_virtualenv + install_requirement +} + +install_app diff --git a/script/server/sysom_cmg/requirements.txt b/script/server/sysom_cmg/requirements.txt new file mode 100644 index 0000000000000000000000000000000000000000..8c2a17dfe74f495037656202e288b21bf01cc996 --- /dev/null +++ b/script/server/sysom_cmg/requirements.txt @@ -0,0 +1,19 @@ +clogger==0.0.1 +channel_job>=0.0.1 +cec_base>=0.0.1 +cec_redis>=0.0.1 +sysom_utils>=0.0.1 +alembic==1.7.7 +anyio==3.6.2 +asyncer==0.0.2 +asyncssh==2.12.0 +fastapi==0.83.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +uvicorn==0.16.0 +gunicorn==20.1.0 +python-multipart==0.0.5 +###################################################################### +# Add your custom python requirements here +###################################################################### \ No newline at end of file diff --git a/script/server/sysom_cmg/start.sh b/script/server/sysom_cmg/start.sh new file mode 100644 index 0000000000000000000000000000000000000000..95b9a42eacae941985dd63349419b873f89d8eb3 --- /dev/null +++ b/script/server/sysom_cmg/start.sh @@ -0,0 +1,29 @@ +#!/bin/bash +SERVICE_NAME=sysom-cmg + +is_start() { + status=`supervisorctl status ${SERVICE_NAME} | awk '{print $2}'` + result=`echo "RUNNING STARTING" | grep $status` + if [[ "$result" != "" ]] + then + return 1 + else + return 0 + fi +} + +start_app() { + is_start + if [[ $? == 0 ]]; then + supervisorctl start $SERVICE_NAME + is_start + if [[ $? == 0 ]]; then + echo "${SERVICE_NAME} service start fail, please check log" + exit 1 + else + echo "supervisorctl start ${SERVICE_NAME} success..." + fi + fi +} + +start_app diff --git a/script/server/sysom_cmg/stop.sh b/script/server/sysom_cmg/stop.sh new file mode 100644 index 0000000000000000000000000000000000000000..7c747e127e7438b2138ef8158a01028e5001bdbc --- /dev/null +++ b/script/server/sysom_cmg/stop.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SERVICE_NAME=sysom-cmg + +stop_app() { + supervisorctl stop $SERVICE_NAME +} + +stop_app diff --git a/script/server/sysom_cmg/sysom-cmg.ini b/script/server/sysom_cmg/sysom-cmg.ini new file mode 100644 index 0000000000000000000000000000000000000000..f4f381b0e6f361687b724a56158195d58e42ad8b --- /dev/null +++ b/script/server/sysom_cmg/sysom-cmg.ini @@ -0,0 +1,9 @@ +[program:sysom-cmg] +directory=/usr/local/sysom/server/sysom_cmg +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +startsecs=3 +autostart=true +autorestart=true +environment=PATH=/usr/local/sysom/virtualenv/bin:%(ENV_PATH)s +stderr_logfile=/var/log/sysom/sysom-cmg-error.log +stdout_logfile=/var/log/sysom/sysom-cmg.log diff --git a/script/server/sysom_cmg/uninstall.sh b/script/server/sysom_cmg/uninstall.sh new file mode 100644 index 0000000000000000000000000000000000000000..0aac04c09286c7f24e98d8a20ce131cdbec36375 --- /dev/null +++ b/script/server/sysom_cmg/uninstall.sh @@ -0,0 +1,9 @@ +#!/bin/bash +BaseDir=$(dirname $(readlink -f "$0")) + +uninstall_app() { + # do nothing + echo "" +} + +uninstall_app \ No newline at end of file diff --git a/sysom_server/sysom_cmg/alembic.ini b/sysom_server/sysom_cmg/alembic.ini new file mode 100644 index 0000000000000000000000000000000000000000..f6ab9febcd93add9d0ea8857f857d7f30f1fe48f --- /dev/null +++ b/sysom_server/sysom_cmg/alembic.ini @@ -0,0 +1,102 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = "" + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/sysom_server/sysom_cmg/alembic/README b/sysom_server/sysom_cmg/alembic/README new file mode 100644 index 0000000000000000000000000000000000000000..98e4f9c44effe479ed38c66ba922e7bcc672916f --- /dev/null +++ b/sysom_server/sysom_cmg/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/sysom_server/sysom_cmg/alembic/env.py b/sysom_server/sysom_cmg/alembic/env.py new file mode 100644 index 0000000000000000000000000000000000000000..19c7e30c64f87d2d16ed5b3ecc2ed32cbbc15154 --- /dev/null +++ b/sysom_server/sysom_cmg/alembic/env.py @@ -0,0 +1,115 @@ +import inspect +import app.models as models +from logging.config import fileConfig +from sqlalchemy import engine_from_config, Table +from sqlalchemy import pool +from app.models import Base +from alembic import context +from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL + +################################################################## +# Load yaml config first +################################################################## +mysql_config = YAML_CONFIG.get_server_config().db.mysql + +################################################################## +# Scan models +################################################################## +service_tables = [] +for name, data in inspect.getmembers(models): + if inspect.isclass(data): + if data.__module__ != "app.models": + continue + if "__tablename__" in data.__dict__: + service_tables.append(data.__dict__["__tablename__"]) + elif "__table__" in data.__dict__: + service_tables.append(data.__dict__["__table__"]) + elif isinstance(data, Table): + service_tables.append(name) + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Update mysql config according config.yml +config.set_main_option( + "sqlalchemy.url", + SQLALCHEMY_DATABASE_URL +) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def include_object(object, name, type_, reflected, compare_to): + if type_ == "table" and name not in service_tables: + return False + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + include_object=include_object, + version_table="cmg_version", + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, + include_object=include_object, + version_table="cmg_version" + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/sysom_server/sysom_cmg/alembic/script.py.mako b/sysom_server/sysom_cmg/alembic/script.py.mako new file mode 100644 index 0000000000000000000000000000000000000000..2c0156303a8df3ffdc9de87765bf801bf6bea4a5 --- /dev/null +++ b/sysom_server/sysom_cmg/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/sysom_server/sysom_cmg/alembic/versions/.gitkeep b/sysom_server/sysom_cmg/alembic/versions/.gitkeep new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/sysom_server/sysom_cmg/app/__init__.py b/sysom_server/sysom_cmg/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..30e79558304b508c489edd0fd7b541ee4af47f58 --- /dev/null +++ b/sysom_server/sysom_cmg/app/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py +Description: +""" \ No newline at end of file diff --git a/sysom_server/sysom_cmg/app/crud.py b/sysom_server/sysom_cmg/app/crud.py new file mode 100644 index 0000000000000000000000000000000000000000..08d529f5276f383912dfa63a096e5f3d0df15621 --- /dev/null +++ b/sysom_server/sysom_cmg/app/crud.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File crud.py +Description: +""" +from typing import Optional, List +from sqlalchemy.orm import Session +from app import models, schemas, query + +################################################################################################ +# Define database crud here +################################################################################################ + +# def get_person_by_name(db: Session, name: str) -> Optional[models.Person]: +# return db.query(models.Person).filter(models.Person.name == name).first() + +# def create_person(db: Session, person: schemas.Person) -> models.Person: +# person = models.Person(**person.dict()) +# db.add(person) +# db.commit() +# db.refresh(person) +# return person + +# def del_person_by_id(db: Session, person_id: int): +# person = db.get(models.Person, person_id) +# db.delete(person) +# db.commit() + +# def get_person_list(db: Session, query_params: query.PersonQueryParams) -> List[models.Person]: +# return ( +# query_params.get_query_exp(db) +# .all() +# ) diff --git a/sysom_server/sysom_cmg/app/database.py b/sysom_server/sysom_cmg/app/database.py new file mode 100644 index 0000000000000000000000000000000000000000..773a7ada72834d1f9e6d6c3be3bfc681ca00877a --- /dev/null +++ b/sysom_server/sysom_cmg/app/database.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File database.py +Description: +""" +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from conf.settings import SQLALCHEMY_DATABASE_URL +from sysom_utils import FastApiResponseHelper + +engine = create_engine( + SQLALCHEMY_DATABASE_URL, connect_args={} +) + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + +Base = declarative_base() + +FastApiResponseHelper.bind_base_class(Base) \ No newline at end of file diff --git a/sysom_server/sysom_cmg/app/models.py b/sysom_server/sysom_cmg/app/models.py new file mode 100644 index 0000000000000000000000000000000000000000..7e2a989f3aa1907d390afd613b1338087af6003c --- /dev/null +++ b/sysom_server/sysom_cmg/app/models.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File models.py +Description: +""" +from sqlalchemy import Column, Integer, String, DateTime +from sqlalchemy.sql import func +from app.database import Base + + +########################################################################### +# Define databse model here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/sql-databases/ +# class Person(Base): +# __tablename__ = "sys_person" +# id = Column(Integer, primary_key=True) +# name = Column(String(254), unique=True) +# age = Column(Integer) +# created_at = Column(DateTime(timezone=True), server_default=func.now()) \ No newline at end of file diff --git a/sysom_server/sysom_cmg/app/query.py b/sysom_server/sysom_cmg/app/query.py new file mode 100644 index 0000000000000000000000000000000000000000..fcbcd0898fc8d3f2be8d64ce694d7aa99ccb2332 --- /dev/null +++ b/sysom_server/sysom_cmg/app/query.py @@ -0,0 +1,28 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/09/19 15:41 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File schemas.py +Description: +""" +from typing import Optional +from app import models +from sysom_utils import BaseQueryParams + + +# class PersonQueryParams(BaseQueryParams): + +# # 1. 指定要查询的模型 +# __modelclass__ = models.Person + +# # 2. 定义排序字段 +# sort: str = "-created_at" + +# # 3. 定义支持用于过滤的参数 +# name: Optional[str] = None +# age: Optional[str] = None + +# # 4. 指定哪些字段是枚举类型,并且指明对应的枚举类 +# __enum_fields__ = { +# } \ No newline at end of file diff --git a/sysom_server/sysom_cmg/app/routers/health.py b/sysom_server/sysom_cmg/app/routers/health.py new file mode 100644 index 0000000000000000000000000000000000000000..8bc7ca8b30990634c2bbe6bddd026b5209a1b08b --- /dev/null +++ b/sysom_server/sysom_cmg/app/routers/health.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get("/check") +async def get_channel_config(): + return { + "code": 0, + "err_msg": "", + "data": "" + } diff --git a/sysom_server/sysom_cmg/app/routers/person.py b/sysom_server/sysom_cmg/app/routers/person.py new file mode 100644 index 0000000000000000000000000000000000000000..822d6e02958587f71db1edb2ed93385c3430ab05 --- /dev/null +++ b/sysom_server/sysom_cmg/app/routers/person.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter, Depends +from sqlalchemy.orm import Session +from app.query import PersonQueryParams +from app.database import get_db +from app.crud import get_person_list, get_person_by_name +from app.schemas import Person +from sysom_utils import StandardListResponse, StandardResponse + + +router = APIRouter() + + +@router.get("/get") +async def get_specific_person( + person_name: str, db: Session = Depends(get_db) +): + person = get_person_by_name(db, person_name) + return StandardResponse(person, Person) + +@router.get("/list") +async def get_persons( + query_params: PersonQueryParams = Depends(), db: Session = Depends(get_db) +): + person_list = get_person_list(db, query_params) + return StandardListResponse(person_list, Person) diff --git a/sysom_server/sysom_cmg/app/routers/serivces.py b/sysom_server/sysom_cmg/app/routers/serivces.py new file mode 100644 index 0000000000000000000000000000000000000000..4f4d648ab924614af5a25d093e1e851df2c9fb69 --- /dev/null +++ b/sysom_server/sysom_cmg/app/routers/serivces.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- +''' +@Author: wb-msm241621 +@Date: 2024-01-31 17:59:13 +@LastEditTime: 2024-01-31 17:59:13 +@Description: +''' +from fastapi import APIRouter +from conf.settings import * +from cmg_base import dispatch_service_discovery + +router = APIRouter() + + +discovery = dispatch_service_discovery(YAML_CONFIG.get_cmg_url()) + + +@router.get("/list") +async def list_services(): + try: + res = {} + services = discovery.get_services() + for service in services: + count = discovery.get_instance_count(service) + res[service] = count + except Exception as exc: + return { + "code": 1, + "data": "", + "err_msg": str(exc) + } + return { + "code": 0, + "data": res, + "err_msg": "" + } diff --git a/sysom_server/sysom_cmg/app/schemas.py b/sysom_server/sysom_cmg/app/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..a02e4b5f516f27badeab965aaf2af4eeda2774ba --- /dev/null +++ b/sysom_server/sysom_cmg/app/schemas.py @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File schemas.py +Description: +""" +from pydantic import BaseModel +from datetime import datetime + +########################################################################### +# Define schemas here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/response-model/ +# class Person(BaseModel): +# id: int +# name: str +# age: int +# created_at: datetime + +# class Config: +# orm_mode = True \ No newline at end of file diff --git a/sysom_server/sysom_cmg/conf/common.py b/sysom_server/sysom_cmg/conf/common.py new file mode 100644 index 0000000000000000000000000000000000000000..e89384c4d99169d6e126c5385d421812b2daa601 --- /dev/null +++ b/sysom_server/sysom_cmg/conf/common.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" +from pathlib import Path +from sysom_utils import ConfigParser, SysomFramework + +BASE_DIR = Path(__file__).resolve().parent.parent + +################################################################## +# Load yaml config first +################################################################## +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" + +YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) + +mysql_config = YAML_CONFIG.get_server_config().db.mysql +service_config = YAML_CONFIG.get_service_config() + +SysomFramework.init(YAML_CONFIG) + +################################################################## +# fastapi config +################################################################## +SQLALCHEMY_DATABASE_URL = ( + f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" + f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" +) \ No newline at end of file diff --git a/sysom_server/sysom_cmg/conf/develop.py b/sysom_server/sysom_cmg/conf/develop.py new file mode 100644 index 0000000000000000000000000000000000000000..dea9392e7c342c168133ac4fbae72ce3306fe0fc --- /dev/null +++ b/sysom_server/sysom_cmg/conf/develop.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File develoop.py +Description: +""" +from .common import * + +''' +开发环境配置项 +''' + +DEBUG = True \ No newline at end of file diff --git a/sysom_server/sysom_cmg/conf/gunicorn.py b/sysom_server/sysom_cmg/conf/gunicorn.py new file mode 100644 index 0000000000000000000000000000000000000000..b7234e4647320b90bf01f4920a6e7752747556aa --- /dev/null +++ b/sysom_server/sysom_cmg/conf/gunicorn.py @@ -0,0 +1,23 @@ +''' +Channel Service Gunicorn Settings +''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + +workers = 2 # 指定工作进程数 + +threads = 3 + +bind = f'{bind}:{port}' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/sysom-cmg-access.log' + +loglevel = 'error' + +proc_name = 'sysom_cmg_service' diff --git a/sysom_server/sysom_cmg/conf/product.py b/sysom_server/sysom_cmg/conf/product.py new file mode 100644 index 0000000000000000000000000000000000000000..2c9c65541bc23c8ab4559c67eb9c2e29525bca6c --- /dev/null +++ b/sysom_server/sysom_cmg/conf/product.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File product.py +Description: +""" +from .common import * + +''' +生产环境配置项 +''' + +DEBUG = False diff --git a/sysom_server/sysom_cmg/conf/settings.py b/sysom_server/sysom_cmg/conf/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..1ceea45f13218128244c79455ab87134eaecee87 --- /dev/null +++ b/sysom_server/sysom_cmg/conf/settings.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File settings.py +Description: +""" +import os + +env = os.environ.get("env", "product") + + +if env == "develop": + from .develop import * +elif env == "testing": + from .testing import * +elif env == "product": + from .product import * \ No newline at end of file diff --git a/sysom_server/sysom_cmg/conf/testing.py b/sysom_server/sysom_cmg/conf/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..46de2e5ce02c6cc90b5aeb9438181a88f2681dfd --- /dev/null +++ b/sysom_server/sysom_cmg/conf/testing.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File testing.py +Description: +""" +from .common import * + +''' +测试环境配置项 +''' +DEBUG = True diff --git a/sysom_server/sysom_cmg/config.yml b/sysom_server/sysom_cmg/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..c108f28ac6e042ac46e8b0351de142c198f0e8ad --- /dev/null +++ b/sysom_server/sysom_cmg/config.yml @@ -0,0 +1,35 @@ +vars: + SERVICE_NAME: &SERVICE_NAME sysom_cmg + SERVICE_CONSUMER_GROUP: + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] + +sysom_server: + cec: + consumer_group: *SERVICE_CONSUMER_GROUP + +sysom_service: + service_name: *SERVICE_NAME + service_dir: *SERVICE_NAME + protocol: http + host: 127.0.0.1 + bind: 127.0.0.1 + port: 7023 + framework: + gcache: + protocol: redis + node_dispatch: + cmg: + tags: + - cmg + - FastApi + # Metadata of service + metadata: + check: + type: http + url: "/api/v1/cmg/health/check" + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false + diff --git a/sysom_server/sysom_cmg/lib/README.md b/sysom_server/sysom_cmg/lib/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ec74424ba6ecde5d035d0c113bafb0ddc6e4cfa --- /dev/null +++ b/sysom_server/sysom_cmg/lib/README.md @@ -0,0 +1 @@ +The current directory holds the public libraries or utils needed for microservices \ No newline at end of file diff --git a/sysom_server/sysom_cmg/main.py b/sysom_server/sysom_cmg/main.py new file mode 100644 index 0000000000000000000000000000000000000000..ad4b9928077a0f9ad651ab84d9c14c7878e1c426 --- /dev/null +++ b/sysom_server/sysom_cmg/main.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- # +""" +Time 2024/01/31 17:58 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File ssh.py +Description: +""" +from clogger import logger +from fastapi import FastAPI +from app.routers import health, serivces +from conf.settings import YAML_CONFIG +from sysom_utils import CmgPlugin, SysomFramework + + +app = FastAPI() + +app.include_router(health.router, prefix="/api/v1/cmg/health") +app.include_router(serivces.router, prefix="/api/v1/cmg/services") +# app.include_router(health.router, prefix="/api/v1/cmg/person") + + +############################################################################# +# Write your API interface here, or add to app/routes +############################################################################# + + +def init_framwork(): + SysomFramework\ + .init(YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() + logger.info("SysomFramework init finished!") + + +@app.on_event("startup") +async def on_start(): + init_framwork() + + ############################################################################# + # Perform some microservice initialization operations over here + ############################################################################# + + +@app.on_event("shutdown") +async def on_shutdown(): + pass \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/serializer.py b/sysom_server/sysom_hotfix/apps/hotfix/serializer.py index 72bbbc4d174e13a23084f38760911abf73946e1f..0e78bb20b16d0b5d952de23c5f9a6be553660ed3 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/serializer.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/serializer.py @@ -1,8 +1,11 @@ import os +import re import tempfile import pandas as pd +from typing import List from datetime import datetime from pandas.core.frame import DataFrame +from pandas._libs.tslibs.timestamps import Timestamp from clogger import logger from django.utils import timezone from django.core.files.uploadedfile import InMemoryUploadedFile @@ -186,31 +189,76 @@ class BulkImportHotfixReleasedSerializer(serializers.Serializer): ) return context + def _kernel_version_or_download_link_map( + self, + kernel_versions: List[str], + urls: List[str] + ) -> List[dict]: + _version_download_link_list = list() + reg_compile = re.compile("(\d+)\.(\d+)\.(\d+)-(\d+)") + + def _filter(marks, url): + flag = True + for mark in marks: + if mark not in url: + flag = False + break + return flag + + for kernel_version in kernel_versions: + _version_map_download_link = {} + version_s = re.sub(reg_compile, "", kernel_version) + marks = version_s.split(".")[1:] + _version_map_download_link['released_kernel_version'] = kernel_version + + for url in filter(lambda x: _filter(marks, x), urls): + _version_map_download_link["download_link"] = url + _version_download_link_list.append(_version_map_download_link) + return _version_download_link_list + def create(self, validated_data): """Save the data to the database""" - def _structure_released_hotfix(data_rows) -> list[ReleasedHotfixListModule]: + def _released_time(released_time): + if isinstance(released_time, str) and released_time != "": + released_time = datetime.strptime( + released_time.split(".")[0], "%Y-%m-%d %H:%M:%S" + ) + released_time = timezone.get_current_timezone().localize(released_time) + elif isinstance(released_time, Timestamp): + released_time = timezone.get_current_timezone().localize(released_time) + else: + released_time = timezone.now() + return released_time + + def _structure_released_hotfix(data_rows: dict) -> List[ReleasedHotfixListModule]: """ initizaer released hotfix model """ - released_hotfix_list = [] - for kernel_version in data_rows['kernel_version'].split(","): - kwargs = dict() - kwargs["hotfix_id"] = data_rows.get("hotfix_id") - kwargs["download_link"] = data_rows.get("download_link", "") - kwargs["released_kernel_version"] = kernel_version - - released_time = data_rows['modified_time'] - if isinstance(released_time, str): - released_time = datetime.strptime( - released_time.split(".")[0], "%Y-%m-%d %H:%M:%S" - ) - kwargs['released_time'] = timezone.get_current_timezone().localize(released_time) - - try: - ReleasedHotfixListModule.objects.get(**kwargs) - except ReleasedHotfixListModule.DoesNotExist: - released_hotfix_list.append(ReleasedHotfixListModule(**kwargs)) + released_hotfix_list = list() + kwargs = dict() + kwargs["hotfix_id"] = data_rows.get("hotfix_id") + kernel_versions = data_rows.get("kernel_version").split(",") + download_links = re.split(" ", data_rows.get("download_link")) + + if len(kernel_versions) == 1: + kwargs['released_kernel_version'] = kernel_versions[0] + if len(download_links) == 1: + kwargs['download_link'] = download_links[0] + else: + for item in self._kernel_version_or_download_link_map( + kernel_versions, download_links + ): kwargs.update(item) + + kwargs['released_time'] = _released_time(data_rows.get("modified_time", "")) + + try: + ReleasedHotfixListModule.objects.get( + released_kernel_version=kwargs['released_kernel_version'], + hotfix_id=kwargs["hotfix_id"] + ) + except ReleasedHotfixListModule.DoesNotExist: + released_hotfix_list.append(ReleasedHotfixListModule(**kwargs)) return released_hotfix_list excel_file_content: DataFrame = validated_data.get("file")