From 20451b1039f765efe9cf3e117c402662addf45bc Mon Sep 17 00:00:00 2001 From: ydzhang Date: Tue, 13 Jun 2023 22:01:58 +0800 Subject: [PATCH 1/4] adjust hotfix_builder to ms change builder info micro service unite the hotfix job topic of builder and server to hotfix_msg add the sys env into ini and renew the usage page of check_env --- script/server/deploy_exclude | 1 - script/server/sysom_hotfix_builder/clear.sh | 21 ++- .../server/sysom_hotfix_builder/db_migrate.sh | 18 +++ script/server/sysom_hotfix_builder/init.sh | 23 ++- script/server/sysom_hotfix_builder/install.sh | 12 +- .../sysom_hotfix_builder/requirements.txt | 15 ++ script/server/sysom_hotfix_builder/start.sh | 9 +- script/server/sysom_hotfix_builder/stop.sh | 14 +- .../sysom-hotfix-builder.ini | 9 -- .../sysom-hotfix_builder.ini | 9 ++ sysom_server/deploy_exclude | 1 - sysom_server/sysom_hotfix/apps/hotfix/apps.py | 7 + .../sysom_hotfix/apps/hotfix/views.py | 10 +- .../{apps/hotfix => lib}/function.py | 16 +- sysom_server/sysom_hotfix_builder/alembic.ini | 102 +++++++++++++ .../sysom_hotfix_builder/alembic/README | 1 + .../sysom_hotfix_builder/alembic/env.py | 110 ++++++++++++++ .../alembic/script.py.mako | 24 +++ .../sysom_hotfix_builder/app/__init__.py | 8 + .../sysom_hotfix_builder/{ => app}/builder.py | 143 ++++++++---------- sysom_server/sysom_hotfix_builder/app/crud.py | 30 ++++ .../sysom_hotfix_builder/app/database.py | 20 +++ .../sysom_hotfix_builder/app/models.py | 22 +++ .../app/routers/health.py | 21 +++ .../sysom_hotfix_builder/app/schemas.py | 22 +++ sysom_server/sysom_hotfix_builder/builder.ini | 19 --- .../sysom_hotfix_builder/conf/common.py | 33 ++++ .../sysom_hotfix_builder/conf/develop.py | 15 ++ .../sysom_hotfix_builder/conf/gunicorn.py | 23 +++ .../{ => conf}/img_list.json | 0 .../sysom_hotfix_builder/conf/product.py | 15 ++ .../sysom_hotfix_builder/conf/settings.py | 19 +++ .../sysom_hotfix_builder/conf/testing.py | 14 ++ sysom_server/sysom_hotfix_builder/config.yml | 54 +++++++ .../sysom_hotfix_builder/lib/README.md | 1 + sysom_server/sysom_hotfix_builder/main.py | 47 ++++++ .../{ => script}/build_hotfix.sh | 0 .../{ => script}/build_rpm.sh | 0 .../{ => script}/check_env.sh | 10 +- sysom_web/yarn.lock | 10 +- 40 files changed, 763 insertions(+), 165 deletions(-) create mode 100644 script/server/sysom_hotfix_builder/db_migrate.sh delete mode 100644 script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini create mode 100644 script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini rename sysom_server/sysom_hotfix/{apps/hotfix => lib}/function.py (98%) create mode 100644 sysom_server/sysom_hotfix_builder/alembic.ini create mode 100644 sysom_server/sysom_hotfix_builder/alembic/README create mode 100644 sysom_server/sysom_hotfix_builder/alembic/env.py create mode 100644 sysom_server/sysom_hotfix_builder/alembic/script.py.mako create mode 100644 sysom_server/sysom_hotfix_builder/app/__init__.py rename sysom_server/sysom_hotfix_builder/{ => app}/builder.py (88%) create mode 100644 sysom_server/sysom_hotfix_builder/app/crud.py create mode 100644 sysom_server/sysom_hotfix_builder/app/database.py create mode 100644 sysom_server/sysom_hotfix_builder/app/models.py create mode 100644 sysom_server/sysom_hotfix_builder/app/routers/health.py create mode 100644 sysom_server/sysom_hotfix_builder/app/schemas.py delete mode 100644 sysom_server/sysom_hotfix_builder/builder.ini create mode 100644 sysom_server/sysom_hotfix_builder/conf/common.py create mode 100644 sysom_server/sysom_hotfix_builder/conf/develop.py create mode 100644 sysom_server/sysom_hotfix_builder/conf/gunicorn.py rename sysom_server/sysom_hotfix_builder/{ => conf}/img_list.json (100%) create mode 100644 sysom_server/sysom_hotfix_builder/conf/product.py create mode 100644 sysom_server/sysom_hotfix_builder/conf/settings.py create mode 100644 sysom_server/sysom_hotfix_builder/conf/testing.py create mode 100644 sysom_server/sysom_hotfix_builder/config.yml create mode 100644 sysom_server/sysom_hotfix_builder/lib/README.md create mode 100644 sysom_server/sysom_hotfix_builder/main.py rename sysom_server/sysom_hotfix_builder/{ => script}/build_hotfix.sh (100%) rename sysom_server/sysom_hotfix_builder/{ => script}/build_rpm.sh (100%) rename sysom_server/sysom_hotfix_builder/{ => script}/check_env.sh (91%) diff --git a/script/server/deploy_exclude b/script/server/deploy_exclude index 5e69798f..e69de29b 100644 --- a/script/server/deploy_exclude +++ b/script/server/deploy_exclude @@ -1 +0,0 @@ -sysom_hotfix_builder \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/clear.sh b/script/server/sysom_hotfix_builder/clear.sh index eb5dded3..6ff21380 100644 --- a/script/server/sysom_hotfix_builder/clear.sh +++ b/script/server/sysom_hotfix_builder/clear.sh @@ -1,21 +1,18 @@ #!/bin/bash BaseDir=$(dirname $(readlink -f "$0")) -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder clear_app() { LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix sed -i '/hotfix_builder/d' /etc/exports - - # kill all kpatch-build process - echo "find : `ps -eo comm,pid | grep "kpatch-build"`" - for each_line in `ps -eo comm,pid | grep "kpatch-build"`; do - echo $each_line - if [[ $each_line =~ "kpatch-build" ]]; then - echo "find kpatch-build" - else - kill -9 $each_line - fi - done + + result=$(systemctl is-active nfs-server) + if [[ $result =~ "inactive" ]]; then + systemctl restart nfs-server + fi + umount -f $LOCAL_NFS_HOME + rm -rf ${SERVER_HOME}/builder + systemctl stop nfs-server rm -rf /etc/supervisord.d/${SERVICE_NAME}.ini ###use supervisorctl update to stop and clear services### supervisorctl update diff --git a/script/server/sysom_hotfix_builder/db_migrate.sh b/script/server/sysom_hotfix_builder/db_migrate.sh new file mode 100644 index 00000000..f530e1f8 --- /dev/null +++ b/script/server/sysom_hotfix_builder/db_migrate.sh @@ -0,0 +1,18 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + alembic upgrade head + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/init.sh b/script/server/sysom_hotfix_builder/init.sh index a06a00e9..bfc69a7b 100644 --- a/script/server/sysom_hotfix_builder/init.sh +++ b/script/server/sysom_hotfix_builder/init.sh @@ -1,16 +1,10 @@ -#! /bin/bash +#!/bin/bash SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} -SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} -VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME BaseDir=$(dirname $(readlink -f "$0")) -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder NFS_SERVER_IP=${SERVER_LOCAL_IP} -init_conf() { - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ -} - mount_nfs() { HOTFIX_NFS_HOME=${SERVER_HOME}/hotfix_builder/hotfix-nfs @@ -18,10 +12,23 @@ mount_nfs() mkdir -p ${LOCAL_NFS_HOME} + result=$(systemctl is-active nfs-server) + if [[ $result =~ "inactive" ]]; then + systemctl restart nfs-server + fi + sudo umount $LOCAL_NFS_HOME # Remove the mounted directory in case it mounted before sudo mount -t nfs ${NFS_SERVER_IP}:${HOTFIX_NFS_HOME} ${LOCAL_NFS_HOME} || exit 1 } +init_conf() { + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=$(cat /proc/cpuinfo | grep processor | wc -l) + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/gunicorn.py +} + init_app() { init_conf mount_nfs diff --git a/script/server/sysom_hotfix_builder/install.sh b/script/server/sysom_hotfix_builder/install.sh index 050d16fd..b391c67e 100644 --- a/script/server/sysom_hotfix_builder/install.sh +++ b/script/server/sysom_hotfix_builder/install.sh @@ -3,7 +3,7 @@ SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder if [ "$UID" -ne 0 ]; then echo "Please run as root" @@ -16,17 +16,17 @@ install_requirement() { popd } -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - install_package() { rpm -q --quiet make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || yum install -y make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || exit 1 rpm -q --quiet docker git || yum install -y docker git || echo "Warngin : Docker is not installed in this machine!" } +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + install_app() { rpm -q --quiet gcc || yum install -y gcc rpm -q --quiet make || yum install -y make diff --git a/script/server/sysom_hotfix_builder/requirements.txt b/script/server/sysom_hotfix_builder/requirements.txt index d3d398e7..ee975fdb 100644 --- a/script/server/sysom_hotfix_builder/requirements.txt +++ b/script/server/sysom_hotfix_builder/requirements.txt @@ -2,4 +2,19 @@ clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 +sysom_utils>=0.0.1 +alembic==1.7.7 +anyio==3.6.2 +asyncer==0.0.2 +asyncssh==2.12.0 +fastapi==0.83.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +uvicorn==0.16.0 +gunicorn==20.1.0 +python-multipart==0.0.5 +###################################################################### +# Add your custom python requirements here +###################################################################### requests==2.27.1 \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/start.sh b/script/server/sysom_hotfix_builder/start.sh index b6464a14..4265ea21 100644 --- a/script/server/sysom_hotfix_builder/start.sh +++ b/script/server/sysom_hotfix_builder/start.sh @@ -1,10 +1,9 @@ #!/bin/bash -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder is_start() { supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then + if [ $? -eq 0 ]; then return 1 else return 0 @@ -13,11 +12,11 @@ is_start() { start_app() { is_start - if [[ $? == 0 ]];then + if [[ $? == 0 ]]; then supervisorctl start $SERVICE_NAME is_start if [[ $? == 0 ]]; then - echo "${SERVICE_NAME} service start fail, please check log" + echo "${SERVICE_NAME} service start fail, please check log" exit 1 else echo "supervisorctl start ${SERVICE_NAME} success..." diff --git a/script/server/sysom_hotfix_builder/stop.sh b/script/server/sysom_hotfix_builder/stop.sh index f1ece2eb..5b953a93 100644 --- a/script/server/sysom_hotfix_builder/stop.sh +++ b/script/server/sysom_hotfix_builder/stop.sh @@ -1,8 +1,18 @@ #!/bin/bash -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder stop_app() { - umount $LOCAL_NFS_HOME + # kill all kpatch-build process + echo "find : `ps -eo comm,pid | grep "kpatch-build"`" + for each_line in `ps -eo comm,pid | grep "kpatch-build"`; do + echo $each_line + if [[ $each_line =~ "kpatch-build" ]]; then + echo "find kpatch-build" + else + kill -9 $each_line + fi + done + supervisorctl stop $SERVICE_NAME } diff --git a/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini b/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini deleted file mode 100644 index 3359d39b..00000000 --- a/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-hotfix-builder] -directory = /usr/local/sysom/server/sysom_hotfix_builder -command=/usr/local/sysom/environment/virtualenv/bin/python3 builder.py -startsecs=3 -autostart=true -autorestart=true -environment=PATH=/usr/local/sysom/environment/virtualenv/bin:%(ENV_PATH)s -stderr_logfile=/var/log/sysom/sysom-hotfix-builder-error.log -stdout_logfile=/var/log/sysom/sysom-hotfix-builder.log diff --git a/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini b/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini new file mode 100644 index 00000000..d1c25359 --- /dev/null +++ b/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini @@ -0,0 +1,9 @@ +[program:sysom-hotfix_builder] +directory=/usr/local/sysom/server/sysom_hotfix_builder +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +startsecs=3 +autostart=true +autorestart=true +stderr_logfile=/var/log/sysom/sysom-hotfix_builder-error.log +stdout_logfile=/var/log/sysom/sysom-hotfix_builder.log +environment=PATH=/usr/local/sysom/virtualenv/bin:%(ENV_PATH)s diff --git a/sysom_server/deploy_exclude b/sysom_server/deploy_exclude index 5e69798f..e69de29b 100644 --- a/sysom_server/deploy_exclude +++ b/sysom_server/deploy_exclude @@ -1 +0,0 @@ -sysom_hotfix_builder \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/apps.py b/sysom_server/sysom_hotfix/apps/hotfix/apps.py index 56f2514b..f97295cf 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/apps.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/apps.py @@ -26,6 +26,13 @@ class HotfixConfig(AppConfig): .load_plugin_cls(CmgPlugin) \ .enable_channel_job() \ .start() + + # create CECListerner object for cec message listening + from lib.function import CECListener + ceclistener = CECListener(settings.SYSOM_CEC_URL, settings.SYSOM_CEC_HOTFIX_SERVER_MSG_TOPIC) + if ceclistener is None: + logger.error("INIT CECListener Failed...") + ceclistener.run() else: # nothing to do when execute database migrations pass diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index b4f89c03..583d8f4f 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -33,7 +33,7 @@ from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer from cec_base.event import Event from django.http import HttpResponse, FileResponse -from apps.hotfix.function import FunctionClass, RedisCache +from lib.function import FunctionClass, RedisCache class SaveUploadFile(APIView): authentication_classes = [] @@ -79,8 +79,7 @@ class SaveUploadFile(APIView): patch_file_repo = os.path.join(settings.HOTFIX_FILE_BRANCH_RULE) if not os.path.exists(patch_file_repo): os.makedirs(patch_file_repo) - # patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + str(time.time()).split(".")[0] + "." + file_upload_type - patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + datetime.now().strftime('%Y%m%d%H%M%S-%f') + "." + file_upload_type + patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + datetime.now().strftime('%Y%m%d%H%M%S') + "." + file_upload_type file_path = os.path.join(patch_file_repo, patch_file_name) try: with open(file_path, 'wb') as f: @@ -191,6 +190,10 @@ class HotfixAPIView(GenericViewSet, if hotfix_object.building_status != self.build_failed: return other_response(message="This Hotfix Job is Not Failed", code=401) + # clear the original build log + hotfix_object.log="" + hotfix_object.save() + try: customize_version_object = KernelVersionModel.objects.all().filter(kernel_version=kernel_version).first() except Exception as e: @@ -278,6 +281,7 @@ class HotfixAPIView(GenericViewSet, def update_building_status(self, request): hotfix = HotfixModel.objects.filter(id=request.data["id"]).first() status = request.data["status"] + print(request.data) if hotfix is None: return other_response(message="No such hotfix id", result={"mgs":"update building status failed"}, code=400) if status == "waiting": diff --git a/sysom_server/sysom_hotfix/apps/hotfix/function.py b/sysom_server/sysom_hotfix/lib/function.py similarity index 98% rename from sysom_server/sysom_hotfix/apps/hotfix/function.py rename to sysom_server/sysom_hotfix/lib/function.py index 8619d416..bf2c32a1 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -2,16 +2,21 @@ import os import re import urllib import subprocess +import sys from clogger import logger from django.conf import settings import threading import redis import requests +import json from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel from cec_base.producer import dispatch_producer, Producer +from cec_base.consumer import Consumer, dispatch_consumer +from cec_base.admin import dispatch_admin from bs4 import BeautifulSoup from lib.utils import human_datetime +from sysom_utils import SysomFramework """ Function class @@ -20,7 +25,8 @@ This class contains the support/tool function class FunctionClass(): def __init__(self): - self.cec = dispatch_producer(settings.SYSOM_CEC_URL) + #self.cec = dispatch_producer(settings.SYSOM_CEC_URL) + self.producer = SysomFramework.cec_producer() def delete_document(self, doc_path, doc_name): document = os.path.join(doc_path, doc_name) @@ -133,7 +139,7 @@ class FunctionClass(): try: if not customize: if re.search('anolis', os_type): - self.cec.produce(cec_topic, { + self.producer.produce(cec_topic, { "hotfix_id" : hotfix_id, "kernel_version" : kernel_version, "patch_name" : patch_file, @@ -145,7 +151,6 @@ class FunctionClass(): "git_repo": "git@gitee.com:anolis/cloud-kernel.git", "customize": 0 }) - return True else: # this is customize kernel source_repo = kwargs['source_repo'] @@ -154,7 +159,7 @@ class FunctionClass(): debuginfo_link = kwargs['debuginfo_link'] image = kwargs['image'] is_src_package = kwargs["is_src_package"] - self.cec.produce(cec_topic, { + self.producer.produce(cec_topic, { "hotfix_id" : hotfix_id, "kernel_version" : kernel_version, "hotfix_name" : hotfix_name, @@ -171,6 +176,7 @@ class FunctionClass(): "image": image, "is_src_package": is_src_package }) + self.producer.flush() return True except Exception as e: logger.exception(e) @@ -446,7 +452,6 @@ class FunctionClass(): logger.error(e) - class CECListener(): def __init__(self, cec_url, listen_topic) -> None: @@ -483,7 +488,6 @@ class CECListener(): hotfix_id = parameters["hotfix_id"] if self.sync_key in parameters: if self.rpm_key in parameters: - self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) else: self.update_hotfix_job_status(hotfix_id, parameters["status"]) diff --git a/sysom_server/sysom_hotfix_builder/alembic.ini b/sysom_server/sysom_hotfix_builder/alembic.ini new file mode 100644 index 00000000..f6ab9feb --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic.ini @@ -0,0 +1,102 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = "" + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/sysom_server/sysom_hotfix_builder/alembic/README b/sysom_server/sysom_hotfix_builder/alembic/README new file mode 100644 index 00000000..98e4f9c4 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/alembic/env.py b/sysom_server/sysom_hotfix_builder/alembic/env.py new file mode 100644 index 00000000..8fcb7206 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/env.py @@ -0,0 +1,110 @@ +import inspect +import app.models as models +from logging.config import fileConfig +from sqlalchemy import engine_from_config +from sqlalchemy import pool +from app.models import Base +from alembic import context +from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL + +################################################################## +# Load yaml config first +################################################################## +mysql_config = YAML_CONFIG.get_server_config().db.mysql + +################################################################## +# Scan models +################################################################## +service_tables = [] +for name, data in inspect.getmembers(models, inspect.isclass): + if data.__module__ != "app.models": + continue + if "__tablename__" in data.__dict__: + service_tables.append(data.__dict__["__tablename__"]) + elif "__table__" in data.__dict__: + service_tables.append(data.__dict__["__table__"]) + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Update mysql config according config.yml +config.set_main_option( + "sqlalchemy.url", + SQLALCHEMY_DATABASE_URL +) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def include_object(object, name, type_, reflected, compare_to): + if type_ == "table" and name not in service_tables: + return False + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + include_object=include_object, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, + include_object=include_object + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/sysom_server/sysom_hotfix_builder/alembic/script.py.mako b/sysom_server/sysom_hotfix_builder/alembic/script.py.mako new file mode 100644 index 00000000..2c015630 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/sysom_server/sysom_hotfix_builder/app/__init__.py b/sysom_server/sysom_hotfix_builder/app/__init__.py new file mode 100644 index 00000000..3fb0576a --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py +Description: +""" \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/builder.py b/sysom_server/sysom_hotfix_builder/app/builder.py similarity index 88% rename from sysom_server/sysom_hotfix_builder/builder.py rename to sysom_server/sysom_hotfix_builder/app/builder.py index 1083e8c0..33a32348 100644 --- a/sysom_server/sysom_hotfix_builder/builder.py +++ b/sysom_server/sysom_hotfix_builder/app/builder.py @@ -22,6 +22,7 @@ import redis from cec_base.consumer import Consumer, dispatch_consumer from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer, Producer +from conf.settings import YAML_CONFIG class ServerConnector(): @@ -30,8 +31,9 @@ class ServerConnector(): self.username = username self.password = password self.token = None - self.cec_url = cec + self.cec_url = cec_url self.cec = dispatch_producer(cec_url) + #self.cec = SysomFramework.cec_consumer(cec_topic) self.cec_produce_topic = cec_topic def get_token(self): @@ -70,6 +72,8 @@ class ServerConnector(): if resp.status_code != 200: logger.error("change_building_status : can not connect to the server correctly..") + + def sync_building_log(self, hotfix_id): url = self.server_ip + "/api/v1/hotfix/sync_building_log/" headers = {'Content-Type': "application/json", 'Authorization': self.token} @@ -84,7 +88,7 @@ class ServerConnector(): logger.error("sync_building_log : can not connect to the server correctly..") return resp - def sync_rpm_name(self, hotfix_id, rpm_name): + def sync_rpm_name_api(self, hotfix_id, rpm_name): url = self.server_ip + "/api/v1/hotfix/update_hotfix_name/" headers = {'Content-Type': "application/json", 'Authorization': self.token} cond = {"id":hotfix_id, "rpm":rpm_name} @@ -108,55 +112,64 @@ class ServerConnector(): }) - def sync_rpm_name(self, hotfix_id, rpm_name): + def sync_rpm_name_cec(self, hotfix_id, rpm_name): logger.info("produce rpm_name ") self.cec.produce(self.cec_produce_topic, { "hotfix_id" : hotfix_id, + "type": "rpm", "sync" : True, "rpm_name" : rpm_name }) + def sync_building_log_cec(self, hotfix_id): + logger.info("sync hotfix build log") + self.cec.produce(self.cec_produce_topic, { + "hotfix_id": hotfix_id, + "type": "log", + "sync": True + }) + """ Object : Cache Function : initalize the Redis Cache Object """ class Cache(): - def __init__(self, con : configparser.ConfigParser): - redis_conn = dict(con.items('redis')) - host = redis_conn['host'] - port = redis_conn['port'] + def __init__(self, con : dict): + host = con.redis.host + port = con.redis.port pool = redis.ConnectionPool(host=host, port=port) self.conn = redis.Redis(connection_pool=pool) def get(self,key): return self.conn.get(key) + + def all_keys(self): + return self.conn.keys() - -class HotfixBuilder(): - - def __init__(self, con : configparser.ConfigParser): - # read init parameters from builder.ini - # cec - cec = dict(con.items('cec')) - cec_url = cec['cec_url'] - cec_hotfix_topic = cec['cec_hotfix_topic'] - hotfix_building_status_topic = cec['job_status_topic'] class HotfixBuilder(): - def __init__(self, nfs_dir_home, hotfix_base, cec_url, cec_topic, server_ip, username, password, packages_repo): + def __init__(self, con: dict): cache = Cache(con) self.cache = cache - self.nfs_dir_home = nfs_dir_home - self.hotfix_base = hotfix_base - self.cec_url = cec_url - self.builder_hotfix_package_repo = packages_repo + ####################### + # Server config + ####################### + # if this builder run in local, we should use the local repo directory instead of the nfs directory + if re.search("127.0.0.1", con.server_ip): + self.nfs_dir_home="/usr/local/sysom/server/builder/hotfix" + else: + self.nfs_dir_home = con.builder.nfs_dir_home + + self.hotfix_base = con.builder.hotfix_base + self.cec_url = con.cec.cec_url + self.builder_hotfix_package_repo = con.builder.package_repo self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder") - self.cec_hotfix_topic = "hotfix_job" + self.cec_hotfix_topic = con.cec.hotfix_topic self.local_arch = os.uname().release.split(".")[-1] - self.connector = ServerConnector(server_ip, username, password, cec_url=cec_url, cec_topic=cec_topic) - self.tmpdir="/hotfix_tmpdir" - self.src_rpm_tmpdir="/tmp/hotfix_src" + self.connector = ServerConnector(con.server_ip, con.username, con.password, cec_url=self.cec_url, cec_topic=con.cec.job_status_topic) + self.tmpdir=con.builder.tmpdir + self.src_rpm_tmpdir=con.builder.src_rpm_tmpdir self.hotfix_id = None self.fd = None self.token = self.connector.get_token() @@ -174,20 +187,16 @@ class HotfixBuilder(): self.thread_runner.start() def change_building_status(self, status): - self.connector.change_building_status(self.hotfix_id, status) + self.connector.update_hotfix_building_status(self.hotfix_id, status) def die(self, msg): self.fd.write("%s \n" % msg) self.change_building_status("failed") def prepare_env(self): - # prepare kernel src and kaptch-build - # cmd = "chmod +x check_env.sh && ./check_env.sh -b %s -n %s" % (self.hotfix_base, self.nfs_dir_home) - # with os.popen(cmd) as process: - # output = process.read() # get the img_list image information and pull them based on machine's kernel arch - image_config_file = open(os.path.join(os.getcwd(), "img_list.json")) + image_config_file = open(os.path.join(os.getcwd(), "conf" , "img_list.json")) config_data = json.load(image_config_file) machine_kernel = platform.uname().release arch = machine_kernel.split(".")[-1] @@ -226,13 +235,13 @@ class HotfixBuilder(): os.makedirs(os.path.join(self.builder_hotfix_package_repo, "src_pack")) # copy build_hotfix.sh to BASE - if os.path.exists("./build_hotfix.sh"): - shutil.copy("./build_hotfix.sh", self.hotfix_base) + if os.path.exists(os.path.join(os.getcwd(), "script", "build_hotfix.sh")): + shutil.copy(os.path.join(os.getcwd(), "script", "build_hotfix.sh"), self.hotfix_base) else: logger.error("ERROR: cannot find build_hotfix.sh") - if os.path.exists("./build_rpm.sh"): - shutil.copy("./build_rpm.sh", self.hotfix_base) + if os.path.exists(os.path.join(os.getcwd(), "script", "build_rpm.sh")): + shutil.copy(os.path.join(os.getcwd(), "script", "build_rpm.sh"), self.hotfix_base) else: logger.error("ERROR: cannot find build_rpm.sh") @@ -240,8 +249,9 @@ class HotfixBuilder(): os.system("rm -rf {} && mkdir {} ".format(self.tmpdir, self.tmpdir)) os.system("rm -rf {}/*patch".format(self.hotfix_base)) - def find_build_rpm(self): - directory = os.path.join(self.tmpdir, "rpmbuild") + def find_build_rpm(self, kernel_version): + arch = kernel_version.split(".")[-1] + directory = os.path.join(self.tmpdir, "rpmbuild", "RPMS", arch) rpms = [] for root, dirs, files in os.walk(directory): for eachfile in files: @@ -334,7 +344,7 @@ class HotfixBuilder(): def get_building_image(self, kernel_version): arch = kernel_version.split(".")[-1] - image_list_file = open('./img_list.json') + image_list_file = open(os.path.join(os.getcwd(), "conf" , "img_list.json")) images = json.load(image_list_file) return images[arch]['anolis'] @@ -435,7 +445,6 @@ class HotfixBuilder(): except Exception as e: self.fd.write(str(e)) logger.error(str(e)) - # self.connector.change_building_status(hotfix_id, "failed") description = self.extract_description_from_patch(local_patch) # run the build hotfix script @@ -453,16 +462,14 @@ class HotfixBuilder(): return_code=p.wait() logger.info("The return code is %d" % return_code) - rpm_names = self.find_build_rpm() + rpm_names = self.find_build_rpm(kernel_version) # when finished building, sync the build log #self.connector.sync_building_log(hotfix_id) # if rpm is more than one, upload it one by one for each_rpm in rpm_names: - resp = self.connector.sync_rpm_name(hotfix_id, each_rpm) - if resp.status_code != 200: - self.connector.insert_log_to_server(hotfix_id, "cannot sync rpm package name %s" % each_rpm) + self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) # check the last output if return_code == 0: @@ -586,16 +593,14 @@ class HotfixBuilder(): return_code=p.wait() logger.info("The return code is %d" % return_code) - rpm_names = self.find_build_rpm() + rpm_names = self.find_build_rpm(kernel_version) # when finished building, sync the build log #self.connector.sync_building_log(hotfix_id) # if rpm is more than one, upload it one by one for each_rpm in rpm_names: - resp = self.connector.sync_rpm_name(hotfix_id, each_rpm) - if resp.status_code != 200: - self.connector.insert_log_to_server(hotfix_id, "cannot sync rpm package name %s" % each_rpm) + self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) # check the last output if return_code == 0: @@ -618,6 +623,7 @@ class HotfixBuilder(): if not admin.is_topic_exist(self.cec_hotfix_topic): admin.create_topic(self.cec_hotfix_topic) consumer_id = Consumer.generate_consumer_id() + if self.local_arch == "x86_64": consumer = dispatch_consumer(self.cec_url, self.cec_hotfix_topic, consumer_id=consumer_id, @@ -626,7 +632,6 @@ class HotfixBuilder(): consumer = dispatch_consumer(self.cec_url, self.cec_hotfix_topic, consumer_id=consumer_id, group_id="hotfix_job_group_arm") - for event in consumer: try: parameters = event.value @@ -640,13 +645,13 @@ class HotfixBuilder(): # this operation aims to clear the previous log if rebuild with open(log_file_path, "w") as f: f.write("=========================================================\n") - f.write("==========*******Sysom Hotfix Building System*******==============\n") + f.write("=================Sysom Hotfix Building System===================\n") f.write("=========================================================\n") self.change_building_status("building") # for each run, update the repo - cmd = "chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) + cmd = "echo \"hello\" && chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) with os.popen(cmd) as process: output = process.read() @@ -670,35 +675,11 @@ class HotfixBuilder(): consumer.ack(event) - -if __name__ == "__main__": - config_file = "builder.ini" - - con = configparser.ConfigParser() - con.read(config_file, encoding='utf-8') +class HotfixBuilderMain(): - # cec - cec = dict(con.items('cec')) - cec_url = cec['cec_url'] - hotfix_building_status_topic = cec['job_status_topic'] - - # server config - sysom_server = dict(con.items('sysom_server')) - server_ip = sysom_server['server_ip'] - server_login_account = sysom_server['account'] - server_login_password = sysom_server['password'] - - # builder config - builder = dict(con.items('builder')) - hotfix_base = builder['hotfix_base'] - nfs_dir_home = builder['nfs_dir_home'] - builder_hotfix_package_repo = builder['package_repo'] + def __init__(self, config: YAML_CONFIG) -> None: + self.parameters = config.get_service_config() - - # if this builder run in local, we should use the local repo directory instead of the nfs directory - if re.search("127.0.0.1", server_ip): - nfs_dir_home="/usr/local/sysom/server/builder/hotfix" - - hotfix_builder = HotfixBuilder(nfs_dir_home, hotfix_base, cec_url, hotfix_building_status_topic, - server_ip, server_login_account, server_login_password, builder_hotfix_package_repo) - hotfix_builder.run() \ No newline at end of file + def start(self): + hotfix_builder = HotfixBuilder(self.parameters) + hotfix_builder.run() diff --git a/sysom_server/sysom_hotfix_builder/app/crud.py b/sysom_server/sysom_hotfix_builder/app/crud.py new file mode 100644 index 00000000..6e0e26c3 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/crud.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File crud.py +Description: +""" +from typing import Optional +from sqlalchemy.orm import Session +from app import models, schemas + +################################################################################################ +# Define database crud here +################################################################################################ + +# def get_person_by_name(db: Session, name: str) -> Optional[models.Person]: +# return db.query(models.Person).filter(models.Person.name == name).first() + +# def create_person(db: Session, person: schemas.Person) -> models.Person: +# person = models.Person(**person.dict()) +# db.add(person) +# db.commit() +# db.refresh(person) +# return person + +# def del_person_by_id(db: Session, person_id: int): +# person = db.get(models.Person, person_id) +# db.delete(person) +# db.commit() diff --git a/sysom_server/sysom_hotfix_builder/app/database.py b/sysom_server/sysom_hotfix_builder/app/database.py new file mode 100644 index 00000000..d1500828 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/database.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File database.py +Description: +""" +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from conf.settings import SQLALCHEMY_DATABASE_URL + +engine = create_engine( + SQLALCHEMY_DATABASE_URL, connect_args={} +) + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/app/models.py b/sysom_server/sysom_hotfix_builder/app/models.py new file mode 100644 index 00000000..60fa74d4 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/models.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File models.py +Description: +""" +from sqlalchemy import Column, Integer, String +from app.database import Base + + +########################################################################### +# Define databse model here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/sql-databases/ +# class Person(Base): +# __tablename__ = "sys_person" +# id = Column(Integer, primary_key=True) +# name = Column(String(254), unique=True) +# age = Column(Integer) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/app/routers/health.py b/sysom_server/sysom_hotfix_builder/app/routers/health.py new file mode 100644 index 00000000..9b6f43fc --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/routers/health.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/04/17 19:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get("/check") +async def get_channel_config(): + return { + "code": 0, + "err_msg": "", + "data": "" + } diff --git a/sysom_server/sysom_hotfix_builder/app/schemas.py b/sysom_server/sysom_hotfix_builder/app/schemas.py new file mode 100644 index 00000000..aa413a9d --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/schemas.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File schemas.py +Description: +""" +from pydantic import BaseModel + +########################################################################### +# Define schemas here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/response-model/ +# class Person(BaseModel): +# id: int +# name: str +# age: int + +# class Config: +# orm_mode = True \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/builder.ini b/sysom_server/sysom_hotfix_builder/builder.ini deleted file mode 100644 index 02aeff21..00000000 --- a/sysom_server/sysom_hotfix_builder/builder.ini +++ /dev/null @@ -1,19 +0,0 @@ -[sysom_server] -server_ip = http://127.0.0.1 -account = account -password = password - -[cec] -cec_url = redis://127.0.0.1:6379 -job_status_topic = hotfix_building_status - -[builder] -hotfix_base = /hotfix_build/hotfix -nfs_dir_home = /usr/local/sysom/server/builder/hotfix -package_repo = /hotfix/packages -tmpdir = /hotfix_tmpdir -src_rpm_tmpdir = /tmp/hotfix_src - -[redis] -host = localhost -port = 6379 diff --git a/sysom_server/sysom_hotfix_builder/conf/common.py b/sysom_server/sysom_hotfix_builder/conf/common.py new file mode 100644 index 00000000..1b38077e --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/common.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" +from pathlib import Path +from sysom_utils import ConfigParser, SysomFramework + +BASE_DIR = Path(__file__).resolve().parent.parent + +################################################################## +# Load yaml config first +################################################################## +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" + +YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) + +mysql_config = YAML_CONFIG.get_server_config().db.mysql +service_config = YAML_CONFIG.get_service_config() + +SysomFramework.init(YAML_CONFIG) + +################################################################## +# fastapi config +################################################################## +SQLALCHEMY_DATABASE_URL = ( + f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" + f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" +) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/develop.py b/sysom_server/sysom_hotfix_builder/conf/develop.py new file mode 100644 index 00000000..bb64bf49 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/develop.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File develoop.py +Description: +""" +from .common import * + +''' +开发环境配置项 +''' + +DEBUG = True \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/gunicorn.py b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py new file mode 100644 index 00000000..ed32b81d --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py @@ -0,0 +1,23 @@ +''' +Channel Service Gunicorn Settings +''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + +workers = 2 # 指定工作进程数 + +threads = 3 + +bind = f'{bind}:{port}' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/sysom-hotfix_builder-access.log' + +loglevel = 'error' + +proc_name = 'channel_service' diff --git a/sysom_server/sysom_hotfix_builder/img_list.json b/sysom_server/sysom_hotfix_builder/conf/img_list.json similarity index 100% rename from sysom_server/sysom_hotfix_builder/img_list.json rename to sysom_server/sysom_hotfix_builder/conf/img_list.json diff --git a/sysom_server/sysom_hotfix_builder/conf/product.py b/sysom_server/sysom_hotfix_builder/conf/product.py new file mode 100644 index 00000000..c748de1d --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/product.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File product.py +Description: +""" +from .common import * + +''' +生产环境配置项 +''' + +DEBUG = False diff --git a/sysom_server/sysom_hotfix_builder/conf/settings.py b/sysom_server/sysom_hotfix_builder/conf/settings.py new file mode 100644 index 00000000..b9156df0 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/settings.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File settings.py +Description: +""" +import os + +env = os.environ.get("env", "product") + + +if env == "develop": + from .develop import * +elif env == "testing": + from .testing import * +elif env == "product": + from .product import * \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/testing.py b/sysom_server/sysom_hotfix_builder/conf/testing.py new file mode 100644 index 00000000..864f53a2 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/testing.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File testing.py +Description: +""" +from .common import * + +''' +测试环境配置项 +''' +DEBUG = True diff --git a/sysom_server/sysom_hotfix_builder/config.yml b/sysom_server/sysom_hotfix_builder/config.yml new file mode 100644 index 00000000..e59fbea9 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/config.yml @@ -0,0 +1,54 @@ +vars: + SERVICE_NAME: &SERVICE_NAME sysom_hotfix_builder + SERVICE_CONSUMER_GROUP: + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] + +sysom_server: + cec: + consumer_group: SYSOM_CEC_CHANNEL_CONSUMER_GROUP + +sysom_service: + service_name: *SERVICE_NAME + service_dir: *SERVICE_NAME + protocol: http + host: 127.0.0.1 + bind: 127.0.0.1 + port: 7011 + server_ip: http://127.0.0.1 + account: admin + password: 123456 + framework: + gcache: + protocol: redis + node_dispatch: + cmg: + tags: + - hotfix_builder + - FastApi + # Metadata of service + metadata: + check: + type: http + url: "/api/v1/hotfix_builder/health/check" + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false + cec: + cec_url: "redis://127.0.0.1:6379" + port: 6379 + job_status_topic: "hotfix_msg" + hotfix_topic: "hotfix_job" + + builder: + hotfix_base: "/hotfix_build/hotfix" + nfs_dir_home: "/usr/local/sysom/server/builder/hotfix" + package_repo: "/hotfix/packages" + tmpdir: "/hotfix_tmpdir" + src_rpm_tmpdir: "/tmp/hotfix_src" + + redis: + host: "127.0.0.1" + port: 6379 + diff --git a/sysom_server/sysom_hotfix_builder/lib/README.md b/sysom_server/sysom_hotfix_builder/lib/README.md new file mode 100644 index 00000000..3ec74424 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/lib/README.md @@ -0,0 +1 @@ +The current directory holds the public libraries or utils needed for microservices \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/main.py b/sysom_server/sysom_hotfix_builder/main.py new file mode 100644 index 00000000..62dcb404 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/main.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File ssh.py +Description: +""" +from clogger import logger +from fastapi import FastAPI +import app.routers.health as health +from app.builder import HotfixBuilderMain +from conf.settings import YAML_CONFIG +from sysom_utils import CmgPlugin, SysomFramework + + +app = FastAPI() + +app.include_router(health.router, prefix="/api/v1/hotfix_builder/health") + + +############################################################################# +# Write your API interface here, or add to app/routes +############################################################################# + + +def init_framwork(): + SysomFramework\ + .init(YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() + logger.info("SysomFramework init finished!") + + +@app.on_event("startup") +async def on_start(): + init_framwork() + ############################################################################# + # Perform some microservice initialization operations over here + ############################################################################# + hotfix_builder_obj = HotfixBuilderMain(YAML_CONFIG) + hotfix_builder_obj.start() + + +@app.on_event("shutdown") +async def on_shutdown(): + pass \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/build_hotfix.sh b/sysom_server/sysom_hotfix_builder/script/build_hotfix.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/build_hotfix.sh rename to sysom_server/sysom_hotfix_builder/script/build_hotfix.sh diff --git a/sysom_server/sysom_hotfix_builder/build_rpm.sh b/sysom_server/sysom_hotfix_builder/script/build_rpm.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/build_rpm.sh rename to sysom_server/sysom_hotfix_builder/script/build_rpm.sh diff --git a/sysom_server/sysom_hotfix_builder/check_env.sh b/sysom_server/sysom_hotfix_builder/script/check_env.sh similarity index 91% rename from sysom_server/sysom_hotfix_builder/check_env.sh rename to sysom_server/sysom_hotfix_builder/script/check_env.sh index 6441ca0f..d7177638 100755 --- a/sysom_server/sysom_hotfix_builder/check_env.sh +++ b/sysom_server/sysom_hotfix_builder/script/check_env.sh @@ -5,15 +5,19 @@ usage() { echo "./check_env.sh [OPTIONS]"; echo ""; echo "Options:"; - echo " -p|--patch"; - echo " The patch (must not null)"; + echo " -b|--hotfix_base"; + echo " The hotfix running base directory"; + echo " -n|--nfs"; + echo " Select the NFS directory home"; + echo " -l|--log"; + echo " Select the log file path"; echo " -k|--kernelversion"; echo " The complete kernel version include architecture (must not null)"; echo " -h|--help"; echo " For help"; echo ""; echo "For example:"; - echo "./kpatch-packager.sh -p \${patch} -k \${kernel} -d \"\${description}\" -v --prefix=\${prefix_patch}"; + echo "./check_env.sh -k {kernel_version} -b {hotfix_base_directory} -n {nfs_directory_home} -l {log_file_path}"; } # this is to output the error msg diff --git a/sysom_web/yarn.lock b/sysom_web/yarn.lock index e7d2ee39..5bae1478 100644 --- a/sysom_web/yarn.lock +++ b/sysom_web/yarn.lock @@ -13961,7 +13961,7 @@ urix@^0.1.0: resolved "https://registry.npmmirror.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg== -url-parse@^1.5.10: +url-parse@^1.5.10, url-parse@^1.5.3: version "1.5.10" resolved "https://mirrors.cloud.tencent.com/npm/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== @@ -13969,14 +13969,6 @@ url-parse@^1.5.10: querystringify "^2.1.1" requires-port "^1.0.0" -url-parse@^1.5.3: - version "1.5.10" - resolved "https://registry.npmmirror.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" - integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== - dependencies: - querystringify "^2.1.1" - requires-port "^1.0.0" - url@^0.11.0: version "0.11.0" resolved "https://mirrors.cloud.tencent.com/npm/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" -- Gitee From d2b77181a9da45f407e161bcb378668e15057dde Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Mon, 19 Jun 2023 17:05:00 +0800 Subject: [PATCH 2/4] fix(rpm): Missing add /etc/sysom to %files --- tools/deploy/sysom.spec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tools/deploy/sysom.spec b/tools/deploy/sysom.spec index d38d5dbe..64273c81 100755 --- a/tools/deploy/sysom.spec +++ b/tools/deploy/sysom.spec @@ -154,7 +154,8 @@ cp -a script %{buildroot}/usr/local/sysom/init_scripts cp -a sysom_web/dist %{buildroot}/usr/local/sysom/web cp -a deps %{buildroot}/usr/local/sysom/deps cp -a environment %{buildroot}/usr/local/sysom/environment -cp -a conf/config.yml %{buildroot}/etc/sysom/config.yml +mkdir -p %{buildroot}/etc +cp -a conf %{buildroot}/etc/sysom #mkdir -p %{buildroot}/usr/local/sysom/server/redis # cp -a sysom_server %{buildroot}/usr/local/sysom/server/target @@ -166,6 +167,7 @@ cp -a conf/config.yml %{buildroot}/etc/sysom/config.yml %files /usr/local/sysom/ +/etc/sysom %post # bash +x /usr/local/sysom/init_scripts/sysom.sh install ALL -- Gitee From fbedb3d4bdb298da92c343e9081b0606327d2092 Mon Sep 17 00:00:00 2001 From: SunnyQjm Date: Tue, 20 Jun 2023 16:45:27 +0800 Subject: [PATCH 3/4] fix(rpm): rpm deployment error --- deps/3_prometheus/install.sh | 1 + deps/4_grafana/install.sh | 1 + environment/1_sdk/sysom_utils/config_parser.py | 2 ++ script/server/sysom_migration/init.sh | 2 +- script/server/sysom_monitor_server/db_migrate.sh | 4 +++- 5 files changed, 8 insertions(+), 2 deletions(-) diff --git a/deps/3_prometheus/install.sh b/deps/3_prometheus/install.sh index 76886c07..06321546 100644 --- a/deps/3_prometheus/install.sh +++ b/deps/3_prometheus/install.sh @@ -24,6 +24,7 @@ install_prometheus() ls | grep ${PROMETHEUS_TAR} 1>/dev/null 2>/dev/null if [ $? -ne 0 ] then + rpm -q --quiet wget || yum install -y wget wget ${OSS_URL}/${PROMETHEUS_TAR} || wget ${PROMETHEUS_DL_URL}/${PROMETHEUS_TAR} ls fi diff --git a/deps/4_grafana/install.sh b/deps/4_grafana/install.sh index 2b24b55f..53a89b8b 100644 --- a/deps/4_grafana/install.sh +++ b/deps/4_grafana/install.sh @@ -14,6 +14,7 @@ install_grafana() ls | grep ${GRAFANA_PKG} 1>/dev/null 2>/dev/null if [ $? -ne 0 ] then + rpm -q --quiet wget || yum install -y wget wget ${OSS_URL}/${GRAFANA_PKG} || wget ${GRAFANA_DL_URL}/${GRAFANA_PKG} ls fi diff --git a/environment/1_sdk/sysom_utils/config_parser.py b/environment/1_sdk/sysom_utils/config_parser.py index 030dd984..69845937 100644 --- a/environment/1_sdk/sysom_utils/config_parser.py +++ b/environment/1_sdk/sysom_utils/config_parser.py @@ -25,6 +25,8 @@ def dict_merge(dct: dict, merge_dct: dict): Return: None """ + if dct is None or merge_dct is None: + return for k, v in merge_dct.items(): if (k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], dict)): # noqa dict_merge(dct[k], merge_dct[k]) diff --git a/script/server/sysom_migration/init.sh b/script/server/sysom_migration/init.sh index d9157ed1..1d9796e0 100644 --- a/script/server/sysom_migration/init.sh +++ b/script/server/sysom_migration/init.sh @@ -1,7 +1,7 @@ #!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME -SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) BaseDir=$(dirname $(readlink -f "$0")) SERVICE_NAME=sysom-migration diff --git a/script/server/sysom_monitor_server/db_migrate.sh b/script/server/sysom_monitor_server/db_migrate.sh index f530e1f8..ac7881c3 100644 --- a/script/server/sysom_monitor_server/db_migrate.sh +++ b/script/server/sysom_monitor_server/db_migrate.sh @@ -10,7 +10,9 @@ source_virtualenv() { db_migrate() { pushd ${SERVICE_HOME} - alembic upgrade head + if [ -d "alembic" ]; then + alembic upgrade head + fi popd } -- Gitee From fa95986aedf04ce0977ba19aef8decd81a1312a4 Mon Sep 17 00:00:00 2001 From: ydzhang Date: Wed, 21 Jun 2023 15:05:21 +0800 Subject: [PATCH 4/4] change building state update method This commit contains: 1. Delete all the API communication with server. Use cec communication instead. 2. Add cec consumer crash recovery. Under the situation that redis is crash, the consumer process will exit. This commit add max_retry_time to sleep for a while, and reload the cec consumer again. 3. Config the work process from 2 to 1, in gunicorn.py remove comment --- .../server/sysom_hotfix_builder/db_migrate.sh | 4 +- script/server/sysom_hotfix_builder/init.sh | 2 +- sysom_server/sysom_hotfix/apps/hotfix/apps.py | 2 +- .../sysom_hotfix/apps/hotfix/views.py | 3 +- sysom_server/sysom_hotfix/config.yml | 3 + sysom_server/sysom_hotfix/lib/function.py | 53 +++-- .../sysom_hotfix_builder/app/builder.py | 197 ++++++------------ .../sysom_hotfix_builder/conf/gunicorn.py | 2 +- sysom_server/sysom_hotfix_builder/config.yml | 7 +- 9 files changed, 110 insertions(+), 163 deletions(-) diff --git a/script/server/sysom_hotfix_builder/db_migrate.sh b/script/server/sysom_hotfix_builder/db_migrate.sh index f530e1f8..f7007941 100644 --- a/script/server/sysom_hotfix_builder/db_migrate.sh +++ b/script/server/sysom_hotfix_builder/db_migrate.sh @@ -10,7 +10,9 @@ source_virtualenv() { db_migrate() { pushd ${SERVICE_HOME} - alembic upgrade head + if [ -d "alembic/versions" ]; then + alembic upgrade head + fi popd } diff --git a/script/server/sysom_hotfix_builder/init.sh b/script/server/sysom_hotfix_builder/init.sh index bfc69a7b..de20386a 100644 --- a/script/server/sysom_hotfix_builder/init.sh +++ b/script/server/sysom_hotfix_builder/init.sh @@ -14,7 +14,7 @@ mount_nfs() result=$(systemctl is-active nfs-server) if [[ $result =~ "inactive" ]]; then - systemctl restart nfs-server + systemctl start nfs-server fi sudo umount $LOCAL_NFS_HOME # Remove the mounted directory in case it mounted before diff --git a/sysom_server/sysom_hotfix/apps/hotfix/apps.py b/sysom_server/sysom_hotfix/apps/hotfix/apps.py index f97295cf..1542c435 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/apps.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/apps.py @@ -29,7 +29,7 @@ class HotfixConfig(AppConfig): # create CECListerner object for cec message listening from lib.function import CECListener - ceclistener = CECListener(settings.SYSOM_CEC_URL, settings.SYSOM_CEC_HOTFIX_SERVER_MSG_TOPIC) + ceclistener = CECListener(settings.YAML_CONFIG.get_service_config(), settings.SYSOM_CEC_URL, settings.SYSOM_CEC_HOTFIX_SERVER_MSG_TOPIC) if ceclistener is None: logger.error("INIT CECListener Failed...") ceclistener.run() diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index 583d8f4f..5d490ad2 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -122,7 +122,7 @@ class HotfixAPIView(GenericViewSet, log_file = "{}-{}.log".format(request.data["hotfix_name"], time.strftime("%Y%m%d%H%M%S")) hotfix_name = request.data["hotfix_name"] hotfix_name.replace(" ","-") - kernel_version = request.data['kernel_version'] + kernel_version = request.data['kernel_version'].replace(" ","") patch_file_name = request.data['patch_file_name'] # check if this kernel_version is customize @@ -281,7 +281,6 @@ class HotfixAPIView(GenericViewSet, def update_building_status(self, request): hotfix = HotfixModel.objects.filter(id=request.data["id"]).first() status = request.data["status"] - print(request.data) if hotfix is None: return other_response(message="No such hotfix id", result={"mgs":"update building status failed"}, code=400) if status == "waiting": diff --git a/sysom_server/sysom_hotfix/config.yml b/sysom_server/sysom_hotfix/config.yml index 08d3e700..54a1e154 100644 --- a/sysom_server/sysom_hotfix/config.yml +++ b/sysom_server/sysom_hotfix/config.yml @@ -36,3 +36,6 @@ sysom_service: deregister: 25 header: tls_skip_verify: false + cec: + max_retry_time: 10 # max retry the recovery of listener + sleep_time: 5 # second \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/lib/function.py b/sysom_server/sysom_hotfix/lib/function.py index bf2c32a1..95f27546 100644 --- a/sysom_server/sysom_hotfix/lib/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -9,6 +9,7 @@ import threading import redis import requests import json +import time from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel from cec_base.producer import dispatch_producer, Producer @@ -25,7 +26,6 @@ This class contains the support/tool function class FunctionClass(): def __init__(self): - #self.cec = dispatch_producer(settings.SYSOM_CEC_URL) self.producer = SysomFramework.cec_producer() def delete_document(self, doc_path, doc_name): @@ -454,14 +454,15 @@ class FunctionClass(): class CECListener(): - def __init__(self, cec_url, listen_topic) -> None: + def __init__(self, con, cec_url, listen_topic) -> None: try: logger.info("Server CECListener init ...") - print(">>>>>>>>>>>>>>>>init CECListener....") + self.parameters = con self.cec_url = cec_url self.listen_topic = listen_topic self.sync_key = "sync" # this key is to tag this message for sync job self.rpm_key = "rpm_name" # this key is to tag this message for sync rpm name + self.log_key = "log" # this key is to tag this message for sync log self.thread_runner = threading.Thread(target=self.listener, name="hotfix_server_listener") except Exception as e: return None @@ -482,20 +483,28 @@ class CECListener(): group_id="server_listener") logger.info("Server CECListener init finished...") - for event in self.consumer: - try: - parameters = event.value - hotfix_id = parameters["hotfix_id"] - if self.sync_key in parameters: - if self.rpm_key in parameters: - self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) - else: - self.update_hotfix_job_status(hotfix_id, parameters["status"]) - except Exception as e: - logger.error(str(e)) - finally: - logger.info("ack one msg from builder...") - self.consumer.ack(event=event) + + retry_time = 0 + while retry_time < self.parameters.cec.max_retry_time: + for event in self.consumer: + logger.info("processing one msg...") + try: + parameters = event.value + hotfix_id = parameters["hotfix_id"] + if self.sync_key in parameters: + if self.rpm_key in parameters: + self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) + if self.log_key in parameters: + self.sync_hotfix_log(hotfix_id) + else: + self.update_hotfix_job_status(hotfix_id, parameters["status"]) + except Exception as e: + logger.error(str(e)) + finally: + logger.info("ack one msg from builder...") + self.consumer.ack(event=event) + time.sleep(self.parameters.cec.sleep_time) + retry_time += 1 def update_hotfix_job_status(self, hotfix_id, status): hotfix = HotfixModel.objects.filter(id=hotfix_id).first() @@ -527,6 +536,16 @@ class CECListener(): logger.error("%s : Exception raised..." % sys._getframe().f_code.co_name) return None + def sync_hotfix_log(self, hotfix_id): + hotfix = HotfixModel.objects.filter(id=hotfix_id).first() + try: + log = "" + for line in open(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): + log = log + str(line) + hotfix.log = log + hotfix.save() + except Exception as e: + logger.error(str(e)) class RedisCache(): def __init__(self): diff --git a/sysom_server/sysom_hotfix_builder/app/builder.py b/sysom_server/sysom_hotfix_builder/app/builder.py index 33a32348..45e77a26 100644 --- a/sysom_server/sysom_hotfix_builder/app/builder.py +++ b/sysom_server/sysom_hotfix_builder/app/builder.py @@ -19,89 +19,20 @@ import sys import subprocess import configparser import redis +import time from cec_base.consumer import Consumer, dispatch_consumer from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer, Producer from conf.settings import YAML_CONFIG +from sysom_utils import CecTarget, SysomFramework class ServerConnector(): - def __init__(self, server_ip, username, password, cec_url=None, cec_topic=None): - self.server_ip = server_ip - self.username = username - self.password = password - self.token = None - self.cec_url = cec_url - self.cec = dispatch_producer(cec_url) + def __init__(self, cec_topic=None): + self.cec = dispatch_producer(YAML_CONFIG.get_cec_url(CecTarget.PRODUCER)) #self.cec = SysomFramework.cec_consumer(cec_topic) self.cec_produce_topic = cec_topic - def get_token(self): - url = self.server_ip + "/api/v1/auth/" - headers = {"Content-Type":"application/json"} - cond = {"username":self.username, "password":self.password} - resp = requests.post(url, data=json.dumps(cond), headers = headers) - data = resp.json().get('data') ## it is a dict - token = data.get("token") - self.token = token - return token - - def insert_log_to_server(self, hotfix_id, logs): - url = self.server_ip + "/api/v1/hotfix/insert_building_log/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "log" : logs} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'content-type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("insert_log_to_server : can not connect to the server correctly..") - - def change_building_status(self, hotfix_id, status): - url = self.server_ip + "/api/v1/hotfix/update_building_status/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "status" : status} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("change_building_status : can not connect to the server correctly..") - - - - def sync_building_log(self, hotfix_id): - url = self.server_ip + "/api/v1/hotfix/sync_building_log/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("sync_building_log : can not connect to the server correctly..") - return resp - - def sync_rpm_name_api(self, hotfix_id, rpm_name): - url = self.server_ip + "/api/v1/hotfix/update_hotfix_name/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "rpm":rpm_name} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("sync_rpm_name : can not connect to the server correctly..") - return resp - # Here, we design to use some status more than building status # in building status , we include : waiting\building\success\failed # but more, we can send : sync to server for log and hotfix name sync @@ -155,24 +86,19 @@ class HotfixBuilder(): ####################### # Server config ####################### - # if this builder run in local, we should use the local repo directory instead of the nfs directory - if re.search("127.0.0.1", con.server_ip): - self.nfs_dir_home="/usr/local/sysom/server/builder/hotfix" - else: - self.nfs_dir_home = con.builder.nfs_dir_home - + self.nfs_dir_home = con.builder.nfs_dir_home + self.max_retry_time = con.cec.max_retry + self.sleep_time = con.cec.retry_sleep_time self.hotfix_base = con.builder.hotfix_base - self.cec_url = con.cec.cec_url self.builder_hotfix_package_repo = con.builder.package_repo - self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder") + self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder", daemon=True) self.cec_hotfix_topic = con.cec.hotfix_topic self.local_arch = os.uname().release.split(".")[-1] - self.connector = ServerConnector(con.server_ip, con.username, con.password, cec_url=self.cec_url, cec_topic=con.cec.job_status_topic) + self.connector = ServerConnector(cec_topic=con.cec.job_status_topic) self.tmpdir=con.builder.tmpdir self.src_rpm_tmpdir=con.builder.src_rpm_tmpdir self.hotfix_id = None self.fd = None - self.token = self.connector.get_token() self.prepare_env() ################################################################## @@ -464,19 +390,18 @@ class HotfixBuilder(): rpm_names = self.find_build_rpm(kernel_version) - # when finished building, sync the build log - #self.connector.sync_building_log(hotfix_id) - # if rpm is more than one, upload it one by one for each_rpm in rpm_names: self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) + self.connector.sync_building_log_cec(self.hotfix_id) + # check the last output if return_code == 0: - self.connector.change_building_status(hotfix_id, "success") + self.change_building_status("success") else: os.system("echo \"BUILD FAILED\" >> %s" % log_file_path) - self.connector.change_building_status(hotfix_id, "failed") + self.change_building_status("failed") """ build the customize kernel from user defined @@ -602,12 +527,13 @@ class HotfixBuilder(): for each_rpm in rpm_names: self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) + self.connector.sync_building_log_cec(self.hotfix_id) # check the last output if return_code == 0: - self.connector.change_building_status(hotfix_id, "success") + self.change_building_status("success") else: os.system("echo \"BUILD FAILED\" >> %s" % log_file_path) - self.connector.change_building_status(hotfix_id, "failed") + self.change_building_status("failed") ''' Each event is an object, the parameter is inside event.value @@ -619,60 +545,61 @@ class HotfixBuilder(): ''' def build(self): - with dispatch_admin(self.cec_url) as admin: - if not admin.is_topic_exist(self.cec_hotfix_topic): - admin.create_topic(self.cec_hotfix_topic) consumer_id = Consumer.generate_consumer_id() if self.local_arch == "x86_64": - consumer = dispatch_consumer(self.cec_url, self.cec_hotfix_topic, + consumer = SysomFramework.cec_consumer(self.cec_hotfix_topic, consumer_id=consumer_id, group_id="hotfix_job_group_x86") else: - consumer = dispatch_consumer(self.cec_url, self.cec_hotfix_topic, + consumer = SysomFramework.cec_consumer(self.cec_hotfix_topic, consumer_id=consumer_id, group_id="hotfix_job_group_arm") - for event in consumer: - try: - parameters = event.value - log_file = parameters['log_file'] - self.hotfix_id = parameters['hotfix_id'] - log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) - is_deleted = self.cache.get(str(self.hotfix_id)) - # if this hotfix_id is not exist in the deleted pool - if not is_deleted: - if parameters['arch'] == self.local_arch: - # this operation aims to clear the previous log if rebuild - with open(log_file_path, "w") as f: - f.write("=========================================================\n") - f.write("=================Sysom Hotfix Building System===================\n") - f.write("=========================================================\n") - - self.change_building_status("building") - - # for each run, update the repo - cmd = "echo \"hello\" && chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) - with os.popen(cmd) as process: - output = process.read() - - customize = parameters['customize'] - # before build one job, clear the tmpdir - self.clear_tmpdir() - if not customize: - self.build_supported_kernel(parameters) - else: - self.build_customize_kernel(parameters) - logger.info(log_file) + retry_time = 0 + while retry_time < self.max_retry_time: + for event in consumer: + try: + parameters = event.value + log_file = parameters['log_file'] + self.hotfix_id = parameters['hotfix_id'] + log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) + is_deleted = self.cache.get(str(self.hotfix_id)) + # if this hotfix_id is not exist in the deleted pool + if not is_deleted: + if parameters['arch'] == self.local_arch: + # this operation aims to clear the previous log if rebuild + with open(log_file_path, "w") as f: + f.write("=========================================================\n") + f.write("=================Sysom Hotfix Building System===================\n") + f.write("=========================================================\n") + + self.change_building_status("building") + + # for each run, update the repo + cmd = "echo \"hello\" && chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) + with os.popen(cmd) as process: + output = process.read() + + customize = parameters['customize'] + # before build one job, clear the tmpdir + self.clear_tmpdir() + if not customize: + self.build_supported_kernel(parameters) + else: + self.build_customize_kernel(parameters) + logger.info(log_file) + self.fd.close() + else: + logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id) + except Exception as e: + logger.error(str(e)) + self.change_building_status("failed") + finally: + if self.fd is not None: self.fd.close() - else: - logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id) - except Exception as e: - logger.error(str(e)) - self.connector.change_building_status(self.hotfix_id, "failed") - finally: - if self.fd is not None: - self.fd.close() - consumer.ack(event) + consumer.ack(event) + time.sleep(self.sleep_time) + retry_time += 1 class HotfixBuilderMain(): diff --git a/sysom_server/sysom_hotfix_builder/conf/gunicorn.py b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py index ed32b81d..c835217f 100644 --- a/sysom_server/sysom_hotfix_builder/conf/gunicorn.py +++ b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py @@ -6,7 +6,7 @@ from conf.common import YAML_CONFIG bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") port = YAML_CONFIG.get_service_config().get("port", "80") -workers = 2 # 指定工作进程数 +workers = 1 # 指定工作进程数, builder应该只开一个工作进程 threads = 3 diff --git a/sysom_server/sysom_hotfix_builder/config.yml b/sysom_server/sysom_hotfix_builder/config.yml index e59fbea9..03efc403 100644 --- a/sysom_server/sysom_hotfix_builder/config.yml +++ b/sysom_server/sysom_hotfix_builder/config.yml @@ -14,9 +14,6 @@ sysom_service: host: 127.0.0.1 bind: 127.0.0.1 port: 7011 - server_ip: http://127.0.0.1 - account: admin - password: 123456 framework: gcache: protocol: redis @@ -36,10 +33,10 @@ sysom_service: header: tls_skip_verify: false cec: - cec_url: "redis://127.0.0.1:6379" - port: 6379 job_status_topic: "hotfix_msg" hotfix_topic: "hotfix_job" + max_retry: 10 # retry consumer listen time + retry_sleep_time: 5 # second builder: hotfix_base: "/hotfix_build/hotfix" -- Gitee