diff --git a/deps/3_prometheus/install.sh b/deps/3_prometheus/install.sh index 76886c0751be70f17ec5943275404538e4068559..0632154628068de0ef4919e6bddf6176193291c1 100644 --- a/deps/3_prometheus/install.sh +++ b/deps/3_prometheus/install.sh @@ -24,6 +24,7 @@ install_prometheus() ls | grep ${PROMETHEUS_TAR} 1>/dev/null 2>/dev/null if [ $? -ne 0 ] then + rpm -q --quiet wget || yum install -y wget wget ${OSS_URL}/${PROMETHEUS_TAR} || wget ${PROMETHEUS_DL_URL}/${PROMETHEUS_TAR} ls fi diff --git a/deps/4_grafana/install.sh b/deps/4_grafana/install.sh index 2b24b55f41a8c6186a0a0129e3f902e9a1db769b..53a89b8b708e0f03d2e69631acd8238c510803ad 100644 --- a/deps/4_grafana/install.sh +++ b/deps/4_grafana/install.sh @@ -14,6 +14,7 @@ install_grafana() ls | grep ${GRAFANA_PKG} 1>/dev/null 2>/dev/null if [ $? -ne 0 ] then + rpm -q --quiet wget || yum install -y wget wget ${OSS_URL}/${GRAFANA_PKG} || wget ${GRAFANA_DL_URL}/${GRAFANA_PKG} ls fi diff --git a/environment/1_sdk/sysom_utils/config_parser.py b/environment/1_sdk/sysom_utils/config_parser.py index 030dd984bdb8a7a7a9b00981fcf55805bac1e169..6984593775f3da10a19354ba5e7961a1bffb15ed 100644 --- a/environment/1_sdk/sysom_utils/config_parser.py +++ b/environment/1_sdk/sysom_utils/config_parser.py @@ -25,6 +25,8 @@ def dict_merge(dct: dict, merge_dct: dict): Return: None """ + if dct is None or merge_dct is None: + return for k, v in merge_dct.items(): if (k in dct and isinstance(dct[k], dict) and isinstance(merge_dct[k], dict)): # noqa dict_merge(dct[k], merge_dct[k]) diff --git a/script/server/deploy_exclude b/script/server/deploy_exclude index 5e69798f6332449420274b3ded29dc33c4891e68..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 --- a/script/server/deploy_exclude +++ b/script/server/deploy_exclude @@ -1 +0,0 @@ -sysom_hotfix_builder \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/clear.sh b/script/server/sysom_hotfix_builder/clear.sh index 3dd3743b0c7961f24c07133c13c62bb575cb2fa1..28381711911f837a1d4889fc4b36a5a3cf924b7e 100644 --- a/script/server/sysom_hotfix_builder/clear.sh +++ b/script/server/sysom_hotfix_builder/clear.sh @@ -1,11 +1,10 @@ #!/bin/bash BaseDir=$(dirname $(readlink -f "$0")) -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder clear_app() { LOCAL_NFS_HOME=${SERVER_HOME}/builder/hotfix sed -i '/hotfix_builder/d' /etc/exports - result=$(systemctl is-active nfs-server) if [[ $result =~ "inactive" ]]; then systemctl restart nfs-server diff --git a/script/server/sysom_hotfix_builder/db_migrate.sh b/script/server/sysom_hotfix_builder/db_migrate.sh new file mode 100644 index 0000000000000000000000000000000000000000..f700794102271d7a3494ac010b9f8174b2850d5f --- /dev/null +++ b/script/server/sysom_hotfix_builder/db_migrate.sh @@ -0,0 +1,20 @@ +#!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) +SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} +VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME + +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + +db_migrate() { + pushd ${SERVICE_HOME} + if [ -d "alembic/versions" ]; then + alembic upgrade head + fi + popd +} + +source_virtualenv +db_migrate \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/init.sh b/script/server/sysom_hotfix_builder/init.sh index a06a00e92b768f2acc99e378acdd84ff52de3d13..de20386af3889a713688a8fef5e171184c28619b 100644 --- a/script/server/sysom_hotfix_builder/init.sh +++ b/script/server/sysom_hotfix_builder/init.sh @@ -1,16 +1,10 @@ -#! /bin/bash +#!/bin/bash SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} -SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} -VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME BaseDir=$(dirname $(readlink -f "$0")) -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder NFS_SERVER_IP=${SERVER_LOCAL_IP} -init_conf() { - cp ${SERVICE_NAME}.ini /etc/supervisord.d/ -} - mount_nfs() { HOTFIX_NFS_HOME=${SERVER_HOME}/hotfix_builder/hotfix-nfs @@ -18,10 +12,23 @@ mount_nfs() mkdir -p ${LOCAL_NFS_HOME} + result=$(systemctl is-active nfs-server) + if [[ $result =~ "inactive" ]]; then + systemctl start nfs-server + fi + sudo umount $LOCAL_NFS_HOME # Remove the mounted directory in case it mounted before sudo mount -t nfs ${NFS_SERVER_IP}:${HOTFIX_NFS_HOME} ${LOCAL_NFS_HOME} || exit 1 } +init_conf() { + cp ${SERVICE_NAME}.ini /etc/supervisord.d/ + ###change the install dir base on param $1### + sed -i "s;/usr/local/sysom;${APP_HOME};g" /etc/supervisord.d/${SERVICE_NAME}.ini + cpu_num=$(cat /proc/cpuinfo | grep processor | wc -l) + sed -i "s/threads = 3/threads = $cpu_num/g" ${SERVICE_HOME}/conf/gunicorn.py +} + init_app() { init_conf mount_nfs diff --git a/script/server/sysom_hotfix_builder/install.sh b/script/server/sysom_hotfix_builder/install.sh index 050d16fd9d87bf771c18af132013d10c910e4493..b391c67edd271233e6895c1388f1618abc772354 100644 --- a/script/server/sysom_hotfix_builder/install.sh +++ b/script/server/sysom_hotfix_builder/install.sh @@ -3,7 +3,7 @@ SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} SERVICE_SCRIPT_HOME=${MICROSERVICE_SCRIPT_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder if [ "$UID" -ne 0 ]; then echo "Please run as root" @@ -16,17 +16,17 @@ install_requirement() { popd } -source_virtualenv() { - echo "INFO: activate virtualenv..." - source ${VIRTUALENV_HOME}/bin/activate || exit 1 -} - install_package() { rpm -q --quiet make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || yum install -y make gcc patch bison flex openssl-devel elfutils elfutils-devel dwarves || exit 1 rpm -q --quiet docker git || yum install -y docker git || echo "Warngin : Docker is not installed in this machine!" } +source_virtualenv() { + echo "INFO: activate virtualenv..." + source ${VIRTUALENV_HOME}/bin/activate || exit 1 +} + install_app() { rpm -q --quiet gcc || yum install -y gcc rpm -q --quiet make || yum install -y make diff --git a/script/server/sysom_hotfix_builder/requirements.txt b/script/server/sysom_hotfix_builder/requirements.txt index d3d398e708ff7dcd40196aeb4bb956c9f8522940..ee975fdb668c3be7c25b75b196851990f76b43f3 100644 --- a/script/server/sysom_hotfix_builder/requirements.txt +++ b/script/server/sysom_hotfix_builder/requirements.txt @@ -2,4 +2,19 @@ clogger==0.0.1 channel_job>=0.0.1 cec_base>=0.0.1 cec_redis>=0.0.1 +sysom_utils>=0.0.1 +alembic==1.7.7 +anyio==3.6.2 +asyncer==0.0.2 +asyncssh==2.12.0 +fastapi==0.83.0 +PyMySQL==1.0.2 +pyyaml==6.0 +pyyaml-include==1.3 +uvicorn==0.16.0 +gunicorn==20.1.0 +python-multipart==0.0.5 +###################################################################### +# Add your custom python requirements here +###################################################################### requests==2.27.1 \ No newline at end of file diff --git a/script/server/sysom_hotfix_builder/start.sh b/script/server/sysom_hotfix_builder/start.sh index b6464a14fa70b679caeec8bb7bf7ad31b50afb73..4265ea21b8db623888ea1f62a9519f8343dc1126 100644 --- a/script/server/sysom_hotfix_builder/start.sh +++ b/script/server/sysom_hotfix_builder/start.sh @@ -1,10 +1,9 @@ #!/bin/bash -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder is_start() { supervisorctl status ${SERVICE_NAME} - if [ $? -eq 0 ] - then + if [ $? -eq 0 ]; then return 1 else return 0 @@ -13,11 +12,11 @@ is_start() { start_app() { is_start - if [[ $? == 0 ]];then + if [[ $? == 0 ]]; then supervisorctl start $SERVICE_NAME is_start if [[ $? == 0 ]]; then - echo "${SERVICE_NAME} service start fail, please check log" + echo "${SERVICE_NAME} service start fail, please check log" exit 1 else echo "supervisorctl start ${SERVICE_NAME} success..." diff --git a/script/server/sysom_hotfix_builder/stop.sh b/script/server/sysom_hotfix_builder/stop.sh index d653543911b79ebb331bf3262414e70ef46b04e7..5b953a93ea42713bdee88dbb7782753ae0fc5eb0 100644 --- a/script/server/sysom_hotfix_builder/stop.sh +++ b/script/server/sysom_hotfix_builder/stop.sh @@ -1,5 +1,5 @@ #!/bin/bash -SERVICE_NAME=sysom-hotfix-builder +SERVICE_NAME=sysom-hotfix_builder stop_app() { # kill all kpatch-build process diff --git a/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini b/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini deleted file mode 100644 index 3359d39bbb665d379f5f615d679944ab72658d24..0000000000000000000000000000000000000000 --- a/script/server/sysom_hotfix_builder/sysom-hotfix-builder.ini +++ /dev/null @@ -1,9 +0,0 @@ -[program:sysom-hotfix-builder] -directory = /usr/local/sysom/server/sysom_hotfix_builder -command=/usr/local/sysom/environment/virtualenv/bin/python3 builder.py -startsecs=3 -autostart=true -autorestart=true -environment=PATH=/usr/local/sysom/environment/virtualenv/bin:%(ENV_PATH)s -stderr_logfile=/var/log/sysom/sysom-hotfix-builder-error.log -stdout_logfile=/var/log/sysom/sysom-hotfix-builder.log diff --git a/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini b/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini new file mode 100644 index 0000000000000000000000000000000000000000..d1c253592b7bff6c9065e93120d8ccf37d2214c8 --- /dev/null +++ b/script/server/sysom_hotfix_builder/sysom-hotfix_builder.ini @@ -0,0 +1,9 @@ +[program:sysom-hotfix_builder] +directory=/usr/local/sysom/server/sysom_hotfix_builder +command=/usr/local/sysom/environment/virtualenv/bin/gunicorn -c ./conf/gunicorn.py main:app +startsecs=3 +autostart=true +autorestart=true +stderr_logfile=/var/log/sysom/sysom-hotfix_builder-error.log +stdout_logfile=/var/log/sysom/sysom-hotfix_builder.log +environment=PATH=/usr/local/sysom/virtualenv/bin:%(ENV_PATH)s diff --git a/script/server/sysom_migration/init.sh b/script/server/sysom_migration/init.sh index d9157ed1d04495ea5da541e395866f244a1e1f80..1d9796e0a824826eb0dbb8c16dc8423fe3e1c0eb 100644 --- a/script/server/sysom_migration/init.sh +++ b/script/server/sysom_migration/init.sh @@ -1,7 +1,7 @@ #!/bin/bash +SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) SERVICE_HOME=${MICROSERVICE_HOME}/${SERVICE_SCRIPT_DIR} VIRTUALENV_HOME=$GLOBAL_VIRTUALENV_HOME -SERVICE_SCRIPT_DIR=$(basename $(dirname $0)) BaseDir=$(dirname $(readlink -f "$0")) SERVICE_NAME=sysom-migration diff --git a/script/server/sysom_monitor_server/db_migrate.sh b/script/server/sysom_monitor_server/db_migrate.sh index f530e1f82f0e3becd05b158b68263bc37fbd0447..ac7881c3205d066fb0b7cfda9394d5e5bf42fee2 100644 --- a/script/server/sysom_monitor_server/db_migrate.sh +++ b/script/server/sysom_monitor_server/db_migrate.sh @@ -10,7 +10,9 @@ source_virtualenv() { db_migrate() { pushd ${SERVICE_HOME} - alembic upgrade head + if [ -d "alembic" ]; then + alembic upgrade head + fi popd } diff --git a/sysom_server/deploy_exclude b/sysom_server/deploy_exclude index 5e69798f6332449420274b3ded29dc33c4891e68..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 100644 --- a/sysom_server/deploy_exclude +++ b/sysom_server/deploy_exclude @@ -1 +0,0 @@ -sysom_hotfix_builder \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/apps.py b/sysom_server/sysom_hotfix/apps/hotfix/apps.py index 56f2514bff134b29e831280fa277d2ab9a2ee954..1542c43576d95fdc9cb738c91aab557f65fba8c4 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/apps.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/apps.py @@ -26,6 +26,13 @@ class HotfixConfig(AppConfig): .load_plugin_cls(CmgPlugin) \ .enable_channel_job() \ .start() + + # create CECListerner object for cec message listening + from lib.function import CECListener + ceclistener = CECListener(settings.YAML_CONFIG.get_service_config(), settings.SYSOM_CEC_URL, settings.SYSOM_CEC_HOTFIX_SERVER_MSG_TOPIC) + if ceclistener is None: + logger.error("INIT CECListener Failed...") + ceclistener.run() else: # nothing to do when execute database migrations pass diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index b4f89c0316b539424e469f2ce4854ecdedbeab67..5d490ad2c9dc2d8dc008969338548c5fe5178a62 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -33,7 +33,7 @@ from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer from cec_base.event import Event from django.http import HttpResponse, FileResponse -from apps.hotfix.function import FunctionClass, RedisCache +from lib.function import FunctionClass, RedisCache class SaveUploadFile(APIView): authentication_classes = [] @@ -79,8 +79,7 @@ class SaveUploadFile(APIView): patch_file_repo = os.path.join(settings.HOTFIX_FILE_BRANCH_RULE) if not os.path.exists(patch_file_repo): os.makedirs(patch_file_repo) - # patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + str(time.time()).split(".")[0] + "." + file_upload_type - patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + datetime.now().strftime('%Y%m%d%H%M%S-%f') + "." + file_upload_type + patch_file_name = patch_file.name.rstrip("."+file_upload_type) + "-" + datetime.now().strftime('%Y%m%d%H%M%S') + "." + file_upload_type file_path = os.path.join(patch_file_repo, patch_file_name) try: with open(file_path, 'wb') as f: @@ -123,7 +122,7 @@ class HotfixAPIView(GenericViewSet, log_file = "{}-{}.log".format(request.data["hotfix_name"], time.strftime("%Y%m%d%H%M%S")) hotfix_name = request.data["hotfix_name"] hotfix_name.replace(" ","-") - kernel_version = request.data['kernel_version'] + kernel_version = request.data['kernel_version'].replace(" ","") patch_file_name = request.data['patch_file_name'] # check if this kernel_version is customize @@ -191,6 +190,10 @@ class HotfixAPIView(GenericViewSet, if hotfix_object.building_status != self.build_failed: return other_response(message="This Hotfix Job is Not Failed", code=401) + # clear the original build log + hotfix_object.log="" + hotfix_object.save() + try: customize_version_object = KernelVersionModel.objects.all().filter(kernel_version=kernel_version).first() except Exception as e: diff --git a/sysom_server/sysom_hotfix/config.yml b/sysom_server/sysom_hotfix/config.yml index 08d3e700ed39ea7731b7884807f12660a62b3660..54a1e1549985b8af1a24ecb7d807797328ebb58f 100644 --- a/sysom_server/sysom_hotfix/config.yml +++ b/sysom_server/sysom_hotfix/config.yml @@ -36,3 +36,6 @@ sysom_service: deregister: 25 header: tls_skip_verify: false + cec: + max_retry_time: 10 # max retry the recovery of listener + sleep_time: 5 # second \ No newline at end of file diff --git a/sysom_server/sysom_hotfix/apps/hotfix/function.py b/sysom_server/sysom_hotfix/lib/function.py similarity index 91% rename from sysom_server/sysom_hotfix/apps/hotfix/function.py rename to sysom_server/sysom_hotfix/lib/function.py index 8619d41696f85fe7161ec440b36f887b3a3ea3d6..95f2754621540cbfc985ce331d302b7cf0f875fa 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/function.py +++ b/sysom_server/sysom_hotfix/lib/function.py @@ -2,16 +2,22 @@ import os import re import urllib import subprocess +import sys from clogger import logger from django.conf import settings import threading import redis import requests +import json +import time from apps.hotfix.models import HotfixModel, OSTypeModel, KernelVersionModel from cec_base.producer import dispatch_producer, Producer +from cec_base.consumer import Consumer, dispatch_consumer +from cec_base.admin import dispatch_admin from bs4 import BeautifulSoup from lib.utils import human_datetime +from sysom_utils import SysomFramework """ Function class @@ -20,7 +26,7 @@ This class contains the support/tool function class FunctionClass(): def __init__(self): - self.cec = dispatch_producer(settings.SYSOM_CEC_URL) + self.producer = SysomFramework.cec_producer() def delete_document(self, doc_path, doc_name): document = os.path.join(doc_path, doc_name) @@ -133,7 +139,7 @@ class FunctionClass(): try: if not customize: if re.search('anolis', os_type): - self.cec.produce(cec_topic, { + self.producer.produce(cec_topic, { "hotfix_id" : hotfix_id, "kernel_version" : kernel_version, "patch_name" : patch_file, @@ -145,7 +151,6 @@ class FunctionClass(): "git_repo": "git@gitee.com:anolis/cloud-kernel.git", "customize": 0 }) - return True else: # this is customize kernel source_repo = kwargs['source_repo'] @@ -154,7 +159,7 @@ class FunctionClass(): debuginfo_link = kwargs['debuginfo_link'] image = kwargs['image'] is_src_package = kwargs["is_src_package"] - self.cec.produce(cec_topic, { + self.producer.produce(cec_topic, { "hotfix_id" : hotfix_id, "kernel_version" : kernel_version, "hotfix_name" : hotfix_name, @@ -171,6 +176,7 @@ class FunctionClass(): "image": image, "is_src_package": is_src_package }) + self.producer.flush() return True except Exception as e: logger.exception(e) @@ -446,17 +452,17 @@ class FunctionClass(): logger.error(e) - class CECListener(): - def __init__(self, cec_url, listen_topic) -> None: + def __init__(self, con, cec_url, listen_topic) -> None: try: logger.info("Server CECListener init ...") - print(">>>>>>>>>>>>>>>>init CECListener....") + self.parameters = con self.cec_url = cec_url self.listen_topic = listen_topic self.sync_key = "sync" # this key is to tag this message for sync job self.rpm_key = "rpm_name" # this key is to tag this message for sync rpm name + self.log_key = "log" # this key is to tag this message for sync log self.thread_runner = threading.Thread(target=self.listener, name="hotfix_server_listener") except Exception as e: return None @@ -477,21 +483,28 @@ class CECListener(): group_id="server_listener") logger.info("Server CECListener init finished...") - for event in self.consumer: - try: - parameters = event.value - hotfix_id = parameters["hotfix_id"] - if self.sync_key in parameters: - if self.rpm_key in parameters: - - self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) - else: - self.update_hotfix_job_status(hotfix_id, parameters["status"]) - except Exception as e: - logger.error(str(e)) - finally: - logger.info("ack one msg from builder...") - self.consumer.ack(event=event) + + retry_time = 0 + while retry_time < self.parameters.cec.max_retry_time: + for event in self.consumer: + logger.info("processing one msg...") + try: + parameters = event.value + hotfix_id = parameters["hotfix_id"] + if self.sync_key in parameters: + if self.rpm_key in parameters: + self.sync_hotfix_job_rpm_name(hotfix_id, parameters[self.rpm_key]) + if self.log_key in parameters: + self.sync_hotfix_log(hotfix_id) + else: + self.update_hotfix_job_status(hotfix_id, parameters["status"]) + except Exception as e: + logger.error(str(e)) + finally: + logger.info("ack one msg from builder...") + self.consumer.ack(event=event) + time.sleep(self.parameters.cec.sleep_time) + retry_time += 1 def update_hotfix_job_status(self, hotfix_id, status): hotfix = HotfixModel.objects.filter(id=hotfix_id).first() @@ -523,6 +536,16 @@ class CECListener(): logger.error("%s : Exception raised..." % sys._getframe().f_code.co_name) return None + def sync_hotfix_log(self, hotfix_id): + hotfix = HotfixModel.objects.filter(id=hotfix_id).first() + try: + log = "" + for line in open(os.path.join(settings.HOTFIX_FILE_STORAGE_REPO, "log", hotfix.log_file)): + log = log + str(line) + hotfix.log = log + hotfix.save() + except Exception as e: + logger.error(str(e)) class RedisCache(): def __init__(self): diff --git a/sysom_server/sysom_hotfix_builder/alembic.ini b/sysom_server/sysom_hotfix_builder/alembic.ini new file mode 100644 index 0000000000000000000000000000000000000000..f6ab9febcd93add9d0ea8857f857d7f30f1fe48f --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic.ini @@ -0,0 +1,102 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# sys.path path, will be prepended to sys.path if present. +# defaults to the current working directory. +prepend_sys_path = . + +# timezone to use when rendering the date within the migration file +# as well as the filename. +# If specified, requires the python-dateutil library that can be +# installed by adding `alembic[tz]` to the pip requirements +# string value is passed to dateutil.tz.gettz() +# leave blank for localtime +# timezone = + +# max length of characters to apply to the +# "slug" field +# truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +# set to 'true' to allow .pyc and .pyo files without +# a source .py file to be detected as revisions in the +# versions/ directory +# sourceless = false + +# version location specification; This defaults +# to alembic/versions. When using multiple version +# directories, initial revisions must be specified with --version-path. +# The path separator used here should be the separator specified by "version_path_separator" below. +# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions + +# version path separator; As mentioned above, this is the character used to split +# version_locations. The default within new alembic.ini files is "os", which uses os.pathsep. +# If this key is omitted entirely, it falls back to the legacy behavior of splitting on spaces and/or commas. +# Valid values for version_path_separator are: +# +# version_path_separator = : +# version_path_separator = ; +# version_path_separator = space +version_path_separator = os # Use os.pathsep. Default configuration used for new projects. + +# the output encoding used when revision files +# are written from script.py.mako +# output_encoding = utf-8 + +sqlalchemy.url = "" + + +[post_write_hooks] +# post_write_hooks defines scripts or Python functions that are run +# on newly generated revision scripts. See the documentation for further +# detail and examples + +# format using "black" - use the console_scripts runner, against the "black" entrypoint +# hooks = black +# black.type = console_scripts +# black.entrypoint = black +# black.options = -l 79 REVISION_SCRIPT_FILENAME + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/sysom_server/sysom_hotfix_builder/alembic/README b/sysom_server/sysom_hotfix_builder/alembic/README new file mode 100644 index 0000000000000000000000000000000000000000..98e4f9c44effe479ed38c66ba922e7bcc672916f --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration. \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/alembic/env.py b/sysom_server/sysom_hotfix_builder/alembic/env.py new file mode 100644 index 0000000000000000000000000000000000000000..8fcb7206bc6bdc4ac01ce88bb5444dbe87e5a4f3 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/env.py @@ -0,0 +1,110 @@ +import inspect +import app.models as models +from logging.config import fileConfig +from sqlalchemy import engine_from_config +from sqlalchemy import pool +from app.models import Base +from alembic import context +from conf.settings import YAML_CONFIG, SQLALCHEMY_DATABASE_URL + +################################################################## +# Load yaml config first +################################################################## +mysql_config = YAML_CONFIG.get_server_config().db.mysql + +################################################################## +# Scan models +################################################################## +service_tables = [] +for name, data in inspect.getmembers(models, inspect.isclass): + if data.__module__ != "app.models": + continue + if "__tablename__" in data.__dict__: + service_tables.append(data.__dict__["__tablename__"]) + elif "__table__" in data.__dict__: + service_tables.append(data.__dict__["__table__"]) + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Update mysql config according config.yml +config.set_main_option( + "sqlalchemy.url", + SQLALCHEMY_DATABASE_URL +) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +# target_metadata = mymodel.Base.metadata +target_metadata = Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + +def include_object(object, name, type_, reflected, compare_to): + if type_ == "table" and name not in service_tables: + return False + return True + + +def run_migrations_offline(): + """Run migrations in 'offline' mode. + + This configures the context with just a URL + and not an Engine, though an Engine is acceptable + here as well. By skipping the Engine creation + we don't even need a DBAPI to be available. + + Calls to context.execute() here emit the given string to the + script output. + + """ + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + include_object=include_object, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + connectable = engine_from_config( + config.get_section(config.config_ini_section), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure( + connection=connection, target_metadata=target_metadata, + include_object=include_object + ) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/sysom_server/sysom_hotfix_builder/alembic/script.py.mako b/sysom_server/sysom_hotfix_builder/alembic/script.py.mako new file mode 100644 index 0000000000000000000000000000000000000000..2c0156303a8df3ffdc9de87765bf801bf6bea4a5 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/alembic/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} diff --git a/sysom_server/sysom_hotfix_builder/app/__init__.py b/sysom_server/sysom_hotfix_builder/app/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..3fb0576aedb7152ee926ff1e4c4cfeb97be1a11e --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/__init__.py @@ -0,0 +1,8 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File __init__.py +Description: +""" \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/builder.py b/sysom_server/sysom_hotfix_builder/app/builder.py similarity index 69% rename from sysom_server/sysom_hotfix_builder/builder.py rename to sysom_server/sysom_hotfix_builder/app/builder.py index 1083e8c04a97f631c29b472be7cc64cda973cbc3..45e77a26132b4ad5d401229356bdb21ada511fef 100644 --- a/sysom_server/sysom_hotfix_builder/builder.py +++ b/sysom_server/sysom_hotfix_builder/app/builder.py @@ -19,85 +19,20 @@ import sys import subprocess import configparser import redis +import time from cec_base.consumer import Consumer, dispatch_consumer from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer, Producer +from conf.settings import YAML_CONFIG +from sysom_utils import CecTarget, SysomFramework class ServerConnector(): - def __init__(self, server_ip, username, password, cec_url=None, cec_topic=None): - self.server_ip = server_ip - self.username = username - self.password = password - self.token = None - self.cec_url = cec - self.cec = dispatch_producer(cec_url) + def __init__(self, cec_topic=None): + self.cec = dispatch_producer(YAML_CONFIG.get_cec_url(CecTarget.PRODUCER)) + #self.cec = SysomFramework.cec_consumer(cec_topic) self.cec_produce_topic = cec_topic - def get_token(self): - url = self.server_ip + "/api/v1/auth/" - headers = {"Content-Type":"application/json"} - cond = {"username":self.username, "password":self.password} - resp = requests.post(url, data=json.dumps(cond), headers = headers) - data = resp.json().get('data') ## it is a dict - token = data.get("token") - self.token = token - return token - - def insert_log_to_server(self, hotfix_id, logs): - url = self.server_ip + "/api/v1/hotfix/insert_building_log/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "log" : logs} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'content-type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("insert_log_to_server : can not connect to the server correctly..") - - def change_building_status(self, hotfix_id, status): - url = self.server_ip + "/api/v1/hotfix/update_building_status/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "status" : status} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("change_building_status : can not connect to the server correctly..") - - def sync_building_log(self, hotfix_id): - url = self.server_ip + "/api/v1/hotfix/sync_building_log/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("sync_building_log : can not connect to the server correctly..") - return resp - - def sync_rpm_name(self, hotfix_id, rpm_name): - url = self.server_ip + "/api/v1/hotfix/update_hotfix_name/" - headers = {'Content-Type': "application/json", 'Authorization': self.token} - cond = {"id":hotfix_id, "rpm":rpm_name} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code == 403: - # signature has expired, reflash the token - self.get_token() - headers = {'Content-Type': "application/json", 'Authorization': self.token} - resp = requests.post(url, data = json.dumps(cond), headers = headers) - if resp.status_code != 200: - logger.error("sync_rpm_name : can not connect to the server correctly..") - return resp - # Here, we design to use some status more than building status # in building status , we include : waiting\building\success\failed # but more, we can send : sync to server for log and hotfix name sync @@ -108,58 +43,62 @@ class ServerConnector(): }) - def sync_rpm_name(self, hotfix_id, rpm_name): + def sync_rpm_name_cec(self, hotfix_id, rpm_name): logger.info("produce rpm_name ") self.cec.produce(self.cec_produce_topic, { "hotfix_id" : hotfix_id, + "type": "rpm", "sync" : True, "rpm_name" : rpm_name }) + def sync_building_log_cec(self, hotfix_id): + logger.info("sync hotfix build log") + self.cec.produce(self.cec_produce_topic, { + "hotfix_id": hotfix_id, + "type": "log", + "sync": True + }) + """ Object : Cache Function : initalize the Redis Cache Object """ class Cache(): - def __init__(self, con : configparser.ConfigParser): - redis_conn = dict(con.items('redis')) - host = redis_conn['host'] - port = redis_conn['port'] + def __init__(self, con : dict): + host = con.redis.host + port = con.redis.port pool = redis.ConnectionPool(host=host, port=port) self.conn = redis.Redis(connection_pool=pool) def get(self,key): return self.conn.get(key) + + def all_keys(self): + return self.conn.keys() - -class HotfixBuilder(): - - def __init__(self, con : configparser.ConfigParser): - # read init parameters from builder.ini - # cec - cec = dict(con.items('cec')) - cec_url = cec['cec_url'] - cec_hotfix_topic = cec['cec_hotfix_topic'] - hotfix_building_status_topic = cec['job_status_topic'] class HotfixBuilder(): - def __init__(self, nfs_dir_home, hotfix_base, cec_url, cec_topic, server_ip, username, password, packages_repo): + def __init__(self, con: dict): cache = Cache(con) self.cache = cache - self.nfs_dir_home = nfs_dir_home - self.hotfix_base = hotfix_base - self.cec_url = cec_url - self.builder_hotfix_package_repo = packages_repo - self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder") - self.cec_hotfix_topic = "hotfix_job" + ####################### + # Server config + ####################### + self.nfs_dir_home = con.builder.nfs_dir_home + self.max_retry_time = con.cec.max_retry + self.sleep_time = con.cec.retry_sleep_time + self.hotfix_base = con.builder.hotfix_base + self.builder_hotfix_package_repo = con.builder.package_repo + self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder", daemon=True) + self.cec_hotfix_topic = con.cec.hotfix_topic self.local_arch = os.uname().release.split(".")[-1] - self.connector = ServerConnector(server_ip, username, password, cec_url=cec_url, cec_topic=cec_topic) - self.tmpdir="/hotfix_tmpdir" - self.src_rpm_tmpdir="/tmp/hotfix_src" + self.connector = ServerConnector(cec_topic=con.cec.job_status_topic) + self.tmpdir=con.builder.tmpdir + self.src_rpm_tmpdir=con.builder.src_rpm_tmpdir self.hotfix_id = None self.fd = None - self.token = self.connector.get_token() self.prepare_env() ################################################################## @@ -174,20 +113,16 @@ class HotfixBuilder(): self.thread_runner.start() def change_building_status(self, status): - self.connector.change_building_status(self.hotfix_id, status) + self.connector.update_hotfix_building_status(self.hotfix_id, status) def die(self, msg): self.fd.write("%s \n" % msg) self.change_building_status("failed") def prepare_env(self): - # prepare kernel src and kaptch-build - # cmd = "chmod +x check_env.sh && ./check_env.sh -b %s -n %s" % (self.hotfix_base, self.nfs_dir_home) - # with os.popen(cmd) as process: - # output = process.read() # get the img_list image information and pull them based on machine's kernel arch - image_config_file = open(os.path.join(os.getcwd(), "img_list.json")) + image_config_file = open(os.path.join(os.getcwd(), "conf" , "img_list.json")) config_data = json.load(image_config_file) machine_kernel = platform.uname().release arch = machine_kernel.split(".")[-1] @@ -226,13 +161,13 @@ class HotfixBuilder(): os.makedirs(os.path.join(self.builder_hotfix_package_repo, "src_pack")) # copy build_hotfix.sh to BASE - if os.path.exists("./build_hotfix.sh"): - shutil.copy("./build_hotfix.sh", self.hotfix_base) + if os.path.exists(os.path.join(os.getcwd(), "script", "build_hotfix.sh")): + shutil.copy(os.path.join(os.getcwd(), "script", "build_hotfix.sh"), self.hotfix_base) else: logger.error("ERROR: cannot find build_hotfix.sh") - if os.path.exists("./build_rpm.sh"): - shutil.copy("./build_rpm.sh", self.hotfix_base) + if os.path.exists(os.path.join(os.getcwd(), "script", "build_rpm.sh")): + shutil.copy(os.path.join(os.getcwd(), "script", "build_rpm.sh"), self.hotfix_base) else: logger.error("ERROR: cannot find build_rpm.sh") @@ -240,8 +175,9 @@ class HotfixBuilder(): os.system("rm -rf {} && mkdir {} ".format(self.tmpdir, self.tmpdir)) os.system("rm -rf {}/*patch".format(self.hotfix_base)) - def find_build_rpm(self): - directory = os.path.join(self.tmpdir, "rpmbuild") + def find_build_rpm(self, kernel_version): + arch = kernel_version.split(".")[-1] + directory = os.path.join(self.tmpdir, "rpmbuild", "RPMS", arch) rpms = [] for root, dirs, files in os.walk(directory): for eachfile in files: @@ -334,7 +270,7 @@ class HotfixBuilder(): def get_building_image(self, kernel_version): arch = kernel_version.split(".")[-1] - image_list_file = open('./img_list.json') + image_list_file = open(os.path.join(os.getcwd(), "conf" , "img_list.json")) images = json.load(image_list_file) return images[arch]['anolis'] @@ -435,7 +371,6 @@ class HotfixBuilder(): except Exception as e: self.fd.write(str(e)) logger.error(str(e)) - # self.connector.change_building_status(hotfix_id, "failed") description = self.extract_description_from_patch(local_patch) # run the build hotfix script @@ -453,23 +388,20 @@ class HotfixBuilder(): return_code=p.wait() logger.info("The return code is %d" % return_code) - rpm_names = self.find_build_rpm() - - # when finished building, sync the build log - #self.connector.sync_building_log(hotfix_id) + rpm_names = self.find_build_rpm(kernel_version) # if rpm is more than one, upload it one by one for each_rpm in rpm_names: - resp = self.connector.sync_rpm_name(hotfix_id, each_rpm) - if resp.status_code != 200: - self.connector.insert_log_to_server(hotfix_id, "cannot sync rpm package name %s" % each_rpm) + self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) + + self.connector.sync_building_log_cec(self.hotfix_id) # check the last output if return_code == 0: - self.connector.change_building_status(hotfix_id, "success") + self.change_building_status("success") else: os.system("echo \"BUILD FAILED\" >> %s" % log_file_path) - self.connector.change_building_status(hotfix_id, "failed") + self.change_building_status("failed") """ build the customize kernel from user defined @@ -586,23 +518,22 @@ class HotfixBuilder(): return_code=p.wait() logger.info("The return code is %d" % return_code) - rpm_names = self.find_build_rpm() + rpm_names = self.find_build_rpm(kernel_version) # when finished building, sync the build log #self.connector.sync_building_log(hotfix_id) # if rpm is more than one, upload it one by one for each_rpm in rpm_names: - resp = self.connector.sync_rpm_name(hotfix_id, each_rpm) - if resp.status_code != 200: - self.connector.insert_log_to_server(hotfix_id, "cannot sync rpm package name %s" % each_rpm) + self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) + self.connector.sync_building_log_cec(self.hotfix_id) # check the last output if return_code == 0: - self.connector.change_building_status(hotfix_id, "success") + self.change_building_status("success") else: os.system("echo \"BUILD FAILED\" >> %s" % log_file_path) - self.connector.change_building_status(hotfix_id, "failed") + self.change_building_status("failed") ''' Each event is an object, the parameter is inside event.value @@ -614,91 +545,68 @@ class HotfixBuilder(): ''' def build(self): - with dispatch_admin(self.cec_url) as admin: - if not admin.is_topic_exist(self.cec_hotfix_topic): - admin.create_topic(self.cec_hotfix_topic) consumer_id = Consumer.generate_consumer_id() + if self.local_arch == "x86_64": - consumer = dispatch_consumer(self.cec_url, self.cec_hotfix_topic, + consumer = SysomFramework.cec_consumer(self.cec_hotfix_topic, consumer_id=consumer_id, group_id="hotfix_job_group_x86") else: - consumer = dispatch_consumer(self.cec_url, self.cec_hotfix_topic, + consumer = SysomFramework.cec_consumer(self.cec_hotfix_topic, consumer_id=consumer_id, group_id="hotfix_job_group_arm") - - for event in consumer: - try: - parameters = event.value - log_file = parameters['log_file'] - self.hotfix_id = parameters['hotfix_id'] - log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) - is_deleted = self.cache.get(str(self.hotfix_id)) - # if this hotfix_id is not exist in the deleted pool - if not is_deleted: - if parameters['arch'] == self.local_arch: - # this operation aims to clear the previous log if rebuild - with open(log_file_path, "w") as f: - f.write("=========================================================\n") - f.write("==========*******Sysom Hotfix Building System*******==============\n") - f.write("=========================================================\n") - - self.change_building_status("building") - - # for each run, update the repo - cmd = "chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) - with os.popen(cmd) as process: - output = process.read() - - customize = parameters['customize'] - # before build one job, clear the tmpdir - self.clear_tmpdir() - if not customize: - self.build_supported_kernel(parameters) - else: - self.build_customize_kernel(parameters) - logger.info(log_file) + retry_time = 0 + while retry_time < self.max_retry_time: + for event in consumer: + try: + parameters = event.value + log_file = parameters['log_file'] + self.hotfix_id = parameters['hotfix_id'] + log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) + is_deleted = self.cache.get(str(self.hotfix_id)) + # if this hotfix_id is not exist in the deleted pool + if not is_deleted: + if parameters['arch'] == self.local_arch: + # this operation aims to clear the previous log if rebuild + with open(log_file_path, "w") as f: + f.write("=========================================================\n") + f.write("=================Sysom Hotfix Building System===================\n") + f.write("=========================================================\n") + + self.change_building_status("building") + + # for each run, update the repo + cmd = "echo \"hello\" && chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) + with os.popen(cmd) as process: + output = process.read() + + customize = parameters['customize'] + # before build one job, clear the tmpdir + self.clear_tmpdir() + if not customize: + self.build_supported_kernel(parameters) + else: + self.build_customize_kernel(parameters) + logger.info(log_file) + self.fd.close() + else: + logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id) + except Exception as e: + logger.error(str(e)) + self.change_building_status("failed") + finally: + if self.fd is not None: self.fd.close() - else: - logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id) - except Exception as e: - logger.error(str(e)) - self.connector.change_building_status(self.hotfix_id, "failed") - finally: - if self.fd is not None: - self.fd.close() - consumer.ack(event) + consumer.ack(event) + time.sleep(self.sleep_time) + retry_time += 1 - -if __name__ == "__main__": - config_file = "builder.ini" - - con = configparser.ConfigParser() - con.read(config_file, encoding='utf-8') +class HotfixBuilderMain(): - # cec - cec = dict(con.items('cec')) - cec_url = cec['cec_url'] - hotfix_building_status_topic = cec['job_status_topic'] - - # server config - sysom_server = dict(con.items('sysom_server')) - server_ip = sysom_server['server_ip'] - server_login_account = sysom_server['account'] - server_login_password = sysom_server['password'] - - # builder config - builder = dict(con.items('builder')) - hotfix_base = builder['hotfix_base'] - nfs_dir_home = builder['nfs_dir_home'] - builder_hotfix_package_repo = builder['package_repo'] + def __init__(self, config: YAML_CONFIG) -> None: + self.parameters = config.get_service_config() - - # if this builder run in local, we should use the local repo directory instead of the nfs directory - if re.search("127.0.0.1", server_ip): - nfs_dir_home="/usr/local/sysom/server/builder/hotfix" - - hotfix_builder = HotfixBuilder(nfs_dir_home, hotfix_base, cec_url, hotfix_building_status_topic, - server_ip, server_login_account, server_login_password, builder_hotfix_package_repo) - hotfix_builder.run() \ No newline at end of file + def start(self): + hotfix_builder = HotfixBuilder(self.parameters) + hotfix_builder.run() diff --git a/sysom_server/sysom_hotfix_builder/app/crud.py b/sysom_server/sysom_hotfix_builder/app/crud.py new file mode 100644 index 0000000000000000000000000000000000000000..6e0e26c397d02049db90528b19f9893430d2a539 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/crud.py @@ -0,0 +1,30 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File crud.py +Description: +""" +from typing import Optional +from sqlalchemy.orm import Session +from app import models, schemas + +################################################################################################ +# Define database crud here +################################################################################################ + +# def get_person_by_name(db: Session, name: str) -> Optional[models.Person]: +# return db.query(models.Person).filter(models.Person.name == name).first() + +# def create_person(db: Session, person: schemas.Person) -> models.Person: +# person = models.Person(**person.dict()) +# db.add(person) +# db.commit() +# db.refresh(person) +# return person + +# def del_person_by_id(db: Session, person_id: int): +# person = db.get(models.Person, person_id) +# db.delete(person) +# db.commit() diff --git a/sysom_server/sysom_hotfix_builder/app/database.py b/sysom_server/sysom_hotfix_builder/app/database.py new file mode 100644 index 0000000000000000000000000000000000000000..d1500828b35a22dcae59d801706e21e651d574d1 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/database.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File database.py +Description: +""" +from sqlalchemy import create_engine +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import sessionmaker +from conf.settings import SQLALCHEMY_DATABASE_URL + +engine = create_engine( + SQLALCHEMY_DATABASE_URL, connect_args={} +) + +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) + +Base = declarative_base() \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/app/models.py b/sysom_server/sysom_hotfix_builder/app/models.py new file mode 100644 index 0000000000000000000000000000000000000000..60fa74d4424b86d61c17b2641d12fb0ff0f459d6 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/models.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File models.py +Description: +""" +from sqlalchemy import Column, Integer, String +from app.database import Base + + +########################################################################### +# Define databse model here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/sql-databases/ +# class Person(Base): +# __tablename__ = "sys_person" +# id = Column(Integer, primary_key=True) +# name = Column(String(254), unique=True) +# age = Column(Integer) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/app/routers/health.py b/sysom_server/sysom_hotfix_builder/app/routers/health.py new file mode 100644 index 0000000000000000000000000000000000000000..9b6f43fcabc9da534bf2bd1f17f9d4115ff409c5 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/routers/health.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- # +""" +Time 2023/04/17 19:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File health.py +Description: +""" +from fastapi import APIRouter + + +router = APIRouter() + + +@router.get("/check") +async def get_channel_config(): + return { + "code": 0, + "err_msg": "", + "data": "" + } diff --git a/sysom_server/sysom_hotfix_builder/app/schemas.py b/sysom_server/sysom_hotfix_builder/app/schemas.py new file mode 100644 index 0000000000000000000000000000000000000000..aa413a9db4750be1fa29a014023c50598544bb68 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/app/schemas.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File schemas.py +Description: +""" +from pydantic import BaseModel + +########################################################################### +# Define schemas here +########################################################################### + +# @reference https://fastapi.tiangolo.com/zh/tutorial/response-model/ +# class Person(BaseModel): +# id: int +# name: str +# age: int + +# class Config: +# orm_mode = True \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/builder.ini b/sysom_server/sysom_hotfix_builder/builder.ini deleted file mode 100644 index 02aeff21bcbb3d65d8aeef9848ddc360fadf6fd5..0000000000000000000000000000000000000000 --- a/sysom_server/sysom_hotfix_builder/builder.ini +++ /dev/null @@ -1,19 +0,0 @@ -[sysom_server] -server_ip = http://127.0.0.1 -account = account -password = password - -[cec] -cec_url = redis://127.0.0.1:6379 -job_status_topic = hotfix_building_status - -[builder] -hotfix_base = /hotfix_build/hotfix -nfs_dir_home = /usr/local/sysom/server/builder/hotfix -package_repo = /hotfix/packages -tmpdir = /hotfix_tmpdir -src_rpm_tmpdir = /tmp/hotfix_src - -[redis] -host = localhost -port = 6379 diff --git a/sysom_server/sysom_hotfix_builder/conf/common.py b/sysom_server/sysom_hotfix_builder/conf/common.py new file mode 100644 index 0000000000000000000000000000000000000000..1b38077e92eee48c371b91b13d2e453bf9942b46 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/common.py @@ -0,0 +1,33 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File common.py +Description: +""" +from pathlib import Path +from sysom_utils import ConfigParser, SysomFramework + +BASE_DIR = Path(__file__).resolve().parent.parent + +################################################################## +# Load yaml config first +################################################################## +YAML_GLOBAL_CONFIG_PATH = f"{BASE_DIR.parent.parent}/conf/config.yml" +YAML_SERVICE_CONFIG_PATH = f"{BASE_DIR}/config.yml" + +YAML_CONFIG = ConfigParser(YAML_GLOBAL_CONFIG_PATH, YAML_SERVICE_CONFIG_PATH) + +mysql_config = YAML_CONFIG.get_server_config().db.mysql +service_config = YAML_CONFIG.get_service_config() + +SysomFramework.init(YAML_CONFIG) + +################################################################## +# fastapi config +################################################################## +SQLALCHEMY_DATABASE_URL = ( + f"{mysql_config.dialect}+{mysql_config.engine}://{mysql_config.user}:{mysql_config.password}@" + f"{mysql_config.host}:{mysql_config.port}/{mysql_config.database}" +) \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/develop.py b/sysom_server/sysom_hotfix_builder/conf/develop.py new file mode 100644 index 0000000000000000000000000000000000000000..bb64bf49cb03f9329d2e9bc3b7ed92b4d2439728 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/develop.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File develoop.py +Description: +""" +from .common import * + +''' +开发环境配置项 +''' + +DEBUG = True \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/gunicorn.py b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py new file mode 100644 index 0000000000000000000000000000000000000000..c835217fc14b0ce0905a2c90f52ff01fc5c0d47d --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/gunicorn.py @@ -0,0 +1,23 @@ +''' +Channel Service Gunicorn Settings +''' +from conf.common import YAML_CONFIG + +bind = YAML_CONFIG.get_service_config().get("bind", "127.0.0.1") +port = YAML_CONFIG.get_service_config().get("port", "80") + +workers = 1 # 指定工作进程数, builder应该只开一个工作进程 + +threads = 3 + +bind = f'{bind}:{port}' + +worker_class = 'uvicorn.workers.UvicornWorker' # 工作模式线程, 默认为sync模式 + +max_requests = 2000 # 设置最大并发数量为2000 (每个worker处理请求的工作线程) + +accesslog = '/var/log/sysom/sysom-hotfix_builder-access.log' + +loglevel = 'error' + +proc_name = 'channel_service' diff --git a/sysom_server/sysom_hotfix_builder/img_list.json b/sysom_server/sysom_hotfix_builder/conf/img_list.json similarity index 100% rename from sysom_server/sysom_hotfix_builder/img_list.json rename to sysom_server/sysom_hotfix_builder/conf/img_list.json diff --git a/sysom_server/sysom_hotfix_builder/conf/product.py b/sysom_server/sysom_hotfix_builder/conf/product.py new file mode 100644 index 0000000000000000000000000000000000000000..c748de1d0546f5bef1d9f4018a86e14bdc171798 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/product.py @@ -0,0 +1,15 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File product.py +Description: +""" +from .common import * + +''' +生产环境配置项 +''' + +DEBUG = False diff --git a/sysom_server/sysom_hotfix_builder/conf/settings.py b/sysom_server/sysom_hotfix_builder/conf/settings.py new file mode 100644 index 0000000000000000000000000000000000000000..b9156df0beb987bc042e7def23a05ad37c1de634 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/settings.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File settings.py +Description: +""" +import os + +env = os.environ.get("env", "product") + + +if env == "develop": + from .develop import * +elif env == "testing": + from .testing import * +elif env == "product": + from .product import * \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/conf/testing.py b/sysom_server/sysom_hotfix_builder/conf/testing.py new file mode 100644 index 0000000000000000000000000000000000000000..864f53a24748a2ad163227e54aacd6b7946914f7 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/conf/testing.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File testing.py +Description: +""" +from .common import * + +''' +测试环境配置项 +''' +DEBUG = True diff --git a/sysom_server/sysom_hotfix_builder/config.yml b/sysom_server/sysom_hotfix_builder/config.yml new file mode 100644 index 0000000000000000000000000000000000000000..03efc403694d5530daace874173f0679bb6452e5 --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/config.yml @@ -0,0 +1,51 @@ +vars: + SERVICE_NAME: &SERVICE_NAME sysom_hotfix_builder + SERVICE_CONSUMER_GROUP: + !concat &SERVICE_CONSUMER_GROUP [*SERVICE_NAME, "_consumer_group"] + +sysom_server: + cec: + consumer_group: SYSOM_CEC_CHANNEL_CONSUMER_GROUP + +sysom_service: + service_name: *SERVICE_NAME + service_dir: *SERVICE_NAME + protocol: http + host: 127.0.0.1 + bind: 127.0.0.1 + port: 7011 + framework: + gcache: + protocol: redis + node_dispatch: + cmg: + tags: + - hotfix_builder + - FastApi + # Metadata of service + metadata: + check: + type: http + url: "/api/v1/hotfix_builder/health/check" + interval: 10 + timeout: 10 + deregister: 25 + header: + tls_skip_verify: false + cec: + job_status_topic: "hotfix_msg" + hotfix_topic: "hotfix_job" + max_retry: 10 # retry consumer listen time + retry_sleep_time: 5 # second + + builder: + hotfix_base: "/hotfix_build/hotfix" + nfs_dir_home: "/usr/local/sysom/server/builder/hotfix" + package_repo: "/hotfix/packages" + tmpdir: "/hotfix_tmpdir" + src_rpm_tmpdir: "/tmp/hotfix_src" + + redis: + host: "127.0.0.1" + port: 6379 + diff --git a/sysom_server/sysom_hotfix_builder/lib/README.md b/sysom_server/sysom_hotfix_builder/lib/README.md new file mode 100644 index 0000000000000000000000000000000000000000..3ec74424ba6ecde5d035d0c113bafb0ddc6e4cfa --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/lib/README.md @@ -0,0 +1 @@ +The current directory holds the public libraries or utils needed for microservices \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/main.py b/sysom_server/sysom_hotfix_builder/main.py new file mode 100644 index 0000000000000000000000000000000000000000..62dcb4042dc971a7ee5c0bf136c6339982bac60d --- /dev/null +++ b/sysom_server/sysom_hotfix_builder/main.py @@ -0,0 +1,47 @@ +# -*- coding: utf-8 -*- # +""" +Time 2022/11/14 14:32 +Author: mingfeng (SunnyQjm) +Email mfeng@linux.alibaba.com +File ssh.py +Description: +""" +from clogger import logger +from fastapi import FastAPI +import app.routers.health as health +from app.builder import HotfixBuilderMain +from conf.settings import YAML_CONFIG +from sysom_utils import CmgPlugin, SysomFramework + + +app = FastAPI() + +app.include_router(health.router, prefix="/api/v1/hotfix_builder/health") + + +############################################################################# +# Write your API interface here, or add to app/routes +############################################################################# + + +def init_framwork(): + SysomFramework\ + .init(YAML_CONFIG) \ + .load_plugin_cls(CmgPlugin) \ + .start() + logger.info("SysomFramework init finished!") + + +@app.on_event("startup") +async def on_start(): + init_framwork() + ############################################################################# + # Perform some microservice initialization operations over here + ############################################################################# + hotfix_builder_obj = HotfixBuilderMain(YAML_CONFIG) + hotfix_builder_obj.start() + + +@app.on_event("shutdown") +async def on_shutdown(): + pass \ No newline at end of file diff --git a/sysom_server/sysom_hotfix_builder/build_hotfix.sh b/sysom_server/sysom_hotfix_builder/script/build_hotfix.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/build_hotfix.sh rename to sysom_server/sysom_hotfix_builder/script/build_hotfix.sh diff --git a/sysom_server/sysom_hotfix_builder/build_rpm.sh b/sysom_server/sysom_hotfix_builder/script/build_rpm.sh similarity index 100% rename from sysom_server/sysom_hotfix_builder/build_rpm.sh rename to sysom_server/sysom_hotfix_builder/script/build_rpm.sh diff --git a/sysom_server/sysom_hotfix_builder/check_env.sh b/sysom_server/sysom_hotfix_builder/script/check_env.sh similarity index 91% rename from sysom_server/sysom_hotfix_builder/check_env.sh rename to sysom_server/sysom_hotfix_builder/script/check_env.sh index 6441ca0ff1db9ab6f9eab875de30830160caec57..d717763869de0138268d454c6d31eb549ffa67e5 100755 --- a/sysom_server/sysom_hotfix_builder/check_env.sh +++ b/sysom_server/sysom_hotfix_builder/script/check_env.sh @@ -5,15 +5,19 @@ usage() { echo "./check_env.sh [OPTIONS]"; echo ""; echo "Options:"; - echo " -p|--patch"; - echo " The patch (must not null)"; + echo " -b|--hotfix_base"; + echo " The hotfix running base directory"; + echo " -n|--nfs"; + echo " Select the NFS directory home"; + echo " -l|--log"; + echo " Select the log file path"; echo " -k|--kernelversion"; echo " The complete kernel version include architecture (must not null)"; echo " -h|--help"; echo " For help"; echo ""; echo "For example:"; - echo "./kpatch-packager.sh -p \${patch} -k \${kernel} -d \"\${description}\" -v --prefix=\${prefix_patch}"; + echo "./check_env.sh -k {kernel_version} -b {hotfix_base_directory} -n {nfs_directory_home} -l {log_file_path}"; } # this is to output the error msg diff --git a/sysom_web/yarn.lock b/sysom_web/yarn.lock index e7d2ee393659991daec5d154ef7fa939747e6ae0..5bae1478dbb42907fab284475d8f7c7b7abaeebe 100644 --- a/sysom_web/yarn.lock +++ b/sysom_web/yarn.lock @@ -13961,7 +13961,7 @@ urix@^0.1.0: resolved "https://registry.npmmirror.com/urix/-/urix-0.1.0.tgz#da937f7a62e21fec1fd18d49b35c2935067a6c72" integrity sha512-Am1ousAhSLBeB9cG/7k7r2R0zj50uDRlZHPGbazid5s9rlF1F/QKYObEKSIunSjIOkJZqwRRLpvewjEkM7pSqg== -url-parse@^1.5.10: +url-parse@^1.5.10, url-parse@^1.5.3: version "1.5.10" resolved "https://mirrors.cloud.tencent.com/npm/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== @@ -13969,14 +13969,6 @@ url-parse@^1.5.10: querystringify "^2.1.1" requires-port "^1.0.0" -url-parse@^1.5.3: - version "1.5.10" - resolved "https://registry.npmmirror.com/url-parse/-/url-parse-1.5.10.tgz#9d3c2f736c1d75dd3bd2be507dcc111f1e2ea9c1" - integrity sha512-WypcfiRhfeUP9vvF0j6rw0J3hrWrw6iZv3+22h6iRMJ/8z1Tj6XfLP4DsUix5MhMPnXpiHDoKyoZ/bdCkwBCiQ== - dependencies: - querystringify "^2.1.1" - requires-port "^1.0.0" - url@^0.11.0: version "0.11.0" resolved "https://mirrors.cloud.tencent.com/npm/url/-/url-0.11.0.tgz#3838e97cfc60521eb73c525a8e55bfdd9e2e28f1" diff --git a/tools/deploy/sysom.spec b/tools/deploy/sysom.spec index d38d5dbe072978b678921a96f040eb47e6265b2d..64273c8186f056f5f9e4d0b7a6edda745077cc7a 100755 --- a/tools/deploy/sysom.spec +++ b/tools/deploy/sysom.spec @@ -154,7 +154,8 @@ cp -a script %{buildroot}/usr/local/sysom/init_scripts cp -a sysom_web/dist %{buildroot}/usr/local/sysom/web cp -a deps %{buildroot}/usr/local/sysom/deps cp -a environment %{buildroot}/usr/local/sysom/environment -cp -a conf/config.yml %{buildroot}/etc/sysom/config.yml +mkdir -p %{buildroot}/etc +cp -a conf %{buildroot}/etc/sysom #mkdir -p %{buildroot}/usr/local/sysom/server/redis # cp -a sysom_server %{buildroot}/usr/local/sysom/server/target @@ -166,6 +167,7 @@ cp -a conf/config.yml %{buildroot}/etc/sysom/config.yml %files /usr/local/sysom/ +/etc/sysom %post # bash +x /usr/local/sysom/init_scripts/sysom.sh install ALL