diff --git a/sysom_server/sysom_hotfix/apps/hotfix/views.py b/sysom_server/sysom_hotfix/apps/hotfix/views.py index a05517d0cba7f50cf50f1260438bde7a9100a163..babea32b7a3b979fffd9aacd567b01bddeb6c443 100644 --- a/sysom_server/sysom_hotfix/apps/hotfix/views.py +++ b/sysom_server/sysom_hotfix/apps/hotfix/views.py @@ -282,7 +282,7 @@ class HotfixAPIView(GenericViewSet, try: if customize_version_object is None: status = self.function.create_message_to_cec(customize=False, cec_topic=settings.SYSOM_CEC_HOTFIX_TOPIC, os_type=hotfix_object.os_type, - hotfix_id=hotfix_object.id, kernel_version=hotfix_object.kernel_version, hotfix_name=hotfix_object.hotfix_name, + hotfix_id=hotfix_object.hotfix_id, hotfix_db_index=hotfix_object.id, kernel_version=hotfix_object.kernel_version, hotfix_name=hotfix_object.hotfix_name, patch_file=hotfix_object.patch_file, patch_path=hotfix_object.patch_path, arch=hotfix_object.arch, log_file=hotfix_object.log_file ) else: @@ -293,7 +293,7 @@ class HotfixAPIView(GenericViewSet, image = self.function.get_image_of_os(os_type) is_src_package = self.function.get_src_pkg_mark_of_os(os_type) status = self.function.create_message_to_cec(customize=True, cec_topic=settings.SYSOM_CEC_HOTFIX_TOPIC, os_type=hotfix_object.os_type, - hotfix_id=hotfix_object.id, kernel_version=hotfix_object.kernel_version, hotfix_name=hotfix_object.hotfix_name, patch_file=hotfix_object.patch_file, + hotfix_id=hotfix_object.hotfix_id, hotfix_db_index=hotfix_object.id, kernel_version=hotfix_object.kernel_version, hotfix_name=hotfix_object.hotfix_name, patch_file=hotfix_object.patch_file, patch_path=hotfix_object.patch_path, arch=hotfix_object.arch, log_file=hotfix_object.log_file, source_repo=source_repo, source=source, devel_link=devel_link,debuginfo_link=debuginfo_link,image=image,is_src_package=is_src_package ) diff --git a/sysom_server/sysom_hotfix_builder/app/builder.py b/sysom_server/sysom_hotfix_builder/app/builder.py index 8b40553e6c48a32c5e52922b18855c861b628433..105890baa222bbce1b07488abdf1d2acc95217a2 100644 --- a/sysom_server/sysom_hotfix_builder/app/builder.py +++ b/sysom_server/sysom_hotfix_builder/app/builder.py @@ -25,6 +25,8 @@ from cec_base.admin import dispatch_admin from cec_base.producer import dispatch_producer, Producer from conf.settings import YAML_CONFIG from sysom_utils import CecTarget, SysomFramework +from cec_base.event import Event +from cec_base.cec_client import MultiConsumer, CecAsyncConsumeTask class ServerConnector(): @@ -34,37 +36,50 @@ class ServerConnector(): self.cec_msg_topic = hotfix_msg_topic self.cec_hotfix_job_received_topic = hotfix_job_received_topic + def gernerate_cec_msg(self, hotfix_id, op, value): + msg = dict() + msg["hotfix_id"] = hotfix_id + msg["op"] = op + msg["value"] = value + return msg + # Here, we design to use some status more than building status # in building status , we include : waiting\building\success\failed # but more, we can send : sync to server for log and hotfix name sync - def update_hotfix_building_status(self, id, status): - self.cec.produce(self.cec_msg_topic, { - "hotfix_id" : id, - "status" : status - }) + def update_hotfix_building_status(self, hotfix_id, status): + self.cec.produce(self.cec_msg_topic, self.gernerate_cec_msg(hotfix_id, "status", status)) + self.cec.flush() def sync_rpm_name_cec(self, hotfix_id, rpm_name): logger.info("produce rpm_name ") - self.cec.produce(self.cec_msg_topic, { - "hotfix_id" : hotfix_id, - "type": "rpm", - "sync" : True, - "rpm_name" : rpm_name - }) - - def sync_building_log_cec(self, hotfix_id): + self.cec.produce(self.cec_msg_topic, self.gernerate_cec_msg(hotfix_id, "rpm_name", rpm_name)) + self.cec.flush() + + def sync_building_log_cec(self, hotfix_id, msg): logger.info("sync hotfix build log") - self.cec.produce(self.cec_msg_topic, { - "hotfix_id": hotfix_id, - "type": "log", - "sync": True - }) + self.cec.produce(self.cec_msg_topic, self.gernerate_cec_msg(hotfix_id, "log", msg)) + self.cec.flush() + + def sync_extern_build_id(self, hotfix_id, extern_build_id): + logger.info("sync jenkins build id") + self.cec.produce(self.cec_msg_topic, self.gernerate_cec_msg(hotfix_id, "extern_build_id", value)) + self.cec.flush() + + # this function used to call server get the build log when build finished + def sync_building_log_cec_finish(self, hotfix_id, log_location): + logger.info("sync hotfix log when finished") + self.cec.produce(self.cec_msg_topic, self.gernerate_cec_msg(hotfix_id, "build_log_finished", log_location)) + self.cec.flush() -class HotfixBuilder(): +class HotfixBuilder(MultiConsumer): def __init__(self, con: dict): + super().__init__( + YAML_CONFIG.get_cec_url(CecTarget.PRODUCER), + custom_callback=self.on_receive_event, + ) cache = SysomFramework.gcache("deleted_hotfix") self.cache = cache ####################### @@ -73,6 +88,7 @@ class HotfixBuilder(): self.nfs_dir_home = con.builder.nfs_dir_home self.max_retry_time = con.cec.max_retry self.sleep_time = con.cec.retry_sleep_time + self.server_base = con.builder.server_base self.hotfix_base = con.builder.hotfix_base self.builder_hotfix_package_repo = con.builder.package_repo self.thread_runner = threading.Thread(target=self.build, name="hotfix_builder", daemon=True) @@ -85,6 +101,12 @@ class HotfixBuilder(): self.hotfix_id = None self.fd = None self.prepare_env() + self.append_group_consume_task( + con.cec.job_topic, + "sysom_hotfix", + Consumer.generate_consumer_id(), + ensure_topic_exist=True, + ) ################################################################## # Logging config @@ -318,7 +340,7 @@ class HotfixBuilder(): """ def build_supported_kernel(self, parameters): # get the hotfix building parametes - hotfix_id = parameters['hotfix_id'] + hotfix_id = parameters['hotfix_db_index'] kernel_version = parameters['kernel_version'] hotfix_name = parameters['hotfix_name'] # find the patch_path in builder local @@ -360,8 +382,8 @@ class HotfixBuilder(): description = self.extract_description_from_patch(local_patch) # run the build hotfix script - cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -r {} -t NULL ".format( - self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image, + cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -r {} -t NULL ".format( + self.server_base, self.server_base, self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image, self.hotfix_base, local_patch, kernel_version, description, self.hotfix_base, hotfix_name, log_file_path, source_code_repo ) self.fd.write(cmd+"\n") @@ -381,7 +403,7 @@ class HotfixBuilder(): logger.info("==> sync rpm : %s" % each_rpm) self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) - self.connector.sync_building_log_cec(self.hotfix_id) + self.connector.sync_building_log_cec_finish(self.hotfix_id, log_file_path) # check the last output if return_code == 0: @@ -395,7 +417,7 @@ class HotfixBuilder(): """ def build_customize_kernel(self, parameters): # get the hotfix building parameters - hotfix_id = parameters['hotfix_id'] + hotfix_id = parameters['hotfix_db_index'] kernel_version = parameters['kernel_version'] hotfix_name = parameters['hotfix_name'] devel_link = parameters['devel_link'] @@ -485,15 +507,15 @@ class HotfixBuilder(): # run the build hotfix script, if is_src_package: # use src.rpm - cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -c {} -v {} -r {} 2>&1 1 >> {} ".format( - self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image, + cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -c {} -v {} -r {} 2>&1 1 >> {} ".format( + self.server_base, self.server_base, self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image, self.hotfix_base, local_patch, kernel_version, description, self.hotfix_base, hotfix_name, log_file_path, kernel_config, vmlinux, source_code_path, log_file_path ) else: # use git branch for source management git_branch = src_origin - cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -c {} -v {} -r {} -t {} 2>&1 1 >> {}".format( - self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image, + cmd = "docker run --rm -v {}:{} -v {}:{} -v {}:{} -v {}:{} -v {}:{} --net=host {} sh {}/build_hotfix.sh -p {} -k {} -d \"{}\" -b {} -n {} -g {} -c {} -v {} -r {} -t {} 2>&1 1 >> {}".format( + self.server_base, self.server_base, self.hotfix_base, self.hotfix_base, self.nfs_dir_home, self.nfs_dir_home, self.builder_hotfix_package_repo, self.builder_hotfix_package_repo, self.tmpdir, self.tmpdir, image, self.hotfix_base, local_patch, kernel_version, description, self.hotfix_base, hotfix_name, log_file_path, kernel_config, vmlinux, source_code_repo, git_branch, log_file_path ) @@ -511,7 +533,7 @@ class HotfixBuilder(): for each_rpm in rpm_names: self.connector.sync_rpm_name_cec(hotfix_id, each_rpm) - self.connector.sync_building_log_cec(self.hotfix_id) + self.connector.sync_building_log_cec_finish(self.hotfix_id, log_file_path) # check the last output if return_code == 0: self.change_building_status("success") @@ -519,6 +541,49 @@ class HotfixBuilder(): os.system("echo \"BUILD FAILED\" >> %s" % log_file_path) self.change_building_status("failed") + def on_receive_event(self, event: Event, task: CecAsyncConsumeTask): + try: + parameters = event.value + log_file = parameters['log_file'] + self.hotfix_id = parameters['hotfix_db_index'] + log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) + is_deleted = self.cache.load(str(self.hotfix_id)) + # if this hotfix_id is not exist in the deleted pool + if not is_deleted: + if parameters['arch'] == self.local_arch: + # this operation aims to clear the previous log if rebuild + with open(log_file_path, "w") as f: + f.write("=========================================================\n") + f.write(" Sysom Hotfix Building System \n") + f.write("=========================================================\n") + + self.change_building_status("building") + + # for each run, update the repo + cmd = "echo \"hello\" && chmod +x ./script/check_env.sh && ./script/check_env.sh -k %s -b %s -n %s -l %s " % (parameters['kernel_version'], self.hotfix_base, self.nfs_dir_home, log_file_path) + with os.popen(cmd) as process: + output = process.read() + logger.error(cmd) + customize = parameters['customize'] + # before build one job, clear the tmpdir + self.clear_tmpdir() + if not customize: + self.build_supported_kernel(parameters) + else: + self.build_customize_kernel(parameters) + logger.info(log_file) + self.fd.close() + else: + logger.info("hotfix : %s is deleted, ignore this job ..." % self.hotfix_id) + except Exception as e: + logger.error(str(e)) + self.change_building_status("failed") + finally: + if self.fd is not None: + self.fd.close() + task.ack(event) + + ''' Each event is an object, the parameter is inside event.value event.value is a dictionary. @@ -545,7 +610,7 @@ class HotfixBuilder(): try: parameters = event.value log_file = parameters['log_file'] - self.hotfix_id = parameters['hotfix_id'] + self.hotfix_id = parameters['hotfix_db_index'] log_file_path = os.path.join(self.nfs_dir_home, "log", log_file) is_deleted = self.cache.load(str(self.hotfix_id)) # if this hotfix_id is not exist in the deleted pool @@ -593,4 +658,4 @@ class HotfixBuilderMain(): def start(self): hotfix_builder = HotfixBuilder(self.parameters) - hotfix_builder.run() + hotfix_builder.start() diff --git a/sysom_server/sysom_hotfix_builder/config.yml b/sysom_server/sysom_hotfix_builder/config.yml index 88841de05db15d16f93b629090e7f1e5d10de89f..dddb0d41a68b2afa95df6b86c5f17401895a6e7e 100644 --- a/sysom_server/sysom_hotfix_builder/config.yml +++ b/sysom_server/sysom_hotfix_builder/config.yml @@ -40,6 +40,7 @@ sysom_service: builder: hotfix_base: "/hotfix_build/hotfix" + server_base: "/usr/local/sysom/server" nfs_dir_home: "/usr/local/sysom/server/builder/hotfix" package_repo: "/hotfix/packages" tmpdir: "/hotfix_tmpdir" diff --git a/sysom_web/src/pages/hotfix/Make/index.jsx b/sysom_web/src/pages/hotfix/Make/index.jsx index 7e359f4d8ef0547711d405fd9027088bc58c23cf..4ffcbc0fd2e394410db6f0f0bf6dec0acad1abe5 100644 --- a/sysom_web/src/pages/hotfix/Make/index.jsx +++ b/sysom_web/src/pages/hotfix/Make/index.jsx @@ -35,6 +35,7 @@ const changeFormal = (record) => { //创建 const submitHotfix = (params) => { const datapar = { + hotfix_id: params.hotfix_id, hotfix_name: params.hotfix_name, kernel_version: params.kernel_version, upload: params.patch[0].response.data.patch_name, @@ -73,6 +74,11 @@ const HotfixList = () => { } const columns = [ + { + title: , + dataIndex: 'hotfix_id', + valueType: 'message', + }, { title: , dataIndex: 'created_at', @@ -104,6 +110,18 @@ const HotfixList = () => { dataIndex: 'patch_file', valueType: 'input', hideInSearch: true, + render: (_, record) => { + const url = record.patch_path; + const segments = url.split('/'); + // 获取最后一个非空段 + const patch_name = segments.pop(); + + return ( + + {patch_name} + + ) + } }, { title: , @@ -217,7 +235,9 @@ const HotfixList = () => { valueType: 'option', hideInSearch: true, render: (_, record) => [ -