From 92b41c2c5dcd50c634d58e12c7f1cb22623ab246 Mon Sep 17 00:00:00 2001 From: yang_feida Date: Mon, 16 Jun 2025 20:20:08 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E6=97=A5=E5=BF=97=E9=BB=98=E8=AE=A4?= =?UTF-8?q?=E4=BF=9D=E7=95=99=E4=B8=89=E4=BD=8D=E5=B0=8F=E6=95=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/common/constant.py | 3 +++ .../src/omniadvisor/interface/config_tuning.py | 10 +++++++--- .../src/omniadvisor/interface/hijack_recommend.py | 11 ++++++++--- omniadvisor/src/omniadvisor/service/retest_service.py | 6 +++++- omniadvisor/src/omniadvisor/utils/utils.py | 10 ++++++++++ 5 files changed, 33 insertions(+), 7 deletions(-) diff --git a/omniadvisor/src/common/constant.py b/omniadvisor/src/common/constant.py index b506eb258..75be2cc51 100644 --- a/omniadvisor/src/common/constant.py +++ b/omniadvisor/src/common/constant.py @@ -75,5 +75,8 @@ class OmniAdvisorConf: spark_exec_timeout_ratio = _common_config.getfloat('spark', 'spark.exec.timeout.ratio') tuning_strategies = json.loads(_common_config.get('common', 'tuning.strategy')) + # 保留小数位数 + decimal_digits = 3 + OA_CONF = OmniAdvisorConf() diff --git a/omniadvisor/src/omniadvisor/interface/config_tuning.py b/omniadvisor/src/omniadvisor/interface/config_tuning.py index 24723db97..551dde8a0 100644 --- a/omniadvisor/src/omniadvisor/interface/config_tuning.py +++ b/omniadvisor/src/omniadvisor/interface/config_tuning.py @@ -20,6 +20,7 @@ from omniadvisor.service.tuning_result.tuning_result_history import ( ) from omniadvisor.utils.logger import global_logger +from omniadvisor.utils.utils import num_format # 异常类暂时用于当前模块,没必要放到公共里 @@ -86,9 +87,12 @@ def unified_tuning(load, retest_way: str, tuning_method: str): tuning_result_history = get_tuning_result_history(load=load) # 更新最优配置 tuning_result_history.refresh_best_config() - global_logger.info(f'Current tuning result: {tuning_result.runtime}, best tuning result ever: ' - f'{tuning_result_history.best_tuning_result.runtime}, best tuning speedup: ' - f'{tuning_result_history.boost_percentage} %') + global_logger.info( + 'Current tuning result: %s, best tuning result ever: %s, best tuning speedup: %s %%', + num_format(tuning_result.runtime), + num_format(tuning_result_history.best_tuning_result.runtime), + num_format(tuning_result_history.boost_percentage) + ) else: # 更新待测试配置即可 diff --git a/omniadvisor/src/omniadvisor/interface/hijack_recommend.py b/omniadvisor/src/omniadvisor/interface/hijack_recommend.py index 7f260c0e1..13070f341 100644 --- a/omniadvisor/src/omniadvisor/interface/hijack_recommend.py +++ b/omniadvisor/src/omniadvisor/interface/hijack_recommend.py @@ -10,6 +10,8 @@ from omniadvisor.service.tuning_result.tuning_result_history import get_tuning_r from omniadvisor.utils.logger import global_logger from omniadvisor.service.spark_service.spark_run import spark_run +from omniadvisor.utils.utils import num_format + def _query_or_create_load(name: str, exec_attr: dict, default_config: dict): """ @@ -80,9 +82,12 @@ def _process_load_config(load: Load): tuning_result_history = get_tuning_result_history(load=load) # 更新最优配置 tuning_result_history.refresh_best_config() - global_logger.info(f'Current tuning result: {tuning_result.runtime}, best tuning result ever: ' - f'{tuning_result_history.best_tuning_result.runtime}, best tuning speedup: ' - f'{tuning_result_history.boost_percentage} %') + global_logger.info( + 'Current tuning result: %s, best tuning result ever: %s, best tuning speedup: %s %%', + num_format(tuning_result.runtime), + num_format(tuning_result_history.best_tuning_result.runtime), + num_format(tuning_result_history.boost_percentage) + ) # 当前test_config达到调优轮次后 置空test_config LoadRepository.update_test_config(load, {}) diff --git a/omniadvisor/src/omniadvisor/service/retest_service.py b/omniadvisor/src/omniadvisor/service/retest_service.py index c92138d9a..6bb07bcce 100644 --- a/omniadvisor/src/omniadvisor/service/retest_service.py +++ b/omniadvisor/src/omniadvisor/service/retest_service.py @@ -3,6 +3,7 @@ from omniadvisor.repository.model.load import Load from omniadvisor.service.spark_service.spark_run import spark_run from omniadvisor.service.tuning_result.tuning_result import get_tuning_result from omniadvisor.utils.logger import global_logger +from omniadvisor.utils.utils import num_format def retest(load: Load, config: dict): @@ -25,7 +26,10 @@ def retest(load: Load, config: dict): raise if exam_record.status == OA_CONF.ExecStatus.success: - global_logger.info('Retest succeeded in round %d. Performance result: %.3f.', i, exam_record.runtime) + global_logger.info( + 'Retest succeeded in round %d. Performance result: %s.', i, + num_format(exam_record.runtime) + ) else: # 若是出现异常配置,也要退出 global_logger.warning('Retest failed in round %d. Exception source: Spark exception: %s.', i, diff --git a/omniadvisor/src/omniadvisor/utils/utils.py b/omniadvisor/src/omniadvisor/utils/utils.py index 8e333afa4..0b527c248 100644 --- a/omniadvisor/src/omniadvisor/utils/utils.py +++ b/omniadvisor/src/omniadvisor/utils/utils.py @@ -6,6 +6,8 @@ from typing import Tuple, List, Dict from omniadvisor.utils.logger import global_logger +from common.constant import OmniAdvisorConf + def run_cmd(submit_cmd) -> Tuple[int, str]: """ @@ -69,3 +71,11 @@ def read_json_file(file_path: str): return data + +def num_format(to_format: float) -> str: + """ + 按 decimal_digits 保留小数位数 + :param to_format: 待格式化的的小数 + :return: + """ + return "{:.{}f}".format(to_format, OmniAdvisorConf.decimal_digits) -- Gitee From f9c23f715d0857ffb46b41467960b7a660c8229f Mon Sep 17 00:00:00 2001 From: yang_feida Date: Mon, 16 Jun 2025 20:38:39 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E8=B0=83=E6=95=B4=E6=8E=A7=E5=88=B6?= =?UTF-8?q?=E5=8F=B0=E7=9A=84=E6=97=A5=E5=BF=97=E6=A0=BC=E5=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/omniadvisor/utils/logger.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/omniadvisor/src/omniadvisor/utils/logger.py b/omniadvisor/src/omniadvisor/utils/logger.py index bf3d2275b..108e5a8ad 100755 --- a/omniadvisor/src/omniadvisor/utils/logger.py +++ b/omniadvisor/src/omniadvisor/utils/logger.py @@ -3,14 +3,16 @@ import logging from logging.config import dictConfig from common.constant import OA_CONF +CONSOLE_FORMAT = '%(asctime)s [%(levelname)s] %(message)s' +FILE_FORMAT = '%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d in %(funcName)s - %(message)s' + LOGGING_CONFIG = { 'version': 1, 'disable_existing_loggers': True, 'formatters': { 'colorFormatter': { '()': 'colorlog.ColoredFormatter', - 'format': '%(log_color)s%(asctime)s [%(levelname)s] ' - '%(filename)s:%(lineno)d in %(funcName)s - %(message)s%(reset)s', + 'format': f'%(log_color)s{CONSOLE_FORMAT}%(reset)s', 'datefmt': '%Y-%m-%d %H:%M:%S', 'log_colors': { 'DEBUG': 'cyan', @@ -21,7 +23,7 @@ LOGGING_CONFIG = { }, }, 'simpleFormatter': { - 'format': '%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d in %(funcName)s - %(message)s', + 'format': FILE_FORMAT, 'datefmt': '%Y-%m-%d %H:%M:%S' } }, -- Gitee From f077e0cb874e02cf450ec80747e2d521dc083fe1 Mon Sep 17 00:00:00 2001 From: yang_feida Date: Mon, 16 Jun 2025 21:14:17 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E6=9B=B4=E6=94=B9=E5=AF=BC=E5=8C=85?= =?UTF-8?q?=E9=A1=BA=E5=BA=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/omniadvisor/utils/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/omniadvisor/src/omniadvisor/utils/utils.py b/omniadvisor/src/omniadvisor/utils/utils.py index 0b527c248..b46c4947b 100644 --- a/omniadvisor/src/omniadvisor/utils/utils.py +++ b/omniadvisor/src/omniadvisor/utils/utils.py @@ -4,9 +4,8 @@ import uuid import subprocess from typing import Tuple, List, Dict -from omniadvisor.utils.logger import global_logger - from common.constant import OmniAdvisorConf +from omniadvisor.utils.logger import global_logger def run_cmd(submit_cmd) -> Tuple[int, str]: -- Gitee From 5564f226051b2bac090f4e668a8b2039331c79b7 Mon Sep 17 00:00:00 2001 From: yang_feida Date: Tue, 17 Jun 2025 11:08:06 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E8=AF=84=E5=AE=A1=E6=84=8F=E8=A7=81?= =?UTF-8?q?=E4=BF=AE=E6=94=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/omniadvisor/interface/config_tuning.py | 8 ++++---- .../src/omniadvisor/interface/hijack_recommend.py | 8 ++++---- omniadvisor/src/omniadvisor/service/retest_service.py | 10 +++++----- omniadvisor/src/omniadvisor/utils/logger.py | 8 ++++---- omniadvisor/src/omniadvisor/utils/utils.py | 2 +- 5 files changed, 18 insertions(+), 18 deletions(-) diff --git a/omniadvisor/src/omniadvisor/interface/config_tuning.py b/omniadvisor/src/omniadvisor/interface/config_tuning.py index 551dde8a0..a8136a3f1 100644 --- a/omniadvisor/src/omniadvisor/interface/config_tuning.py +++ b/omniadvisor/src/omniadvisor/interface/config_tuning.py @@ -20,7 +20,7 @@ from omniadvisor.service.tuning_result.tuning_result_history import ( ) from omniadvisor.utils.logger import global_logger -from omniadvisor.utils.utils import num_format +from omniadvisor.utils.utils import float_format # 异常类暂时用于当前模块,没必要放到公共里 @@ -89,9 +89,9 @@ def unified_tuning(load, retest_way: str, tuning_method: str): tuning_result_history.refresh_best_config() global_logger.info( 'Current tuning result: %s, best tuning result ever: %s, best tuning speedup: %s %%', - num_format(tuning_result.runtime), - num_format(tuning_result_history.best_tuning_result.runtime), - num_format(tuning_result_history.boost_percentage) + float_format(tuning_result.runtime), + float_format(tuning_result_history.best_tuning_result.runtime), + float_format(tuning_result_history.boost_percentage) ) else: diff --git a/omniadvisor/src/omniadvisor/interface/hijack_recommend.py b/omniadvisor/src/omniadvisor/interface/hijack_recommend.py index 13070f341..ef6a67f4d 100644 --- a/omniadvisor/src/omniadvisor/interface/hijack_recommend.py +++ b/omniadvisor/src/omniadvisor/interface/hijack_recommend.py @@ -10,7 +10,7 @@ from omniadvisor.service.tuning_result.tuning_result_history import get_tuning_r from omniadvisor.utils.logger import global_logger from omniadvisor.service.spark_service.spark_run import spark_run -from omniadvisor.utils.utils import num_format +from omniadvisor.utils.utils import float_format def _query_or_create_load(name: str, exec_attr: dict, default_config: dict): @@ -84,9 +84,9 @@ def _process_load_config(load: Load): tuning_result_history.refresh_best_config() global_logger.info( 'Current tuning result: %s, best tuning result ever: %s, best tuning speedup: %s %%', - num_format(tuning_result.runtime), - num_format(tuning_result_history.best_tuning_result.runtime), - num_format(tuning_result_history.boost_percentage) + float_format(tuning_result.runtime), + float_format(tuning_result_history.best_tuning_result.runtime), + float_format(tuning_result_history.boost_percentage) ) # 当前test_config达到调优轮次后 置空test_config LoadRepository.update_test_config(load, {}) diff --git a/omniadvisor/src/omniadvisor/service/retest_service.py b/omniadvisor/src/omniadvisor/service/retest_service.py index 6bb07bcce..061319584 100644 --- a/omniadvisor/src/omniadvisor/service/retest_service.py +++ b/omniadvisor/src/omniadvisor/service/retest_service.py @@ -3,7 +3,7 @@ from omniadvisor.repository.model.load import Load from omniadvisor.service.spark_service.spark_run import spark_run from omniadvisor.service.tuning_result.tuning_result import get_tuning_result from omniadvisor.utils.logger import global_logger -from omniadvisor.utils.utils import num_format +from omniadvisor.utils.utils import float_format def retest(load: Load, config: dict): @@ -27,13 +27,13 @@ def retest(load: Load, config: dict): if exam_record.status == OA_CONF.ExecStatus.success: global_logger.info( - 'Retest succeeded in round %d. Performance result: %s.', i, - num_format(exam_record.runtime) + 'Retest succeeded in round %d. Performance result: %s.', + i, + float_format(exam_record.runtime) ) else: # 若是出现异常配置,也要退出 - global_logger.warning('Retest failed in round %d. Exception source: Spark exception: %s.', i, - spark_output) + global_logger.warning('Retest failed in round %d. Exception source: Spark exception: %s.', i, spark_output) tuning_result = get_tuning_result(load, config) if tuning_result.failed_times >= OA_CONF.config_fail_threshold: global_logger.info('The number of retest failures has reached the failure threshold.') diff --git a/omniadvisor/src/omniadvisor/utils/logger.py b/omniadvisor/src/omniadvisor/utils/logger.py index 108e5a8ad..937547623 100755 --- a/omniadvisor/src/omniadvisor/utils/logger.py +++ b/omniadvisor/src/omniadvisor/utils/logger.py @@ -10,7 +10,7 @@ LOGGING_CONFIG = { 'version': 1, 'disable_existing_loggers': True, 'formatters': { - 'colorFormatter': { + 'consoleFormatter': { '()': 'colorlog.ColoredFormatter', 'format': f'%(log_color)s{CONSOLE_FORMAT}%(reset)s', 'datefmt': '%Y-%m-%d %H:%M:%S', @@ -22,7 +22,7 @@ LOGGING_CONFIG = { 'CRITICAL': 'bold_red', }, }, - 'simpleFormatter': { + 'fileFormatter': { 'format': FILE_FORMAT, 'datefmt': '%Y-%m-%d %H:%M:%S' } @@ -31,13 +31,13 @@ LOGGING_CONFIG = { 'consoleHandler': { 'class': 'logging.StreamHandler', 'level': 'DEBUG', - 'formatter': 'colorFormatter', + 'formatter': 'consoleFormatter', 'stream': 'ext://sys.stdout', }, 'fileHandler': { 'class': 'logging.handlers.TimedRotatingFileHandler', 'level': 'DEBUG', - 'formatter': 'simpleFormatter', + 'formatter': 'fileFormatter', 'filename': OA_CONF.log_file_path, 'when': 'midnight', 'interval': 1, diff --git a/omniadvisor/src/omniadvisor/utils/utils.py b/omniadvisor/src/omniadvisor/utils/utils.py index b46c4947b..fea781692 100644 --- a/omniadvisor/src/omniadvisor/utils/utils.py +++ b/omniadvisor/src/omniadvisor/utils/utils.py @@ -71,7 +71,7 @@ def read_json_file(file_path: str): return data -def num_format(to_format: float) -> str: +def float_format(to_format: float) -> str: """ 按 decimal_digits 保留小数位数 :param to_format: 待格式化的的小数 -- Gitee From 04e9caa6846705e4a6d9c8f13790c3e4ae6cfa24 Mon Sep 17 00:00:00 2001 From: yang_feida Date: Tue, 17 Jun 2025 11:08:38 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E6=B5=8B=E8=AF=95=E7=94=A8=E4=BE=8B?= =?UTF-8?q?=E4=BF=AE=E5=A4=8D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/omniadvisor/interface/test_config_tuning.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py b/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py index 83dfe32ff..660db9049 100644 --- a/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py +++ b/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py @@ -28,7 +28,9 @@ class TestTuning: patch('omniadvisor.service.retest_service.spark_run') as mock_spark_run, \ patch('omniadvisor.interface.config_tuning.get_tuning_result'), \ patch('omniadvisor.interface.config_tuning.get_tuning_result_history') as mock_get_history, \ - patch('algo.iterative.tuning.SmacAppendTuning.tune') as mock_smac_tuning: + patch('algo.iterative.tuning.SmacAppendTuning.tune') as mock_smac_tuning, \ + patch('omniadvisor.interface.config_tuning.float_format'), \ + patch('omniadvisor.service.retest_service.float_format'): mock_exam_record = MagicMock() mock_smac_tuning.return_value = self.tune_return_val mock_exam_record.status = OA_CONF.ExecStatus.success @@ -54,7 +56,9 @@ class TestTuning: patch('omniadvisor.service.retest_service.get_tuning_result') as mock_get_tuning_result, \ patch('omniadvisor.interface.config_tuning.remove_tuning_result') as mock_remove_tuning_result, \ patch('algo.iterative.tuning.SmacAppendTuning.tune') as mock_smac_tuning, \ - patch('omniadvisor.repository.load_repository.LoadRepository.update_best_config') as mock_update_best: + patch('omniadvisor.repository.load_repository.LoadRepository.update_best_config') as mock_update_best, \ + patch('omniadvisor.interface.config_tuning.float_format'), \ + patch('omniadvisor.service.retest_service.float_format'): mock_exam_record = MagicMock() mock_exam_record.status = OA_CONF.ExecStatus.fail spark_output = self.empty_str -- Gitee From d80e51492c594e259e230f5ee447065c98ef122f Mon Sep 17 00:00:00 2001 From: yang_feida Date: Tue, 17 Jun 2025 11:14:19 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E8=A1=A5=E5=85=85=E5=8D=95=E5=85=83?= =?UTF-8?q?=E6=B5=8B=E8=AF=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../tests/omniadvisor/utils/test_utils.py | 27 +++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/omniadvisor/tests/omniadvisor/utils/test_utils.py b/omniadvisor/tests/omniadvisor/utils/test_utils.py index ae50ad5ed..81f7ab29d 100644 --- a/omniadvisor/tests/omniadvisor/utils/test_utils.py +++ b/omniadvisor/tests/omniadvisor/utils/test_utils.py @@ -1,6 +1,6 @@ import subprocess from unittest.mock import patch,mock_open -from omniadvisor.utils.utils import run_cmd, save_trace_data # 假设你的函数定义在 'your_module' 中 +from omniadvisor.utils.utils import run_cmd, save_trace_data, float_format # 假设你的函数定义在 'your_module' 中 class TestRunCmd: @@ -75,4 +75,27 @@ class TestTraceDataSaver: save_trace_data(data, data_dir) assert False, "Expected an Exception to be raised." except Exception as e: - assert str(e) == "保存过程中出现错误: Unexpected error", f"Unexpected error message: {str(e)}" \ No newline at end of file + assert str(e) == "保存过程中出现错误: Unexpected error", f"Unexpected error message: {str(e)}" + + +class TestFloatFormat: + + def test_basic_formatting(self): + # 正常四舍五入 + assert float_format(3.14159) == "3.142" + # 补零 + assert float_format(2.5) == "2.500" + # 整数输入 + assert float_format(100) == "100.000" + + def test_boundary_values(self): + # 零值 + assert float_format(0) == "0.000" + # 负数 + assert float_format(-3.14159) == "-3.142" + + def test_special_floats(self): + # 无穷大 + assert float_format(float('inf')) == "inf" + # 负无穷大 + assert float_format(float('-inf')) == "-inf" -- Gitee From f860fb6326230892823e717ff360305cc814b482 Mon Sep 17 00:00:00 2001 From: yang_feida Date: Wed, 18 Jun 2025 11:22:44 +0800 Subject: [PATCH 7/7] =?UTF-8?q?=E5=90=8C=E6=AD=A5=E5=AE=89=E5=85=A8?= =?UTF-8?q?=E7=BC=96=E8=AF=91=E6=96=87=E4=BB=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/omniadvisor/src/setup.py b/omniadvisor/src/setup.py index ecdb8ddfd..d23c92eb7 100644 --- a/omniadvisor/src/setup.py +++ b/omniadvisor/src/setup.py @@ -4,6 +4,7 @@ """ import os import py_compile +import sysconfig from Cython.Build import cythonize from setuptools import setup, Extension, find_packages @@ -11,6 +12,11 @@ from setuptools import setup, Extension, find_packages # 要编译成 .pyc的文件(几个入口文件) pyc_files = ['__init__.py', 'init.py', 'hijack.py', 'tuning.py'] +sysconfig.get_config_vars()['CCSHARED'] = '-O2 -pipe -Wall -Wtrampolines -D_FORTIFY_SOURCE=2 -O2 -fPIC ' \ + '-finline-functions -fstack-protector-strong ' \ + '-s -Wl,-z,noexecstack -Wl,-z,relro,-z,now ' \ + '-ftrapv -fstack-check' + # 1 需要编译的文件 def find_py_files(root): @@ -41,7 +47,7 @@ for py_file in find_py_files("."): Extension( module_name, [py_file], - extra_compile_args=["-O3"], # 优化选项 + extra_compile_args=[], # 优化选项 ) ) -- Gitee