diff --git a/omniadvisor/src/common/constant.py b/omniadvisor/src/common/constant.py index b506eb25827eaf03e2be7b867da1337e71ca4cf2..75be2cc51f4d30f9eeb5052fb3e84cdc07cb8786 100644 --- a/omniadvisor/src/common/constant.py +++ b/omniadvisor/src/common/constant.py @@ -75,5 +75,8 @@ class OmniAdvisorConf: spark_exec_timeout_ratio = _common_config.getfloat('spark', 'spark.exec.timeout.ratio') tuning_strategies = json.loads(_common_config.get('common', 'tuning.strategy')) + # 保留小数位数 + decimal_digits = 3 + OA_CONF = OmniAdvisorConf() diff --git a/omniadvisor/src/omniadvisor/interface/config_tuning.py b/omniadvisor/src/omniadvisor/interface/config_tuning.py index 24723db97654f9b6bda1ee03a53578710433bccd..a8136a3f13b170e082f132891ba6a6cd17642e9f 100644 --- a/omniadvisor/src/omniadvisor/interface/config_tuning.py +++ b/omniadvisor/src/omniadvisor/interface/config_tuning.py @@ -20,6 +20,7 @@ from omniadvisor.service.tuning_result.tuning_result_history import ( ) from omniadvisor.utils.logger import global_logger +from omniadvisor.utils.utils import float_format # 异常类暂时用于当前模块，没必要放到公共里 @@ -86,9 +87,12 @@ def unified_tuning(load, retest_way: str, tuning_method: str): tuning_result_history = get_tuning_result_history(load=load) # 更新最优配置 tuning_result_history.refresh_best_config() - global_logger.info(f'Current tuning result: {tuning_result.runtime}, best tuning result ever: ' - f'{tuning_result_history.best_tuning_result.runtime}, best tuning speedup: ' - f'{tuning_result_history.boost_percentage} %') + global_logger.info( + 'Current tuning result: %s, best tuning result ever: %s, best tuning speedup: %s %%', + float_format(tuning_result.runtime), + float_format(tuning_result_history.best_tuning_result.runtime), + float_format(tuning_result_history.boost_percentage) + ) else: # 更新待测试配置即可 diff --git a/omniadvisor/src/omniadvisor/interface/hijack_recommend.py b/omniadvisor/src/omniadvisor/interface/hijack_recommend.py index 7f260c0e192cef30d0e1d83f0ff32ccc89ed80a9..ef6a67f4d61aa21931dd8aee83a7119f68cd239d 100644 --- a/omniadvisor/src/omniadvisor/interface/hijack_recommend.py +++ b/omniadvisor/src/omniadvisor/interface/hijack_recommend.py @@ -10,6 +10,8 @@ from omniadvisor.service.tuning_result.tuning_result_history import get_tuning_r from omniadvisor.utils.logger import global_logger from omniadvisor.service.spark_service.spark_run import spark_run +from omniadvisor.utils.utils import float_format + def _query_or_create_load(name: str, exec_attr: dict, default_config: dict): """ @@ -80,9 +82,12 @@ def _process_load_config(load: Load): tuning_result_history = get_tuning_result_history(load=load) # 更新最优配置 tuning_result_history.refresh_best_config() - global_logger.info(f'Current tuning result: {tuning_result.runtime}, best tuning result ever: ' - f'{tuning_result_history.best_tuning_result.runtime}, best tuning speedup: ' - f'{tuning_result_history.boost_percentage} %') + global_logger.info( + 'Current tuning result: %s, best tuning result ever: %s, best tuning speedup: %s %%', + float_format(tuning_result.runtime), + float_format(tuning_result_history.best_tuning_result.runtime), + float_format(tuning_result_history.boost_percentage) + ) # 当前test_config达到调优轮次后 置空test_config LoadRepository.update_test_config(load, {}) diff --git a/omniadvisor/src/omniadvisor/service/retest_service.py b/omniadvisor/src/omniadvisor/service/retest_service.py index c92138d9a6fc5b48be5d5b8f3551c71f8f8be62c..061319584679388c097b388f5b40120bfb0a4ec3 100644 --- a/omniadvisor/src/omniadvisor/service/retest_service.py +++ b/omniadvisor/src/omniadvisor/service/retest_service.py @@ -3,6 +3,7 @@ from omniadvisor.repository.model.load import Load from omniadvisor.service.spark_service.spark_run import spark_run from omniadvisor.service.tuning_result.tuning_result import get_tuning_result from omniadvisor.utils.logger import global_logger +from omniadvisor.utils.utils import float_format def retest(load: Load, config: dict): @@ -25,11 +26,14 @@ def retest(load: Load, config: dict): raise if exam_record.status == OA_CONF.ExecStatus.success: - global_logger.info('Retest succeeded in round %d. Performance result: %.3f.', i, exam_record.runtime) + global_logger.info( + 'Retest succeeded in round %d. Performance result: %s.', + i, + float_format(exam_record.runtime) + ) else: # 若是出现异常配置，也要退出 - global_logger.warning('Retest failed in round %d. Exception source: Spark exception: %s.', i, - spark_output) + global_logger.warning('Retest failed in round %d. Exception source: Spark exception: %s.', i, spark_output) tuning_result = get_tuning_result(load, config) if tuning_result.failed_times >= OA_CONF.config_fail_threshold: global_logger.info('The number of retest failures has reached the failure threshold.') diff --git a/omniadvisor/src/omniadvisor/utils/logger.py b/omniadvisor/src/omniadvisor/utils/logger.py index bf3d2275b12cf216912479126daf413d1ba0590b..93754762337f3fca6c4d2fdda15ecad944953cfe 100755 --- a/omniadvisor/src/omniadvisor/utils/logger.py +++ b/omniadvisor/src/omniadvisor/utils/logger.py @@ -3,14 +3,16 @@ import logging from logging.config import dictConfig from common.constant import OA_CONF +CONSOLE_FORMAT = '%(asctime)s [%(levelname)s] %(message)s' +FILE_FORMAT = '%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d in %(funcName)s - %(message)s' + LOGGING_CONFIG = { 'version': 1, 'disable_existing_loggers': True, 'formatters': { - 'colorFormatter': { + 'consoleFormatter': { '()': 'colorlog.ColoredFormatter', - 'format': '%(log_color)s%(asctime)s [%(levelname)s] ' - '%(filename)s:%(lineno)d in %(funcName)s - %(message)s%(reset)s', + 'format': f'%(log_color)s{CONSOLE_FORMAT}%(reset)s', 'datefmt': '%Y-%m-%d %H:%M:%S', 'log_colors': { 'DEBUG': 'cyan', @@ -20,8 +22,8 @@ LOGGING_CONFIG = { 'CRITICAL': 'bold_red', }, }, - 'simpleFormatter': { - 'format': '%(asctime)s [%(levelname)s] %(filename)s:%(lineno)d in %(funcName)s - %(message)s', + 'fileFormatter': { + 'format': FILE_FORMAT, 'datefmt': '%Y-%m-%d %H:%M:%S' } }, @@ -29,13 +31,13 @@ LOGGING_CONFIG = { 'consoleHandler': { 'class': 'logging.StreamHandler', 'level': 'DEBUG', - 'formatter': 'colorFormatter', + 'formatter': 'consoleFormatter', 'stream': 'ext://sys.stdout', }, 'fileHandler': { 'class': 'logging.handlers.TimedRotatingFileHandler', 'level': 'DEBUG', - 'formatter': 'simpleFormatter', + 'formatter': 'fileFormatter', 'filename': OA_CONF.log_file_path, 'when': 'midnight', 'interval': 1, diff --git a/omniadvisor/src/omniadvisor/utils/utils.py b/omniadvisor/src/omniadvisor/utils/utils.py index 8e333afa411735e9637fb3ef8580c032035d72f9..fea781692b3a9bd3aa5a1af7b0f2efa8b3140dda 100644 --- a/omniadvisor/src/omniadvisor/utils/utils.py +++ b/omniadvisor/src/omniadvisor/utils/utils.py @@ -4,6 +4,7 @@ import uuid import subprocess from typing import Tuple, List, Dict +from common.constant import OmniAdvisorConf from omniadvisor.utils.logger import global_logger @@ -69,3 +70,11 @@ def read_json_file(file_path: str): return data + +def float_format(to_format: float) -> str: + """ + 按 decimal_digits 保留小数位数 + :param to_format: 待格式化的的小数 + :return: + """ + return "{:.{}f}".format(to_format, OmniAdvisorConf.decimal_digits) diff --git a/omniadvisor/src/setup.py b/omniadvisor/src/setup.py index ecdb8ddfd7eedc3bb627ee6282945ee0f7573719..d23c92eb77bd7ff0f2a0b983462bb8edd155163b 100644 --- a/omniadvisor/src/setup.py +++ b/omniadvisor/src/setup.py @@ -4,6 +4,7 @@ """ import os import py_compile +import sysconfig from Cython.Build import cythonize from setuptools import setup, Extension, find_packages @@ -11,6 +12,11 @@ from setuptools import setup, Extension, find_packages # 要编译成 .pyc的文件（几个入口文件） pyc_files = ['__init__.py', 'init.py', 'hijack.py', 'tuning.py'] +sysconfig.get_config_vars()['CCSHARED'] = '-O2 -pipe -Wall -Wtrampolines -D_FORTIFY_SOURCE=2 -O2 -fPIC ' \ + '-finline-functions -fstack-protector-strong ' \ + '-s -Wl,-z,noexecstack -Wl,-z,relro,-z,now ' \ + '-ftrapv -fstack-check' + # 1 需要编译的文件 def find_py_files(root): @@ -41,7 +47,7 @@ for py_file in find_py_files("."): Extension( module_name, [py_file], - extra_compile_args=["-O3"], # 优化选项 + extra_compile_args=[], # 优化选项 ) ) diff --git a/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py b/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py index 83dfe32ff561c28dd16b1aa2779756c6639becfc..660db90490247e341be32210d7f83d13fcd29e9a 100644 --- a/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py +++ b/omniadvisor/tests/omniadvisor/interface/test_config_tuning.py @@ -28,7 +28,9 @@ class TestTuning: patch('omniadvisor.service.retest_service.spark_run') as mock_spark_run, \ patch('omniadvisor.interface.config_tuning.get_tuning_result'), \ patch('omniadvisor.interface.config_tuning.get_tuning_result_history') as mock_get_history, \ - patch('algo.iterative.tuning.SmacAppendTuning.tune') as mock_smac_tuning: + patch('algo.iterative.tuning.SmacAppendTuning.tune') as mock_smac_tuning, \ + patch('omniadvisor.interface.config_tuning.float_format'), \ + patch('omniadvisor.service.retest_service.float_format'): mock_exam_record = MagicMock() mock_smac_tuning.return_value = self.tune_return_val mock_exam_record.status = OA_CONF.ExecStatus.success @@ -54,7 +56,9 @@ class TestTuning: patch('omniadvisor.service.retest_service.get_tuning_result') as mock_get_tuning_result, \ patch('omniadvisor.interface.config_tuning.remove_tuning_result') as mock_remove_tuning_result, \ patch('algo.iterative.tuning.SmacAppendTuning.tune') as mock_smac_tuning, \ - patch('omniadvisor.repository.load_repository.LoadRepository.update_best_config') as mock_update_best: + patch('omniadvisor.repository.load_repository.LoadRepository.update_best_config') as mock_update_best, \ + patch('omniadvisor.interface.config_tuning.float_format'), \ + patch('omniadvisor.service.retest_service.float_format'): mock_exam_record = MagicMock() mock_exam_record.status = OA_CONF.ExecStatus.fail spark_output = self.empty_str diff --git a/omniadvisor/tests/omniadvisor/utils/test_utils.py b/omniadvisor/tests/omniadvisor/utils/test_utils.py index ae50ad5edcebb0219cf8ad81584cc00603e1e0e8..81f7ab29d64dc9df42ff459e5ba8788b932750ed 100644 --- a/omniadvisor/tests/omniadvisor/utils/test_utils.py +++ b/omniadvisor/tests/omniadvisor/utils/test_utils.py @@ -1,6 +1,6 @@ import subprocess from unittest.mock import patch,mock_open -from omniadvisor.utils.utils import run_cmd, save_trace_data # 假设你的函数定义在 'your_module' 中 +from omniadvisor.utils.utils import run_cmd, save_trace_data, float_format # 假设你的函数定义在 'your_module' 中 class TestRunCmd: @@ -75,4 +75,27 @@ class TestTraceDataSaver: save_trace_data(data, data_dir) assert False, "Expected an Exception to be raised." except Exception as e: - assert str(e) == "保存过程中出现错误: Unexpected error", f"Unexpected error message: {str(e)}" \ No newline at end of file + assert str(e) == "保存过程中出现错误: Unexpected error", f"Unexpected error message: {str(e)}" + + +class TestFloatFormat: + + def test_basic_formatting(self): + # 正常四舍五入 + assert float_format(3.14159) == "3.142" + # 补零 + assert float_format(2.5) == "2.500" + # 整数输入 + assert float_format(100) == "100.000" + + def test_boundary_values(self): + # 零值 + assert float_format(0) == "0.000" + # 负数 + assert float_format(-3.14159) == "-3.142" + + def test_special_floats(self): + # 无穷大 + assert float_format(float('inf')) == "inf" + # 负无穷大 + assert float_format(float('-inf')) == "-inf"