From 5cf797eab79101064c9836abfafd644f500eb1f2 Mon Sep 17 00:00:00 2001 From: wtingkai <330445001@qq.com> Date: Thu, 29 May 2025 09:57:11 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat(OmniAdvisor):=20=E5=B1=8F=E8=94=BD?= =?UTF-8?q?=E7=AC=AC=E4=B8=89=E6=96=B9=E5=BA=93=E4=B8=AD=E7=9A=84log?= =?UTF-8?q?=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/src/omniadvisor/utils/logger.py | 13 ++- .../tests/omniadvisor/utils/test_logger.py | 99 ++++++++++++------- 2 files changed, 75 insertions(+), 37 deletions(-) diff --git a/omniadvisor/src/omniadvisor/utils/logger.py b/omniadvisor/src/omniadvisor/utils/logger.py index f1e070a31..3f7774e02 100755 --- a/omniadvisor/src/omniadvisor/utils/logger.py +++ b/omniadvisor/src/omniadvisor/utils/logger.py @@ -33,7 +33,7 @@ LOGGING_CONFIG = { }, 'fileHandler': { 'class': 'logging.handlers.TimedRotatingFileHandler', - 'level': 'INFO', + 'level': 'DEBUG', 'formatter': 'simpleFormatter', 'filename': OA_CONF.log_file_path, 'when': 'midnight', @@ -43,8 +43,8 @@ LOGGING_CONFIG = { } }, 'loggers': { - 'project_logger': { - 'level': 'DEBUG', + 'omniadvisor': { + 'level': 'INFO', 'handlers': ['consoleHandler', 'fileHandler'], } } @@ -58,7 +58,12 @@ if not os.path.exists(OA_CONF.log_dir): dictConfig(LOGGING_CONFIG) # 获取logger并使用 -global_logger = logging.getLogger('project_logger') +global_logger = logging.getLogger('omniadvisor') + +# 屏蔽部分第三方库中的Logger +modules_setting_error = ['smac'] +for model in modules_setting_error: + logging.getLogger(model).setLevel(logging.ERROR) def disable_console_handler(logger): diff --git a/omniadvisor/tests/omniadvisor/utils/test_logger.py b/omniadvisor/tests/omniadvisor/utils/test_logger.py index 5ed1206a6..b22ffbed6 100755 --- a/omniadvisor/tests/omniadvisor/utils/test_logger.py +++ b/omniadvisor/tests/omniadvisor/utils/test_logger.py @@ -1,50 +1,83 @@ import logging -import os - +import io import pytest -from pathlib import Path - -from omniadvisor.utils.logger import global_logger - - -@pytest.fixture(scope="class", autouse=True) -def class_fixture(): - global tmplog - tmplog = "tmp.log" - # 如果原有日志文件存在 删除文件 - if os.path.exists(tmplog): - os.remove(tmplog) - # 临时替换FileHandler 用于测试 - orihandler = "" - orihandler_index = "" - for handler in global_logger.handlers: - if isinstance(handler, logging.FileHandler): - orihandler = handler - new_file_handler = logging.FileHandler(tmplog) - new_file_handler.setLevel(logging.INFO) - orihandler_index = global_logger.handlers.index(handler) - global_logger.handlers[orihandler_index] = new_file_handler - - yield - global_logger.handlers[orihandler_index] = orihandler - os.remove(tmplog) + +from omniadvisor.utils.logger import global_logger, disable_console_handler class TestLogger: + """ + Logger测试类 + """ + def test_logger_output(self): + """ + 测试软件logger输出 + """ + # 创建内存缓冲区 + log_stream = io.StringIO() + handler = logging.StreamHandler(log_stream) + # 屏蔽其他handler，使用内存IO作为测试使用 + global_logger.handlers = [handler] - def test_log(self): + # log输出 global_logger.debug("This is a debug message") global_logger.info("This is an info message") global_logger.warning("This is a warning message") global_logger.error("This is an error message") global_logger.critical("This is a critical message") - assert Path(tmplog).stat().st_size > 0, f"文件 {tmplog} 大小为 0 字节,写入失败" # 验证日志文件内容 - with open(tmplog, 'r') as file: - content = file.read() - + content = log_stream.getvalue() + assert global_logger.level == logging.INFO + assert "This is an debug message" not in content, "Debug message should not found in log file" assert "This is an info message" in content, "Info message not found in log file" assert "This is a warning message" in content, "Warning message not found in log file" assert "This is an error message" in content, "Error message not found in log file" assert "This is a critical message" in content, "Critical message not found in log file" + + def test_third_party_logger_suppression(self): + """ + 测试第三方库日志级别是否设置为 ERROR + """ + # 以smac logger作为示例 + smac_logger = logging.getLogger('smac') + # 创建内存缓冲区 + log_stream = io.StringIO() + handler = logging.StreamHandler(log_stream) + # 屏蔽其他handler，使用内存IO作为测试使用 + smac_logger.handlers = [handler] + + # log输出 + smac_logger.debug("This is a debug message") + smac_logger.info("This is an info message") + smac_logger.warning("This is a warning message") + smac_logger.error("This is an error message") + smac_logger.critical("This is a critical message") + + # 验证日志文件内容 + content = log_stream.getvalue() + assert smac_logger.level == logging.ERROR + assert "This is an debug message" not in content, "Debug message should not found in log file" + assert "This is an info message" not in content, "Info message should not found in log file" + assert "This is a warning message" not in content, "Warning message should not found in log file" + assert "This is an error message" in content, "Error message not found in log file" + assert "This is a critical message" in content, "Critical message not found in log file" + + def test_disable_console_handler(self): + """ + 测试禁用控制台处理器的功能 + """ + # 创建一个测试记录器并添加控制台处理器 + test_logger = logging.getLogger('test_logger') + console_handler = logging.StreamHandler() + console_handler.name = 'consoleHandler' + test_logger.addHandler(console_handler) + + # 验证处理器存在 + assert console_handler in test_logger.handlers + + # 调用禁用函数 + disable_console_handler(test_logger) + + # 验证处理器已被移除 + assert console_handler not in test_logger.handlers -- Gitee From 6f67eb7479261f54e8c467363913940293c0054b Mon Sep 17 00:00:00 2001 From: wtingkai <330445001@qq.com> Date: Thu, 29 May 2025 15:14:06 +0800 Subject: [PATCH 2/2] =?UTF-8?q?feat(OmniAdvisor):=20=E4=BF=AE=E6=94=B9?= =?UTF-8?q?=E5=8A=AB=E6=8C=81=E8=84=9A=E6=9C=AC=E4=B8=AD=E7=9A=84=E8=BE=93?= =?UTF-8?q?=E5=87=BA=E4=BF=A1=E6=81=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omniadvisor/script/spark-submit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/omniadvisor/script/spark-submit b/omniadvisor/script/spark-submit index f29028e62..58193f6ae 100644 --- a/omniadvisor/script/spark-submit +++ b/omniadvisor/script/spark-submit @@ -26,7 +26,7 @@ export PYTHONHASHSEED=0 # 使能OmniAdvisor劫持优化 if [[ -v enable_omniadvisor && "$enable_omniadvisor" = "true" ]]; then - echo "enable_omniadvisor" + echo "*** OmniAdvisor is enabled! We will find best config to SPEEDUP! ***" # 根据特性实际的部署路径进行修改 hijack_path="" # OmniAdvisor劫持优化 -- Gitee