From 9b6ca0fb989b48218f149fec029c3e263bf028c1 Mon Sep 17 00:00:00 2001 From: Bai <12577555+wch_study@user.noreply.gitee.com> Date: Fri, 25 Jul 2025 15:06:19 +0800 Subject: [PATCH 1/5] =?UTF-8?q?feat(chore):=20"=E5=A2=9E=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jiuwen/extensions/common/configs/_init_.py | 0 jiuwen/extensions/common/configs/base.py | 23 +++ jiuwen/extensions/common/exception/_init_.py | 0 jiuwen/extensions/common/exception/base.py | 41 ++++ jiuwen/extensions/common/log/_init_.py | 0 jiuwen/extensions/common/log/base.py | 188 +++++++++++++++++++ jiuwen/extensions/common/log/utils.py | 42 +++++ jiuwen/extensions/config.yaml | 45 +++++ tests/unit_tests/common/log/__init__.py | 0 tests/unit_tests/common/log/test_base.py | 28 +++ 10 files changed, 367 insertions(+) create mode 100644 jiuwen/extensions/common/configs/_init_.py create mode 100644 jiuwen/extensions/common/configs/base.py create mode 100644 jiuwen/extensions/common/exception/_init_.py create mode 100644 jiuwen/extensions/common/exception/base.py create mode 100644 jiuwen/extensions/common/log/_init_.py create mode 100644 jiuwen/extensions/common/log/base.py create mode 100644 jiuwen/extensions/common/log/utils.py create mode 100644 jiuwen/extensions/config.yaml create mode 100644 tests/unit_tests/common/log/__init__.py create mode 100644 tests/unit_tests/common/log/test_base.py diff --git a/jiuwen/extensions/common/configs/_init_.py b/jiuwen/extensions/common/configs/_init_.py new file mode 100644 index 0000000..e69de29 diff --git a/jiuwen/extensions/common/configs/base.py b/jiuwen/extensions/common/configs/base.py new file mode 100644 index 0000000..37fd4b8 --- /dev/null +++ b/jiuwen/extensions/common/configs/base.py @@ -0,0 +1,23 @@ +import os +import yaml + +class SingletonConfig: + _instance: yaml.constructor = None + + def __init__(self): + if SingletonConfig._instance is not None: + raise Exception("This class is a singleton!") + + log_config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config.yaml") + with open(log_config_path, encoding="utf-8") as file: + # 加载配置文件 + SingletonConfig._instance = yaml.safe_load(file) + + + @staticmethod + def get_config(): + if SingletonConfig._instance is None: + SingletonConfig() + return SingletonConfig._instance + +config = SingletonConfig.get_config() \ No newline at end of file diff --git a/jiuwen/extensions/common/exception/_init_.py b/jiuwen/extensions/common/exception/_init_.py new file mode 100644 index 0000000..e69de29 diff --git a/jiuwen/extensions/common/exception/base.py b/jiuwen/extensions/common/exception/base.py new file mode 100644 index 0000000..f9b98d6 --- /dev/null +++ b/jiuwen/extensions/common/exception/base.py @@ -0,0 +1,41 @@ + + +MAGIC_CODE = "\t" + +class JiuWenException(Exception): + def __init__( + self, + message:str, + *args, + **kwargs + )->None: + super().__init__(message, args, kwargs) + + +class JiuWenBaseException(Exception): + + def __init__(self, error_code: int, message:str, node_id : str = None, node_name : str = None, + node_type : str = None) -> None: + super().__init__(error_code, message) + self._error_code = error_code + self._message = message + self.node_id = node_id + self.node_name = node_name + self.node_type = node_type + + def __str__(self): + return f"[{self._error_code}]{self._message}{MAGIC_CODE}" + + @property + def error_code(self): + return self._error_code + + @property + def message(self): + return self._message + +class InterruptException(JiuWenBaseException): + def __init__(self, error_code: int, message:str) -> None: + super().__init__(error_code, message) + self.__error_code = error_code + self.__message = message \ No newline at end of file diff --git a/jiuwen/extensions/common/log/_init_.py b/jiuwen/extensions/common/log/_init_.py new file mode 100644 index 0000000..e69de29 diff --git a/jiuwen/extensions/common/log/base.py b/jiuwen/extensions/common/log/base.py new file mode 100644 index 0000000..6f9a5ef --- /dev/null +++ b/jiuwen/extensions/common/log/base.py @@ -0,0 +1,188 @@ +"""logger for common and interface.""" +__all__ = ("logger", "interface_logger", "prompt_builder_interface_logger",\ + "performance_logger", "get_thread_session", "set_thread_session") + +import ast +import os +import sys +import threading +import logging +from logging.handlers import RotatingFileHandler + +from jiuwen.extensions.common.configs.base import config, SingletonConfig +from jiuwen.extensions.common.exception.base import JiuWenBaseException + +CRITICAL = 50 +FATAL = CRITICAL +ERROR = 40 +WARNING = 30 +WARN = WARNING +INFO = 20 +DEBUG = 10 +NOTSET = 0 +DEFAULT_LOG_MAX_BYTES = 100 * 1024 * 1024 #100MB +CONFIG_LOG_MAX_BYTES = 20 * 1024 * 1024 #20MB +DEFAULT_BACKUP_COUNT = 20 + + +name_to_level = { + 'CRITICAL': CRITICAL, + 'FATAL': FATAL, + 'ERROR': ERROR, + 'WARNING': WARNING, + 'WARN': WARNING, + 'INFO': INFO, + 'DEBUG': DEBUG, + 'NOTSET': NOTSET, +} +# Log output method, output to the console +CONSOLE = 'console' +# Unique identifier of the session, user tag log +TRACE_ID = 'trace_id' + +# 使用方式，在需要注入session_id的参数地方，使用thread_local_data.trace_id = trace_id +_thread_log_instance = threading.local() + +def set_thread_session(trace_id): + _thread_log_instance.trace_id = trace_id + +def get_thread_session() -> str: + return getattr(_thread_log_instance, 'trace_id', '') + +def get_log_max_bytes(max_bytes_config): + try: + max_bytes = int(max_bytes_config) + except ValueError as e: + raise JiuWenBaseException( + error_code=-1, message="-1" + ) from e + + if max_bytes <= 0 or max_bytes > DEFAULT_LOG_MAX_BYTES: + max_bytes = DEFAULT_LOG_MAX_BYTES #小于0或者超出默认最大限制，使用默认最大值 + + return max_bytes + + +class SafeRotatingFileHandler(RotatingFileHandler): + def __init__(self, filename, *args, **kwargs): + pid = os.getpid() + filename = f"{filename}-{pid}" + super().__init__(filename, *args, **kwargs) + #设置文件的初始权限为 640 + os.chmod(self.baseFilename, 0o640) + + def doRollover(self): + # Rotate the file first + # 日志文件（正在记录）权限为640， 日志文件（记录完毕或者已经归档）权限为440 + RotatingFileHandler.doRollover(self) + for i in range(self.backupCount, 0, -1): + sfn = f"{self.baseFilename}.{i}" + if os.path.exists(sfn): + os.chmod(sfn, 0o440) + os.chmod(self.baseFilename, 0o640) + +class SingletonLogger: + _instance : logging.Logger = None + _interface_instance : logging.Logger = None + _prompt_builder_interface_instance : logging.Logger = None + _performance_instance : logging.Logger = None + + def __init__(self): + SingletonFlag = SingletonLogger._instance is not None or SingletonLogger._interface_instance is not None + if SingletonFlag or SingletonLogger._prompt_builder_interface_instance \ + is not None or SingletonLogger._performance_instance is not None: + raise Exception("This class is a singleton!") + log_config = config["log"] + if log_config is None: + return + jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path')) + if not os.path.exists(os.path.dirname(jiuwen_log_path)): + # 日志文件目录权限为750 + os.makedirs(os.path.dirname(jiuwen_log_path), mode=0o750) + log_file = os.path.join(jiuwen_log_path, log_config.get('log_file')) + performance_log_file = os.path.join(jiuwen_log_path, log_config.get('performance_log_file')) + output = log_config.get('output', CONSOLE) + + interface_log_file = os.path.join(jiuwen_log_path, log_config.get('interface_log_file')) + prompt_builder_interface_log_file = os.path.join(jiuwen_log_path, log_config.get('prompt_builder_interface_log_file')) + interface_output = log_config.get('interface_output', CONSOLE) + performance_output = log_config.get('performance_output', CONSOLE) + + SingletonLogger._instance = SingletonLogger._get_logger('common', log_config, output, log_file) + SingletonLogger._interface_instance = SingletonLogger._get_logger('interface', log_config, interface_output, interface_log_file) + + SingletonLogger._prompt_builder_interface_instance = SingletonLogger._get_logger('interface_prompt_builder', log_config, interface_output, prompt_builder_interface_log_file) + SingletonLogger._performance_instance = SingletonLogger._get_logger('performance', log_config, performance_output, performance_log_file) + + + + @staticmethod + def get_logger(): + + SingletonFlag = SingletonLogger._instance is not None or SingletonLogger._interface_instance + if SingletonFlag or SingletonLogger._prompt_builder_interface_instance \ + or SingletonLogger._performance_instance: + SingletonLogger() + return SingletonLogger._instance, SingletonLogger._interface_instance, \ + SingletonLogger._prompt_builder_interface_instance, \ + SingletonLogger._performance_instance + + + @classmethod + def _get_logger(cls, log_type, log_config, output, log_file): + level = os.environ.get("JIUWEN_LOG_LEVEL", log_config.get('level','WARNING')) + env_backup_count = os.environ.get("JIUWEN_LOG_BACKUP_COUNT") + backup_count = ast.literal_eval(env_backup_count) if env_backup_count else log_config.get('backup_count', DEFAULT_BACKUP_COUNT) + env_max_bytes = os.environ.get("JIUWEN_LOG_MAX_BYTES") + max_bytes = get_log_max_bytes(ast.literal_eval(env_max_bytes) if env_max_bytes else get_log_max_bytes( + log_config.get('max_bytes', CONFIG_LOG_MAX_BYTES))) + log_format = '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s' + if log_type == 'common': + log_format = log_config.get('format', + '%(asctime)s | %(log_type)s | %(filename)s | %(lineno)d | %(funcName)s | %(trace_id)s' + '| %(levelname)s | %(message)s') + + common_logger = logging.getLogger(log_type) + common_logger.setLevel(level) + if CONSOLE in output: + stream_handler = logging.StreamHandler(stream=sys.stdout) + if TRACE_ID in log_format: + stream_handler.addFilter(ThreadContextFilter(log_type)) + stream_handler.setFormatter(logging.Formatter(log_format)) + common_logger.addHandler(stream_handler) + + if 'log' in output: + if not os.path.exists(os.path.dirname(log_file)): + # 日志文件目录权限为750 + os.makedirs(os.path.dirname(log_file), mode=0o750) + file_handler = SafeRotatingFileHandler( + filename=log_file, + maxBytes=max_bytes, + backupCount=backup_count, + encoding='utf-8' + ) + if TRACE_ID in log_format: + file_handler.addFilter(ThreadContextFilter(log_type)) + file_handler.setFormatter(logging.Formatter(log_format)) + common_logger.addHandler(file_handler) + + return common_logger + + +class ThreadContextFilter(logging.Filter): + """ + 线程上下文过滤器 + 其主要用途是过滤日志记录，添加线程相关信息，如线程ID等。 + """ + def __init__(self, log_type): + super().__init__() + self.log_type = log_type + + def filter(self, record): + # 搜寻线程本地存储以获取trace_id如果存在的话 + record.trace_id = getattr(_thread_log_instance, 'trace_id', '') + record.log_type = "perf" if self.log_type == 'performance' else self.log_type + return True + +# 使用示例 +logger, interface_logger, prompt_builder_interface_logger, performance_logger = SingletonLogger.get_logger() diff --git a/jiuwen/extensions/common/log/utils.py b/jiuwen/extensions/common/log/utils.py new file mode 100644 index 0000000..4830470 --- /dev/null +++ b/jiuwen/extensions/common/log/utils.py @@ -0,0 +1,42 @@ +__all__ = ("LazyStr", "blur", "blur_later") + +from typing import Any, Callable + +from pydantic import BaseModel + +class LazyStr: + """Call 'f' while calling its '_str_'.""" + def __init__(self, f:Callable[[],str]): + self._f = f + self.__val = None + + def __str__(self): + if self.__val is None: + self.__val = self._f() + return self.__val + + + +def blur_later(obj): + return LazyStr(lambda: blur(obj)) + + + +def blur(obj) -> str: + + if obj is None: + return "None" + for t, f in _blur_map: + if isinstance(obj, t): + return f(obj) + return f"...({type(obj).__name__})" + +def _blur_dict(d : dict) -> str: + contents = ','.join(f"{k!r} : {blur(v)}" for k, v in d.items()) + return '{' + contents + '}' + +_blur_map : list[tuple[type, Callable[[Any], str]]] = [ + (dict, _blur_dict), + (list, lambda li : '[' + ','.join(blur(v) for v in li) + ']'), + (BaseModel, lambda b : blur(b.model_dump(by_alias=True))), +] \ No newline at end of file diff --git a/jiuwen/extensions/config.yaml b/jiuwen/extensions/config.yaml new file mode 100644 index 0000000..e6b2d3b --- /dev/null +++ b/jiuwen/extensions/config.yaml @@ -0,0 +1,45 @@ +logging: + level: INFO + backup_count: 20 + max_bytes: 20971520 + format: '%(asctime)s | %(log_type)s | %(filename)s | %(lineno)d | %(funcName)s | %(trace_id)s | %(levelname)s | %(message)s' + log_file: "run/jiuwen.log" + output: console + interface_log_file : "interface/jiuwen_interface.log" + prompt_builder_interface_log_file: "interface/jiuwen_prompt_builder_interface.log" + performance_log_file: "performance/jiuwen_performance/log" + interface_output: log + performance_output: console + log_path: "./logs/" + +plugin: + store: "gaussdb" + cache_maxsize: "1000" + default_dir: "./plugins" + recall: + emb_class: + emb_service_url: "" + app_code: "" + emb_dim: 256 + infer_fields: [ "func_description" ] + +prompt: + default_dir: "./prompts" + +executor: + worker: + timeout: 300 + conn_timeout: 300 + read_timeout: 5 + resume_interval: 15000 + max_retry: 2 + thread_num: 100 + stream_out_mode: mq + +planner_config: + finish_triggers: + - "yes" + - "是" + +rails: + custom_rails_path: #自定义护栏路径 \ No newline at end of file diff --git a/tests/unit_tests/common/log/__init__.py b/tests/unit_tests/common/log/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit_tests/common/log/test_base.py b/tests/unit_tests/common/log/test_base.py new file mode 100644 index 0000000..0271d5f --- /dev/null +++ b/tests/unit_tests/common/log/test_base.py @@ -0,0 +1,28 @@ +import threading +from unittest import TestCase + +from jiuwen.extensions.common.log.base import logger, set_thread_session + +def set_session_id(session_id): + set_thread_session(session_id) + + +def thread_function(session_id): + set_thread_session(session_id) + logger.info('Thread started with session id {}'.format(session_id)) + + +class Test(TestCase): + def test_logger(self): + thread1 = threading.Thread(target=thread_function, args=('10001',)) + thread2 = threading.Thread(target=thread_function, args=('10002',)) + thread3 = threading.Thread(target=thread_function, args=('10003',)) + + + thread1.start() + thread2.start() + thread3.start() + + thread1.join() + thread2.join() + thread3.join() \ No newline at end of file -- Gitee From 1e814bd930379ee2f7f90b0823e5c80a01921bfe Mon Sep 17 00:00:00 2001 From: Bai <12577555+wch_study@user.noreply.gitee.com> Date: Fri, 25 Jul 2025 17:40:34 +0800 Subject: [PATCH 2/5] =?UTF-8?q?feat:=20"=E5=A2=9E=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jiuwen/extensions/common/log/base.py | 59 ++++++++++++++++++++++------ 1 file changed, 47 insertions(+), 12 deletions(-) diff --git a/jiuwen/extensions/common/log/base.py b/jiuwen/extensions/common/log/base.py index 6f9a5ef..8d17489 100644 --- a/jiuwen/extensions/common/log/base.py +++ b/jiuwen/extensions/common/log/base.py @@ -12,6 +12,7 @@ from logging.handlers import RotatingFileHandler from jiuwen.extensions.common.configs.base import config, SingletonConfig from jiuwen.extensions.common.exception.base import JiuWenBaseException +# 日志级别常量 CRITICAL = 50 FATAL = CRITICAL ERROR = 40 @@ -20,11 +21,13 @@ WARN = WARNING INFO = 20 DEBUG = 10 NOTSET = 0 + +# 日志文件大小限制 DEFAULT_LOG_MAX_BYTES = 100 * 1024 * 1024 #100MB CONFIG_LOG_MAX_BYTES = 20 * 1024 * 1024 #20MB -DEFAULT_BACKUP_COUNT = 20 - +DEFAULT_BACKUP_COUNT = 20 # 默认备份文件数量 +# 日志级别名称到数值的映射 name_to_level = { 'CRITICAL': CRITICAL, 'FATAL': FATAL, @@ -35,28 +38,31 @@ name_to_level = { 'DEBUG': DEBUG, 'NOTSET': NOTSET, } -# Log output method, output to the console -CONSOLE = 'console' -# Unique identifier of the session, user tag log -TRACE_ID = 'trace_id' + +# 日志输出方式常量 +CONSOLE = 'console' # 控制台输出 +TRACE_ID = 'trace_id' # 会话追踪ID # 使用方式，在需要注入session_id的参数地方，使用thread_local_data.trace_id = trace_id -_thread_log_instance = threading.local() +_thread_log_instance = threading.local() # 线程局部存储，用于保存trace_id def set_thread_session(trace_id): + """设置当前线程的trace_id""" _thread_log_instance.trace_id = trace_id def get_thread_session() -> str: + """获取当前线程的trace_id""" return getattr(_thread_log_instance, 'trace_id', '') def get_log_max_bytes(max_bytes_config): + """验证并获取有效的日志文件大小限制""" try: max_bytes = int(max_bytes_config) except ValueError as e: raise JiuWenBaseException( error_code=-1, message="-1" ) from e - + # 限制文件大小再合理范围内 if max_bytes <= 0 or max_bytes > DEFAULT_LOG_MAX_BYTES: max_bytes = DEFAULT_LOG_MAX_BYTES #小于0或者超出默认最大限制，使用默认最大值 @@ -64,6 +70,7 @@ def get_log_max_bytes(max_bytes_config): class SafeRotatingFileHandler(RotatingFileHandler): + """安全轮转文件处理器，添加进程ID和文件权限控制""" def __init__(self, filename, *args, **kwargs): pid = os.getpid() filename = f"{filename}-{pid}" @@ -72,52 +79,68 @@ class SafeRotatingFileHandler(RotatingFileHandler): os.chmod(self.baseFilename, 0o640) def doRollover(self): + """执行日志轮转并设置文件权限""" # Rotate the file first # 日志文件（正在记录）权限为640， 日志文件（记录完毕或者已经归档）权限为440 RotatingFileHandler.doRollover(self) + # 设置归档日志文件权限为440 for i in range(self.backupCount, 0, -1): sfn = f"{self.baseFilename}.{i}" if os.path.exists(sfn): os.chmod(sfn, 0o440) + # 设置当前日志文件权限为640 os.chmod(self.baseFilename, 0o640) class SingletonLogger: + """单例日志管理器，创建四种类型的日志记录器""" _instance : logging.Logger = None _interface_instance : logging.Logger = None _prompt_builder_interface_instance : logging.Logger = None _performance_instance : logging.Logger = None def __init__(self): + # 单例检查 SingletonFlag = SingletonLogger._instance is not None or SingletonLogger._interface_instance is not None if SingletonFlag or SingletonLogger._prompt_builder_interface_instance \ is not None or SingletonLogger._performance_instance is not None: raise Exception("This class is a singleton!") + + # 从配置获取日志设置 log_config = config["log"] if log_config is None: return + + # 获取日志路径（环境变量优先） jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path')) + + # 创建日志目录（如果不存在） if not os.path.exists(os.path.dirname(jiuwen_log_path)): # 日志文件目录权限为750 os.makedirs(os.path.dirname(jiuwen_log_path), mode=0o750) + + # 构建各类型日志文件路径 log_file = os.path.join(jiuwen_log_path, log_config.get('log_file')) performance_log_file = os.path.join(jiuwen_log_path, log_config.get('performance_log_file')) - output = log_config.get('output', CONSOLE) - interface_log_file = os.path.join(jiuwen_log_path, log_config.get('interface_log_file')) prompt_builder_interface_log_file = os.path.join(jiuwen_log_path, log_config.get('prompt_builder_interface_log_file')) + + # 获取输出方式配置 + output = log_config.get('output', CONSOLE) interface_output = log_config.get('interface_output', CONSOLE) performance_output = log_config.get('performance_output', CONSOLE) + # 创建各类型日志记录器 SingletonLogger._instance = SingletonLogger._get_logger('common', log_config, output, log_file) SingletonLogger._interface_instance = SingletonLogger._get_logger('interface', log_config, interface_output, interface_log_file) - SingletonLogger._prompt_builder_interface_instance = SingletonLogger._get_logger('interface_prompt_builder', log_config, interface_output, prompt_builder_interface_log_file) SingletonLogger._performance_instance = SingletonLogger._get_logger('performance', log_config, performance_output, performance_log_file) + @staticmethod def get_logger(): + """获取日志记录器实例(单例入口)""" SingletonFlag = SingletonLogger._instance is not None or SingletonLogger._interface_instance if SingletonFlag or SingletonLogger._prompt_builder_interface_instance \ @@ -130,20 +153,29 @@ class SingletonLogger: @classmethod def _get_logger(cls, log_type, log_config, output, log_file): + """创建并配置指定类型的日志记录器""" + # 获取日志级别（环境变量优先） level = os.environ.get("JIUWEN_LOG_LEVEL", log_config.get('level','WARNING')) + + # 获取备份数量和文件大小限制 env_backup_count = os.environ.get("JIUWEN_LOG_BACKUP_COUNT") backup_count = ast.literal_eval(env_backup_count) if env_backup_count else log_config.get('backup_count', DEFAULT_BACKUP_COUNT) env_max_bytes = os.environ.get("JIUWEN_LOG_MAX_BYTES") max_bytes = get_log_max_bytes(ast.literal_eval(env_max_bytes) if env_max_bytes else get_log_max_bytes( log_config.get('max_bytes', CONFIG_LOG_MAX_BYTES))) + + # 设置日志格式 log_format = '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s' if log_type == 'common': log_format = log_config.get('format', '%(asctime)s | %(log_type)s | %(filename)s | %(lineno)d | %(funcName)s | %(trace_id)s' '| %(levelname)s | %(message)s') + # 创建日志记录器 common_logger = logging.getLogger(log_type) common_logger.setLevel(level) + + # 添加控制台处理器 if CONSOLE in output: stream_handler = logging.StreamHandler(stream=sys.stdout) if TRACE_ID in log_format: @@ -151,10 +183,13 @@ class SingletonLogger: stream_handler.setFormatter(logging.Formatter(log_format)) common_logger.addHandler(stream_handler) + + # 添加文件处理器 if 'log' in output: if not os.path.exists(os.path.dirname(log_file)): # 日志文件目录权限为750 os.makedirs(os.path.dirname(log_file), mode=0o750) + # 创建安全轮转文件处理器 file_handler = SafeRotatingFileHandler( filename=log_file, maxBytes=max_bytes, @@ -179,7 +214,7 @@ class ThreadContextFilter(logging.Filter): self.log_type = log_type def filter(self, record): - # 搜寻线程本地存储以获取trace_id如果存在的话 + # 搜寻线程本地存储以获取trace_id如果存在的话，添加自定义日志字段 record.trace_id = getattr(_thread_log_instance, 'trace_id', '') record.log_type = "perf" if self.log_type == 'performance' else self.log_type return True -- Gitee From 9ecdc4981040bc1fe1649a310dc081f21a4088d2 Mon Sep 17 00:00:00 2001 From: Bai <12577555+wch_study@user.noreply.gitee.com> Date: Fri, 25 Jul 2025 17:52:54 +0800 Subject: [PATCH 3/5] =?UTF-8?q?feat:=20"=E5=A2=9E=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jiuwen/extensions/common/log/base.py | 98 +++++++++++++++++++--------- 1 file changed, 68 insertions(+), 30 deletions(-) diff --git a/jiuwen/extensions/common/log/base.py b/jiuwen/extensions/common/log/base.py index 8d17489..85b8403 100644 --- a/jiuwen/extensions/common/log/base.py +++ b/jiuwen/extensions/common/log/base.py @@ -97,6 +97,8 @@ class SingletonLogger: _interface_instance : logging.Logger = None _prompt_builder_interface_instance : logging.Logger = None _performance_instance : logging.Logger = None + _lock = threading.Lock() + _initialized = False def __init__(self): # 单例检查 @@ -106,23 +108,34 @@ class SingletonLogger: raise Exception("This class is a singleton!") # 从配置获取日志设置 - log_config = config["log"] - if log_config is None: - return + # log_config = config["log"] + # if log_config is None: + # return + log_config = config.get("log", {}) # 获取日志路径（环境变量优先） - jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path')) + # jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path')) + jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path', '/var/log/jiuwen')) # 创建日志目录（如果不存在） - if not os.path.exists(os.path.dirname(jiuwen_log_path)): - # 日志文件目录权限为750 - os.makedirs(os.path.dirname(jiuwen_log_path), mode=0o750) + # if not os.path.exists(os.path.dirname(jiuwen_log_path)): + # # 日志文件目录权限为750 + # os.makedirs(os.path.dirname(jiuwen_log_path), mode=0o750) + log_dir = os.path.dirname(jiuwen_log_path) + if not os.path.exists(log_dir): + try: + os.makedirs(log_dir, mode=0o750) + except OSError as e: + sys.stderr.write(f"无法创建日志目录 {log_dir}: {str(e)}\n") + # 回退到临时目录 + jiuwen_log_path = '/tmp/jiuwen.log' # 构建各类型日志文件路径 - log_file = os.path.join(jiuwen_log_path, log_config.get('log_file')) - performance_log_file = os.path.join(jiuwen_log_path, log_config.get('performance_log_file')) - interface_log_file = os.path.join(jiuwen_log_path, log_config.get('interface_log_file')) - prompt_builder_interface_log_file = os.path.join(jiuwen_log_path, log_config.get('prompt_builder_interface_log_file')) + log_file = os.path.join(jiuwen_log_path, log_config.get('log_file', 'common.log')) + performance_log_file = os.path.join(jiuwen_log_path, log_config.get('performance_log_file', 'performance.log')) + interface_log_file = os.path.join(jiuwen_log_path, log_config.get('interface_log_file', 'interface.log')) + prompt_builder_log_file = os.path.join(jiuwen_log_path, log_config.get('prompt_builder_interface_log_file', + 'prompt_builder.log')) # 获取输出方式配置 output = log_config.get('output', CONSOLE) @@ -130,22 +143,31 @@ class SingletonLogger: performance_output = log_config.get('performance_output', CONSOLE) # 创建各类型日志记录器 - SingletonLogger._instance = SingletonLogger._get_logger('common', log_config, output, log_file) - SingletonLogger._interface_instance = SingletonLogger._get_logger('interface', log_config, interface_output, interface_log_file) - SingletonLogger._prompt_builder_interface_instance = SingletonLogger._get_logger('interface_prompt_builder', log_config, interface_output, prompt_builder_interface_log_file) - SingletonLogger._performance_instance = SingletonLogger._get_logger('performance', log_config, performance_output, performance_log_file) - - - + # SingletonLogger._instance = SingletonLogger._get_logger('common', log_config, output, log_file) + # SingletonLogger._interface_instance = SingletonLogger._get_logger('interface', log_config, interface_output, interface_log_file) + # SingletonLogger._prompt_builder_interface_instance = SingletonLogger._get_logger('interface_prompt_builder', log_config, interface_output, prompt_builder_interface_log_file) + # SingletonLogger._performance_instance = SingletonLogger._get_logger('performance', log_config, performance_output, performance_log_file) + self._instance = self._get_logger('common', log_config, output, log_file) + self._interface_instance = self._get_logger('interface', log_config, interface_output, interface_log_file) + self._prompt_builder_interface_instance = self._get_logger( + 'interface_prompt_builder', log_config, interface_output, prompt_builder_log_file) + self._performance_instance = self._get_logger('performance', log_config, performance_output, + performance_log_file) @staticmethod def get_logger(): """获取日志记录器实例(单例入口)""" - SingletonFlag = SingletonLogger._instance is not None or SingletonLogger._interface_instance - if SingletonFlag or SingletonLogger._prompt_builder_interface_instance \ - or SingletonLogger._performance_instance: - SingletonLogger() + # 双重检查锁定模式 + if not SingletonLogger._initialized: + with SingletonLogger._lock: # 加锁确保线程安全 + if not SingletonLogger._initialized: + SingletonLogger() # 创建单例实例 + + # SingletonFlag = SingletonLogger._instance is not None or SingletonLogger._interface_instance + # if SingletonFlag or SingletonLogger._prompt_builder_interface_instance \ + # or SingletonLogger._performance_instance: + # SingletonLogger() return SingletonLogger._instance, SingletonLogger._interface_instance, \ SingletonLogger._prompt_builder_interface_instance, \ SingletonLogger._performance_instance @@ -185,21 +207,37 @@ class SingletonLogger: # 添加文件处理器 + # if 'log' in output: + # if not os.path.exists(os.path.dirname(log_file)): + # # 日志文件目录权限为750 + # os.makedirs(os.path.dirname(log_file), mode=0o750) + # # 创建安全轮转文件处理器 + # file_handler = SafeRotatingFileHandler( + # filename=log_file, + # maxBytes=max_bytes, + # backupCount=backup_count, + # encoding='utf-8' + # ) + # if TRACE_ID in log_format: + # file_handler.addFilter(ThreadContextFilter(log_type)) + # file_handler.setFormatter(logging.Formatter(log_format)) + # common_logger.addHandler(file_handler) if 'log' in output: - if not os.path.exists(os.path.dirname(log_file)): - # 日志文件目录权限为750 - os.makedirs(os.path.dirname(log_file), mode=0o750) + # 确保日志目录存在 + log_dir = os.path.dirname(log_file) + if not os.path.exists(log_dir): + os.makedirs(log_dir, mode=0o750) + # 创建安全轮转文件处理器 file_handler = SafeRotatingFileHandler( filename=log_file, maxBytes=max_bytes, backupCount=backup_count, encoding='utf-8' - ) - if TRACE_ID in log_format: - file_handler.addFilter(ThreadContextFilter(log_type)) - file_handler.setFormatter(logging.Formatter(log_format)) - common_logger.addHandler(file_handler) + ) + file_handler.addFilter(ThreadContextFilter(log_type)) # 添加上下文过滤器 + file_handler.setFormatter(logging.Formatter(log_format)) + logger.addHandler(file_handler) return common_logger -- Gitee From dff8a3aeaf6a3df83ff4dad074d1d9b907db3d82 Mon Sep 17 00:00:00 2001 From: Bai <12577555+wch_study@user.noreply.gitee.com> Date: Fri, 25 Jul 2025 17:55:26 +0800 Subject: [PATCH 4/5] =?UTF-8?q?feat:=20"=E5=A2=9E=E5=8A=A0=E4=BA=86?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- jiuwen/extensions/common/log/base.py | 32 ++-------------------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/jiuwen/extensions/common/log/base.py b/jiuwen/extensions/common/log/base.py index 85b8403..7f8cdde 100644 --- a/jiuwen/extensions/common/log/base.py +++ b/jiuwen/extensions/common/log/base.py @@ -108,19 +108,13 @@ class SingletonLogger: raise Exception("This class is a singleton!") # 从配置获取日志设置 - # log_config = config["log"] - # if log_config is None: - # return log_config = config.get("log", {}) # 获取日志路径（环境变量优先） - # jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path')) jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path', '/var/log/jiuwen')) # 创建日志目录（如果不存在） - # if not os.path.exists(os.path.dirname(jiuwen_log_path)): - # # 日志文件目录权限为750 - # os.makedirs(os.path.dirname(jiuwen_log_path), mode=0o750) + log_dir = os.path.dirname(jiuwen_log_path) if not os.path.exists(log_dir): try: @@ -143,10 +137,7 @@ class SingletonLogger: performance_output = log_config.get('performance_output', CONSOLE) # 创建各类型日志记录器 - # SingletonLogger._instance = SingletonLogger._get_logger('common', log_config, output, log_file) - # SingletonLogger._interface_instance = SingletonLogger._get_logger('interface', log_config, interface_output, interface_log_file) - # SingletonLogger._prompt_builder_interface_instance = SingletonLogger._get_logger('interface_prompt_builder', log_config, interface_output, prompt_builder_interface_log_file) - # SingletonLogger._performance_instance = SingletonLogger._get_logger('performance', log_config, performance_output, performance_log_file) + self._instance = self._get_logger('common', log_config, output, log_file) self._interface_instance = self._get_logger('interface', log_config, interface_output, interface_log_file) self._prompt_builder_interface_instance = self._get_logger( @@ -164,10 +155,6 @@ class SingletonLogger: if not SingletonLogger._initialized: SingletonLogger() # 创建单例实例 - # SingletonFlag = SingletonLogger._instance is not None or SingletonLogger._interface_instance - # if SingletonFlag or SingletonLogger._prompt_builder_interface_instance \ - # or SingletonLogger._performance_instance: - # SingletonLogger() return SingletonLogger._instance, SingletonLogger._interface_instance, \ SingletonLogger._prompt_builder_interface_instance, \ SingletonLogger._performance_instance @@ -207,21 +194,6 @@ class SingletonLogger: # 添加文件处理器 - # if 'log' in output: - # if not os.path.exists(os.path.dirname(log_file)): - # # 日志文件目录权限为750 - # os.makedirs(os.path.dirname(log_file), mode=0o750) - # # 创建安全轮转文件处理器 - # file_handler = SafeRotatingFileHandler( - # filename=log_file, - # maxBytes=max_bytes, - # backupCount=backup_count, - # encoding='utf-8' - # ) - # if TRACE_ID in log_format: - # file_handler.addFilter(ThreadContextFilter(log_type)) - # file_handler.setFormatter(logging.Formatter(log_format)) - # common_logger.addHandler(file_handler) if 'log' in output: # 确保日志目录存在 log_dir = os.path.dirname(log_file) -- Gitee From 5a0b615e1109f5e55a36cea586d95a046ea8ee2f Mon Sep 17 00:00:00 2001 From: Bai <12577555+wch_study@user.noreply.gitee.com> Date: Mon, 28 Jul 2025 20:44:21 +0800 Subject: [PATCH 5/5] =?UTF-8?q?feat:=20"=E9=87=8D=E6=9E=84=E4=BA=86?= =?UTF-8?q?=E6=97=A5=E5=BF=97=E5=8A=9F=E8=83=BD"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../{ => common/configs}/__init__.py | 0 jiuwen/extensions/common/configs/_init_.py | 0 jiuwen/extensions/common/configs/base.py | 45 ++- jiuwen/extensions/common/log/__init__.py | 46 +++ jiuwen/extensions/common/log/_init_.py | 0 jiuwen/extensions/common/log/base.py | 233 ------------- jiuwen/extensions/common/log/log_handlers.py | 37 +++ jiuwen/extensions/common/log/log_manager.py | 115 +++++++ jiuwen/extensions/common/log/log_utils.py | 35 ++ jiuwen/extensions/common/log/logger_impl.py | 116 +++++++ .../extensions/common/log/logger_protocol.py | 64 ++++ jiuwen/extensions/config.yaml | 8 +- tests/unit_tests/common/log/test_base.py | 28 -- tests/unit_tests/common/log/test_logger.py | 311 ++++++++++++++++++ tests/unit_tests/tracer/test_workflow.py | 2 +- tests/unit_tests/workflow/test_checkpoint.py | 2 +- .../workflow/test_intent_detection_comp.py | 2 +- tests/unit_tests/workflow/test_llm_comp.py | 2 +- 18 files changed, 773 insertions(+), 273 deletions(-) rename jiuwen/extensions/{ => common/configs}/__init__.py (100%) delete mode 100644 jiuwen/extensions/common/configs/_init_.py create mode 100644 jiuwen/extensions/common/log/__init__.py delete mode 100644 jiuwen/extensions/common/log/_init_.py delete mode 100644 jiuwen/extensions/common/log/base.py create mode 100644 jiuwen/extensions/common/log/log_handlers.py create mode 100644 jiuwen/extensions/common/log/log_manager.py create mode 100644 jiuwen/extensions/common/log/log_utils.py create mode 100644 jiuwen/extensions/common/log/logger_impl.py create mode 100644 jiuwen/extensions/common/log/logger_protocol.py delete mode 100644 tests/unit_tests/common/log/test_base.py create mode 100644 tests/unit_tests/common/log/test_logger.py diff --git a/jiuwen/extensions/__init__.py b/jiuwen/extensions/common/configs/__init__.py similarity index 100% rename from jiuwen/extensions/__init__.py rename to jiuwen/extensions/common/configs/__init__.py diff --git a/jiuwen/extensions/common/configs/_init_.py b/jiuwen/extensions/common/configs/_init_.py deleted file mode 100644 index e69de29..0000000 diff --git a/jiuwen/extensions/common/configs/base.py b/jiuwen/extensions/common/configs/base.py index 37fd4b8..506690d 100644 --- a/jiuwen/extensions/common/configs/base.py +++ b/jiuwen/extensions/common/configs/base.py @@ -1,18 +1,46 @@ +# jiuwen/extensions/common/configs/base.py import os import yaml +# --- 添加日志级别映射 --- +CRITICAL = 50 +FATAL = CRITICAL +ERROR = 40 +WARNING = 30 +WARN = WARNING +INFO = 20 +DEBUG = 10 +NOTSET = 0 + +name_to_level = { + 'CRITICAL': CRITICAL, + 'FATAL': FATAL, + 'ERROR': ERROR, + 'WARNING': WARNING, + 'WARN': WARN, + 'INFO': INFO, + 'DEBUG': DEBUG, + 'NOTSET': NOTSET, +} +# --- 结束添加 --- + class SingletonConfig: _instance: yaml.constructor = None def __init__(self): if SingletonConfig._instance is not None: raise Exception("This class is a singleton!") - log_config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config.yaml") with open(log_config_path, encoding="utf-8") as file: # 加载配置文件 - SingletonConfig._instance = yaml.safe_load(file) - + raw_instance = yaml.safe_load(file) + # --- 关键修复：在加载后立即转换日志级别 --- + if 'log' in raw_instance: + level_str = raw_instance['log'].get('level', 'WARNING').upper() + raw_instance['log']['level'] = name_to_level.get(level_str, WARNING) + # --- 关键修复结束 --- + # 将转换后的配置存入 _instance + SingletonConfig._instance = raw_instance @staticmethod def get_config(): @@ -20,4 +48,13 @@ class SingletonConfig: SingletonConfig() return SingletonConfig._instance -config = SingletonConfig.get_config() \ No newline at end of file +# --- 关键修复：定义一个可调用的字典 --- +class ConfigDict(dict): + """一个既像字典又可调用的类，用于兼容旧代码""" + def __call__(self): + return self + +# --- 关键修复：创建 config 实例 --- +# config = SingletonConfig.get_config() # <--- 注释掉这一行 +config = ConfigDict(SingletonConfig.get_config()) # <--- 用 ConfigDict 包装 +# --- 关键修复结束 --- \ No newline at end of file diff --git a/jiuwen/extensions/common/log/__init__.py b/jiuwen/extensions/common/log/__init__.py new file mode 100644 index 0000000..460e745 --- /dev/null +++ b/jiuwen/extensions/common/log/__init__.py @@ -0,0 +1,46 @@ +# jiuwen/extensions/common/log/__init__.py +"""logger for common and interface.""" +__all__ = ("logger", "interface_logger", "prompt_builder_interface_logger", + "performance_logger", "get_thread_session", "set_thread_session", + "LoggerProtocol", "LogManager", "initialize_loggers") + +from .log_manager import LogManager +from .log_utils import set_thread_session, get_thread_session +from .logger_protocol import LoggerProtocol + + +# 将 logger 等变量定义为延迟加载的属性 +class _LazyLogger: + def __init__(self, log_type): + self.log_type = log_type + self._logger = None + + @property + def logger(self): + if self._logger is None: + # 确保 LogManager 已初始化 + if not LogManager._initialized: + LogManager.initialize() + # 获取 logger + self._logger = LogManager.get_logger(self.log_type) + return self._logger + + def __getattr__(self, name): + return getattr(self.logger, name) + +# 创建惰性 logger 实例 +_common_logger = _LazyLogger('common') +_interface_logger = _LazyLogger('interface') +_prompt_builder_interface_logger = _LazyLogger('prompt_builder') +_performance_logger = _LazyLogger('performance') + +# 全局变量指向惰性实例 +logger = _common_logger +interface_logger = _interface_logger +prompt_builder_interface_logger = _prompt_builder_interface_logger +performance_logger = _performance_logger + +# 提供一个显式初始化函数 +def initialize_loggers(): + """强制初始化日志系统""" + LogManager.initialize() \ No newline at end of file diff --git a/jiuwen/extensions/common/log/_init_.py b/jiuwen/extensions/common/log/_init_.py deleted file mode 100644 index e69de29..0000000 diff --git a/jiuwen/extensions/common/log/base.py b/jiuwen/extensions/common/log/base.py deleted file mode 100644 index 7f8cdde..0000000 --- a/jiuwen/extensions/common/log/base.py +++ /dev/null @@ -1,233 +0,0 @@ -"""logger for common and interface.""" -__all__ = ("logger", "interface_logger", "prompt_builder_interface_logger",\ - "performance_logger", "get_thread_session", "set_thread_session") - -import ast -import os -import sys -import threading -import logging -from logging.handlers import RotatingFileHandler - -from jiuwen.extensions.common.configs.base import config, SingletonConfig -from jiuwen.extensions.common.exception.base import JiuWenBaseException - -# 日志级别常量 -CRITICAL = 50 -FATAL = CRITICAL -ERROR = 40 -WARNING = 30 -WARN = WARNING -INFO = 20 -DEBUG = 10 -NOTSET = 0 - -# 日志文件大小限制 -DEFAULT_LOG_MAX_BYTES = 100 * 1024 * 1024 #100MB -CONFIG_LOG_MAX_BYTES = 20 * 1024 * 1024 #20MB -DEFAULT_BACKUP_COUNT = 20 # 默认备份文件数量 - -# 日志级别名称到数值的映射 -name_to_level = { - 'CRITICAL': CRITICAL, - 'FATAL': FATAL, - 'ERROR': ERROR, - 'WARNING': WARNING, - 'WARN': WARNING, - 'INFO': INFO, - 'DEBUG': DEBUG, - 'NOTSET': NOTSET, -} - -# 日志输出方式常量 -CONSOLE = 'console' # 控制台输出 -TRACE_ID = 'trace_id' # 会话追踪ID - -# 使用方式，在需要注入session_id的参数地方，使用thread_local_data.trace_id = trace_id -_thread_log_instance = threading.local() # 线程局部存储，用于保存trace_id - -def set_thread_session(trace_id): - """设置当前线程的trace_id""" - _thread_log_instance.trace_id = trace_id - -def get_thread_session() -> str: - """获取当前线程的trace_id""" - return getattr(_thread_log_instance, 'trace_id', '') - -def get_log_max_bytes(max_bytes_config): - """验证并获取有效的日志文件大小限制""" - try: - max_bytes = int(max_bytes_config) - except ValueError as e: - raise JiuWenBaseException( - error_code=-1, message="-1" - ) from e - # 限制文件大小再合理范围内 - if max_bytes <= 0 or max_bytes > DEFAULT_LOG_MAX_BYTES: - max_bytes = DEFAULT_LOG_MAX_BYTES #小于0或者超出默认最大限制，使用默认最大值 - - return max_bytes - - -class SafeRotatingFileHandler(RotatingFileHandler): - """安全轮转文件处理器，添加进程ID和文件权限控制""" - def __init__(self, filename, *args, **kwargs): - pid = os.getpid() - filename = f"{filename}-{pid}" - super().__init__(filename, *args, **kwargs) - #设置文件的初始权限为 640 - os.chmod(self.baseFilename, 0o640) - - def doRollover(self): - """执行日志轮转并设置文件权限""" - # Rotate the file first - # 日志文件（正在记录）权限为640， 日志文件（记录完毕或者已经归档）权限为440 - RotatingFileHandler.doRollover(self) - # 设置归档日志文件权限为440 - for i in range(self.backupCount, 0, -1): - sfn = f"{self.baseFilename}.{i}" - if os.path.exists(sfn): - os.chmod(sfn, 0o440) - # 设置当前日志文件权限为640 - os.chmod(self.baseFilename, 0o640) - -class SingletonLogger: - """单例日志管理器，创建四种类型的日志记录器""" - _instance : logging.Logger = None - _interface_instance : logging.Logger = None - _prompt_builder_interface_instance : logging.Logger = None - _performance_instance : logging.Logger = None - _lock = threading.Lock() - _initialized = False - - def __init__(self): - # 单例检查 - SingletonFlag = SingletonLogger._instance is not None or SingletonLogger._interface_instance is not None - if SingletonFlag or SingletonLogger._prompt_builder_interface_instance \ - is not None or SingletonLogger._performance_instance is not None: - raise Exception("This class is a singleton!") - - # 从配置获取日志设置 - log_config = config.get("log", {}) - - # 获取日志路径（环境变量优先） - jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path', '/var/log/jiuwen')) - - # 创建日志目录（如果不存在） - - log_dir = os.path.dirname(jiuwen_log_path) - if not os.path.exists(log_dir): - try: - os.makedirs(log_dir, mode=0o750) - except OSError as e: - sys.stderr.write(f"无法创建日志目录 {log_dir}: {str(e)}\n") - # 回退到临时目录 - jiuwen_log_path = '/tmp/jiuwen.log' - - # 构建各类型日志文件路径 - log_file = os.path.join(jiuwen_log_path, log_config.get('log_file', 'common.log')) - performance_log_file = os.path.join(jiuwen_log_path, log_config.get('performance_log_file', 'performance.log')) - interface_log_file = os.path.join(jiuwen_log_path, log_config.get('interface_log_file', 'interface.log')) - prompt_builder_log_file = os.path.join(jiuwen_log_path, log_config.get('prompt_builder_interface_log_file', - 'prompt_builder.log')) - - # 获取输出方式配置 - output = log_config.get('output', CONSOLE) - interface_output = log_config.get('interface_output', CONSOLE) - performance_output = log_config.get('performance_output', CONSOLE) - - # 创建各类型日志记录器 - - self._instance = self._get_logger('common', log_config, output, log_file) - self._interface_instance = self._get_logger('interface', log_config, interface_output, interface_log_file) - self._prompt_builder_interface_instance = self._get_logger( - 'interface_prompt_builder', log_config, interface_output, prompt_builder_log_file) - self._performance_instance = self._get_logger('performance', log_config, performance_output, - performance_log_file) - - @staticmethod - def get_logger(): - """获取日志记录器实例(单例入口)""" - - # 双重检查锁定模式 - if not SingletonLogger._initialized: - with SingletonLogger._lock: # 加锁确保线程安全 - if not SingletonLogger._initialized: - SingletonLogger() # 创建单例实例 - - return SingletonLogger._instance, SingletonLogger._interface_instance, \ - SingletonLogger._prompt_builder_interface_instance, \ - SingletonLogger._performance_instance - - - @classmethod - def _get_logger(cls, log_type, log_config, output, log_file): - """创建并配置指定类型的日志记录器""" - # 获取日志级别（环境变量优先） - level = os.environ.get("JIUWEN_LOG_LEVEL", log_config.get('level','WARNING')) - - # 获取备份数量和文件大小限制 - env_backup_count = os.environ.get("JIUWEN_LOG_BACKUP_COUNT") - backup_count = ast.literal_eval(env_backup_count) if env_backup_count else log_config.get('backup_count', DEFAULT_BACKUP_COUNT) - env_max_bytes = os.environ.get("JIUWEN_LOG_MAX_BYTES") - max_bytes = get_log_max_bytes(ast.literal_eval(env_max_bytes) if env_max_bytes else get_log_max_bytes( - log_config.get('max_bytes', CONFIG_LOG_MAX_BYTES))) - - # 设置日志格式 - log_format = '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s' - if log_type == 'common': - log_format = log_config.get('format', - '%(asctime)s | %(log_type)s | %(filename)s | %(lineno)d | %(funcName)s | %(trace_id)s' - '| %(levelname)s | %(message)s') - - # 创建日志记录器 - common_logger = logging.getLogger(log_type) - common_logger.setLevel(level) - - # 添加控制台处理器 - if CONSOLE in output: - stream_handler = logging.StreamHandler(stream=sys.stdout) - if TRACE_ID in log_format: - stream_handler.addFilter(ThreadContextFilter(log_type)) - stream_handler.setFormatter(logging.Formatter(log_format)) - common_logger.addHandler(stream_handler) - - - # 添加文件处理器 - if 'log' in output: - # 确保日志目录存在 - log_dir = os.path.dirname(log_file) - if not os.path.exists(log_dir): - os.makedirs(log_dir, mode=0o750) - - # 创建安全轮转文件处理器 - file_handler = SafeRotatingFileHandler( - filename=log_file, - maxBytes=max_bytes, - backupCount=backup_count, - encoding='utf-8' - ) - file_handler.addFilter(ThreadContextFilter(log_type)) # 添加上下文过滤器 - file_handler.setFormatter(logging.Formatter(log_format)) - logger.addHandler(file_handler) - - return common_logger - - -class ThreadContextFilter(logging.Filter): - """ - 线程上下文过滤器 - 其主要用途是过滤日志记录，添加线程相关信息，如线程ID等。 - """ - def __init__(self, log_type): - super().__init__() - self.log_type = log_type - - def filter(self, record): - # 搜寻线程本地存储以获取trace_id如果存在的话，添加自定义日志字段 - record.trace_id = getattr(_thread_log_instance, 'trace_id', '') - record.log_type = "perf" if self.log_type == 'performance' else self.log_type - return True - -# 使用示例 -logger, interface_logger, prompt_builder_interface_logger, performance_logger = SingletonLogger.get_logger() diff --git a/jiuwen/extensions/common/log/log_handlers.py b/jiuwen/extensions/common/log/log_handlers.py new file mode 100644 index 0000000..ec222c2 --- /dev/null +++ b/jiuwen/extensions/common/log/log_handlers.py @@ -0,0 +1,37 @@ +# jiuwen/extensions/common/log/log_handlers.py +import os +import logging +from logging.handlers import RotatingFileHandler +from .log_utils import get_thread_session + +class SafeRotatingFileHandler(RotatingFileHandler): + """安全轮转文件处理器，添加进程ID和文件权限控制""" + def __init__(self, filename, *args, **kwargs): + pid = os.getpid() + filename = f"{filename}-{pid}" + super().__init__(filename, *args, **kwargs) + # 设置文件的初始权限为 640 + os.chmod(self.baseFilename, 0o640) + + def doRollover(self): + """执行日志轮转并设置文件权限""" + super().doRollover() + # 设置归档日志文件权限为440 + for i in range(self.backupCount, 0, -1): + sfn = f"{self.baseFilename}.{i}" + if os.path.exists(sfn): + os.chmod(sfn, 0o440) + # 设置当前日志文件权限为640 + os.chmod(self.baseFilename, 0o640) + +class ThreadContextFilter(logging.Filter): + """线程上下文过滤器，添加trace_id""" + def __init__(self, log_type: str): + super().__init__() + self.log_type = log_type + + def filter(self, record): + # 添加自定义日志字段 + record.trace_id = get_thread_session() + record.log_type = "perf" if self.log_type == 'performance' else self.log_type + return True diff --git a/jiuwen/extensions/common/log/log_manager.py b/jiuwen/extensions/common/log/log_manager.py new file mode 100644 index 0000000..123f37d --- /dev/null +++ b/jiuwen/extensions/common/log/log_manager.py @@ -0,0 +1,115 @@ +# jiuwen/extensions/common/log/log_manager.py +import threading +import os +from typing import Dict, Tuple, Optional + +from .logger_protocol import LoggerProtocol +from .logger_impl import DefaultLogger +from jiuwen.extensions.common.configs.base import config + + +class LogManager: + """日志管理器，支持多种日志类型和自定义实现""" + _loggers: Dict[str, LoggerProtocol] = {} + _lock = threading.RLock() + _initialized = False + + @classmethod + def initialize(cls) -> None: + """初始化日志系统""" + with cls._lock: + if cls._initialized: + return + + # 从配置获取日志设置 + log_config = config.get("log", {}) + + # 获取日志路径（环境变量优先） + jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path', '/var/log/jiuwen')) + + # 构建各类型日志文件路径 + common_config = { + 'log_file': os.path.join(jiuwen_log_path, log_config.get('log_file', 'common.log')), + 'output': log_config.get('output', ['console']), + 'level': log_config.get('level', 'WARNING'), + 'backup_count': log_config.get('backup_count', 20), + 'max_bytes': log_config.get('max_bytes', 20 * 1024 * 1024), + 'format': log_config.get('format', + '%(asctime)s | %(log_type)s | %(filename)s | %(lineno)d | %(funcName)s | %(trace_id)s | %(levelname)s | %(message)s') + } + + interface_config = { + 'log_file': os.path.join(jiuwen_log_path, log_config.get('interface_log_file', 'interface.log')), + 'output': log_config.get('interface_output', ['console']), + 'level': log_config.get('level', 'WARNING'), + 'backup_count': log_config.get('backup_count', 20), + 'max_bytes': log_config.get('max_bytes', 20 * 1024 * 1024), + 'format': log_config.get('format', + '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s') + } + + prompt_builder_config = { + 'log_file': os.path.join(jiuwen_log_path, + log_config.get('prompt_builder_interface_log_file', 'prompt_builder.log')), + 'output': log_config.get('interface_output', ['console']), + 'level': log_config.get('level', 'WARNING'), + 'backup_count': log_config.get('backup_count', 20), + 'max_bytes': log_config.get('max_bytes', 20 * 1024 * 1024), + 'format': log_config.get('format', + '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s') + } + + performance_config = { + 'log_file': os.path.join(jiuwen_log_path, log_config.get('performance_log_file', 'performance.log')), + 'output': log_config.get('performance_output', ['console']), + 'level': log_config.get('level', 'WARNING'), + 'backup_count': log_config.get('backup_count', 20), + 'max_bytes': log_config.get('max_bytes', 20 * 1024 * 1024), + 'format': log_config.get('format', + '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s') + } + + # 创建默认日志记录器 + cls.register_logger('common', DefaultLogger('common', common_config)) + cls.register_logger('interface', DefaultLogger('interface', interface_config)) + cls.register_logger('prompt_builder', DefaultLogger('prompt_builder', prompt_builder_config)) + cls.register_logger('performance', DefaultLogger('performance', performance_config)) + + cls._initialized = True + + @classmethod + def register_logger(cls, log_type: str, logger: LoggerProtocol) -> None: + """注册自定义日志记录器""" + if not isinstance(logger, LoggerProtocol): + raise TypeError(f"Logger must implement LoggerProtocol, got {type(logger)}") + + with cls._lock: + cls._loggers[log_type] = logger + + @classmethod + def get_logger(cls, log_type: str) -> LoggerProtocol: + + """获取指定类型的日志记录器""" + if not cls._initialized: + cls.initialize() + + with cls._lock: + if log_type not in cls._loggers: + # 动态创建默认日志记录器 + cls._loggers[log_type] = DefaultLogger(log_type, {}) + + return cls._loggers[log_type] + + @classmethod + def get_all_loggers(cls) -> Dict[str, LoggerProtocol]: + """获取所有已注册的日志记录器""" + if not cls._initialized: + cls.initialize() + return cls._loggers.copy() + + @classmethod + def reset(cls): + """重置日志管理器状态（用于测试）""" + with cls._lock: + cls._loggers = {} + cls._initialized = False diff --git a/jiuwen/extensions/common/log/log_utils.py b/jiuwen/extensions/common/log/log_utils.py new file mode 100644 index 0000000..ce2f6ee --- /dev/null +++ b/jiuwen/extensions/common/log/log_utils.py @@ -0,0 +1,35 @@ +# jiuwen/extensions/common/log/log_utils.py +import os +import threading +import ast +from jiuwen.extensions.common.exception.base import JiuWenBaseException + +# 使用方式，在需要注入session_id的参数地方，使用thread_local_data.trace_id = trace_id +_thread_log_instance = threading.local() # 线程局部存储，用于保存trace_id + + +def set_thread_session(trace_id: str) -> None: + """设置当前线程的trace_id""" + _thread_log_instance.trace_id = trace_id + + +def get_thread_session() -> str: + """获取当前线程的trace_id""" + return getattr(_thread_log_instance, 'trace_id', '') + + +def get_log_max_bytes(max_bytes_config) -> int: + """验证并获取有效的日志文件大小限制""" + try: + max_bytes = int(max_bytes_config) + except ValueError as e: + raise JiuWenBaseException( + error_code=-1, message="-1" + ) from e + + # 限制文件大小在合理范围内 + DEFAULT_LOG_MAX_BYTES = 100 * 1024 * 1024 # 100MB + if max_bytes <= 0 or max_bytes > DEFAULT_LOG_MAX_BYTES: + max_bytes = DEFAULT_LOG_MAX_BYTES # 小于0或者超出默认最大限制，使用默认最大值 + + return max_bytes diff --git a/jiuwen/extensions/common/log/logger_impl.py b/jiuwen/extensions/common/log/logger_impl.py new file mode 100644 index 0000000..469b86a --- /dev/null +++ b/jiuwen/extensions/common/log/logger_impl.py @@ -0,0 +1,116 @@ +# jiuwen/extensions/common/log/logger_impl.py +import os +import sys +import ast +import logging +from typing import Dict, Any, Optional + +from .log_handlers import SafeRotatingFileHandler, ThreadContextFilter +from .log_utils import get_log_max_bytes, set_thread_session, get_thread_session +from .logger_protocol import LoggerProtocol + + +class DefaultLogger(LoggerProtocol): + """默认日志实现，符合LoggerProtocol协议""" + + def __init__(self, log_type: str, config: Dict[str, Any]): + self.log_type = log_type + self.config = config + self._logger = logging.getLogger(log_type) + self._setup_logger() + + def _setup_logger(self): + """配置日志记录器""" + # 从配置获取设置 + level_str = self.config.get('level', 'WARNING') + level = getattr(logging, level_str.upper(), logging.WARNING) + self._logger.setLevel(level) + + output = self.config.get('output', ['console']) + log_file = self.config.get('log_file', f'{self.log_type}.log') + + # 清除现有处理器 + for handler in self._logger.handlers[:]: + self._logger.removeHandler(handler) + + # 添加控制台处理器 + if 'console' in output: + stream_handler = logging.StreamHandler(stream=sys.stdout) # 明确指定 sys.stdout + stream_handler.addFilter(ThreadContextFilter(self.log_type)) + stream_handler.setFormatter(self._get_formatter()) + self._logger.addHandler(stream_handler) + + # 添加文件处理器 + if 'file' in output: + # 确保日志目录存在 + log_dir = os.path.dirname(log_file) + if log_dir: # 只有当 log_dir 非空时才尝试创建 + # 使用 exist_ok=True 防止父目录不存在时报错 + os.makedirs(log_dir, mode=0o750, exist_ok=True) + + # 获取备份数量和文件大小限制 + backup_count = self.config.get('backup_count', 20) + max_bytes = get_log_max_bytes(self.config.get('max_bytes', 20 * 1024 * 1024)) + + # 创建安全轮转文件处理器 + file_handler = SafeRotatingFileHandler( + filename=log_file, + maxBytes=max_bytes, + backupCount=backup_count, + encoding='utf-8' + ) + file_handler.addFilter(ThreadContextFilter(self.log_type)) + file_handler.setFormatter(self._get_formatter()) + self._logger.addHandler(file_handler) + + def _get_formatter(self) -> logging.Formatter: + """获取日志格式化器""" + log_format = self.config.get( + 'format') or '%(asctime)s.%(msecs)03d | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s' + return logging.Formatter(log_format, datefmt='%Y-%m-%d %H:%M:%S') + + # 实现协议要求的所有方法 + def debug(self, msg: str, *args, **kwargs) -> None: + self._logger.debug(msg, *args, **kwargs) + + def info(self, msg: str, *args, **kwargs) -> None: + self._logger.info(msg, *args, **kwargs) + + def warning(self, msg: str, *args, **kwargs) -> None: + self._logger.warning(msg, *args, **kwargs) + + def error(self, msg: str, *args, **kwargs) -> None: + self._logger.error(msg, *args, **kwargs) + + def critical(self, msg: str, *args, **kwargs) -> None: + self._logger.critical(msg, *args, **kwargs) + + def exception(self, msg: str, *args, **kwargs) -> None: + self._logger.exception(msg, *args, **kwargs) + + def log(self, level: int, msg: str, *args, **kwargs) -> None: + self._logger.log(level, msg, *args, **kwargs) + + def setLevel(self, level: int) -> None: + self._logger.setLevel(level) + + def addHandler(self, handler: logging.Handler) -> None: + self._logger.addHandler(handler) + + def removeHandler(self, handler: logging.Handler) -> None: + self._logger.removeHandler(handler) + + def addFilter(self, filter) -> None: + self._logger.addFilter(filter) + + def removeFilter(self, filter) -> None: + self._logger.removeFilter(filter) + + def get_config(self) -> Dict[str, Any]: + """获取日志配置""" + return self.config.copy() + + def reconfigure(self, config: Dict[str, Any]) -> None: + """重新配置日志记录器""" + self.config = config + self._setup_logger() diff --git a/jiuwen/extensions/common/log/logger_protocol.py b/jiuwen/extensions/common/log/logger_protocol.py new file mode 100644 index 0000000..f9c9566 --- /dev/null +++ b/jiuwen/extensions/common/log/logger_protocol.py @@ -0,0 +1,64 @@ +# jiuwen/extensions/common/log/logger_protocol.py +from typing import Protocol, runtime_checkable, Dict, Any, Optional +import logging + + +@runtime_checkable +class LoggerProtocol(Protocol): + """日志记录器协议，定义所有日志实现必须提供的方法""" + + def debug(self, msg: str, *args, **kwargs) -> None: + """记录调试级别日志""" + ... + + def info(self, msg: str, *args, **kwargs) -> None: + """记录信息级别日志""" + ... + + def warning(self, msg: str, *args, **kwargs) -> None: + """记录警告级别日志""" + ... + + def error(self, msg: str, *args, **kwargs) -> None: + """记录错误级别日志""" + ... + + def critical(self, msg: str, *args, **kwargs) -> None: + """记录严重级别日志""" + ... + + def exception(self, msg: str, *args, **kwargs) -> None: + """记录异常信息（包含堆栈跟踪）""" + ... + + def log(self, level: int, msg: str, *args, **kwargs) -> None: + """通用日志记录方法""" + ... + + def setLevel(self, level: int) -> None: + """设置日志级别""" + ... + + def addHandler(self, handler: logging.Handler) -> None: + """添加日志处理器""" + ... + + def removeHandler(self, handler: logging.Handler) -> None: + """移除日志处理器""" + ... + + def addFilter(self, filter) -> None: + """添加过滤器""" + ... + + def removeFilter(self, filter) -> None: + """移除过滤器""" + ... + + def get_config(self) -> Dict[str, Any]: + """获取日志配置""" + ... + + def reconfigure(self, config: Dict[str, Any]) -> None: + """重新配置日志记录器""" + ... diff --git a/jiuwen/extensions/config.yaml b/jiuwen/extensions/config.yaml index e6b2d3b..a92b945 100644 --- a/jiuwen/extensions/config.yaml +++ b/jiuwen/extensions/config.yaml @@ -4,12 +4,12 @@ logging: max_bytes: 20971520 format: '%(asctime)s | %(log_type)s | %(filename)s | %(lineno)d | %(funcName)s | %(trace_id)s | %(levelname)s | %(message)s' log_file: "run/jiuwen.log" - output: console + output: ["console"] interface_log_file : "interface/jiuwen_interface.log" prompt_builder_interface_log_file: "interface/jiuwen_prompt_builder_interface.log" - performance_log_file: "performance/jiuwen_performance/log" - interface_output: log - performance_output: console + performance_log_file: "performance/jiuwen_performance.log" + interface_output: ["console", "file"] # 同时输出到控制台和文件 + performance_output: ["console"] log_path: "./logs/" plugin: diff --git a/tests/unit_tests/common/log/test_base.py b/tests/unit_tests/common/log/test_base.py deleted file mode 100644 index 0271d5f..0000000 --- a/tests/unit_tests/common/log/test_base.py +++ /dev/null @@ -1,28 +0,0 @@ -import threading -from unittest import TestCase - -from jiuwen.extensions.common.log.base import logger, set_thread_session - -def set_session_id(session_id): - set_thread_session(session_id) - - -def thread_function(session_id): - set_thread_session(session_id) - logger.info('Thread started with session id {}'.format(session_id)) - - -class Test(TestCase): - def test_logger(self): - thread1 = threading.Thread(target=thread_function, args=('10001',)) - thread2 = threading.Thread(target=thread_function, args=('10002',)) - thread3 = threading.Thread(target=thread_function, args=('10003',)) - - - thread1.start() - thread2.start() - thread3.start() - - thread1.join() - thread2.join() - thread3.join() \ No newline at end of file diff --git a/tests/unit_tests/common/log/test_logger.py b/tests/unit_tests/common/log/test_logger.py new file mode 100644 index 0000000..12cd405 --- /dev/null +++ b/tests/unit_tests/common/log/test_logger.py @@ -0,0 +1,311 @@ + +# test_logger.py +import os +import sys +import threading +import tempfile +import logging +from io import StringIO +from unittest import TestCase, mock +from jiuwen.extensions.common.log.log_manager import LogManager +# 导入日志模块 + + +# 模拟配置模块，避免实际依赖 +import jiuwen.extensions.common.configs.base as config_base + + +def thread_function(session_id, log_list): + from jiuwen.extensions.common.log import set_thread_session + logger = LogManager.get_logger('common') + + # 确保 logger 有处理器且级别足够 + if not logger._logger.handlers: + handler = logging.StreamHandler(sys.stdout) + logger._logger.addHandler(handler) + logger.setLevel(logging.INFO) + + set_thread_session(session_id) + logger.info(f'Thread started with session id {session_id}') + + # 强制刷新 + for handler in logger._logger.handlers: + handler.flush() + + from jiuwen.extensions.common.log import get_thread_session + log_list.append((session_id, get_thread_session())) + +class LoggerBaseTest(TestCase): + """日志测试基类，处理环境设置和清理""" + def setUp(self): + # --- 修改 setUp 逻辑 --- + # 1. 先 mock 配置，再初始化 LogManager + self.mock_config_data = { + "log": { + "log_path": "/default/log/path", + "log_file": "common.log", + "interface_log_file": "interface.log", + "prompt_builder_interface_log_file": "prompt_builder.log", + "performance_log_file": "performance.log", + "output": ["console"], + "console_output": "stdout", # 明确指定使用 stdout + "interface_output": ["console"], + "performance_output": ["console"], + "level": "INFO", # 使用 INFO 级别确保 info 日志能输出 + "backup_count": 5, + "max_bytes": 1024 * 1024, + "format": '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s' + } + } + # 2. Mock config 模块中的 config 变量 + self.config_patcher = mock.patch.object(config_base, 'config', self.mock_config_data) + self.mock_config = self.config_patcher.start() + self.addCleanup(self.config_patcher.stop) # 确保清理 + + # 3. 创建临时日志目录 + self.temp_log_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_log_dir.cleanup) + + # 4. 设置环境变量 + self.original_env = os.environ.get('JIUWEN_LOG_PATH') + os.environ['JIUWEN_LOG_PATH'] = self.temp_log_dir.name + self.addCleanup(lambda: os.environ.pop('JIUWEN_LOG_PATH', None) or (os.environ.update({'JIUWEN_LOG_PATH': self.original_env}) if self.original_env else None)) + + # 5. 捕获标准输出 (必须在 LogManager.initialize() 之前) + self.stdout_capture = StringIO() + self.original_stdout = sys.stdout + sys.stdout = self.stdout_capture + + self.stderr_capture = StringIO() + self.original_stderr = sys.stderr + sys.stderr = self.stderr_capture + self.addCleanup(lambda: setattr(sys, 'stdout', self.original_stdout)) + self.addCleanup(lambda: setattr(sys, 'stderr', self.original_stderr)) + # ****************************************************************** + + # 6. 重置 LogManager 状态 (使用 RLock 后更安全) + with LogManager._lock: # 获取锁再修改 + LogManager._loggers.clear() + LogManager._initialized = False + + # 7. 初始化 LogManager (会使用 mock 的 config 和 重定向后的 sys.stdout) + LogManager.initialize() + + # 8. 确保日志处理器已刷新 + for log in LogManager.get_all_loggers().values(): + if not log._logger.handlers: + handler = logging.StreamHandler(sys.stdout) + log._logger.addHandler(handler) + + + + def tearDown(self): + # --- 关键：重置主线程的 trace_id --- + from jiuwen.extensions.common.log import set_thread_session + set_thread_session('') # 将 trace_id 重置为空字符串 + # --- 关键结束 --- + # 所有清理已在 addCleanup 中处理 + pass + + +class ThreadSafetyTest(LoggerBaseTest): + def test_log_output_contains_trace_id(self): + test_id = "TRACE-12345" + from jiuwen.extensions.common.log import set_thread_session + + # 获取 logger 并设置级别 + logger = LogManager.get_logger('common') + logger.setLevel(logging.INFO) # 确保级别足够低 + + # 清空缓冲区 + self.stdout_capture.truncate(0) + self.stdout_capture.seek(0) + self.stderr_capture.truncate(0) + self.stderr_capture.seek(0) + + set_thread_session(test_id) + logger.info("Test log message with trace_id") + + # 强制刷新 + for handler in logger._logger.handlers: + handler.flush() + + # 获取输出 + stdout_output = self.stdout_capture.getvalue() + stderr_output = self.stderr_capture.getvalue() + combined_output = stdout_output + stderr_output + + # 调试输出 + print("Actual output:", repr(combined_output)) + print("Logger handlers:", logger._logger.handlers) + + # 验证 + self.assertIn(test_id, combined_output) + self.assertRegex( + combined_output, + r'.*TRACE-12345.*Test log message with trace_id' + ) + + def test_thread_trace_id_isolation(self): + log_list = [] + threads = [ + threading.Thread(target=thread_function, args=('10001', log_list)), + threading.Thread(target=thread_function, args=('10002', log_list)), + threading.Thread(target=thread_function, args=('10003', log_list)) + ] + + # 清空缓冲区 + self.stdout_capture.truncate(0) + self.stdout_capture.seek(0) + + for t in threads: + t.start() + for t in threads: + t.join() + + # 验证线程局部变量 + for session_id, recorded_id in log_list: + self.assertEqual(session_id, recorded_id) + + # 验证主线程 trace_id 被重置 + from jiuwen.extensions.common.log import get_thread_session + self.assertEqual(get_thread_session(), '') + + # 获取输出并验证 + output = self.stdout_capture.getvalue() + print("Actual output:", repr(output)) # 调试输出 + + # 验证所有线程ID都出现在输出中 + self.assertIn('10001', output) + self.assertIn('10002', output) + self.assertIn('10003', output) + +class LogManagerTest(LoggerBaseTest): + """测试日志管理器功能""" + + def test_custom_logger_registration_and_usage(self): + """测试注册和使用自定义日志记录器""" + # 创建一个符合协议的简单自定义记录器 + class CustomLogger: + def __init__(self): + self.messages = [] + def info(self, msg): + formatted_msg = f"CUSTOM LOGGER INFO: {msg}" + print(formatted_msg) # 模拟输出到 stdout + self.messages.append(formatted_msg) + # 实现协议所需的其他方法（简化处理） + def debug(self, msg, *args, **kwargs): pass + def warning(self, msg, *args, **kwargs): pass + def error(self, msg, *args, **kwargs): pass + def critical(self, msg, *args, **kwargs): pass + def exception(self, msg, *args, **kwargs): pass + def log(self, level, msg, *args, **kwargs): pass + def setLevel(self, level): pass + def addHandler(self, handler): pass + def removeHandler(self, handler): pass + def addFilter(self, filter): pass + def removeFilter(self, filter): pass + def get_config(self): return {} + def reconfigure(self, config): pass + + custom_logger_instance = CustomLogger() + + # 注册自定义日志记录器 + LogManager.register_logger('custom', custom_logger_instance) + + # 获取并使用自定义日志记录器 + retrieved_logger = LogManager.get_logger('custom') + self.assertIs(retrieved_logger, custom_logger_instance) # 应该返回同一个实例 + retrieved_logger.info("Test custom logger") + + # 验证输出 (通过 CustomLogger 内部捕获或 stdout) + # 方法1: 通过内部列表 + self.assertIn("CUSTOM LOGGER INFO: Test custom logger", custom_logger_instance.messages) + # 方法2: 通过捕获的 stdout + output = self.stdout_capture.getvalue() + self.assertIn("CUSTOM LOGGER INFO: Test custom logger", output) + + def test_default_logger_creation(self): + """测试动态创建默认日志记录器""" + # 获取未预先注册的日志类型 + new_logger = LogManager.get_logger('new_type_test') + + # 验证返回的是 DefaultLogger 实例 + from jiuwen.extensions.common.log.logger_impl import DefaultLogger + self.assertIsInstance(new_logger, DefaultLogger) + + # 使用日志记录器 + new_logger.warning("Test new logger type") + + # 验证输出 (注意 log_type 是 'new_type_test') + output = self.stdout_capture.getvalue() + self.assertIn("Test new logger type", output) + # 格式验证 + self.assertRegex(output, + r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \| new_type_test \| \| WARNING \| Test new logger type\n') + + def test_get_all_loggers(self): + """测试获取所有日志记录器""" + all_loggers = LogManager.get_all_loggers() + expected_types = {'common', 'interface', 'prompt_builder', 'performance'} + self.assertTrue(expected_types.issubset(set(all_loggers.keys()))) + for log_instance in all_loggers.values(): + from jiuwen.extensions.common.log import LoggerProtocol + self.assertIsInstance(log_instance, LoggerProtocol) # 检查是否符合协议 + + +class LogLevelTest(LoggerBaseTest): + """测试日志级别功能""" + + def test_log_level_filtering(self): + """测试日志级别过滤功能""" + # LogManager.initialize() 后，级别是 WARNING (来自 mock_config) + # 我们需要先获取 logger 实例，然后修改它的级别 + test_logger_instance = LogManager.get_logger('level_test') + # 或者使用已有的 logger + # logger_instance = logger # 这个是 common logger + + # 设置为 DEBUG 级别 + test_logger_instance.setLevel(logging.DEBUG) + self.stdout_capture.truncate(0) + self.stdout_capture.seek(0) + + # 记录不同级别的日志 + test_logger_instance.debug("Debug message") + test_logger_instance.info("Info message") + test_logger_instance.warning("Warning message") + test_logger_instance.error("Error message") + + # 获取输出 + output = self.stdout_capture.getvalue() + + # 验证所有级别日志都被记录 (因为当前级别是 DEBUG) + self.assertIn("Debug message", output) + self.assertIn("Info message", output) + self.assertIn("Warning message", output) + self.assertIn("Error message", output) + + # 设置高级别日志 ERROR + test_logger_instance.setLevel(logging.ERROR) + self.stdout_capture.truncate(0) # 清空缓冲区 + self.stdout_capture.seek(0) + + # 记录日志 + test_logger_instance.debug("Should not appear debug") + test_logger_instance.info("Should not appear info") + test_logger_instance.warning("Should not appear warning") + test_logger_instance.error("Should appear error") + + # 获取输出 + output = self.stdout_capture.getvalue() + + # 验证只有错误级别被记录 + self.assertNotIn("Should not appear debug", output) + self.assertNotIn("Should not appear info", output) + self.assertNotIn("Should not appear warning", output) + self.assertIn("Should appear error", output) + + + + + diff --git a/tests/unit_tests/tracer/test_workflow.py b/tests/unit_tests/tracer/test_workflow.py index b21b690..ffba6e6 100644 --- a/tests/unit_tests/tracer/test_workflow.py +++ b/tests/unit_tests/tracer/test_workflow.py @@ -18,7 +18,7 @@ fake_base.logger = Mock() fake_exception_module = types.ModuleType("base") fake_exception_module.JiuWenBaseException = Mock() -sys.modules["jiuwen.core.common.logging.base"] = fake_base +sys.modules["jiuwen.core.common.log.base"] = fake_base sys.modules["jiuwen.core.common.exception.base"] = fake_exception_module from tests.unit_tests.tracer.test_mock_node_with_tracer import StreamNodeWithTracer, CompositeWorkflowNode diff --git a/tests/unit_tests/workflow/test_checkpoint.py b/tests/unit_tests/workflow/test_checkpoint.py index 43d87a3..3a9a948 100644 --- a/tests/unit_tests/workflow/test_checkpoint.py +++ b/tests/unit_tests/workflow/test_checkpoint.py @@ -16,7 +16,7 @@ fake_base.logger = Mock() fake_exception_module = types.ModuleType("base") fake_exception_module.JiuWenBaseException = Mock() -sys.modules["jiuwen.core.common.logging.base"] = fake_base +sys.modules["jiuwen.core.common.log.base"] = fake_base sys.modules["jiuwen.core.common.exception.base"] = fake_exception_module import asyncio diff --git a/tests/unit_tests/workflow/test_intent_detection_comp.py b/tests/unit_tests/workflow/test_intent_detection_comp.py index 9d94529..544e788 100644 --- a/tests/unit_tests/workflow/test_intent_detection_comp.py +++ b/tests/unit_tests/workflow/test_intent_detection_comp.py @@ -11,7 +11,7 @@ from jiuwen.core.context.context import Context fake_base = types.ModuleType("base") fake_base.logger = Mock() -sys.modules["jiuwen.core.common.logging.base"] = fake_base +sys.modules["jiuwen.core.common.log.base"] = fake_base # ------------------------------------------------ diff --git a/tests/unit_tests/workflow/test_llm_comp.py b/tests/unit_tests/workflow/test_llm_comp.py index adf602e..d143cb6 100644 --- a/tests/unit_tests/workflow/test_llm_comp.py +++ b/tests/unit_tests/workflow/test_llm_comp.py @@ -13,7 +13,7 @@ fake_base.logger = Mock() fake_exception_module = types.ModuleType("base") fake_exception_module.JiuWenBaseException = Mock() -sys.modules["jiuwen.core.common.logging.base"] = fake_base +sys.modules["jiuwen.core.common.log.base"] = fake_base sys.modules["jiuwen.core.common.exception.base"] = fake_exception_module from tests.unit_tests.workflow.test_mock_node import MockStartNode, MockEndNode -- Gitee