diff --git a/jiuwen/extensions/__init__.py b/jiuwen/extensions/common/configs/__init__.py similarity index 100% rename from jiuwen/extensions/__init__.py rename to jiuwen/extensions/common/configs/__init__.py diff --git a/jiuwen/extensions/common/configs/base.py b/jiuwen/extensions/common/configs/base.py new file mode 100644 index 0000000000000000000000000000000000000000..506690dead56706496d070dca7494290797aa4a1 --- /dev/null +++ b/jiuwen/extensions/common/configs/base.py @@ -0,0 +1,60 @@ +# jiuwen/extensions/common/configs/base.py +import os +import yaml + +# --- 添加日志级别映射 --- +CRITICAL = 50 +FATAL = CRITICAL +ERROR = 40 +WARNING = 30 +WARN = WARNING +INFO = 20 +DEBUG = 10 +NOTSET = 0 + +name_to_level = { + 'CRITICAL': CRITICAL, + 'FATAL': FATAL, + 'ERROR': ERROR, + 'WARNING': WARNING, + 'WARN': WARN, + 'INFO': INFO, + 'DEBUG': DEBUG, + 'NOTSET': NOTSET, +} +# --- 结束添加 --- + +class SingletonConfig: + _instance: yaml.constructor = None + + def __init__(self): + if SingletonConfig._instance is not None: + raise Exception("This class is a singleton!") + log_config_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), "../../config.yaml") + with open(log_config_path, encoding="utf-8") as file: + # 加载配置文件 + raw_instance = yaml.safe_load(file) + # --- 关键修复：在加载后立即转换日志级别 --- + if 'log' in raw_instance: + level_str = raw_instance['log'].get('level', 'WARNING').upper() + raw_instance['log']['level'] = name_to_level.get(level_str, WARNING) + # --- 关键修复结束 --- + # 将转换后的配置存入 _instance + SingletonConfig._instance = raw_instance + + @staticmethod + def get_config(): + if SingletonConfig._instance is None: + SingletonConfig() + return SingletonConfig._instance + +# --- 关键修复：定义一个可调用的字典 --- +class ConfigDict(dict): + """一个既像字典又可调用的类，用于兼容旧代码""" + def __call__(self): + return self + +# --- 关键修复：创建 config 实例 --- +# config = SingletonConfig.get_config() # <--- 注释掉这一行 +config = ConfigDict(SingletonConfig.get_config()) # <--- 用 ConfigDict 包装 +# --- 关键修复结束 --- \ No newline at end of file diff --git a/jiuwen/extensions/common/exception/_init_.py b/jiuwen/extensions/common/exception/_init_.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/jiuwen/extensions/common/exception/base.py b/jiuwen/extensions/common/exception/base.py new file mode 100644 index 0000000000000000000000000000000000000000..f9b98d6aa9ca431fb11c816d4e96d4ad9ef035a8 --- /dev/null +++ b/jiuwen/extensions/common/exception/base.py @@ -0,0 +1,41 @@ + + +MAGIC_CODE = "\t" + +class JiuWenException(Exception): + def __init__( + self, + message:str, + *args, + **kwargs + )->None: + super().__init__(message, args, kwargs) + + +class JiuWenBaseException(Exception): + + def __init__(self, error_code: int, message:str, node_id : str = None, node_name : str = None, + node_type : str = None) -> None: + super().__init__(error_code, message) + self._error_code = error_code + self._message = message + self.node_id = node_id + self.node_name = node_name + self.node_type = node_type + + def __str__(self): + return f"[{self._error_code}]{self._message}{MAGIC_CODE}" + + @property + def error_code(self): + return self._error_code + + @property + def message(self): + return self._message + +class InterruptException(JiuWenBaseException): + def __init__(self, error_code: int, message:str) -> None: + super().__init__(error_code, message) + self.__error_code = error_code + self.__message = message \ No newline at end of file diff --git a/jiuwen/extensions/common/log/__init__.py b/jiuwen/extensions/common/log/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..460e745fd7cfcdde4e4fab0d27e3d97323adbe09 --- /dev/null +++ b/jiuwen/extensions/common/log/__init__.py @@ -0,0 +1,46 @@ +# jiuwen/extensions/common/log/__init__.py +"""logger for common and interface.""" +__all__ = ("logger", "interface_logger", "prompt_builder_interface_logger", + "performance_logger", "get_thread_session", "set_thread_session", + "LoggerProtocol", "LogManager", "initialize_loggers") + +from .log_manager import LogManager +from .log_utils import set_thread_session, get_thread_session +from .logger_protocol import LoggerProtocol + + +# 将 logger 等变量定义为延迟加载的属性 +class _LazyLogger: + def __init__(self, log_type): + self.log_type = log_type + self._logger = None + + @property + def logger(self): + if self._logger is None: + # 确保 LogManager 已初始化 + if not LogManager._initialized: + LogManager.initialize() + # 获取 logger + self._logger = LogManager.get_logger(self.log_type) + return self._logger + + def __getattr__(self, name): + return getattr(self.logger, name) + +# 创建惰性 logger 实例 +_common_logger = _LazyLogger('common') +_interface_logger = _LazyLogger('interface') +_prompt_builder_interface_logger = _LazyLogger('prompt_builder') +_performance_logger = _LazyLogger('performance') + +# 全局变量指向惰性实例 +logger = _common_logger +interface_logger = _interface_logger +prompt_builder_interface_logger = _prompt_builder_interface_logger +performance_logger = _performance_logger + +# 提供一个显式初始化函数 +def initialize_loggers(): + """强制初始化日志系统""" + LogManager.initialize() \ No newline at end of file diff --git a/jiuwen/extensions/common/log/log_handlers.py b/jiuwen/extensions/common/log/log_handlers.py new file mode 100644 index 0000000000000000000000000000000000000000..ec222c2154a6f9dd9e43851266bc741528841e0b --- /dev/null +++ b/jiuwen/extensions/common/log/log_handlers.py @@ -0,0 +1,37 @@ +# jiuwen/extensions/common/log/log_handlers.py +import os +import logging +from logging.handlers import RotatingFileHandler +from .log_utils import get_thread_session + +class SafeRotatingFileHandler(RotatingFileHandler): + """安全轮转文件处理器，添加进程ID和文件权限控制""" + def __init__(self, filename, *args, **kwargs): + pid = os.getpid() + filename = f"{filename}-{pid}" + super().__init__(filename, *args, **kwargs) + # 设置文件的初始权限为 640 + os.chmod(self.baseFilename, 0o640) + + def doRollover(self): + """执行日志轮转并设置文件权限""" + super().doRollover() + # 设置归档日志文件权限为440 + for i in range(self.backupCount, 0, -1): + sfn = f"{self.baseFilename}.{i}" + if os.path.exists(sfn): + os.chmod(sfn, 0o440) + # 设置当前日志文件权限为640 + os.chmod(self.baseFilename, 0o640) + +class ThreadContextFilter(logging.Filter): + """线程上下文过滤器，添加trace_id""" + def __init__(self, log_type: str): + super().__init__() + self.log_type = log_type + + def filter(self, record): + # 添加自定义日志字段 + record.trace_id = get_thread_session() + record.log_type = "perf" if self.log_type == 'performance' else self.log_type + return True diff --git a/jiuwen/extensions/common/log/log_manager.py b/jiuwen/extensions/common/log/log_manager.py new file mode 100644 index 0000000000000000000000000000000000000000..123f37d78bf4ec279912328f33150bd962ab9f2f --- /dev/null +++ b/jiuwen/extensions/common/log/log_manager.py @@ -0,0 +1,115 @@ +# jiuwen/extensions/common/log/log_manager.py +import threading +import os +from typing import Dict, Tuple, Optional + +from .logger_protocol import LoggerProtocol +from .logger_impl import DefaultLogger +from jiuwen.extensions.common.configs.base import config + + +class LogManager: + """日志管理器，支持多种日志类型和自定义实现""" + _loggers: Dict[str, LoggerProtocol] = {} + _lock = threading.RLock() + _initialized = False + + @classmethod + def initialize(cls) -> None: + """初始化日志系统""" + with cls._lock: + if cls._initialized: + return + + # 从配置获取日志设置 + log_config = config.get("log", {}) + + # 获取日志路径（环境变量优先） + jiuwen_log_path = os.environ.get('JIUWEN_LOG_PATH', log_config.get('log_path', '/var/log/jiuwen')) + + # 构建各类型日志文件路径 + common_config = { + 'log_file': os.path.join(jiuwen_log_path, log_config.get('log_file', 'common.log')), + 'output': log_config.get('output', ['console']), + 'level': log_config.get('level', 'WARNING'), + 'backup_count': log_config.get('backup_count', 20), + 'max_bytes': log_config.get('max_bytes', 20 * 1024 * 1024), + 'format': log_config.get('format', + '%(asctime)s | %(log_type)s | %(filename)s | %(lineno)d | %(funcName)s | %(trace_id)s | %(levelname)s | %(message)s') + } + + interface_config = { + 'log_file': os.path.join(jiuwen_log_path, log_config.get('interface_log_file', 'interface.log')), + 'output': log_config.get('interface_output', ['console']), + 'level': log_config.get('level', 'WARNING'), + 'backup_count': log_config.get('backup_count', 20), + 'max_bytes': log_config.get('max_bytes', 20 * 1024 * 1024), + 'format': log_config.get('format', + '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s') + } + + prompt_builder_config = { + 'log_file': os.path.join(jiuwen_log_path, + log_config.get('prompt_builder_interface_log_file', 'prompt_builder.log')), + 'output': log_config.get('interface_output', ['console']), + 'level': log_config.get('level', 'WARNING'), + 'backup_count': log_config.get('backup_count', 20), + 'max_bytes': log_config.get('max_bytes', 20 * 1024 * 1024), + 'format': log_config.get('format', + '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s') + } + + performance_config = { + 'log_file': os.path.join(jiuwen_log_path, log_config.get('performance_log_file', 'performance.log')), + 'output': log_config.get('performance_output', ['console']), + 'level': log_config.get('level', 'WARNING'), + 'backup_count': log_config.get('backup_count', 20), + 'max_bytes': log_config.get('max_bytes', 20 * 1024 * 1024), + 'format': log_config.get('format', + '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s') + } + + # 创建默认日志记录器 + cls.register_logger('common', DefaultLogger('common', common_config)) + cls.register_logger('interface', DefaultLogger('interface', interface_config)) + cls.register_logger('prompt_builder', DefaultLogger('prompt_builder', prompt_builder_config)) + cls.register_logger('performance', DefaultLogger('performance', performance_config)) + + cls._initialized = True + + @classmethod + def register_logger(cls, log_type: str, logger: LoggerProtocol) -> None: + """注册自定义日志记录器""" + if not isinstance(logger, LoggerProtocol): + raise TypeError(f"Logger must implement LoggerProtocol, got {type(logger)}") + + with cls._lock: + cls._loggers[log_type] = logger + + @classmethod + def get_logger(cls, log_type: str) -> LoggerProtocol: + + """获取指定类型的日志记录器""" + if not cls._initialized: + cls.initialize() + + with cls._lock: + if log_type not in cls._loggers: + # 动态创建默认日志记录器 + cls._loggers[log_type] = DefaultLogger(log_type, {}) + + return cls._loggers[log_type] + + @classmethod + def get_all_loggers(cls) -> Dict[str, LoggerProtocol]: + """获取所有已注册的日志记录器""" + if not cls._initialized: + cls.initialize() + return cls._loggers.copy() + + @classmethod + def reset(cls): + """重置日志管理器状态（用于测试）""" + with cls._lock: + cls._loggers = {} + cls._initialized = False diff --git a/jiuwen/extensions/common/log/log_utils.py b/jiuwen/extensions/common/log/log_utils.py new file mode 100644 index 0000000000000000000000000000000000000000..ce2f6ee2367615a51f15a5e0325612af64dbf112 --- /dev/null +++ b/jiuwen/extensions/common/log/log_utils.py @@ -0,0 +1,35 @@ +# jiuwen/extensions/common/log/log_utils.py +import os +import threading +import ast +from jiuwen.extensions.common.exception.base import JiuWenBaseException + +# 使用方式，在需要注入session_id的参数地方，使用thread_local_data.trace_id = trace_id +_thread_log_instance = threading.local() # 线程局部存储，用于保存trace_id + + +def set_thread_session(trace_id: str) -> None: + """设置当前线程的trace_id""" + _thread_log_instance.trace_id = trace_id + + +def get_thread_session() -> str: + """获取当前线程的trace_id""" + return getattr(_thread_log_instance, 'trace_id', '') + + +def get_log_max_bytes(max_bytes_config) -> int: + """验证并获取有效的日志文件大小限制""" + try: + max_bytes = int(max_bytes_config) + except ValueError as e: + raise JiuWenBaseException( + error_code=-1, message="-1" + ) from e + + # 限制文件大小在合理范围内 + DEFAULT_LOG_MAX_BYTES = 100 * 1024 * 1024 # 100MB + if max_bytes <= 0 or max_bytes > DEFAULT_LOG_MAX_BYTES: + max_bytes = DEFAULT_LOG_MAX_BYTES # 小于0或者超出默认最大限制，使用默认最大值 + + return max_bytes diff --git a/jiuwen/extensions/common/log/logger_impl.py b/jiuwen/extensions/common/log/logger_impl.py new file mode 100644 index 0000000000000000000000000000000000000000..469b86a206e4a543612d412f7b57ddb1f38b97ad --- /dev/null +++ b/jiuwen/extensions/common/log/logger_impl.py @@ -0,0 +1,116 @@ +# jiuwen/extensions/common/log/logger_impl.py +import os +import sys +import ast +import logging +from typing import Dict, Any, Optional + +from .log_handlers import SafeRotatingFileHandler, ThreadContextFilter +from .log_utils import get_log_max_bytes, set_thread_session, get_thread_session +from .logger_protocol import LoggerProtocol + + +class DefaultLogger(LoggerProtocol): + """默认日志实现，符合LoggerProtocol协议""" + + def __init__(self, log_type: str, config: Dict[str, Any]): + self.log_type = log_type + self.config = config + self._logger = logging.getLogger(log_type) + self._setup_logger() + + def _setup_logger(self): + """配置日志记录器""" + # 从配置获取设置 + level_str = self.config.get('level', 'WARNING') + level = getattr(logging, level_str.upper(), logging.WARNING) + self._logger.setLevel(level) + + output = self.config.get('output', ['console']) + log_file = self.config.get('log_file', f'{self.log_type}.log') + + # 清除现有处理器 + for handler in self._logger.handlers[:]: + self._logger.removeHandler(handler) + + # 添加控制台处理器 + if 'console' in output: + stream_handler = logging.StreamHandler(stream=sys.stdout) # 明确指定 sys.stdout + stream_handler.addFilter(ThreadContextFilter(self.log_type)) + stream_handler.setFormatter(self._get_formatter()) + self._logger.addHandler(stream_handler) + + # 添加文件处理器 + if 'file' in output: + # 确保日志目录存在 + log_dir = os.path.dirname(log_file) + if log_dir: # 只有当 log_dir 非空时才尝试创建 + # 使用 exist_ok=True 防止父目录不存在时报错 + os.makedirs(log_dir, mode=0o750, exist_ok=True) + + # 获取备份数量和文件大小限制 + backup_count = self.config.get('backup_count', 20) + max_bytes = get_log_max_bytes(self.config.get('max_bytes', 20 * 1024 * 1024)) + + # 创建安全轮转文件处理器 + file_handler = SafeRotatingFileHandler( + filename=log_file, + maxBytes=max_bytes, + backupCount=backup_count, + encoding='utf-8' + ) + file_handler.addFilter(ThreadContextFilter(self.log_type)) + file_handler.setFormatter(self._get_formatter()) + self._logger.addHandler(file_handler) + + def _get_formatter(self) -> logging.Formatter: + """获取日志格式化器""" + log_format = self.config.get( + 'format') or '%(asctime)s.%(msecs)03d | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s' + return logging.Formatter(log_format, datefmt='%Y-%m-%d %H:%M:%S') + + # 实现协议要求的所有方法 + def debug(self, msg: str, *args, **kwargs) -> None: + self._logger.debug(msg, *args, **kwargs) + + def info(self, msg: str, *args, **kwargs) -> None: + self._logger.info(msg, *args, **kwargs) + + def warning(self, msg: str, *args, **kwargs) -> None: + self._logger.warning(msg, *args, **kwargs) + + def error(self, msg: str, *args, **kwargs) -> None: + self._logger.error(msg, *args, **kwargs) + + def critical(self, msg: str, *args, **kwargs) -> None: + self._logger.critical(msg, *args, **kwargs) + + def exception(self, msg: str, *args, **kwargs) -> None: + self._logger.exception(msg, *args, **kwargs) + + def log(self, level: int, msg: str, *args, **kwargs) -> None: + self._logger.log(level, msg, *args, **kwargs) + + def setLevel(self, level: int) -> None: + self._logger.setLevel(level) + + def addHandler(self, handler: logging.Handler) -> None: + self._logger.addHandler(handler) + + def removeHandler(self, handler: logging.Handler) -> None: + self._logger.removeHandler(handler) + + def addFilter(self, filter) -> None: + self._logger.addFilter(filter) + + def removeFilter(self, filter) -> None: + self._logger.removeFilter(filter) + + def get_config(self) -> Dict[str, Any]: + """获取日志配置""" + return self.config.copy() + + def reconfigure(self, config: Dict[str, Any]) -> None: + """重新配置日志记录器""" + self.config = config + self._setup_logger() diff --git a/jiuwen/extensions/common/log/logger_protocol.py b/jiuwen/extensions/common/log/logger_protocol.py new file mode 100644 index 0000000000000000000000000000000000000000..f9c9566f7b50be4b8eff429fd98cd548200172bc --- /dev/null +++ b/jiuwen/extensions/common/log/logger_protocol.py @@ -0,0 +1,64 @@ +# jiuwen/extensions/common/log/logger_protocol.py +from typing import Protocol, runtime_checkable, Dict, Any, Optional +import logging + + +@runtime_checkable +class LoggerProtocol(Protocol): + """日志记录器协议，定义所有日志实现必须提供的方法""" + + def debug(self, msg: str, *args, **kwargs) -> None: + """记录调试级别日志""" + ... + + def info(self, msg: str, *args, **kwargs) -> None: + """记录信息级别日志""" + ... + + def warning(self, msg: str, *args, **kwargs) -> None: + """记录警告级别日志""" + ... + + def error(self, msg: str, *args, **kwargs) -> None: + """记录错误级别日志""" + ... + + def critical(self, msg: str, *args, **kwargs) -> None: + """记录严重级别日志""" + ... + + def exception(self, msg: str, *args, **kwargs) -> None: + """记录异常信息（包含堆栈跟踪）""" + ... + + def log(self, level: int, msg: str, *args, **kwargs) -> None: + """通用日志记录方法""" + ... + + def setLevel(self, level: int) -> None: + """设置日志级别""" + ... + + def addHandler(self, handler: logging.Handler) -> None: + """添加日志处理器""" + ... + + def removeHandler(self, handler: logging.Handler) -> None: + """移除日志处理器""" + ... + + def addFilter(self, filter) -> None: + """添加过滤器""" + ... + + def removeFilter(self, filter) -> None: + """移除过滤器""" + ... + + def get_config(self) -> Dict[str, Any]: + """获取日志配置""" + ... + + def reconfigure(self, config: Dict[str, Any]) -> None: + """重新配置日志记录器""" + ... diff --git a/jiuwen/extensions/common/log/utils.py b/jiuwen/extensions/common/log/utils.py new file mode 100644 index 0000000000000000000000000000000000000000..483047008141752398ec66ef56a59903ca2b44f8 --- /dev/null +++ b/jiuwen/extensions/common/log/utils.py @@ -0,0 +1,42 @@ +__all__ = ("LazyStr", "blur", "blur_later") + +from typing import Any, Callable + +from pydantic import BaseModel + +class LazyStr: + """Call 'f' while calling its '_str_'.""" + def __init__(self, f:Callable[[],str]): + self._f = f + self.__val = None + + def __str__(self): + if self.__val is None: + self.__val = self._f() + return self.__val + + + +def blur_later(obj): + return LazyStr(lambda: blur(obj)) + + + +def blur(obj) -> str: + + if obj is None: + return "None" + for t, f in _blur_map: + if isinstance(obj, t): + return f(obj) + return f"...({type(obj).__name__})" + +def _blur_dict(d : dict) -> str: + contents = ','.join(f"{k!r} : {blur(v)}" for k, v in d.items()) + return '{' + contents + '}' + +_blur_map : list[tuple[type, Callable[[Any], str]]] = [ + (dict, _blur_dict), + (list, lambda li : '[' + ','.join(blur(v) for v in li) + ']'), + (BaseModel, lambda b : blur(b.model_dump(by_alias=True))), +] \ No newline at end of file diff --git a/jiuwen/extensions/config.yaml b/jiuwen/extensions/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a92b945927c074acc008711713b61d464a7fd9c5 --- /dev/null +++ b/jiuwen/extensions/config.yaml @@ -0,0 +1,45 @@ +logging: + level: INFO + backup_count: 20 + max_bytes: 20971520 + format: '%(asctime)s | %(log_type)s | %(filename)s | %(lineno)d | %(funcName)s | %(trace_id)s | %(levelname)s | %(message)s' + log_file: "run/jiuwen.log" + output: ["console"] + interface_log_file : "interface/jiuwen_interface.log" + prompt_builder_interface_log_file: "interface/jiuwen_prompt_builder_interface.log" + performance_log_file: "performance/jiuwen_performance.log" + interface_output: ["console", "file"] # 同时输出到控制台和文件 + performance_output: ["console"] + log_path: "./logs/" + +plugin: + store: "gaussdb" + cache_maxsize: "1000" + default_dir: "./plugins" + recall: + emb_class: + emb_service_url: "" + app_code: "" + emb_dim: 256 + infer_fields: [ "func_description" ] + +prompt: + default_dir: "./prompts" + +executor: + worker: + timeout: 300 + conn_timeout: 300 + read_timeout: 5 + resume_interval: 15000 + max_retry: 2 + thread_num: 100 + stream_out_mode: mq + +planner_config: + finish_triggers: + - "yes" + - "是" + +rails: + custom_rails_path: #自定义护栏路径 \ No newline at end of file diff --git a/tests/unit_tests/common/log/__init__.py b/tests/unit_tests/common/log/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/unit_tests/common/log/test_logger.py b/tests/unit_tests/common/log/test_logger.py new file mode 100644 index 0000000000000000000000000000000000000000..8b3a26d00bf5bb29d949491dcaeb67770a43e4ec --- /dev/null +++ b/tests/unit_tests/common/log/test_logger.py @@ -0,0 +1,311 @@ + +# test_logger.py +import os +import sys +import threading +import tempfile +import logging +from io import StringIO +from unittest import TestCase, mock +from jiuwen.extensions.common.log.log_manager import LogManager +# 导入日志模块 + + +# 模拟配置模块，避免实际依赖 +import jiuwen.extensions.common.configs.base as config_base + + +def thread_function(session_id, log_list): + from jiuwen.extensions.common.log import set_thread_session + logger = LogManager.get_logger('common') + + # 确保 logger 有处理器且级别足够 + if not logger._logger.handlers: + handler = logging.StreamHandler(sys.stdout) + logger._logger.addHandler(handler) + logger.setLevel(logging.INFO) + + set_thread_session(session_id) + logger.info(f'Thread started with session id {session_id}') + + # 强制刷新 + for handler in logger._logger.handlers: + handler.flush() + + from jiuwen.extensions.common.log import get_thread_session + log_list.append((session_id, get_thread_session())) + +class LoggerBaseTest(TestCase): + """日志测试基类，处理环境设置和清理""" + def setUp(self): + # --- 修改 setUp 逻辑 --- + # 1. 先 mock 配置，再初始化 LogManager + self.mock_config_data = { + "log": { + "log_path": "/default/log/path", + "log_file": "common.log", + "interface_log_file": "interface.log", + "prompt_builder_interface_log_file": "prompt_builder.log", + "performance_log_file": "performance.log", + "output": ["console"], + "console_output": "stdout", # 明确指定使用 stdout + "interface_output": ["console"], + "performance_output": ["console"], + "level": "INFO", + "backup_count": 5, + "max_bytes": 1024 * 1024, + "format": '%(asctime)s | %(log_type)s | %(trace_id)s | %(levelname)s | %(message)s' + } + } + # 2. Mock config 模块中的 config 变量 + self.config_patcher = mock.patch.object(config_base, 'config', self.mock_config_data) + self.mock_config = self.config_patcher.start() + self.addCleanup(self.config_patcher.stop) # 确保清理 + + # 3. 创建临时日志目录 + self.temp_log_dir = tempfile.TemporaryDirectory() + self.addCleanup(self.temp_log_dir.cleanup) + + # 4. 设置环境变量 + self.original_env = os.environ.get('JIUWEN_LOG_PATH') + os.environ['JIUWEN_LOG_PATH'] = self.temp_log_dir.name + self.addCleanup(lambda: os.environ.pop('JIUWEN_LOG_PATH', None) or (os.environ.update({'JIUWEN_LOG_PATH': self.original_env}) if self.original_env else None)) + + # 5. 捕获标准输出 (必须在 LogManager.initialize() 之前) + self.stdout_capture = StringIO() + self.original_stdout = sys.stdout + sys.stdout = self.stdout_capture + + self.stderr_capture = StringIO() + self.original_stderr = sys.stderr + sys.stderr = self.stderr_capture + self.addCleanup(lambda: setattr(sys, 'stdout', self.original_stdout)) + self.addCleanup(lambda: setattr(sys, 'stderr', self.original_stderr)) + # ****************************************************************** + + # 6. 重置 LogManager 状态 (使用 RLock 后更安全) + with LogManager._lock: # 获取锁再修改 + LogManager._loggers.clear() + LogManager._initialized = False + + # 7. 初始化 LogManager (会使用 mock 的 config 和 重定向后的 sys.stdout) + LogManager.initialize() + + # 8. 确保日志处理器已刷新 + for log in LogManager.get_all_loggers().values(): + if not log._logger.handlers: + handler = logging.StreamHandler(sys.stdout) + log._logger.addHandler(handler) + + + + def tearDown(self): + # --- 关键：重置主线程的 trace_id --- + from jiuwen.extensions.common.log import set_thread_session + set_thread_session('') # 将 trace_id 重置为空字符串 + # --- 关键结束 --- + # 所有清理已在 addCleanup 中处理 + pass + + +class ThreadSafetyTest(LoggerBaseTest): + def test_log_output_contains_trace_id(self): + test_id = "TRACE-12345" + from jiuwen.extensions.common.log import set_thread_session + + # 获取 logger 并设置级别 + logger = LogManager.get_logger('common') + logger.setLevel(logging.INFO) # 确保级别足够低 + + # 清空缓冲区 + self.stdout_capture.truncate(0) + self.stdout_capture.seek(0) + self.stderr_capture.truncate(0) + self.stderr_capture.seek(0) + + set_thread_session(test_id) + logger.info("Test log message with trace_id") + + # 强制刷新 + for handler in logger._logger.handlers: + handler.flush() + + # 获取输出 + stdout_output = self.stdout_capture.getvalue() + stderr_output = self.stderr_capture.getvalue() + combined_output = stdout_output + stderr_output + + # 调试输出 + print("Actual output:", repr(combined_output)) + print("Logger handlers:", logger._logger.handlers) + + # 验证 + self.assertIn(test_id, combined_output) + self.assertRegex( + combined_output, + r'.*TRACE-12345.*Test log message with trace_id' + ) + + def test_thread_trace_id_isolation(self): + log_list = [] + threads = [ + threading.Thread(target=thread_function, args=('10001', log_list)), + threading.Thread(target=thread_function, args=('10002', log_list)), + threading.Thread(target=thread_function, args=('10003', log_list)) + ] + + # 清空缓冲区 + self.stdout_capture.truncate(0) + self.stdout_capture.seek(0) + + for t in threads: + t.start() + for t in threads: + t.join() + + # 验证线程局部变量 + for session_id, recorded_id in log_list: + self.assertEqual(session_id, recorded_id) + + # 验证主线程 trace_id 被重置 + from jiuwen.extensions.common.log import get_thread_session + self.assertEqual(get_thread_session(), '') + + # 获取输出并验证 + output = self.stdout_capture.getvalue() + print("Actual output:", repr(output)) # 调试输出 + + # 验证所有线程ID都出现在输出中 + self.assertIn('10001', output) + self.assertIn('10002', output) + self.assertIn('10003', output) + +class LogManagerTest(LoggerBaseTest): + """测试日志管理器功能""" + + def test_custom_logger_registration_and_usage(self): + """测试注册和使用自定义日志记录器""" + # 创建一个符合协议的简单自定义记录器 + class CustomLogger: + def __init__(self): + self.messages = [] + def info(self, msg): + formatted_msg = f"CUSTOM LOGGER INFO: {msg}" + print(formatted_msg) # 模拟输出到 stdout + self.messages.append(formatted_msg) + # 实现协议所需的其他方法（简化处理） + def debug(self, msg, *args, **kwargs): pass + def warning(self, msg, *args, **kwargs): pass + def error(self, msg, *args, **kwargs): pass + def critical(self, msg, *args, **kwargs): pass + def exception(self, msg, *args, **kwargs): pass + def log(self, level, msg, *args, **kwargs): pass + def setLevel(self, level): pass + def addHandler(self, handler): pass + def removeHandler(self, handler): pass + def addFilter(self, filter): pass + def removeFilter(self, filter): pass + def get_config(self): return {} + def reconfigure(self, config): pass + + custom_logger_instance = CustomLogger() + + # 注册自定义日志记录器 + LogManager.register_logger('custom', custom_logger_instance) + + # 获取并使用自定义日志记录器 + retrieved_logger = LogManager.get_logger('custom') + self.assertIs(retrieved_logger, custom_logger_instance) # 应该返回同一个实例 + retrieved_logger.info("Test custom logger") + + # 验证输出 (通过 CustomLogger 内部捕获或 stdout) + # 方法1: 通过内部列表 + self.assertIn("CUSTOM LOGGER INFO: Test custom logger", custom_logger_instance.messages) + # 方法2: 通过捕获的 stdout + output = self.stdout_capture.getvalue() + self.assertIn("CUSTOM LOGGER INFO: Test custom logger", output) + + def test_default_logger_creation(self): + """测试动态创建默认日志记录器""" + # 获取未预先注册的日志类型 + new_logger = LogManager.get_logger('new_type_test') + + # 验证返回的是 DefaultLogger 实例 + from jiuwen.extensions.common.log.logger_impl import DefaultLogger + self.assertIsInstance(new_logger, DefaultLogger) + + # 使用日志记录器 + new_logger.warning("Test new logger type") + + # 验证输出 (注意 log_type 是 'new_type_test') + output = self.stdout_capture.getvalue() + self.assertIn("Test new logger type", output) + # 格式验证 + self.assertRegex(output, + r'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3} \| new_type_test \| \| WARNING \| Test new logger type\n') + + def test_get_all_loggers(self): + """测试获取所有日志记录器""" + all_loggers = LogManager.get_all_loggers() + expected_types = {'common', 'interface', 'prompt_builder', 'performance'} + self.assertTrue(expected_types.issubset(set(all_loggers.keys()))) + for log_instance in all_loggers.values(): + from jiuwen.extensions.common.log import LoggerProtocol + self.assertIsInstance(log_instance, LoggerProtocol) # 检查是否符合协议 + + +class LogLevelTest(LoggerBaseTest): + """测试日志级别功能""" + + def test_log_level_filtering(self): + """测试日志级别过滤功能""" + # LogManager.initialize() 后，级别是 WARNING (来自 mock_config) + # 我们需要先获取 logger 实例，然后修改它的级别 + test_logger_instance = LogManager.get_logger('level_test') + # 或者使用已有的 logger + # logger_instance = logger # 这个是 common logger + + # 设置为 DEBUG 级别 + test_logger_instance.setLevel(logging.DEBUG) + self.stdout_capture.truncate(0) + self.stdout_capture.seek(0) + + # 记录不同级别的日志 + test_logger_instance.debug("Debug message") + test_logger_instance.info("Info message") + test_logger_instance.warning("Warning message") + test_logger_instance.error("Error message") + + # 获取输出 + output = self.stdout_capture.getvalue() + + # 验证所有级别日志都被记录 (因为当前级别是 DEBUG) + self.assertIn("Debug message", output) + self.assertIn("Info message", output) + self.assertIn("Warning message", output) + self.assertIn("Error message", output) + + # 设置高级别日志 ERROR + test_logger_instance.setLevel(logging.ERROR) + self.stdout_capture.truncate(0) # 清空缓冲区 + self.stdout_capture.seek(0) + + # 记录日志 + test_logger_instance.debug("Should not appear debug") + test_logger_instance.info("Should not appear info") + test_logger_instance.warning("Should not appear warning") + test_logger_instance.error("Should appear error") + + # 获取输出 + output = self.stdout_capture.getvalue() + + # 验证只有错误级别被记录 + self.assertNotIn("Should not appear debug", output) + self.assertNotIn("Should not appear info", output) + self.assertNotIn("Should not appear warning", output) + self.assertIn("Should appear error", output) + + + + + diff --git a/tests/unit_tests/tracer/test_workflow.py b/tests/unit_tests/tracer/test_workflow.py index 7b09af023b24f9082c1900f39619b35899fbd0c1..3672923e1055e6f5257af0d1ced1a76503844d00 100644 --- a/tests/unit_tests/tracer/test_workflow.py +++ b/tests/unit_tests/tracer/test_workflow.py @@ -18,7 +18,7 @@ fake_base.logger = Mock() fake_exception_module = types.ModuleType("base") fake_exception_module.JiuWenBaseException = Mock() -sys.modules["jiuwen.core.common.logging.base"] = fake_base +sys.modules["jiuwen.core.common.log.base"] = fake_base sys.modules["jiuwen.core.common.exception.base"] = fake_exception_module from tests.unit_tests.tracer.test_mock_node_with_tracer import StreamNodeWithTracer diff --git a/tests/unit_tests/workflow/test_checkpoint.py b/tests/unit_tests/workflow/test_checkpoint.py index ca5646a15041bee7d2521f07369fc91b4d57e09c..23fdb4d3d62a6fbd6f99bbd2fce0451df6156243 100644 --- a/tests/unit_tests/workflow/test_checkpoint.py +++ b/tests/unit_tests/workflow/test_checkpoint.py @@ -16,7 +16,7 @@ fake_base.logger = Mock() fake_exception_module = types.ModuleType("base") fake_exception_module.JiuWenBaseException = Mock() -sys.modules["jiuwen.core.common.logging.base"] = fake_base +sys.modules["jiuwen.core.common.log.base"] = fake_base sys.modules["jiuwen.core.common.exception.base"] = fake_exception_module import asyncio diff --git a/tests/unit_tests/workflow/test_intent_detection_comp.py b/tests/unit_tests/workflow/test_intent_detection_comp.py index 9d9452975ad7e26225e13be01854a8b83e1143c3..544e788035dc2a44ba61a6985787a425e1d11ea6 100644 --- a/tests/unit_tests/workflow/test_intent_detection_comp.py +++ b/tests/unit_tests/workflow/test_intent_detection_comp.py @@ -11,7 +11,7 @@ from jiuwen.core.context.context import Context fake_base = types.ModuleType("base") fake_base.logger = Mock() -sys.modules["jiuwen.core.common.logging.base"] = fake_base +sys.modules["jiuwen.core.common.log.base"] = fake_base # ------------------------------------------------ diff --git a/tests/unit_tests/workflow/test_llm_comp.py b/tests/unit_tests/workflow/test_llm_comp.py index 35312a56feae820e6322e87036dee26d222a9b47..9767281801ef0be2a18d44c3d3ae79f4070443b4 100644 --- a/tests/unit_tests/workflow/test_llm_comp.py +++ b/tests/unit_tests/workflow/test_llm_comp.py @@ -14,7 +14,7 @@ fake_base.logger = Mock() fake_exception_module = types.ModuleType("base") fake_exception_module.JiuWenBaseException = Mock() -sys.modules["jiuwen.core.common.logging.base"] = fake_base +sys.modules["jiuwen.core.common.log.base"] = fake_base sys.modules["jiuwen.core.common.exception.base"] = fake_exception_module from tests.unit_tests.workflow.test_mock_node import MockStartNode, MockEndNode