feat: Enhance logging configuration with support for multiple log files and formats; improve CLI help output

2025-09-02 15:27:20 +03:00
parent 79c8f127ca
commit fb38853427
10 changed files with 584 additions and 72 deletions
@@ -1,9 +1,9 @@
 """
-PyServe - HTTP веб-сервер с функционалом nginx
+PyServe - HTTP web server written on Python
 """

 __version__ = "0.6.0"
-__author__ = "Илья Глазунов"
+__author__ = "Ilya Glazunov"

 from .server import PyServeServer
 from .config import Config
@@ -8,7 +8,7 @@ from . import PyServeServer, Config, __version__
 def main() -> None:
    parser = argparse.ArgumentParser(
        description="PyServe - HTTP web server",
-        prog="pyserve"
+        prog="pyserve",
    )
    parser.add_argument(
        "-c", "--config",
@@ -1,6 +1,6 @@
 import yaml
 import os
-from typing import Dict, Any, List
+from typing import Dict, Any, List, cast
 from dataclasses import dataclass, field
 import logging
 from .logging_utils import setup_logging
@@ -28,11 +28,37 @@ class SSLConfig:
    key_file: str = "./ssl/key.pem"


+@dataclass
+class LogFormatConfig:
+    type: str = "standard"
+    use_colors: bool = True
+    show_module: bool = True
+    timestamp_format: str = "%Y-%m-%d %H:%M:%S"
+
+
+@dataclass
+class LogFileConfig:
+    path: str
+    level: str = "INFO"
+    format: LogFormatConfig = field(default_factory=LogFormatConfig)
+    loggers: List[str] = field(default_factory=list)  # Список logger'ов для этого файла
+    max_bytes: int = 10 * 1024 * 1024  # 10MB
+    backup_count: int = 5
+
+
+@dataclass
+class LogHandlerConfig:
+    level: str = "INFO"
+    format: LogFormatConfig = field(default_factory=LogFormatConfig)
+
+
@dataclass
 class LoggingConfig:
    level: str = "INFO"
    console_output: bool = True
-    log_file: str = "./logs/pyserve.log"
+    format: LogFormatConfig = field(default_factory=LogFormatConfig)
+    console: LogHandlerConfig = field(default_factory=LogHandlerConfig)
+    files: List[LogFileConfig] = field(default_factory=list)


@dataclass
@@ -99,10 +125,85 @@ class Config:

        if 'logging' in data:
            log_data = data['logging']
+            format_data = log_data.get('format', {})
+            global_format = LogFormatConfig(
+                type=format_data.get('type', 'standard'),
+                use_colors=format_data.get('use_colors', True),
+                show_module=format_data.get('show_module', True),
+                timestamp_format=format_data.get('timestamp_format', '%Y-%m-%d %H:%M:%S')
+            )
+            console_data = log_data.get('console', {})
+            console_format_data = console_data.get('format', {})
+            console_format = LogFormatConfig(
+                type=console_format_data.get('type', global_format.type),
+                use_colors=console_format_data.get('use_colors', global_format.use_colors),
+                show_module=console_format_data.get('show_module', global_format.show_module),
+                timestamp_format=console_format_data.get('timestamp_format', global_format.timestamp_format)
+            )
+            console_config = LogHandlerConfig(
+                level=console_data.get('level', log_data.get('level', 'INFO')),
+                format=console_format
+            )
+            files_config = []
+            if 'log_file' in log_data:
+                default_file_format = LogFormatConfig(
+                    type=global_format.type,
+                    use_colors=False,
+                    show_module=global_format.show_module,
+                    timestamp_format=global_format.timestamp_format
+                )
+                default_file = LogFileConfig(
+                    path=log_data['log_file'],
+                    level=log_data.get('level', 'INFO'),
+                    format=default_file_format,
+                    loggers=[],  # Empty list means including all loggers
+                    max_bytes=10 * 1024 * 1024,
+                    backup_count=5
+                )
+                files_config.append(default_file)
+
+            if 'files' in log_data:
+                for file_data in log_data['files']:
+                    file_format_data = file_data.get('format', {})
+                    file_format = LogFormatConfig(
+                        type=file_format_data.get('type', global_format.type),
+                        use_colors=file_format_data.get('use_colors', False),
+                        show_module=file_format_data.get('show_module', global_format.show_module),
+                        timestamp_format=file_format_data.get('timestamp_format', global_format.timestamp_format)
+                    )
+                    file_config = LogFileConfig(
+                        path=file_data.get('path', './logs/pyserve.log'),
+                        level=file_data.get('level', log_data.get('level', 'INFO')),
+                        format=file_format,
+                        loggers=file_data.get('loggers', []),
+                        max_bytes=file_data.get('max_bytes', 10 * 1024 * 1024),
+                        backup_count=file_data.get('backup_count', 5)
+                    )
+                    files_config.append(file_config)
+
+            if not files_config:
+                default_file_format = LogFormatConfig(
+                    type=global_format.type,
+                    use_colors=False,
+                    show_module=global_format.show_module,
+                    timestamp_format=global_format.timestamp_format
+                )
+                default_file = LogFileConfig(
+                    path='./logs/pyserve.log',
+                    level=log_data.get('level', 'INFO'),
+                    format=default_file_format,
+                    loggers=[],
+                    max_bytes=10 * 1024 * 1024,
+                    backup_count=5
+                )
+                files_config.append(default_file)
+
            config.logging = LoggingConfig(
-                level=log_data.get('level', config.logging.level),
-                console_output=log_data.get('console_output', config.logging.console_output),
-                log_file=log_data.get('log_file', config.logging.log_file)
+                level=log_data.get('level', 'INFO'),
+                console_output=log_data.get('console_output', True),
+                format=global_format,
+                console=console_config,
+                files=files_config
            )

        if 'extensions' in data:
@@ -130,12 +231,34 @@ class Config:
        if not (1 <= self.server.port <= 65535):
            errors.append(f"Invalid port: {self.server.port}")

-        log_dir = os.path.dirname(self.logging.log_file)
-        if log_dir and not os.path.exists(log_dir):
-            try:
-                os.makedirs(log_dir, exist_ok=True)
-            except OSError as e:
-                errors.append(f"Unable to create log directory: {e}")
+        valid_log_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
+        if self.logging.level.upper() not in valid_log_levels:
+            errors.append(f"Invalid logging level: {self.logging.level}")
+
+        if self.logging.console.level.upper() not in valid_log_levels:
+            errors.append(f"Invalid console logging level: {self.logging.console.level}")
+
+        valid_format_types = ['standard', 'json']
+
+        if self.logging.format.type not in valid_format_types:
+            errors.append(f"Invalid logging format type: {self.logging.format.type}")
+
+        if self.logging.console.format.type not in valid_format_types:
+            errors.append(f"Invalid console format type: {self.logging.console.format.type}")
+
+        for i, file_config in enumerate(self.logging.files):
+            if file_config.level.upper() not in valid_log_levels:
+                errors.append(f"Invalid file[{i}] logging level: {file_config.level}")
+
+            if file_config.format.type not in valid_format_types:
+                errors.append(f"Invalid file[{i}] format type: {file_config.format.type}")
+
+            log_dir = os.path.dirname(file_config.path)
+            if log_dir and not os.path.exists(log_dir):
+                try:
+                    os.makedirs(log_dir, exist_ok=True)
+                except OSError as e:
+                    errors.append(f"Unable to create log directory for {file_config.path}: {e}")

        if errors:
            for error in errors:
@@ -148,6 +271,38 @@ class Config:
        config_dict = {
            'level': self.logging.level,
            'console_output': self.logging.console_output,
-            'log_file': self.logging.log_file
+            'format': {
+                'type': self.logging.format.type,
+                'use_colors': self.logging.format.use_colors,
+                'show_module': self.logging.format.show_module,
+                'timestamp_format': self.logging.format.timestamp_format
+            },
+            'console': {
+                'level': self.logging.console.level,
+                'format': {
+                    'type': self.logging.console.format.type,
+                    'use_colors': self.logging.console.format.use_colors,
+                    'show_module': self.logging.console.format.show_module,
+                    'timestamp_format': self.logging.console.format.timestamp_format
+                }
+            },
+            'files': []
        }
+
+        for file_config in self.logging.files:
+            file_dict = {
+                'path': file_config.path,
+                'level': file_config.level,
+                'loggers': file_config.loggers,
+                'max_bytes': file_config.max_bytes,
+                'backup_count': file_config.backup_count,
+                'format': {
+                    'type': file_config.format.type,
+                    'use_colors': file_config.format.use_colors,
+                    'show_module': file_config.format.show_module,
+                    'timestamp_format': file_config.format.timestamp_format
+                }
+            }
+            cast(List[Dict[str, Any]], config_dict['files']).append(file_dict)
+
        setup_logging(config_dict)
@@ -152,7 +152,7 @@ class ExtensionManager:

    def load_extension(self, extension_type: str, config: Dict[str, Any]) -> None:
        if extension_type not in self.extension_registry:
-            logger.error(f"Неизвестный тип расширения: {extension_type}")
+            logger.error(f"Unknown extension type: {extension_type}")
            return

        try:
@@ -160,9 +160,9 @@ class ExtensionManager:
            extension = extension_class(config)
            extension.initialize()
            self.extensions.append(extension)
-            logger.info(f"Загружено расширение: {extension_type}")
+            logger.info(f"Loaded extension: {extension_type}")
        except Exception as e:
-            logger.error(f"Ошибка загрузки расширения {extension_type}: {e}")
+            logger.error(f"Error loading extension {extension_type}: {e}")

    async def process_request(self, request: Request) -> Optional[Response]:
        for extension in self.extensions:
@@ -174,7 +174,7 @@ class ExtensionManager:
                if response is not None:
                    return response
            except Exception as e:
-                logger.error(f"Ошибка в расширении {type(extension).__name__}: {e}")
+                logger.error(f"Error in extension {type(extension).__name__}: {e}")

        return None

@@ -186,7 +186,7 @@ class ExtensionManager:
            try:
                response = await extension.process_response(request, response)
            except Exception as e:
-                logger.error(f"Ошибка в расширении {type(extension).__name__}: {e}")
+                logger.error(f"Error in extension {type(extension).__name__}: {e}")

        return response

@@ -195,6 +195,6 @@ class ExtensionManager:
            try:
                extension.cleanup()
            except Exception as e:
-                logger.error(f"Ошибка при очистке расширения {type(extension).__name__}: {e}")
+                logger.error(f"Error cleaning up extension {type(extension).__name__}: {e}")

        self.extensions.clear()
@@ -2,12 +2,30 @@ import logging
 import logging.handlers
 import sys
 import time
+import json
 from pathlib import Path
 from typing import Dict, Any, List

 from . import __version__


+class LoggerFilter(logging.Filter):
+    def __init__(self, logger_names: List[str]):
+        super().__init__()
+        self.logger_names = logger_names
+        self.accept_all = len(logger_names) == 0
+
+    def filter(self, record: logging.LogRecord) -> bool:
+        if self.accept_all:
+            return True
+
+        for logger_name in self.logger_names:
+            if record.name == logger_name or record.name.startswith(logger_name + '.'):
+                return True
+
+        return False
+
+
 class UvicornLogFilter(logging.Filter):
    def filter(self, record: logging.LogRecord) -> bool:
        if hasattr(record, 'name') and 'uvicorn.access' in record.name:
@@ -36,10 +54,12 @@ class PyServeFormatter(logging.Formatter):
        'RESET': '\033[0m'        # Reset
    }

-    def __init__(self, use_colors: bool = True, show_module: bool = True, *args: Any, **kwargs: Any):
+    def __init__(self, use_colors: bool = True, show_module: bool = True,
+                 timestamp_format: str = "%Y-%m-%d %H:%M:%S", *args: Any, **kwargs: Any):
        super().__init__(*args, **kwargs)
        self.use_colors = use_colors and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty()
        self.show_module = show_module
+        self.timestamp_format = timestamp_format

    def format(self, record: logging.LogRecord) -> str:
        if self.use_colors:
@@ -59,6 +79,37 @@ class PyServeFormatter(logging.Formatter):
        return super().format(record)


+class PyServeJSONFormatter(logging.Formatter):
+    def __init__(self, timestamp_format: str = "%Y-%m-%d %H:%M:%S", *args: Any, **kwargs: Any):
+        super().__init__(*args, **kwargs)
+        self.timestamp_format = timestamp_format
+
+    def format(self, record: logging.LogRecord) -> str:
+        log_entry = {
+            'timestamp': time.strftime(self.timestamp_format, time.localtime(record.created)),
+            'level': record.levelname,
+            'logger': record.name,
+            'message': record.getMessage(),
+            'module': record.module,
+            'function': record.funcName,
+            'line': record.lineno,
+            'thread': record.thread,
+            'thread_name': record.threadName,
+        }
+
+        if record.exc_info:
+            log_entry['exception'] = self.formatException(record.exc_info)
+
+        for key, value in record.__dict__.items():
+            if key not in ['name', 'msg', 'args', 'levelname', 'levelno', 'pathname',
+                           'filename', 'module', 'lineno', 'funcName', 'created',
+                           'msecs', 'relativeCreated', 'thread', 'threadName',
+                           'processName', 'process', 'getMessage', 'exc_info', 'exc_text', 'stack_info']:
+                log_entry[key] = value
+
+        return json.dumps(log_entry, ensure_ascii=False, default=str)
+
+
 class AccessLogHandler(logging.Handler):
    def __init__(self, logger_name: str = 'pyserve.access'):
        super().__init__()
@@ -75,68 +126,154 @@ class PyServeLogManager:
        self.loggers: Dict[str, logging.Logger] = {}
        self.original_handlers: Dict[str, List[logging.Handler]] = {}

+    def _create_formatter(self, format_config: Dict[str, Any]) -> logging.Formatter:
+        format_type = format_config.get('type', 'standard').lower()
+        use_colors = format_config.get('use_colors', True)
+        show_module = format_config.get('show_module', True)
+        timestamp_format = format_config.get('timestamp_format', '%Y-%m-%d %H:%M:%S')
+
+        if format_type == 'json':
+            return PyServeJSONFormatter(timestamp_format=timestamp_format)
+        else:
+            if format_type == 'json':
+                fmt = None
+            else:
+                fmt = '%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s'
+
+            return PyServeFormatter(
+                use_colors=use_colors,
+                show_module=show_module,
+                timestamp_format=timestamp_format,
+                fmt=fmt
+            )
+
    def setup_logging(self, config: Dict[str, Any]) -> None:
        if self.configured:
            return

-        level = config.get('level', 'INFO').upper()
+        if 'format' not in config and 'console' not in config and 'files' not in config:
+            level = config.get('level', 'INFO').upper()
+            console_output = config.get('console_output', True)
+            log_file = config.get('log_file', './logs/pyserve.log')
+            config = {
+                'level': level,
+                'console_output': console_output,
+                'format': {
+                    'type': 'standard',
+                    'use_colors': True,
+                    'show_module': True,
+                    'timestamp_format': '%Y-%m-%d %H:%M:%S'
+                },
+                'files': [{
+                    'path': log_file,
+                    'level': level,
+                    'loggers': [],
+                    'max_bytes': 10 * 1024 * 1024,
+                    'backup_count': 5,
+                    'format': {
+                        'type': 'standard',
+                        'use_colors': False,
+                        'show_module': True,
+                        'timestamp_format': '%Y-%m-%d %H:%M:%S'
+                    }
+                }]
+            }
+
+        main_level = config.get('level', 'INFO').upper()
        console_output = config.get('console_output', True)
-        log_file = config.get('log_file', './logs/pyserve.log')
+
+        global_format = config.get('format', {})
+        console_config = config.get('console', {})
+        files_config = config.get('files', [])
+
+        console_format = {**global_format, **console_config.get('format', {})}
+        console_level = console_config.get('level', main_level)
+
        self._save_original_handlers()
        self._clear_all_handlers()
+
        root_logger = logging.getLogger()
        root_logger.setLevel(logging.DEBUG)

-        detailed_formatter = PyServeFormatter(
-            use_colors=False,
-            show_module=True,
-            fmt='%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s'
-        )
-
-        console_formatter = PyServeFormatter(
-            use_colors=True,
-            show_module=True,
-            fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
-        )
-
        if console_output:
            console_handler = logging.StreamHandler(sys.stdout)
-            console_handler.setLevel(getattr(logging, level))
+            console_handler.setLevel(getattr(logging, console_level))
+
+            if console_format.get('type') == 'json':
+                console_formatter = self._create_formatter(console_format)
+            else:
+                console_formatter = PyServeFormatter(
+                    use_colors=console_format.get('use_colors', True),
+                    show_module=console_format.get('show_module', True),
+                    timestamp_format=console_format.get('timestamp_format', '%Y-%m-%d %H:%M:%S'),
+                    fmt='%(asctime)s - %(name)s - %(levelname)s - %(message)s'
+                )
+
            console_handler.setFormatter(console_formatter)
-
            console_handler.addFilter(UvicornLogFilter())
-
            root_logger.addHandler(console_handler)
            self.handlers['console'] = console_handler

-        if log_file:
-            self._ensure_log_directory(log_file)
+        for i, file_config in enumerate(files_config):
+            file_path = file_config.get('path', './logs/pyserve.log')
+            file_level = file_config.get('level', main_level)
+            file_loggers = file_config.get('loggers', [])
+            max_bytes = file_config.get('max_bytes', 10 * 1024 * 1024)
+            backup_count = file_config.get('backup_count', 5)
+            file_format = {**global_format, **file_config.get('format', {})}
+
+            self._ensure_log_directory(file_path)

            file_handler = logging.handlers.RotatingFileHandler(
-                log_file,
-                maxBytes=10 * 1024 * 1024,  # 10MB
-                backupCount=5,
+                file_path,
+                maxBytes=max_bytes,
+                backupCount=backup_count,
                encoding='utf-8'
            )
-            file_handler.setLevel(logging.DEBUG)
-            file_handler.setFormatter(detailed_formatter)
+            file_handler.setLevel(getattr(logging, file_level))
+
+            if file_format.get('type') == 'json':
+                file_formatter = self._create_formatter(file_format)
+            else:
+                file_formatter = PyServeFormatter(
+                    use_colors=file_format.get('use_colors', False),
+                    show_module=file_format.get('show_module', True),
+                    timestamp_format=file_format.get('timestamp_format', '%Y-%m-%d %H:%M:%S'),
+                    fmt='%(asctime)s - %(name)s - %(levelname)s - [%(filename)s:%(lineno)d] - %(message)s'
+                )
+
+            file_handler.setFormatter(file_formatter)
            file_handler.addFilter(UvicornLogFilter())
+            if file_loggers:
+                logger_filter = LoggerFilter(file_loggers)
+                file_handler.addFilter(logger_filter)

            root_logger.addHandler(file_handler)
-            self.handlers['file'] = file_handler
-
-        self._configure_library_loggers(level)
+            self.handlers[f'file_{i}'] = file_handler

+        self._configure_library_loggers(main_level)
        self._intercept_uvicorn_logging()

        pyserve_logger = logging.getLogger('pyserve')
-        pyserve_logger.setLevel(getattr(logging, level))
+        pyserve_logger.setLevel(getattr(logging, main_level))
        self.loggers['pyserve'] = pyserve_logger

        pyserve_logger.info(f"PyServe v{__version__} - Logger initialized")
-        pyserve_logger.info(f"Logging level: {level}")
+        pyserve_logger.info(f"Logging level: {main_level}")
        pyserve_logger.info(f"Console output: {'enabled' if console_output else 'disabled'}")
-        pyserve_logger.info(f"Log file: {log_file if log_file else 'disabled'}")
+        pyserve_logger.info(f"Console format: {console_format.get('type', 'standard')}")
+
+        for i, file_config in enumerate(files_config):
+            file_path = file_config.get('path', './logs/pyserve.log')
+            file_loggers = file_config.get('loggers', [])
+            file_format = file_config.get('format', {})
+
+            pyserve_logger.info(f"Log file[{i}]: {file_path}")
+            pyserve_logger.info(f"File format[{i}]: {file_format.get('type', 'standard')}")
+            if file_loggers:
+                pyserve_logger.info(f"File loggers[{i}]: {', '.join(file_loggers)}")
+            else:
+                pyserve_logger.info(f"File loggers[{i}]: all loggers")

        self.configured = True

@@ -168,13 +305,13 @@ class PyServeLogManager:

    def _configure_library_loggers(self, main_level: str) -> None:
        library_configs = {
-            # Uvicorn и связанные - только в DEBUG режиме
+            # Uvicorn and related - only in DEBUG mode
            'uvicorn': 'DEBUG' if main_level == 'DEBUG' else 'WARNING',
            'uvicorn.access': 'DEBUG' if main_level == 'DEBUG' else 'WARNING',
            'uvicorn.error': 'DEBUG' if main_level == 'DEBUG' else 'ERROR',
            'uvicorn.asgi': 'DEBUG' if main_level == 'DEBUG' else 'WARNING',

-            # Starlette - только в DEBUG режиме
+            # Starlette - only in DEBUG mode
            'starlette': 'DEBUG' if main_level == 'DEBUG' else 'WARNING',

            'asyncio': 'WARNING',
@@ -166,7 +166,7 @@ class RequestHandler:

    async def _handle_proxy(self, request: Request, config: Dict[str, Any],
                            params: Dict[str, str]) -> Response:
-        # TODO: Реализовать полноценное проксирование
+        # TODO: implement real proxying
        proxy_url = config["proxy_pass"]

        for key, value in params.items():
@@ -208,9 +208,10 @@ class PyServeServer:
            self.config.http.templates_dir,
        ]

-        log_dir = Path(self.config.logging.log_file).parent
-        if log_dir != Path("."):
-            directories.append(str(log_dir))
+        for file_config in self.config.logging.files:
+            log_dir = Path(file_config.path).parent
+            if log_dir != Path("."):
+                directories.append(str(log_dir))

        for directory in directories:
            Path(directory).mkdir(parents=True, exist_ok=True)