cython path_matcher added to reduce time on hot operations

This commit is contained in:
Илья Глазунов
2025-12-03 12:54:45 +03:00
parent 6c50a35aa3
commit 5d863bc97c
19 changed files with 1387 additions and 392 deletions
+3 -3
View File
@@ -5,17 +5,17 @@ PyServe - HTTP web server written on Python
__version__ = "0.8.0"
__author__ = "Ilya Glazunov"
from .server import PyServeServer
from .config import Config
from .asgi_mount import (
ASGIAppLoader,
ASGIMountManager,
MountedApp,
create_django_app,
create_fastapi_app,
create_flask_app,
create_django_app,
create_starlette_app,
)
from .config import Config
from .server import PyServeServer
__all__ = [
"PyServeServer",
+225
View File
@@ -0,0 +1,225 @@
# cython: language_level=3
# cython: boundscheck=False
# cython: wraparound=False
# cython: cdivision=True
"""
Fast path matching module for PyServe.
This Cython module provides optimized path matching operations
for ASGI mount routing, significantly reducing overhead on hot paths.
"""
from cpython.object cimport PyObject
cdef class FastMountedPath:
cdef:
str _path
str _path_with_slash
Py_ssize_t _path_len
bint _is_root
public str name
public bint strip_path
def __cinit__(self):
self._path = ""
self._path_with_slash = "/"
self._path_len = 0
self._is_root = 1
self.name = ""
self.strip_path = 1
def __init__(self, str path, str name="", bint strip_path=True):
cdef Py_ssize_t path_len
path_len = len(path)
if path_len > 1 and path[path_len - 1] == '/':
path = path[:path_len - 1]
self._path = path
self._path_len = len(path)
self._is_root = 1 if (path == "" or path == "/") else 0
self._path_with_slash = path + "/" if self._is_root == 0 else "/"
self.name = name if name else path
self.strip_path = 1 if strip_path else 0
@property
def path(self) -> str:
return self._path
cpdef bint matches(self, str request_path):
cdef Py_ssize_t req_len
if self._is_root:
return 1
req_len = len(request_path)
if req_len < self._path_len:
return 0
if req_len == self._path_len:
return 1 if request_path == self._path else 0
if request_path[self._path_len] == '/':
return 1 if request_path[:self._path_len] == self._path else 0
return 0
cpdef str get_modified_path(self, str original_path):
cdef str new_path
if not self.strip_path:
return original_path
if self._is_root:
return original_path
new_path = original_path[self._path_len:]
if not new_path:
return "/"
return new_path
def __repr__(self):
return f"FastMountedPath(path={self._path!r}, name={self.name!r})"
def _get_path_len_neg(mount):
return -len(mount.path)
cdef class FastMountManager:
cdef:
list _mounts
int _mount_count
def __cinit__(self):
self._mounts = []
self._mount_count = 0
def __init__(self):
self._mounts = []
self._mount_count = 0
cpdef void add_mount(self, FastMountedPath mount):
self._mounts.append(mount)
self._mounts = sorted(self._mounts, key=_get_path_len_neg, reverse=False)
self._mount_count = len(self._mounts)
cpdef FastMountedPath get_mount(self, str request_path):
cdef:
int i
FastMountedPath mount
for i in range(self._mount_count):
mount = <FastMountedPath>self._mounts[i]
if mount.matches(request_path):
return mount
return None
cpdef bint remove_mount(self, str path):
cdef:
int i
Py_ssize_t path_len
FastMountedPath mount
path_len = len(path)
if path_len > 1 and path[path_len - 1] == '/':
path = path[:path_len - 1]
for i in range(self._mount_count):
mount = <FastMountedPath>self._mounts[i]
if mount._path == path:
del self._mounts[i]
self._mount_count -= 1
return 1
return 0
@property
def mounts(self) -> list:
return list(self._mounts)
@property
def mount_count(self) -> int:
return self._mount_count
cpdef list list_mounts(self):
cdef:
list result = []
FastMountedPath mount
for mount in self._mounts:
result.append({
"path": mount._path,
"name": mount.name,
"strip_path": mount.strip_path,
})
return result
cpdef bint path_matches_prefix(str request_path, str mount_path):
cdef:
Py_ssize_t mount_len = len(mount_path)
Py_ssize_t req_len = len(request_path)
if mount_len == 0 or mount_path == "/":
return 1
if req_len < mount_len:
return 0
if req_len == mount_len:
return 1 if request_path == mount_path else 0
if request_path[mount_len] == '/':
return 1 if request_path[:mount_len] == mount_path else 0
return 0
cpdef str strip_path_prefix(str original_path, str mount_path):
cdef:
Py_ssize_t mount_len = len(mount_path)
str result
if mount_len == 0 or mount_path == "/":
return original_path
result = original_path[mount_len:]
if not result:
return "/"
return result
cpdef tuple match_and_modify_path(str request_path, str mount_path, bint strip_path=True):
cdef:
Py_ssize_t mount_len = len(mount_path)
Py_ssize_t req_len = len(request_path)
bint is_root = 1 if (mount_len == 0 or mount_path == "/") else 0
str modified
if is_root:
return (True, request_path if strip_path else request_path)
if req_len < mount_len:
return (False, None)
if req_len == mount_len:
if request_path == mount_path:
return (True, "/" if strip_path else request_path)
return (False, None)
if request_path[mount_len] == '/' and request_path[:mount_len] == mount_path:
if strip_path:
modified = request_path[mount_len:]
return (True, modified if modified else "/")
return (True, request_path)
return (False, None)
+168
View File
@@ -0,0 +1,168 @@
"""
Pure Python fallback for _path_matcher when Cython is not available.
This module provides the same interface as the Cython _path_matcher module,
allowing the application to run without compilation.
"""
from typing import Any, Dict, List, Optional, Tuple
class FastMountedPath:
__slots__ = ("_path", "_path_with_slash", "_path_len", "_is_root", "name", "strip_path")
def __init__(self, path: str, name: str = "", strip_path: bool = True):
if path.endswith("/") and len(path) > 1:
path = path[:-1]
self._path = path
self._path_len = len(path)
self._is_root = path == "" or path == "/"
self._path_with_slash = path + "/" if not self._is_root else "/"
self.name = name or path
self.strip_path = strip_path
@property
def path(self) -> str:
return self._path
def matches(self, request_path: str) -> bool:
if self._is_root:
return True
req_len = len(request_path)
if req_len < self._path_len:
return False
if req_len == self._path_len:
return request_path == self._path
if request_path[self._path_len] == "/":
return request_path[: self._path_len] == self._path
return False
def get_modified_path(self, original_path: str) -> str:
if not self.strip_path:
return original_path
if self._is_root:
return original_path
new_path = original_path[self._path_len :]
if not new_path:
return "/"
return new_path
def __repr__(self) -> str:
return f"FastMountedPath(path={self._path!r}, name={self.name!r})"
class FastMountManager:
__slots__ = ("_mounts", "_mount_count")
def __init__(self) -> None:
self._mounts: List[FastMountedPath] = []
self._mount_count: int = 0
def add_mount(self, mount: FastMountedPath) -> None:
self._mounts.append(mount)
self._mounts.sort(key=lambda m: len(m.path), reverse=True)
self._mount_count = len(self._mounts)
def get_mount(self, request_path: str) -> Optional[FastMountedPath]:
for mount in self._mounts:
if mount.matches(request_path):
return mount
return None
def remove_mount(self, path: str) -> bool:
if path.endswith("/") and len(path) > 1:
path = path[:-1]
for i, mount in enumerate(self._mounts):
if mount._path == path:
del self._mounts[i]
self._mount_count -= 1
return True
return False
@property
def mounts(self) -> List[FastMountedPath]:
return self._mounts.copy()
@property
def mount_count(self) -> int:
return self._mount_count
def list_mounts(self) -> List[Dict[str, Any]]:
return [
{
"path": mount._path,
"name": mount.name,
"strip_path": mount.strip_path,
}
for mount in self._mounts
]
def path_matches_prefix(request_path: str, mount_path: str) -> bool:
mount_len = len(mount_path)
req_len = len(request_path)
if mount_len == 0 or mount_path == "/":
return True
if req_len < mount_len:
return False
if req_len == mount_len:
return request_path == mount_path
if request_path[mount_len] == "/":
return request_path[:mount_len] == mount_path
return False
def strip_path_prefix(original_path: str, mount_path: str) -> str:
mount_len = len(mount_path)
if mount_len == 0 or mount_path == "/":
return original_path
result = original_path[mount_len:]
if not result:
return "/"
return result
def match_and_modify_path(request_path: str, mount_path: str, strip_path: bool = True) -> Tuple[bool, Optional[str]]:
mount_len = len(mount_path)
req_len = len(request_path)
is_root = mount_len == 0 or mount_path == "/"
if is_root:
return (True, request_path)
if req_len < mount_len:
return (False, None)
if req_len == mount_len:
if request_path == mount_path:
return (True, "/" if strip_path else request_path)
return (False, None)
if request_path[mount_len] == "/" and request_path[:mount_len] == mount_path:
if strip_path:
modified = request_path[mount_len:]
return (True, modified if modified else "/")
return (True, request_path)
return (False, None)
+10 -14
View File
@@ -8,7 +8,8 @@ This module provides functionality to mount external ASGI/WSGI applications
import importlib
import sys
from pathlib import Path
from typing import Dict, Any, Optional, Callable, cast
from typing import Any, Callable, Dict, Optional, cast
from starlette.types import ASGIApp, Receive, Scope, Send
from .logging_utils import get_logger
@@ -74,20 +75,17 @@ class ASGIAppLoader:
def _wrap_wsgi(self, wsgi_app: Callable) -> ASGIApp:
try:
from a2wsgi import WSGIMiddleware
return cast(ASGIApp, WSGIMiddleware(wsgi_app))
except ImportError:
logger.warning("a2wsgi not installed, trying asgiref")
try:
from asgiref.wsgi import WsgiToAsgi
return cast(ASGIApp, WsgiToAsgi(wsgi_app))
except ImportError:
logger.error(
"Neither a2wsgi nor asgiref installed. "
"Install with: pip install a2wsgi or pip install asgiref"
)
raise ImportError(
"WSGI adapter not available. Install a2wsgi or asgiref."
)
logger.error("Neither a2wsgi nor asgiref installed. " "Install with: pip install a2wsgi or pip install asgiref")
raise ImportError("WSGI adapter not available. Install a2wsgi or asgiref.")
def get_app(self, app_path: str) -> Optional[ASGIApp]:
return self._apps.get(app_path)
@@ -132,7 +130,7 @@ class MountedApp:
if self.path == "":
return original_path
new_path = original_path[len(self.path):]
new_path = original_path[len(self.path) :]
return new_path if new_path else "/"
@@ -215,10 +213,7 @@ class ASGIMountManager:
modified_scope["path"] = mount.get_modified_path(path)
modified_scope["root_path"] = scope.get("root_path", "") + mount.path
logger.debug(
f"Routing request to mounted app '{mount.name}': "
f"{path} -> {modified_scope['path']}"
)
logger.debug(f"Routing request to mounted app '{mount.name}': " f"{path} -> {modified_scope['path']}")
try:
await mount.app(modified_scope, receive, send)
@@ -288,7 +283,8 @@ def create_django_app(
os.environ.setdefault("DJANGO_SETTINGS_MODULE", settings_module)
try:
from django.core.asgi import get_asgi_application # type: ignore[import-untyped]
from django.core.asgi import get_asgi_application
return cast(ASGIApp, get_asgi_application())
except ImportError as e:
logger.error(f"Failed to load Django application: {e}")
+7 -26
View File
@@ -1,8 +1,8 @@
import sys
import argparse
import sys
from pathlib import Path
from . import PyServeServer, Config, __version__
from . import Config, PyServeServer, __version__
def main() -> None:
@@ -10,30 +10,11 @@ def main() -> None:
description="PyServe - HTTP web server",
prog="pyserve",
)
parser.add_argument(
"-c", "--config",
default="config.yaml",
help="Path to configuration file (default: config.yaml)"
)
parser.add_argument(
"--host",
help="Host to bind the server to"
)
parser.add_argument(
"--port",
type=int,
help="Port to bind the server to"
)
parser.add_argument(
"--debug",
action="store_true",
help="Enable debug mode"
)
parser.add_argument(
"--version",
action="version",
version=f"%(prog)s {__version__}"
)
parser.add_argument("-c", "--config", default="config.yaml", help="Path to configuration file (default: config.yaml)")
parser.add_argument("--host", help="Host to bind the server to")
parser.add_argument("--port", type=int, help="Port to bind the server to")
parser.add_argument("--debug", action="store_true", help="Enable debug mode")
parser.add_argument("--version", action="version", version=f"%(prog)s {__version__}")
args = parser.parse_args()
+101 -115
View File
@@ -1,8 +1,10 @@
import yaml
import os
from typing import Dict, Any, List, cast
from dataclasses import dataclass, field
import logging
import os
from dataclasses import dataclass, field
from typing import Any, Dict, List, cast
import yaml
from .logging_utils import setup_logging
@@ -84,7 +86,7 @@ class Config:
@classmethod
def from_yaml(cls, file_path: str) -> "Config":
try:
with open(file_path, 'r', encoding='utf-8') as f:
with open(file_path, "r", encoding="utf-8") as f:
data = yaml.safe_load(f)
return cls._from_dict(data)
@@ -99,133 +101,117 @@ class Config:
def _from_dict(cls, data: Dict[str, Any]) -> "Config":
config = cls()
if 'http' in data:
http_data = data['http']
if "http" in data:
http_data = data["http"]
config.http = HttpConfig(
static_dir=http_data.get('static_dir', config.http.static_dir),
templates_dir=http_data.get('templates_dir', config.http.templates_dir)
static_dir=http_data.get("static_dir", config.http.static_dir),
templates_dir=http_data.get("templates_dir", config.http.templates_dir),
)
if 'server' in data:
server_data = data['server']
if "server" in data:
server_data = data["server"]
config.server = ServerConfig(
host=server_data.get('host', config.server.host),
port=server_data.get('port', config.server.port),
backlog=server_data.get('backlog', config.server.backlog),
default_root=server_data.get('default_root', config.server.default_root),
proxy_timeout=server_data.get('proxy_timeout', config.server.proxy_timeout),
redirect_instructions=server_data.get('redirect_instructions', {})
host=server_data.get("host", config.server.host),
port=server_data.get("port", config.server.port),
backlog=server_data.get("backlog", config.server.backlog),
default_root=server_data.get("default_root", config.server.default_root),
proxy_timeout=server_data.get("proxy_timeout", config.server.proxy_timeout),
redirect_instructions=server_data.get("redirect_instructions", {}),
)
if 'ssl' in data:
ssl_data = data['ssl']
if "ssl" in data:
ssl_data = data["ssl"]
config.ssl = SSLConfig(
enabled=ssl_data.get('enabled', config.ssl.enabled),
cert_file=ssl_data.get('cert_file', config.ssl.cert_file),
key_file=ssl_data.get('key_file', config.ssl.key_file)
enabled=ssl_data.get("enabled", config.ssl.enabled),
cert_file=ssl_data.get("cert_file", config.ssl.cert_file),
key_file=ssl_data.get("key_file", config.ssl.key_file),
)
if 'logging' in data:
log_data = data['logging']
format_data = log_data.get('format', {})
if "logging" in data:
log_data = data["logging"]
format_data = log_data.get("format", {})
global_format = LogFormatConfig(
type=format_data.get('type', 'standard'),
use_colors=format_data.get('use_colors', True),
show_module=format_data.get('show_module', True),
timestamp_format=format_data.get('timestamp_format', '%Y-%m-%d %H:%M:%S')
type=format_data.get("type", "standard"),
use_colors=format_data.get("use_colors", True),
show_module=format_data.get("show_module", True),
timestamp_format=format_data.get("timestamp_format", "%Y-%m-%d %H:%M:%S"),
)
console_data = log_data.get('console', {})
console_format_data = console_data.get('format', {})
console_data = log_data.get("console", {})
console_format_data = console_data.get("format", {})
console_format = LogFormatConfig(
type=console_format_data.get('type', global_format.type),
use_colors=console_format_data.get('use_colors', global_format.use_colors),
show_module=console_format_data.get('show_module', global_format.show_module),
timestamp_format=console_format_data.get('timestamp_format', global_format.timestamp_format)
)
console_config = LogHandlerConfig(
level=console_data.get('level', log_data.get('level', 'INFO')),
format=console_format
type=console_format_data.get("type", global_format.type),
use_colors=console_format_data.get("use_colors", global_format.use_colors),
show_module=console_format_data.get("show_module", global_format.show_module),
timestamp_format=console_format_data.get("timestamp_format", global_format.timestamp_format),
)
console_config = LogHandlerConfig(level=console_data.get("level", log_data.get("level", "INFO")), format=console_format)
files_config = []
if 'log_file' in log_data:
if "log_file" in log_data:
default_file_format = LogFormatConfig(
type=global_format.type,
use_colors=False,
show_module=global_format.show_module,
timestamp_format=global_format.timestamp_format
type=global_format.type, use_colors=False, show_module=global_format.show_module, timestamp_format=global_format.timestamp_format
)
default_file = LogFileConfig(
path=log_data['log_file'],
level=log_data.get('level', 'INFO'),
path=log_data["log_file"],
level=log_data.get("level", "INFO"),
format=default_file_format,
loggers=[], # Empty list means including all loggers
max_bytes=10 * 1024 * 1024,
backup_count=5
backup_count=5,
)
files_config.append(default_file)
if 'files' in log_data:
for file_data in log_data['files']:
file_format_data = file_data.get('format', {})
if "files" in log_data:
for file_data in log_data["files"]:
file_format_data = file_data.get("format", {})
file_format = LogFormatConfig(
type=file_format_data.get('type', global_format.type),
use_colors=file_format_data.get('use_colors', False),
show_module=file_format_data.get('show_module', global_format.show_module),
timestamp_format=file_format_data.get('timestamp_format', global_format.timestamp_format)
type=file_format_data.get("type", global_format.type),
use_colors=file_format_data.get("use_colors", False),
show_module=file_format_data.get("show_module", global_format.show_module),
timestamp_format=file_format_data.get("timestamp_format", global_format.timestamp_format),
)
file_config = LogFileConfig(
path=file_data.get('path', './logs/pyserve.log'),
level=file_data.get('level', log_data.get('level', 'INFO')),
path=file_data.get("path", "./logs/pyserve.log"),
level=file_data.get("level", log_data.get("level", "INFO")),
format=file_format,
loggers=file_data.get('loggers', []),
max_bytes=file_data.get('max_bytes', 10 * 1024 * 1024),
backup_count=file_data.get('backup_count', 5)
loggers=file_data.get("loggers", []),
max_bytes=file_data.get("max_bytes", 10 * 1024 * 1024),
backup_count=file_data.get("backup_count", 5),
)
files_config.append(file_config)
if 'show_module' in console_format_data:
print(
"\033[33mWARNING: Parameter 'show_module' in console.format in development and may work incorrectly\033[0m"
)
console_config.format.show_module = console_format_data.get('show_module')
if "show_module" in console_format_data:
print("\033[33mWARNING: Parameter 'show_module' in console.format in development and may work incorrectly\033[0m")
console_config.format.show_module = console_format_data.get("show_module")
for i, file_data in enumerate(log_data.get('files', [])):
if 'format' in file_data and 'show_module' in file_data['format']:
print(
f"\033[33mWARNING: Parameter 'show_module' in files[{i}].format in development and may work incorrectly\033[0m"
)
for i, file_data in enumerate(log_data.get("files", [])):
if "format" in file_data and "show_module" in file_data["format"]:
print(f"\033[33mWARNING: Parameter 'show_module' in files[{i}].format in development and may work incorrectly\033[0m")
if not files_config:
default_file_format = LogFormatConfig(
type=global_format.type,
use_colors=False,
show_module=global_format.show_module,
timestamp_format=global_format.timestamp_format
type=global_format.type, use_colors=False, show_module=global_format.show_module, timestamp_format=global_format.timestamp_format
)
default_file = LogFileConfig(
path='./logs/pyserve.log',
level=log_data.get('level', 'INFO'),
path="./logs/pyserve.log",
level=log_data.get("level", "INFO"),
format=default_file_format,
loggers=[],
max_bytes=10 * 1024 * 1024,
backup_count=5
backup_count=5,
)
files_config.append(default_file)
config.logging = LoggingConfig(
level=log_data.get('level', 'INFO'),
console_output=log_data.get('console_output', True),
level=log_data.get("level", "INFO"),
console_output=log_data.get("console_output", True),
format=global_format,
console=console_config,
files=files_config
files=files_config,
)
if 'extensions' in data:
for ext_data in data['extensions']:
extension = ExtensionConfig(
type=ext_data.get('type', ''),
config=ext_data.get('config', {})
)
if "extensions" in data:
for ext_data in data["extensions"]:
extension = ExtensionConfig(type=ext_data.get("type", ""), config=ext_data.get("config", {}))
config.extensions.append(extension)
return config
@@ -245,14 +231,14 @@ class Config:
if not (1 <= self.server.port <= 65535):
errors.append(f"Invalid port: {self.server.port}")
valid_log_levels = ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']
valid_log_levels = ["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"]
if self.logging.level.upper() not in valid_log_levels:
errors.append(f"Invalid logging level: {self.logging.level}")
if self.logging.console.level.upper() not in valid_log_levels:
errors.append(f"Invalid console logging level: {self.logging.console.level}")
valid_format_types = ['standard', 'json']
valid_format_types = ["standard", "json"]
if self.logging.format.type not in valid_format_types:
errors.append(f"Invalid logging format type: {self.logging.format.type}")
@@ -283,40 +269,40 @@ class Config:
def setup_logging(self) -> None:
config_dict = {
'level': self.logging.level,
'console_output': self.logging.console_output,
'format': {
'type': self.logging.format.type,
'use_colors': self.logging.format.use_colors,
'show_module': self.logging.format.show_module,
'timestamp_format': self.logging.format.timestamp_format
"level": self.logging.level,
"console_output": self.logging.console_output,
"format": {
"type": self.logging.format.type,
"use_colors": self.logging.format.use_colors,
"show_module": self.logging.format.show_module,
"timestamp_format": self.logging.format.timestamp_format,
},
'console': {
'level': self.logging.console.level,
'format': {
'type': self.logging.console.format.type,
'use_colors': self.logging.console.format.use_colors,
'show_module': self.logging.console.format.show_module,
'timestamp_format': self.logging.console.format.timestamp_format
}
"console": {
"level": self.logging.console.level,
"format": {
"type": self.logging.console.format.type,
"use_colors": self.logging.console.format.use_colors,
"show_module": self.logging.console.format.show_module,
"timestamp_format": self.logging.console.format.timestamp_format,
},
},
'files': []
"files": [],
}
for file_config in self.logging.files:
file_dict = {
'path': file_config.path,
'level': file_config.level,
'loggers': file_config.loggers,
'max_bytes': file_config.max_bytes,
'backup_count': file_config.backup_count,
'format': {
'type': file_config.format.type,
'use_colors': file_config.format.use_colors,
'show_module': file_config.format.show_module,
'timestamp_format': file_config.format.timestamp_format
}
"path": file_config.path,
"level": file_config.level,
"loggers": file_config.loggers,
"max_bytes": file_config.max_bytes,
"backup_count": file_config.backup_count,
"format": {
"type": file_config.format.type,
"use_colors": file_config.format.use_colors,
"show_module": file_config.format.show_module,
"timestamp_format": file_config.format.timestamp_format,
},
}
cast(List[Dict[str, Any]], config_dict['files']).append(file_dict)
cast(List[Dict[str, Any]], config_dict["files"]).append(file_dict)
setup_logging(config_dict)
+15 -15
View File
@@ -1,7 +1,9 @@
from abc import ABC, abstractmethod
from typing import Dict, Any, List, Optional, Type
from typing import Any, Dict, List, Optional, Type
from starlette.requests import Request
from starlette.responses import Response
from .logging_utils import get_logger
logger = get_logger(__name__)
@@ -36,6 +38,7 @@ class RoutingExtension(Extension):
default_proxy_timeout = config.get("default_proxy_timeout", 30.0)
self.router = create_router_from_config(regex_locations)
from .routing import RequestHandler
self.handler = RequestHandler(self.router, default_proxy_timeout=default_proxy_timeout)
async def process_request(self, request: Request) -> Optional[Response]:
@@ -54,11 +57,9 @@ class SecurityExtension(Extension):
super().__init__(config)
self.allowed_ips = config.get("allowed_ips", [])
self.blocked_ips = config.get("blocked_ips", [])
self.security_headers = config.get("security_headers", {
"X-Content-Type-Options": "nosniff",
"X-Frame-Options": "DENY",
"X-XSS-Protection": "1; mode=block"
})
self.security_headers = config.get(
"security_headers", {"X-Content-Type-Options": "nosniff", "X-Frame-Options": "DENY", "X-XSS-Protection": "1; mode=block"}
)
async def process_request(self, request: Request) -> Optional[Response]:
client_ip = request.client.host if request.client else "unknown"
@@ -66,11 +67,13 @@ class SecurityExtension(Extension):
if self.blocked_ips and client_ip in self.blocked_ips:
logger.warning(f"Blocked request from IP: {client_ip}")
from starlette.responses import PlainTextResponse
return PlainTextResponse("403 Forbidden", status_code=403)
if self.allowed_ips and client_ip not in self.allowed_ips:
logger.warning(f"Access denied for IP: {client_ip}")
from starlette.responses import PlainTextResponse
return PlainTextResponse("403 Forbidden", status_code=403)
return None
@@ -108,33 +111,30 @@ class MonitoringExtension(Extension):
async def process_request(self, request: Request) -> Optional[Response]:
if self.enable_metrics:
self.request_count += 1
request.state.start_time = __import__('time').time()
request.state.start_time = __import__("time").time()
return None
async def process_response(self, request: Request, response: Response) -> Response:
if self.enable_metrics and hasattr(request.state, 'start_time'):
response_time = __import__('time').time() - request.state.start_time
if self.enable_metrics and hasattr(request.state, "start_time"):
response_time = __import__("time").time() - request.state.start_time
self.response_times.append(response_time)
if response.status_code >= 400:
self.error_count += 1
logger.info(f"Request: {request.method} {request.url.path} - "
f"Status: {response.status_code} - "
f"Time: {response_time:.3f}s")
logger.info(f"Request: {request.method} {request.url.path} - " f"Status: {response.status_code} - " f"Time: {response_time:.3f}s")
return response
def get_metrics(self) -> Dict[str, Any]:
avg_response_time = (sum(self.response_times) / len(self.response_times)
if self.response_times else 0)
avg_response_time = sum(self.response_times) / len(self.response_times) if self.response_times else 0
return {
"request_count": self.request_count,
"error_count": self.error_count,
"error_rate": self.error_count / max(self.request_count, 1),
"avg_response_time": avg_response_time,
"total_response_times": len(self.response_times)
"total_response_times": len(self.response_times),
}
+93 -129
View File
@@ -3,9 +3,10 @@ import logging.handlers
import sys
import time
from pathlib import Path
from typing import Dict, Any, List, cast, Callable
from typing import Any, Callable, Dict, List, cast
import structlog
from structlog.types import FilteringBoundLogger, EventDict
from structlog.types import EventDict, FilteringBoundLogger
from . import __version__
@@ -21,15 +22,15 @@ class StructlogFilter(logging.Filter):
return True
for logger_name in self.logger_names:
if record.name == logger_name or record.name.startswith(logger_name + '.'):
if record.name == logger_name or record.name.startswith(logger_name + "."):
return True
return False
class UvicornStructlogFilter(logging.Filter):
def filter(self, record: logging.LogRecord) -> bool:
if hasattr(record, 'name') and 'uvicorn.access' in record.name:
if hasattr(record, 'getMessage'):
if hasattr(record, "name") and "uvicorn.access" in record.name:
if hasattr(record, "getMessage"):
msg = record.getMessage()
if ' - "' in msg and '" ' in msg:
parts = msg.split(' - "')
@@ -56,14 +57,14 @@ def add_log_level(logger: FilteringBoundLogger, method_name: str, event_dict: Ev
def add_module_info(logger: FilteringBoundLogger, method_name: str, event_dict: EventDict) -> EventDict:
if hasattr(logger, '_context') and 'logger_name' in logger._context:
logger_name = logger._context['logger_name']
if logger_name.startswith('pyserve'):
if hasattr(logger, "_context") and "logger_name" in logger._context:
logger_name = logger._context["logger_name"]
if logger_name.startswith("pyserve"):
event_dict["module"] = logger_name
elif logger_name.startswith('uvicorn'):
event_dict["module"] = 'uvicorn'
elif logger_name.startswith('starlette'):
event_dict["module"] = 'starlette'
elif logger_name.startswith("uvicorn"):
event_dict["module"] = "uvicorn"
elif logger_name.startswith("starlette"):
event_dict["module"] = "starlette"
else:
event_dict["module"] = logger_name
return event_dict
@@ -74,18 +75,19 @@ def filter_module_info(show_module: bool) -> Callable[[FilteringBoundLogger, str
if not show_module and "module" in event_dict:
del event_dict["module"]
return event_dict
return processor
def colored_console_renderer(use_colors: bool = True, show_module: bool = True) -> structlog.dev.ConsoleRenderer:
return structlog.dev.ConsoleRenderer(
colors=use_colors and hasattr(sys.stderr, 'isatty') and sys.stderr.isatty(),
colors=use_colors and hasattr(sys.stderr, "isatty") and sys.stderr.isatty(),
level_styles={
"critical": "\033[35m", # Magenta
"error": "\033[31m", # Red
"warning": "\033[33m", # Yellow
"info": "\033[32m", # Green
"debug": "\033[36m", # Cyan
"error": "\033[31m", # Red
"warning": "\033[33m", # Yellow
"info": "\033[32m", # Green
"debug": "\033[36m", # Cyan
},
pad_event=25,
)
@@ -113,43 +115,35 @@ class PyServeLogManager:
if self.configured:
return
if 'format' not in config and 'console' not in config and 'files' not in config:
level = config.get('level', 'INFO').upper()
console_output = config.get('console_output', True)
log_file = config.get('log_file', './logs/pyserve.log')
if "format" not in config and "console" not in config and "files" not in config:
level = config.get("level", "INFO").upper()
console_output = config.get("console_output", True)
log_file = config.get("log_file", "./logs/pyserve.log")
config = {
'level': level,
'console_output': console_output,
'format': {
'type': 'standard',
'use_colors': True,
'show_module': True,
'timestamp_format': '%Y-%m-%d %H:%M:%S'
},
'files': [{
'path': log_file,
'level': level,
'loggers': [],
'max_bytes': 10 * 1024 * 1024,
'backup_count': 5,
'format': {
'type': 'standard',
'use_colors': False,
'show_module': True,
'timestamp_format': '%Y-%m-%d %H:%M:%S'
"level": level,
"console_output": console_output,
"format": {"type": "standard", "use_colors": True, "show_module": True, "timestamp_format": "%Y-%m-%d %H:%M:%S"},
"files": [
{
"path": log_file,
"level": level,
"loggers": [],
"max_bytes": 10 * 1024 * 1024,
"backup_count": 5,
"format": {"type": "standard", "use_colors": False, "show_module": True, "timestamp_format": "%Y-%m-%d %H:%M:%S"},
}
}]
],
}
main_level = config.get('level', 'INFO').upper()
console_output = config.get('console_output', True)
main_level = config.get("level", "INFO").upper()
console_output = config.get("console_output", True)
global_format = config.get('format', {})
console_config = config.get('console', {})
files_config = config.get('files', [])
global_format = config.get("format", {})
console_config = config.get("console", {})
files_config = config.get("files", [])
console_format = {**global_format, **console_config.get('format', {})}
console_level = console_config.get('level', main_level)
console_format = {**global_format, **console_config.get("format", {})}
console_level = console_config.get("level", main_level)
self._save_original_handlers()
self._clear_all_handlers()
@@ -159,38 +153,33 @@ class PyServeLogManager:
console_output=console_output,
console_format=console_format,
console_level=console_level,
files_config=files_config
files_config=files_config,
)
self._configure_stdlib_loggers(main_level)
logger = self.get_logger('pyserve')
logger = self.get_logger("pyserve")
logger.info(
"PyServe logger initialized",
version=__version__,
level=main_level,
console_output=console_output,
console_format=console_format.get('type', 'standard')
console_format=console_format.get("type", "standard"),
)
for i, file_config in enumerate(files_config):
logger.info(
"File logging configured",
file_index=i,
path=file_config.get('path'),
level=file_config.get('level', main_level),
format_type=file_config.get('format', {}).get('type', 'standard')
path=file_config.get("path"),
level=file_config.get("level", main_level),
format_type=file_config.get("format", {}).get("type", "standard"),
)
self.configured = True
def _configure_structlog(
self,
main_level: str,
console_output: bool,
console_format: Dict[str, Any],
console_level: str,
files_config: List[Dict[str, Any]]
self, main_level: str, console_output: bool, console_format: Dict[str, Any], console_level: str, files_config: List[Dict[str, Any]]
) -> None:
shared_processors = [
structlog.stdlib.filter_by_level,
@@ -202,57 +191,46 @@ class PyServeLogManager:
]
if console_output:
console_show_module = console_format.get('show_module', True)
console_show_module = console_format.get("show_module", True)
console_processors = shared_processors.copy()
console_processors.append(filter_module_info(console_show_module))
if console_format.get('type') == 'json':
if console_format.get("type") == "json":
console_processors.append(json_renderer())
else:
console_processors.append(
colored_console_renderer(
console_format.get('use_colors', True),
console_show_module
)
)
console_processors.append(colored_console_renderer(console_format.get("use_colors", True), console_show_module))
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setLevel(getattr(logging, console_level))
console_handler.addFilter(UvicornStructlogFilter())
console_formatter = structlog.stdlib.ProcessorFormatter(
processor=colored_console_renderer(
console_format.get('use_colors', True),
console_show_module
)
if console_format.get('type') != 'json'
else json_renderer(),
processor=(
colored_console_renderer(console_format.get("use_colors", True), console_show_module)
if console_format.get("type") != "json"
else json_renderer()
),
)
console_handler.setFormatter(console_formatter)
root_logger = logging.getLogger()
root_logger.setLevel(logging.DEBUG)
root_logger.addHandler(console_handler)
self.handlers['console'] = console_handler
self.handlers["console"] = console_handler
for i, file_config in enumerate(files_config):
file_path = file_config.get('path', './logs/pyserve.log')
file_level = file_config.get('level', main_level)
file_loggers = file_config.get('loggers', [])
max_bytes = file_config.get('max_bytes', 10 * 1024 * 1024)
backup_count = file_config.get('backup_count', 5)
file_format = file_config.get('format', {})
file_show_module = file_format.get('show_module', True)
file_path = file_config.get("path", "./logs/pyserve.log")
file_level = file_config.get("level", main_level)
file_loggers = file_config.get("loggers", [])
max_bytes = file_config.get("max_bytes", 10 * 1024 * 1024)
backup_count = file_config.get("backup_count", 5)
file_format = file_config.get("format", {})
file_show_module = file_format.get("show_module", True)
self._ensure_log_directory(file_path)
file_handler = logging.handlers.RotatingFileHandler(
file_path,
maxBytes=max_bytes,
backupCount=backup_count,
encoding='utf-8'
)
file_handler = logging.handlers.RotatingFileHandler(file_path, maxBytes=max_bytes, backupCount=backup_count, encoding="utf-8")
file_handler.setLevel(getattr(logging, file_level))
if file_loggers:
@@ -262,15 +240,13 @@ class PyServeLogManager:
file_processors.append(filter_module_info(file_show_module))
file_formatter = structlog.stdlib.ProcessorFormatter(
processor=json_renderer()
if file_format.get('type') == 'json'
else plain_console_renderer(file_show_module),
processor=json_renderer() if file_format.get("type") == "json" else plain_console_renderer(file_show_module),
)
file_handler.setFormatter(file_formatter)
root_logger = logging.getLogger()
root_logger.addHandler(file_handler)
self.handlers[f'file_{i}'] = file_handler
self.handlers[f"file_{i}"] = file_handler
base_processors = [
structlog.stdlib.filter_by_level,
@@ -293,14 +269,14 @@ class PyServeLogManager:
def _configure_stdlib_loggers(self, main_level: str) -> None:
library_configs = {
'uvicorn': 'DEBUG' if main_level == 'DEBUG' else 'WARNING',
'uvicorn.access': 'DEBUG' if main_level == 'DEBUG' else 'WARNING',
'uvicorn.error': 'DEBUG' if main_level == 'DEBUG' else 'ERROR',
'uvicorn.asgi': 'DEBUG' if main_level == 'DEBUG' else 'WARNING',
'starlette': 'DEBUG' if main_level == 'DEBUG' else 'WARNING',
'asyncio': 'WARNING',
'concurrent.futures': 'WARNING',
'multiprocessing': 'WARNING',
"uvicorn": "DEBUG" if main_level == "DEBUG" else "WARNING",
"uvicorn.access": "DEBUG" if main_level == "DEBUG" else "WARNING",
"uvicorn.error": "DEBUG" if main_level == "DEBUG" else "ERROR",
"uvicorn.asgi": "DEBUG" if main_level == "DEBUG" else "WARNING",
"starlette": "DEBUG" if main_level == "DEBUG" else "WARNING",
"asyncio": "WARNING",
"concurrent.futures": "WARNING",
"multiprocessing": "WARNING",
}
for logger_name, level in library_configs.items():
@@ -309,7 +285,7 @@ class PyServeLogManager:
logger.propagate = True
def _save_original_handlers(self) -> None:
logger_names = ['', 'uvicorn', 'uvicorn.access', 'uvicorn.error', 'starlette']
logger_names = ["", "uvicorn", "uvicorn.access", "uvicorn.error", "starlette"]
for name in logger_names:
logger = logging.getLogger(name)
@@ -320,7 +296,7 @@ class PyServeLogManager:
for handler in root_logger.handlers[:]:
root_logger.removeHandler(handler)
logger_names = ['uvicorn', 'uvicorn.access', 'uvicorn.error', 'starlette']
logger_names = ["uvicorn", "uvicorn.access", "uvicorn.error", "starlette"]
for name in logger_names:
logger = logging.getLogger(name)
for handler in logger.handlers[:]:
@@ -335,14 +311,17 @@ class PyServeLogManager:
def get_logger(self, name: str) -> structlog.stdlib.BoundLogger:
if not self._structlog_configured:
structlog.configure(
processors=cast(Any, [
structlog.stdlib.filter_by_level,
add_timestamp,
add_log_level,
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
]),
processors=cast(
Any,
[
structlog.stdlib.filter_by_level,
add_timestamp,
add_log_level,
structlog.processors.StackInfoRenderer(),
structlog.processors.format_exc_info,
structlog.stdlib.ProcessorFormatter.wrap_for_formatter,
],
),
context_class=dict,
logger_factory=structlog.stdlib.LoggerFactory(),
wrapper_class=structlog.stdlib.BoundLogger,
@@ -370,16 +349,8 @@ class PyServeLogManager:
handler.close()
del self.handlers[name]
def create_access_log(
self,
method: str,
path: str,
status_code: int,
response_time: float,
client_ip: str,
user_agent: str = ""
) -> None:
access_logger = self.get_logger('pyserve.access')
def create_access_log(self, method: str, path: str, status_code: int, response_time: float, client_ip: str, user_agent: str = "") -> None:
access_logger = self.get_logger("pyserve.access")
access_logger.info(
"HTTP access",
method=method,
@@ -388,7 +359,7 @@ class PyServeLogManager:
response_time_ms=round(response_time * 1000, 2),
client_ip=client_ip,
user_agent=user_agent,
timestamp_format="access"
timestamp_format="access",
)
def shutdown(self) -> None:
@@ -416,14 +387,7 @@ def get_logger(name: str) -> structlog.stdlib.BoundLogger:
return log_manager.get_logger(name)
def create_access_log(
method: str,
path: str,
status_code: int,
response_time: float,
client_ip: str,
user_agent: str = ""
) -> None:
def create_access_log(method: str, path: str, status_code: int, response_time: float, client_ip: str, user_agent: str = "") -> None:
log_manager.create_access_log(method, path, status_code, response_time, client_ip, user_agent)
+33
View File
@@ -0,0 +1,33 @@
"""
Path matcher module - uses Cython implementation if available, falls back to pure Python.
"""
try:
from pyserve._path_matcher import (
FastMountedPath,
FastMountManager,
match_and_modify_path,
path_matches_prefix,
strip_path_prefix,
)
CYTHON_AVAILABLE = True
except ImportError:
from pyserve._path_matcher_py import (
FastMountedPath,
FastMountManager,
match_and_modify_path,
path_matches_prefix,
strip_path_prefix,
)
CYTHON_AVAILABLE = False
__all__ = [
"FastMountedPath",
"FastMountManager",
"path_matches_prefix",
"strip_path_prefix",
"match_and_modify_path",
"CYTHON_AVAILABLE",
]
+16 -10
View File
@@ -1,11 +1,13 @@
import re
import mimetypes
import re
from pathlib import Path
from typing import Dict, Any, Optional, Pattern
from typing import Any, Dict, Optional, Pattern
from urllib.parse import urlparse
import httpx
from starlette.requests import Request
from starlette.responses import Response, FileResponse, PlainTextResponse
from starlette.responses import FileResponse, PlainTextResponse, Response
from .logging_utils import get_logger
logger = get_logger(__name__)
@@ -100,8 +102,7 @@ class RequestHandler:
text = ""
content_type = config.get("content_type", "text/plain")
return PlainTextResponse(text, status_code=status_code,
media_type=content_type)
return PlainTextResponse(text, status_code=status_code, media_type=content_type)
if "proxy_pass" in config:
return await self._handle_proxy(request, config, route_match.params)
@@ -171,8 +172,7 @@ class RequestHandler:
return PlainTextResponse("404 Not Found", status_code=404)
async def _handle_proxy(self, request: Request, config: Dict[str, Any],
params: Dict[str, str]) -> Response:
async def _handle_proxy(self, request: Request, config: Dict[str, Any], params: Dict[str, str]) -> Response:
proxy_url = config["proxy_pass"]
for key, value in params.items():
@@ -197,9 +197,15 @@ class RequestHandler:
proxy_headers = dict(request.headers)
hop_by_hop_headers = [
"connection", "keep-alive", "proxy-authenticate",
"proxy-authorization", "te", "trailers", "transfer-encoding",
"upgrade", "host"
"connection",
"keep-alive",
"proxy-authenticate",
"proxy-authorization",
"te",
"trailers",
"transfer-encoding",
"upgrade",
"host",
]
for header in hop_by_hop_headers:
proxy_headers.pop(header, None)
+33 -68
View File
@@ -1,18 +1,19 @@
import ssl
import uvicorn
import time
from pathlib import Path
from typing import Any, Dict, Optional
import uvicorn
from starlette.applications import Starlette
from starlette.requests import Request
from starlette.responses import Response, PlainTextResponse
from starlette.responses import PlainTextResponse, Response
from starlette.routing import Route
from starlette.types import ASGIApp, Receive, Scope, Send
from pathlib import Path
from typing import Optional, Dict, Any
from .config import Config
from .extensions import ExtensionManager, ASGIExtension
from .logging_utils import get_logger
from . import __version__
from .config import Config
from .extensions import ASGIExtension, ExtensionManager
from .logging_utils import get_logger
logger = get_logger(__name__)
@@ -21,7 +22,7 @@ class PyServeMiddleware:
def __init__(self, app: ASGIApp, extension_manager: ExtensionManager):
self.app = app
self.extension_manager = extension_manager
self.access_logger = get_logger('pyserve.access')
self.access_logger = get_logger("pyserve.access")
async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None:
if scope["type"] != "http":
@@ -48,14 +49,7 @@ class PyServeMiddleware:
await response(scope, receive, send)
async def _try_asgi_mount(
self,
scope: Scope,
receive: Receive,
send: Send,
request: Request,
start_time: float
) -> bool:
async def _try_asgi_mount(self, scope: Scope, receive: Receive, send: Send, request: Request, start_time: float) -> bool:
for extension in self.extension_manager.extensions:
if isinstance(extension, ASGIExtension):
mount = extension.get_asgi_handler(request)
@@ -65,10 +59,7 @@ class PyServeMiddleware:
modified_scope["path"] = mount.get_modified_path(request.url.path)
modified_scope["root_path"] = scope.get("root_path", "") + mount.path
logger.debug(
f"Routing to ASGI mount '{mount.name}': "
f"{request.url.path} -> {modified_scope['path']}"
)
logger.debug(f"Routing to ASGI mount '{mount.name}': " f"{request.url.path} -> {modified_scope['path']}")
try:
response_started = False
@@ -92,15 +83,12 @@ class PyServeMiddleware:
mount=mount.name,
status_code=status_code,
process_time_ms=process_time,
user_agent=request.headers.get("user-agent", "")
user_agent=request.headers.get("user-agent", ""),
)
return True
except Exception as e:
logger.error(f"Error in ASGI mount '{mount.name}': {e}")
error_response = PlainTextResponse(
"500 Internal Server Error",
status_code=500
)
error_response = PlainTextResponse("500 Internal Server Error", status_code=500)
await error_response(scope, receive, send)
return True
return False
@@ -122,7 +110,7 @@ class PyServeMiddleware:
path=path,
status_code=status_code,
process_time_ms=process_time,
user_agent=request.headers.get("user-agent", "")
user_agent=request.headers.get("user-agent", ""),
)
@@ -145,27 +133,13 @@ class PyServeServer:
if ext_config.type == "routing":
config.setdefault("default_proxy_timeout", self.config.server.proxy_timeout)
self.extension_manager.load_extension(
ext_config.type,
config
)
self.extension_manager.load_extension(ext_config.type, config)
def _create_app(self) -> None:
routes = [
Route("/health", self._health_check, methods=["GET"]),
Route("/metrics", self._metrics, methods=["GET"]),
Route(
"/{path:path}",
self._catch_all,
methods=[
"GET",
"POST",
"PUT",
"DELETE",
"PATCH",
"OPTIONS"
]
),
Route("/{path:path}", self._catch_all, methods=["GET", "POST", "PUT", "DELETE", "PATCH", "OPTIONS"]),
]
self.app = Starlette(routes=routes)
@@ -178,19 +152,16 @@ class PyServeServer:
metrics = {}
for extension in self.extension_manager.extensions:
if hasattr(extension, 'get_metrics'):
if hasattr(extension, "get_metrics"):
try:
ext_metrics = getattr(extension, 'get_metrics')()
ext_metrics = getattr(extension, "get_metrics")()
metrics.update(ext_metrics)
except Exception as e:
logger.error("Error getting metrics from extension",
extension=type(extension).__name__, error=str(e))
logger.error("Error getting metrics from extension", extension=type(extension).__name__, error=str(e))
import json
return Response(
json.dumps(metrics, ensure_ascii=False, indent=2),
media_type="application/json"
)
return Response(json.dumps(metrics, ensure_ascii=False, indent=2), media_type="application/json")
async def _catch_all(self, request: Request) -> Response:
return PlainTextResponse("404 Not Found", status_code=404)
@@ -209,10 +180,7 @@ class PyServeServer:
try:
context = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH)
context.load_cert_chain(
self.config.ssl.cert_file,
self.config.ssl.key_file
)
context.load_cert_chain(self.config.ssl.cert_file, self.config.ssl.key_file)
logger.info("SSL context created successfully")
return context
except Exception as e:
@@ -237,20 +205,17 @@ class PyServeServer:
}
if ssl_context:
uvicorn_config.update({
"ssl_keyfile": self.config.ssl.key_file,
"ssl_certfile": self.config.ssl.cert_file,
})
uvicorn_config.update(
{
"ssl_keyfile": self.config.ssl.key_file,
"ssl_certfile": self.config.ssl.cert_file,
}
)
protocol = "https"
else:
protocol = "http"
logger.info(
"Starting PyServe server",
protocol=protocol,
host=self.config.server.host,
port=self.config.server.port
)
logger.info("Starting PyServe server", protocol=protocol, host=self.config.server.host, port=self.config.server.port)
try:
assert self.app is not None, "App not initialized"
@@ -306,6 +271,7 @@ class PyServeServer:
self.extension_manager.cleanup()
from .logging_utils import shutdown_logging
shutdown_logging()
logger.info("Server stopped")
@@ -317,13 +283,12 @@ class PyServeServer:
metrics = {"server_status": "running"}
for extension in self.extension_manager.extensions:
if hasattr(extension, 'get_metrics'):
if hasattr(extension, "get_metrics"):
try:
ext_metrics = getattr(extension, 'get_metrics')()
ext_metrics = getattr(extension, "get_metrics")()
metrics.update(ext_metrics)
except Exception as e:
logger.error("Error getting metrics from extension",
extension=type(extension).__name__, error=str(e))
logger.error("Error getting metrics from extension", extension=type(extension).__name__, error=str(e))
return metrics