531 lines
18 KiB
Python
531 lines
18 KiB
Python
"""Structured logging helpers shared by HTTP, WebSocket and Celery code."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import contextvars
|
|
import json
|
|
import logging
|
|
import re
|
|
from datetime import datetime, timezone
|
|
from ipaddress import ip_address, ip_network
|
|
from time import monotonic
|
|
from typing import Any, Mapping, Sequence
|
|
from urllib.parse import unquote_plus
|
|
from uuid import uuid4
|
|
|
|
from django.conf import settings
|
|
|
|
_LOG_CONTEXT: contextvars.ContextVar[dict[str, Any]] = contextvars.ContextVar(
|
|
"eusrr_log_context", default={}
|
|
)
|
|
|
|
_REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$")
|
|
_SENSITIVE_KEY_RE = re.compile(
|
|
r"(?:authorization|proxy-authorization|password|passwd|token|secret|"
|
|
r"cookie|set-cookie|api[-_]?key|session|ticket)",
|
|
re.IGNORECASE,
|
|
)
|
|
_BEARER_RE = re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+")
|
|
_EMAIL_RE = re.compile(r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b")
|
|
_LDAP_DN_RE = re.compile(
|
|
r"(?i)\b(?:cn|ou|uid|dc)=[^,\r\n]+"
|
|
r"(?:,(?:cn|ou|uid)=[^,\r\n]+)*"
|
|
r",dc=[^,\s\r\n]+(?:,dc=[^,\s\r\n]+)*"
|
|
)
|
|
_PHONE_RE = re.compile(
|
|
r"(?<!\w)(?:\+7|8)[\s()-]*\d{3}[\s()-]*\d{3}[\s-]*\d{2}[\s-]*\d{2}(?!\w)"
|
|
)
|
|
_ANSI_ESCAPE_RE = re.compile(r"\x1B(?:[@-Z\\-_]|\[[0-?]*[ -/]*[@-~])")
|
|
_LOGGER_NAME_RE = re.compile(r"[^a-z0-9_]+")
|
|
_SENSITIVE_VALUE_RE = re.compile(
|
|
r"(?i)(authorization|password|passwd|[\w-]*token|secret|cookie|api[-_]?key|ticket)"
|
|
r"([\"']?\s*[=:]\s*)(?:\"[^\"\r\n]*\"|'[^'\r\n]*'|[^\s,;&]+)"
|
|
)
|
|
_QUERY_VALUE_RE = re.compile(r"([?&;])([^?#&;=\s\"']+)=([^&#;\s\"']*)")
|
|
_MAX_LOG_VALUE_LENGTH = 2048
|
|
_MAX_EXCEPTION_LENGTH = 16384
|
|
_CORRELATION_ID_FIELDS = {"request_id", "connection_id", "task_id"}
|
|
|
|
_STANDARD_RECORD_FIELDS = {
|
|
"args",
|
|
"asctime",
|
|
"created",
|
|
"exc_info",
|
|
"exc_text",
|
|
"filename",
|
|
"funcName",
|
|
"levelname",
|
|
"levelno",
|
|
"lineno",
|
|
"message",
|
|
"module",
|
|
"msecs",
|
|
"msg",
|
|
"name",
|
|
"pathname",
|
|
"process",
|
|
"processName",
|
|
"relativeCreated",
|
|
"stack_info",
|
|
"thread",
|
|
"threadName",
|
|
"taskName",
|
|
}
|
|
_OUTPUT_RECORD_FIELDS = {
|
|
"timestamp",
|
|
"level",
|
|
"logger",
|
|
"message",
|
|
"source",
|
|
"exception",
|
|
}
|
|
|
|
_CONSOLE_CONTEXT_FIELDS = (
|
|
"request_id",
|
|
"connection_id",
|
|
"task_id",
|
|
"user_id",
|
|
"client_ip",
|
|
)
|
|
_CONSOLE_HTTP_FIELDS = (
|
|
"http_method",
|
|
"path",
|
|
"status_code",
|
|
"duration_ms",
|
|
"response_bytes",
|
|
)
|
|
_CONSOLE_WEBSOCKET_FIELDS = (
|
|
"connection_id",
|
|
"path",
|
|
"accepted",
|
|
"close_code",
|
|
"duration_ms",
|
|
)
|
|
_CONSOLE_CELERY_FIELDS = (
|
|
"task_id",
|
|
"task_name",
|
|
"task_state",
|
|
"duration_ms",
|
|
)
|
|
_CONSOLE_DOMAIN_LABELS = {
|
|
"celery": "CELERY",
|
|
"framework": "SYSTEM",
|
|
"http": "HTTP",
|
|
"websocket": "WS",
|
|
}
|
|
_CONSOLE_CONTEXT_ALIASES = {
|
|
"request_id": "req",
|
|
"connection_id": "conn",
|
|
"task_id": "task",
|
|
"user_id": "user",
|
|
"client_ip": "ip",
|
|
}
|
|
|
|
|
|
def new_request_id(candidate: str | None = None) -> str:
|
|
"""Return a safe external request ID or generate a new one."""
|
|
if candidate and _REQUEST_ID_RE.fullmatch(candidate):
|
|
return candidate
|
|
return uuid4().hex
|
|
|
|
|
|
def resolve_client_ip(
|
|
remote_addr: str | None,
|
|
*,
|
|
forwarded_for: str | None = None,
|
|
real_ip: str | None = None,
|
|
trust_proxy: bool = False,
|
|
trusted_proxies: Sequence[str] = (),
|
|
) -> str | None:
|
|
"""Peel only configured proxy hops from the right of X-Forwarded-For.
|
|
|
|
Merely enabling proxy support does not trust arbitrary peers or the
|
|
attacker-controlled leftmost value. Invalid chains fail closed to the peer.
|
|
"""
|
|
try:
|
|
peer = ip_address(remote_addr.strip()) if remote_addr else None
|
|
except ValueError:
|
|
peer = None
|
|
if peer is None or not trust_proxy:
|
|
return str(peer) if peer is not None else None
|
|
|
|
networks = []
|
|
for cidr in trusted_proxies:
|
|
try:
|
|
networks.append(ip_network(cidr, strict=False))
|
|
except (TypeError, ValueError):
|
|
continue
|
|
|
|
def trusted(address):
|
|
return any(address in network for network in networks)
|
|
|
|
if not trusted(peer):
|
|
return str(peer)
|
|
chain = forwarded_for.split(",") if forwarded_for else [real_ip] if real_ip else []
|
|
current = peer
|
|
for candidate in reversed(chain):
|
|
if not trusted(current):
|
|
break
|
|
try:
|
|
current = ip_address(candidate.strip())
|
|
except (TypeError, ValueError, AttributeError):
|
|
return str(peer)
|
|
return str(current)
|
|
|
|
|
|
def bind_log_context(**values: Any) -> contextvars.Token:
|
|
"""Merge values into the current async/thread-local log context."""
|
|
context = dict(_LOG_CONTEXT.get())
|
|
context.update({key: value for key, value in values.items() if value is not None})
|
|
return _LOG_CONTEXT.set(context)
|
|
|
|
|
|
def update_log_context(**values: Any) -> None:
|
|
"""Update the current context when information becomes available later."""
|
|
context = dict(_LOG_CONTEXT.get())
|
|
for key, value in values.items():
|
|
if value is None:
|
|
context.pop(key, None)
|
|
else:
|
|
context[key] = value
|
|
_LOG_CONTEXT.set(context)
|
|
|
|
|
|
def reset_log_context(token: contextvars.Token) -> None:
|
|
_LOG_CONTEXT.reset(token)
|
|
|
|
|
|
def get_log_context() -> dict[str, Any]:
|
|
"""Expose a copy for diagnostics and unit tests."""
|
|
return dict(_LOG_CONTEXT.get())
|
|
|
|
|
|
def _redact_query_value(match: re.Match) -> str:
|
|
prefix, raw_key, value = match.groups()
|
|
decoded_key = raw_key
|
|
for _ in range(3):
|
|
decoded_key = unquote_plus(decoded_key)
|
|
if _SENSITIVE_KEY_RE.search(decoded_key) or decoded_key.lower() in {
|
|
"code",
|
|
"state",
|
|
}:
|
|
value = "[REDACTED]"
|
|
return f"{prefix}{raw_key}={value}"
|
|
|
|
|
|
def _redact_text(
|
|
value: Any, *, key: str = "", max_length: int = _MAX_LOG_VALUE_LENGTH
|
|
) -> str:
|
|
text = _ANSI_ESCAPE_RE.sub("", str(value))
|
|
text = _QUERY_VALUE_RE.sub(_redact_query_value, text)
|
|
text = _BEARER_RE.sub("Bearer [REDACTED]", text)
|
|
text = _SENSITIVE_VALUE_RE.sub(r"\1\2[REDACTED]", text)
|
|
text = _EMAIL_RE.sub("[REDACTED_EMAIL]", text)
|
|
text = _LDAP_DN_RE.sub("[REDACTED_DN]", text)
|
|
if not (key in _CORRELATION_ID_FIELDS and _REQUEST_ID_RE.fullmatch(text)):
|
|
text = _PHONE_RE.sub("[REDACTED_PHONE]", text)
|
|
text = text.replace("\r", "\\r").replace("\n", "\\n")
|
|
if len(text) <= max_length:
|
|
return text
|
|
marker = "...[TRUNCATED]..."
|
|
if max_length == _MAX_EXCEPTION_LENGTH:
|
|
# The final frame and exception/cause are more useful than the middle of
|
|
# a long stack. Preserve both ends, after redacting the entire value.
|
|
head_length = (max_length - len(marker)) // 2
|
|
tail_length = max_length - len(marker) - head_length
|
|
return f"{text[:head_length]}{marker}{text[-tail_length:]}"
|
|
return f"{text[:max_length]}...[TRUNCATED]"
|
|
|
|
|
|
def redact(value: Any, *, key: str = "") -> Any:
|
|
"""Mask secrets and make arbitrary values safe for log serialization."""
|
|
if _SENSITIVE_KEY_RE.search(key):
|
|
return "[REDACTED]"
|
|
if value is None or isinstance(value, (bool, int, float)):
|
|
return value
|
|
if isinstance(value, Mapping):
|
|
return {str(k): redact(v, key=str(k)) for k, v in value.items()}
|
|
if isinstance(value, (list, tuple, set)):
|
|
return [redact(item) for item in value]
|
|
|
|
return _redact_text(
|
|
value,
|
|
key=key,
|
|
max_length=_MAX_EXCEPTION_LENGTH
|
|
if key == "failure_traceback"
|
|
else _MAX_LOG_VALUE_LENGTH,
|
|
)
|
|
|
|
|
|
def _record_fields(record: logging.LogRecord) -> dict[str, Any]:
|
|
fields = dict(get_log_context())
|
|
for key, value in record.__dict__.items():
|
|
if key not in _STANDARD_RECORD_FIELDS and not key.startswith("_"):
|
|
fields[key] = value
|
|
fields.pop("event", None)
|
|
for key in _OUTPUT_RECORD_FIELDS:
|
|
fields.pop(key, None)
|
|
safe_fields = {}
|
|
for key, value in fields.items():
|
|
if value is None:
|
|
continue
|
|
if key == "request" and hasattr(value, "path"):
|
|
value = {
|
|
"method": getattr(value, "method", None),
|
|
"path": getattr(value, "path", None),
|
|
}
|
|
safe_fields[key] = redact(value, key=key)
|
|
return safe_fields
|
|
|
|
|
|
def _exception_text(formatter: logging.Formatter, record: logging.LogRecord):
|
|
if not record.exc_info:
|
|
return None
|
|
try:
|
|
return _redact_text(
|
|
formatter.formatException(record.exc_info), max_length=_MAX_EXCEPTION_LENGTH
|
|
)
|
|
except (AttributeError, TypeError, ValueError):
|
|
# Celery/billiard may occasionally provide a serialized traceback where
|
|
# stdlib logging expects a traceback object. Never hide the real failure
|
|
# behind a formatter exception.
|
|
if isinstance(record.exc_info, tuple):
|
|
error_type, error, traceback_value = (*record.exc_info, None, None)[:3]
|
|
return _redact_text(
|
|
f"{getattr(error_type, '__name__', error_type)}: {error}; "
|
|
f"traceback={traceback_value}",
|
|
max_length=_MAX_EXCEPTION_LENGTH,
|
|
)
|
|
return _redact_text(record.exc_info, max_length=_MAX_EXCEPTION_LENGTH)
|
|
|
|
|
|
class ContextFilter(logging.Filter):
|
|
"""Attach context fields for third-party and project log records alike."""
|
|
|
|
def filter(self, record: logging.LogRecord) -> bool:
|
|
if not hasattr(record, "event"):
|
|
logger_name = (
|
|
_LOGGER_NAME_RE.sub("_", record.name.lower().replace(".", "_")).strip(
|
|
"_"
|
|
)
|
|
or "unknown"
|
|
)
|
|
record.event = f"framework.{logger_name}.{record.levelname.lower()}"
|
|
for key, value in get_log_context().items():
|
|
if not hasattr(record, key):
|
|
setattr(record, key, value)
|
|
return True
|
|
|
|
|
|
class ConsoleFormatter(logging.Formatter):
|
|
"""Human-readable grouped output for local development, without ANSI."""
|
|
|
|
@staticmethod
|
|
def _take_group(fields: dict[str, Any], keys: tuple[str, ...]) -> dict[str, Any]:
|
|
return {key: fields.pop(key) for key in keys if key in fields}
|
|
|
|
@staticmethod
|
|
def _format_pairs(
|
|
fields: Mapping[str, Any], aliases: Mapping[str, str] | None = None
|
|
) -> str | None:
|
|
if not fields:
|
|
return None
|
|
aliases = aliases or {}
|
|
values = " ".join(
|
|
f"{aliases.get(key, key)}={json.dumps(value, ensure_ascii=False)}"
|
|
for key, value in fields.items()
|
|
)
|
|
return f"[{values}]"
|
|
|
|
@staticmethod
|
|
def _format_http(fields: Mapping[str, Any]) -> str | None:
|
|
if not fields:
|
|
return None
|
|
values = []
|
|
if method := fields.get("http_method"):
|
|
values.append(str(method))
|
|
if path := fields.get("path"):
|
|
values.append(json.dumps(path, ensure_ascii=False))
|
|
if "status_code" in fields:
|
|
values.extend(("→", str(fields["status_code"])))
|
|
if "duration_ms" in fields:
|
|
values.append(f"{fields['duration_ms']}ms")
|
|
if "response_bytes" in fields:
|
|
values.append(f"bytes={fields['response_bytes']}")
|
|
return f"[{' '.join(values)}]" if values else None
|
|
|
|
@classmethod
|
|
def _format_domain(cls, event_domain: str, fields: Mapping[str, Any]) -> str | None:
|
|
if event_domain == "http":
|
|
return cls._format_http(fields)
|
|
aliases = {
|
|
"connection_id": "conn",
|
|
"task_id": "task",
|
|
"task_name": "name",
|
|
"task_state": "state",
|
|
"close_code": "close",
|
|
"duration_ms": "duration",
|
|
}
|
|
return cls._format_pairs(fields, aliases)
|
|
|
|
def format(self, record: logging.LogRecord) -> str:
|
|
timestamp = datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat(
|
|
timespec="milliseconds"
|
|
)
|
|
event = redact(getattr(record, "event", "log.message"))
|
|
event_domain = str(event).split(".", 1)[0].lower()
|
|
domain_label = _CONSOLE_DOMAIN_LABELS.get(
|
|
event_domain, event_domain.upper() or "LOG"
|
|
)
|
|
fields = _record_fields(record)
|
|
parts = [
|
|
timestamp,
|
|
record.levelname,
|
|
f"[{domain_label}]",
|
|
str(event),
|
|
f"@{redact(record.name)}",
|
|
]
|
|
|
|
domain_fields = ()
|
|
if event_domain == "http":
|
|
domain_fields = _CONSOLE_HTTP_FIELDS
|
|
elif event_domain == "websocket":
|
|
domain_fields = _CONSOLE_WEBSOCKET_FIELDS
|
|
elif event_domain == "celery":
|
|
domain_fields = _CONSOLE_CELERY_FIELDS
|
|
|
|
domain_values = self._take_group(fields, domain_fields)
|
|
request_values = {}
|
|
if event_domain != "http" and ("http_method" in fields or "path" in fields):
|
|
request_values = self._take_group(
|
|
fields, ("http_method", "path", "status_code", "duration_ms")
|
|
)
|
|
context_values = self._take_group(fields, _CONSOLE_CONTEXT_FIELDS)
|
|
if event_domain == "websocket" and domain_values.get(
|
|
"connection_id"
|
|
) == context_values.get("request_id"):
|
|
context_values.pop("request_id", None)
|
|
|
|
groups = [
|
|
self._format_domain(event_domain, domain_values),
|
|
self._format_http(request_values),
|
|
self._format_pairs(context_values, _CONSOLE_CONTEXT_ALIASES),
|
|
self._format_pairs(dict(sorted(fields.items()))),
|
|
]
|
|
parts.extend(group for group in groups if group)
|
|
parts.append(json.dumps(redact(record.getMessage()), ensure_ascii=False))
|
|
exception = _exception_text(self, record)
|
|
if exception:
|
|
parts.append(f"[exception {json.dumps(exception, ensure_ascii=False)}]")
|
|
return " ".join(parts)
|
|
|
|
|
|
class JsonFormatter(logging.Formatter):
|
|
"""One JSON object per line for production log aggregation."""
|
|
|
|
def format(self, record: logging.LogRecord) -> str:
|
|
payload = {
|
|
"timestamp": datetime.fromtimestamp(
|
|
record.created, tz=timezone.utc
|
|
).isoformat(timespec="milliseconds"),
|
|
"level": record.levelname,
|
|
"logger": redact(record.name),
|
|
"event": redact(getattr(record, "event", "log.message")),
|
|
"message": redact(record.getMessage()),
|
|
"source": f"{record.filename}:{record.lineno}",
|
|
**_record_fields(record),
|
|
}
|
|
exception = _exception_text(self, record)
|
|
if exception:
|
|
payload["exception"] = exception
|
|
return json.dumps(payload, ensure_ascii=False, default=str)
|
|
|
|
|
|
class WebSocketLoggingMiddleware:
|
|
"""Bind correlation context and log the WebSocket connection lifecycle."""
|
|
|
|
def __init__(self, app):
|
|
self.app = app
|
|
self.logger = logging.getLogger("eusrr.websocket")
|
|
|
|
@staticmethod
|
|
def _header(scope, name: bytes) -> str | None:
|
|
for header_name, value in scope.get("headers", []):
|
|
if header_name.lower() == name:
|
|
return value.decode("utf-8", errors="replace")
|
|
return None
|
|
|
|
async def __call__(self, scope, receive, send):
|
|
if scope.get("type") != "websocket":
|
|
return await self.app(scope, receive, send)
|
|
|
|
request_id = new_request_id(self._header(scope, b"x-request-id"))
|
|
scope["request_id"] = request_id
|
|
token = bind_log_context(
|
|
request_id=request_id,
|
|
connection_id=request_id,
|
|
path=scope.get("path"),
|
|
client_ip=resolve_client_ip(
|
|
(scope.get("client") or (None,))[0],
|
|
forwarded_for=self._header(scope, b"x-forwarded-for"),
|
|
real_ip=self._header(scope, b"x-real-ip"),
|
|
trust_proxy=getattr(settings, "LOG_TRUST_X_FORWARDED_FOR", False),
|
|
trusted_proxies=getattr(settings, "LOG_TRUSTED_PROXY_CIDRS", ()),
|
|
),
|
|
)
|
|
started = monotonic()
|
|
accepted = False
|
|
requested_close_code = None
|
|
disconnect_code = None
|
|
|
|
async def logged_send(message):
|
|
nonlocal accepted, requested_close_code
|
|
message_type = message.get("type")
|
|
if message_type == "websocket.accept":
|
|
accepted = True
|
|
self.logger.info(
|
|
"WebSocket connection accepted",
|
|
extra={"event": "websocket.connection.accepted"},
|
|
)
|
|
elif message_type == "websocket.close":
|
|
requested_close_code = message.get("code", 1000)
|
|
await send(message)
|
|
|
|
async def logged_receive():
|
|
nonlocal disconnect_code
|
|
message = await receive()
|
|
if message.get("type") == "websocket.disconnect":
|
|
disconnect_code = message.get("code", 1000)
|
|
return message
|
|
|
|
try:
|
|
return await self.app(scope, logged_receive, logged_send)
|
|
except Exception:
|
|
self.logger.exception(
|
|
"WebSocket connection failed",
|
|
extra={
|
|
"event": "websocket.connection.failed",
|
|
"duration_ms": round((monotonic() - started) * 1000, 2),
|
|
},
|
|
)
|
|
raise
|
|
finally:
|
|
duration_ms = round((monotonic() - started) * 1000, 2)
|
|
close_code = (
|
|
disconnect_code if disconnect_code is not None else requested_close_code
|
|
)
|
|
level = logging.WARNING if close_code not in (None, 1000) else logging.INFO
|
|
self.logger.log(
|
|
level,
|
|
"WebSocket connection closed",
|
|
extra={
|
|
"event": "websocket.connection.closed",
|
|
"accepted": accepted,
|
|
"close_code": close_code,
|
|
"requested_close_code": requested_close_code,
|
|
"disconnect_code": disconnect_code,
|
|
"duration_ms": duration_ms,
|
|
},
|
|
)
|
|
reset_log_context(token)
|