"""Structured logging helpers shared by HTTP, WebSocket and Celery code.""" from __future__ import annotations import contextvars import json import logging import re from datetime import datetime, timezone from ipaddress import ip_address, ip_network from time import monotonic from typing import Any, Mapping, Sequence from urllib.parse import unquote_plus from uuid import uuid4 from django.conf import settings _LOG_CONTEXT: contextvars.ContextVar[dict[str, Any]] = contextvars.ContextVar( "eusrr_log_context", default={} ) _REQUEST_ID_RE = re.compile(r"^[A-Za-z0-9._:-]{1,128}$") _SENSITIVE_KEY_RE = re.compile( r"(?:authorization|proxy-authorization|password|passwd|token|secret|" r"cookie|set-cookie|api[-_]?key|session|ticket)", re.IGNORECASE, ) _BEARER_RE = re.compile(r"(?i)\bBearer\s+[A-Za-z0-9._~+/=-]+") _EMAIL_RE = re.compile(r"(?i)\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b") _LDAP_DN_RE = re.compile( r"(?i)\b(?:cn|ou|uid|dc)=[^,\r\n]+" r"(?:,(?:cn|ou|uid)=[^,\r\n]+)*" r",dc=[^,\s\r\n]+(?:,dc=[^,\s\r\n]+)*" ) _PHONE_RE = re.compile( r"(? str: """Return a safe external request ID or generate a new one.""" if candidate and _REQUEST_ID_RE.fullmatch(candidate): return candidate return uuid4().hex def resolve_client_ip( remote_addr: str | None, *, forwarded_for: str | None = None, real_ip: str | None = None, trust_proxy: bool = False, trusted_proxies: Sequence[str] = (), ) -> str | None: """Peel only configured proxy hops from the right of X-Forwarded-For. Merely enabling proxy support does not trust arbitrary peers or the attacker-controlled leftmost value. Invalid chains fail closed to the peer. """ try: peer = ip_address(remote_addr.strip()) if remote_addr else None except ValueError: peer = None if peer is None or not trust_proxy: return str(peer) if peer is not None else None networks = [] for cidr in trusted_proxies: try: networks.append(ip_network(cidr, strict=False)) except (TypeError, ValueError): continue def trusted(address): return any(address in network for network in networks) if not trusted(peer): return str(peer) chain = forwarded_for.split(",") if forwarded_for else [real_ip] if real_ip else [] current = peer for candidate in reversed(chain): if not trusted(current): break try: current = ip_address(candidate.strip()) except (TypeError, ValueError, AttributeError): return str(peer) return str(current) def bind_log_context(**values: Any) -> contextvars.Token: """Merge values into the current async/thread-local log context.""" context = dict(_LOG_CONTEXT.get()) context.update({key: value for key, value in values.items() if value is not None}) return _LOG_CONTEXT.set(context) def update_log_context(**values: Any) -> None: """Update the current context when information becomes available later.""" context = dict(_LOG_CONTEXT.get()) for key, value in values.items(): if value is None: context.pop(key, None) else: context[key] = value _LOG_CONTEXT.set(context) def reset_log_context(token: contextvars.Token) -> None: _LOG_CONTEXT.reset(token) def get_log_context() -> dict[str, Any]: """Expose a copy for diagnostics and unit tests.""" return dict(_LOG_CONTEXT.get()) def _redact_query_value(match: re.Match) -> str: prefix, raw_key, value = match.groups() decoded_key = raw_key for _ in range(3): decoded_key = unquote_plus(decoded_key) if _SENSITIVE_KEY_RE.search(decoded_key) or decoded_key.lower() in { "code", "state", }: value = "[REDACTED]" return f"{prefix}{raw_key}={value}" def _redact_text( value: Any, *, key: str = "", max_length: int = _MAX_LOG_VALUE_LENGTH ) -> str: text = _ANSI_ESCAPE_RE.sub("", str(value)) text = _QUERY_VALUE_RE.sub(_redact_query_value, text) text = _BEARER_RE.sub("Bearer [REDACTED]", text) text = _SENSITIVE_VALUE_RE.sub(r"\1\2[REDACTED]", text) text = _EMAIL_RE.sub("[REDACTED_EMAIL]", text) text = _LDAP_DN_RE.sub("[REDACTED_DN]", text) if not (key in _CORRELATION_ID_FIELDS and _REQUEST_ID_RE.fullmatch(text)): text = _PHONE_RE.sub("[REDACTED_PHONE]", text) text = text.replace("\r", "\\r").replace("\n", "\\n") if len(text) <= max_length: return text marker = "...[TRUNCATED]..." if max_length == _MAX_EXCEPTION_LENGTH: # The final frame and exception/cause are more useful than the middle of # a long stack. Preserve both ends, after redacting the entire value. head_length = (max_length - len(marker)) // 2 tail_length = max_length - len(marker) - head_length return f"{text[:head_length]}{marker}{text[-tail_length:]}" return f"{text[:max_length]}...[TRUNCATED]" def redact(value: Any, *, key: str = "") -> Any: """Mask secrets and make arbitrary values safe for log serialization.""" if _SENSITIVE_KEY_RE.search(key): return "[REDACTED]" if value is None or isinstance(value, (bool, int, float)): return value if isinstance(value, Mapping): return {str(k): redact(v, key=str(k)) for k, v in value.items()} if isinstance(value, (list, tuple, set)): return [redact(item) for item in value] return _redact_text( value, key=key, max_length=_MAX_EXCEPTION_LENGTH if key == "failure_traceback" else _MAX_LOG_VALUE_LENGTH, ) def _record_fields(record: logging.LogRecord) -> dict[str, Any]: fields = dict(get_log_context()) for key, value in record.__dict__.items(): if key not in _STANDARD_RECORD_FIELDS and not key.startswith("_"): fields[key] = value fields.pop("event", None) for key in _OUTPUT_RECORD_FIELDS: fields.pop(key, None) safe_fields = {} for key, value in fields.items(): if value is None: continue if key == "request" and hasattr(value, "path"): value = { "method": getattr(value, "method", None), "path": getattr(value, "path", None), } safe_fields[key] = redact(value, key=key) return safe_fields def _exception_text(formatter: logging.Formatter, record: logging.LogRecord): if not record.exc_info: return None try: return _redact_text( formatter.formatException(record.exc_info), max_length=_MAX_EXCEPTION_LENGTH ) except (AttributeError, TypeError, ValueError): # Celery/billiard may occasionally provide a serialized traceback where # stdlib logging expects a traceback object. Never hide the real failure # behind a formatter exception. if isinstance(record.exc_info, tuple): error_type, error, traceback_value = (*record.exc_info, None, None)[:3] return _redact_text( f"{getattr(error_type, '__name__', error_type)}: {error}; " f"traceback={traceback_value}", max_length=_MAX_EXCEPTION_LENGTH, ) return _redact_text(record.exc_info, max_length=_MAX_EXCEPTION_LENGTH) class ContextFilter(logging.Filter): """Attach context fields for third-party and project log records alike.""" def filter(self, record: logging.LogRecord) -> bool: if not hasattr(record, "event"): logger_name = ( _LOGGER_NAME_RE.sub("_", record.name.lower().replace(".", "_")).strip( "_" ) or "unknown" ) record.event = f"framework.{logger_name}.{record.levelname.lower()}" for key, value in get_log_context().items(): if not hasattr(record, key): setattr(record, key, value) return True class ConsoleFormatter(logging.Formatter): """Human-readable grouped output for local development, without ANSI.""" @staticmethod def _take_group(fields: dict[str, Any], keys: tuple[str, ...]) -> dict[str, Any]: return {key: fields.pop(key) for key in keys if key in fields} @staticmethod def _format_pairs( fields: Mapping[str, Any], aliases: Mapping[str, str] | None = None ) -> str | None: if not fields: return None aliases = aliases or {} values = " ".join( f"{aliases.get(key, key)}={json.dumps(value, ensure_ascii=False)}" for key, value in fields.items() ) return f"[{values}]" @staticmethod def _format_http(fields: Mapping[str, Any]) -> str | None: if not fields: return None values = [] if method := fields.get("http_method"): values.append(str(method)) if path := fields.get("path"): values.append(json.dumps(path, ensure_ascii=False)) if "status_code" in fields: values.extend(("→", str(fields["status_code"]))) if "duration_ms" in fields: values.append(f"{fields['duration_ms']}ms") if "response_bytes" in fields: values.append(f"bytes={fields['response_bytes']}") return f"[{' '.join(values)}]" if values else None @classmethod def _format_domain(cls, event_domain: str, fields: Mapping[str, Any]) -> str | None: if event_domain == "http": return cls._format_http(fields) aliases = { "connection_id": "conn", "task_id": "task", "task_name": "name", "task_state": "state", "close_code": "close", "duration_ms": "duration", } return cls._format_pairs(fields, aliases) def format(self, record: logging.LogRecord) -> str: timestamp = datetime.fromtimestamp(record.created, tz=timezone.utc).isoformat( timespec="milliseconds" ) event = redact(getattr(record, "event", "log.message")) event_domain = str(event).split(".", 1)[0].lower() domain_label = _CONSOLE_DOMAIN_LABELS.get( event_domain, event_domain.upper() or "LOG" ) fields = _record_fields(record) parts = [ timestamp, record.levelname, f"[{domain_label}]", str(event), f"@{redact(record.name)}", ] domain_fields = () if event_domain == "http": domain_fields = _CONSOLE_HTTP_FIELDS elif event_domain == "websocket": domain_fields = _CONSOLE_WEBSOCKET_FIELDS elif event_domain == "celery": domain_fields = _CONSOLE_CELERY_FIELDS domain_values = self._take_group(fields, domain_fields) request_values = {} if event_domain != "http" and ("http_method" in fields or "path" in fields): request_values = self._take_group( fields, ("http_method", "path", "status_code", "duration_ms") ) context_values = self._take_group(fields, _CONSOLE_CONTEXT_FIELDS) if event_domain == "websocket" and domain_values.get( "connection_id" ) == context_values.get("request_id"): context_values.pop("request_id", None) groups = [ self._format_domain(event_domain, domain_values), self._format_http(request_values), self._format_pairs(context_values, _CONSOLE_CONTEXT_ALIASES), self._format_pairs(dict(sorted(fields.items()))), ] parts.extend(group for group in groups if group) parts.append(json.dumps(redact(record.getMessage()), ensure_ascii=False)) exception = _exception_text(self, record) if exception: parts.append(f"[exception {json.dumps(exception, ensure_ascii=False)}]") return " ".join(parts) class JsonFormatter(logging.Formatter): """One JSON object per line for production log aggregation.""" def format(self, record: logging.LogRecord) -> str: payload = { "timestamp": datetime.fromtimestamp( record.created, tz=timezone.utc ).isoformat(timespec="milliseconds"), "level": record.levelname, "logger": redact(record.name), "event": redact(getattr(record, "event", "log.message")), "message": redact(record.getMessage()), "source": f"{record.filename}:{record.lineno}", **_record_fields(record), } exception = _exception_text(self, record) if exception: payload["exception"] = exception return json.dumps(payload, ensure_ascii=False, default=str) class WebSocketLoggingMiddleware: """Bind correlation context and log the WebSocket connection lifecycle.""" def __init__(self, app): self.app = app self.logger = logging.getLogger("eusrr.websocket") @staticmethod def _header(scope, name: bytes) -> str | None: for header_name, value in scope.get("headers", []): if header_name.lower() == name: return value.decode("utf-8", errors="replace") return None async def __call__(self, scope, receive, send): if scope.get("type") != "websocket": return await self.app(scope, receive, send) request_id = new_request_id(self._header(scope, b"x-request-id")) scope["request_id"] = request_id token = bind_log_context( request_id=request_id, connection_id=request_id, path=scope.get("path"), client_ip=resolve_client_ip( (scope.get("client") or (None,))[0], forwarded_for=self._header(scope, b"x-forwarded-for"), real_ip=self._header(scope, b"x-real-ip"), trust_proxy=getattr(settings, "LOG_TRUST_X_FORWARDED_FOR", False), trusted_proxies=getattr(settings, "LOG_TRUSTED_PROXY_CIDRS", ()), ), ) started = monotonic() accepted = False requested_close_code = None disconnect_code = None async def logged_send(message): nonlocal accepted, requested_close_code message_type = message.get("type") if message_type == "websocket.accept": accepted = True self.logger.info( "WebSocket connection accepted", extra={"event": "websocket.connection.accepted"}, ) elif message_type == "websocket.close": requested_close_code = message.get("code", 1000) await send(message) async def logged_receive(): nonlocal disconnect_code message = await receive() if message.get("type") == "websocket.disconnect": disconnect_code = message.get("code", 1000) return message try: return await self.app(scope, logged_receive, logged_send) except Exception: self.logger.exception( "WebSocket connection failed", extra={ "event": "websocket.connection.failed", "duration_ms": round((monotonic() - started) * 1000, 2), }, ) raise finally: duration_ms = round((monotonic() - started) * 1000, 2) close_code = ( disconnect_code if disconnect_code is not None else requested_close_code ) level = logging.WARNING if close_code not in (None, 1000) else logging.INFO self.logger.log( level, "WebSocket connection closed", extra={ "event": "websocket.connection.closed", "accepted": accepted, "close_code": close_code, "requested_close_code": requested_close_code, "disconnect_code": disconnect_code, "duration_ms": duration_ms, }, ) reset_log_context(token)