from dataclasses import dataclass, field from datetime import timedelta from django.db import transaction from django.db.models import Q from django.utils import timezone from commerce.models import IntegrationStatus, IntegrationSyncRun STALE_QUEUED_ERROR = "Задача не была получена обработчиком и снята с очереди." STALE_RUNNING_ERROR = ( "Синхронизация перестала передавать прогресс и была завершена как ошибочная." ) @dataclass class SyncRunReconciliationResult: queued_run_ids: list[str] = field(default_factory=list) running_run_ids: list[str] = field(default_factory=list) connection_ids: list[str] = field(default_factory=list) @property def recovered(self) -> int: return len(self.queued_run_ids) + len(self.running_run_ids) def reconcile_stale_sync_runs( *, queued_timeout: timedelta, running_timeout: timedelta, now=None, dry_run: bool = False, ) -> SyncRunReconciliationResult: """Mark stale runs as errors without deleting history or silently retrying them.""" now = now or timezone.now() queued_cutoff = now - queued_timeout running_cutoff = now - running_timeout queued_ids = list( IntegrationSyncRun.objects.filter( status="queued", created_at__lte=queued_cutoff, ).values_list("pk", flat=True) ) running_ids = list( IntegrationSyncRun.objects.filter(status="running") .filter( Q(last_progress_at__lte=running_cutoff) | Q(last_progress_at__isnull=True, started_at__lte=running_cutoff) | Q( last_progress_at__isnull=True, started_at__isnull=True, created_at__lte=running_cutoff, ) ) .values_list("pk", flat=True) ) candidates = [ *( (run_id, "queued", queued_cutoff, STALE_QUEUED_ERROR, "queued") for run_id in queued_ids ), *( (run_id, "running", running_cutoff, STALE_RUNNING_ERROR, "running") for run_id in running_ids ), ] result = SyncRunReconciliationResult() for run_id, expected_status, cutoff, message, target in candidates: with transaction.atomic(): run = ( IntegrationSyncRun.objects.select_for_update() .select_related("connection") .get(pk=run_id) ) heartbeat = run.created_at if expected_status == "running": heartbeat = run.last_progress_at or run.started_at or run.created_at if run.status != expected_status or heartbeat > cutoff: continue getattr(result, f"{target}_run_ids").append(str(run.pk)) connection = run.connection owns_connection = ( connection.task_id == run.task_id and connection.status in { IntegrationStatus.QUEUED, IntegrationStatus.SYNCING, } ) if owns_connection: result.connection_ids.append(str(connection.pk)) if dry_run: continue run.status = "error" run.error = message run.finished_at = now run.save(update_fields=["status", "error", "finished_at"]) if owns_connection: connection.status = IntegrationStatus.ERROR connection.last_error = message connection.last_sync_finished_at = now connection.next_sync_at = ( now + timedelta(minutes=connection.sync_interval_minutes) if connection.auto_sync else None ) connection.save( update_fields=[ "status", "last_error", "last_sync_finished_at", "next_sync_at", "updated_at", ] ) return result