120 lines
4.1 KiB
Python
120 lines
4.1 KiB
Python
from dataclasses import dataclass, field
|
||
from datetime import timedelta
|
||
|
||
from django.db import transaction
|
||
from django.db.models import Q
|
||
from django.utils import timezone
|
||
|
||
from commerce.models import IntegrationStatus, IntegrationSyncRun
|
||
|
||
|
||
STALE_QUEUED_ERROR = "Задача не была получена обработчиком и снята с очереди."
|
||
STALE_RUNNING_ERROR = (
|
||
"Синхронизация перестала передавать прогресс и была завершена как ошибочная."
|
||
)
|
||
|
||
|
||
@dataclass
|
||
class SyncRunReconciliationResult:
|
||
queued_run_ids: list[str] = field(default_factory=list)
|
||
running_run_ids: list[str] = field(default_factory=list)
|
||
connection_ids: list[str] = field(default_factory=list)
|
||
|
||
@property
|
||
def recovered(self) -> int:
|
||
return len(self.queued_run_ids) + len(self.running_run_ids)
|
||
|
||
|
||
def reconcile_stale_sync_runs(
|
||
*,
|
||
queued_timeout: timedelta,
|
||
running_timeout: timedelta,
|
||
now=None,
|
||
dry_run: bool = False,
|
||
) -> SyncRunReconciliationResult:
|
||
"""Mark stale runs as errors without deleting history or silently retrying them."""
|
||
|
||
now = now or timezone.now()
|
||
queued_cutoff = now - queued_timeout
|
||
running_cutoff = now - running_timeout
|
||
queued_ids = list(
|
||
IntegrationSyncRun.objects.filter(
|
||
status="queued",
|
||
created_at__lte=queued_cutoff,
|
||
).values_list("pk", flat=True)
|
||
)
|
||
running_ids = list(
|
||
IntegrationSyncRun.objects.filter(status="running")
|
||
.filter(
|
||
Q(last_progress_at__lte=running_cutoff)
|
||
| Q(last_progress_at__isnull=True, started_at__lte=running_cutoff)
|
||
| Q(
|
||
last_progress_at__isnull=True,
|
||
started_at__isnull=True,
|
||
created_at__lte=running_cutoff,
|
||
)
|
||
)
|
||
.values_list("pk", flat=True)
|
||
)
|
||
candidates = [
|
||
*(
|
||
(run_id, "queued", queued_cutoff, STALE_QUEUED_ERROR, "queued")
|
||
for run_id in queued_ids
|
||
),
|
||
*(
|
||
(run_id, "running", running_cutoff, STALE_RUNNING_ERROR, "running")
|
||
for run_id in running_ids
|
||
),
|
||
]
|
||
result = SyncRunReconciliationResult()
|
||
for run_id, expected_status, cutoff, message, target in candidates:
|
||
with transaction.atomic():
|
||
run = (
|
||
IntegrationSyncRun.objects.select_for_update()
|
||
.select_related("connection")
|
||
.get(pk=run_id)
|
||
)
|
||
heartbeat = run.created_at
|
||
if expected_status == "running":
|
||
heartbeat = run.last_progress_at or run.started_at or run.created_at
|
||
if run.status != expected_status or heartbeat > cutoff:
|
||
continue
|
||
getattr(result, f"{target}_run_ids").append(str(run.pk))
|
||
connection = run.connection
|
||
owns_connection = (
|
||
connection.task_id == run.task_id
|
||
and connection.status
|
||
in {
|
||
IntegrationStatus.QUEUED,
|
||
IntegrationStatus.SYNCING,
|
||
}
|
||
)
|
||
if owns_connection:
|
||
result.connection_ids.append(str(connection.pk))
|
||
if dry_run:
|
||
continue
|
||
|
||
run.status = "error"
|
||
run.error = message
|
||
run.finished_at = now
|
||
run.save(update_fields=["status", "error", "finished_at"])
|
||
if owns_connection:
|
||
connection.status = IntegrationStatus.ERROR
|
||
connection.last_error = message
|
||
connection.last_sync_finished_at = now
|
||
connection.next_sync_at = (
|
||
now + timedelta(minutes=connection.sync_interval_minutes)
|
||
if connection.auto_sync
|
||
else None
|
||
)
|
||
connection.save(
|
||
update_fields=[
|
||
"status",
|
||
"last_error",
|
||
"last_sync_finished_at",
|
||
"next_sync_at",
|
||
"updated_at",
|
||
]
|
||
)
|
||
return result
|