Switch production deploy to blue-green slots

2026-09-04 16:00:47 +03:00 · 2026-09-04 16:00:47 +03:00 · cd509c4290
commit cd509c4290
parent 6bb9811cd6
8 changed files with 561 additions and 159 deletions
--- a/.gitattributes
+++ b/.gitattributes
@ -1,2 +1,3 @@
 ops/deploy/* text eol=lf
+ops/deploy/systemd/*.service text eol=lf
 .gitea/workflows/*.yml text eol=lf
--- a/ops/deploy/README.md
+++ b/ops/deploy/README.md
@ -1,30 +1,44 @@
-# CORP production deployment
+# CORP blue-green production deployment

-Production is deployed from the current `master` commit in the Gitea repository
-`Robotail/Corp`. The Gitea Actions runner only opens a restricted SSH connection;
-all validation, backup, build, migration, restart, health checks, and rollback are
-performed on the CORP server.
+Production is deployed from the current `master` commit in Gitea repository
+`Robotail/Corp`. The Gitea runner only opens a restricted SSH connection. Build,
+backup, migrations, health checks, and traffic switching run on the CORP server.
+
+Two native slots are used instead of containers:
+
+- `blue`: backend `9001`, frontend `3001`;
+- `green`: backend `9002`, frontend `3002`;
+- releases: `/srv/eusrr-releases/<commit-sha>`;
+- slot links: `/srv/eusrr-slots/{blue,green}`;
+- active release link: `/srv/eusrr-current`.
+
+The inactive release is built and checked while the active release continues to
+serve users. Nginx then reloads an atomic upstream change. Existing WebSocket
+connections may reconnect when the old backend is drained, but HTTP traffic does
+not require a maintenance window. Old and new Next.js static assets are retained
+in `/srv/eusrr-shared/next-static` so already-open pages keep working.
+
+Database migrations run before traffic switches and therefore must be backward
+compatible with the currently active application. Do not combine destructive
+schema removal with the release that stops using the schema. PostgreSQL snapshots
+are stored in `/var/backups/eusrr/deploy` for 14 days, but they are not restored
+automatically while the old release is accepting writes.

 ## Bootstrap

-1. Generate a dedicated Ed25519 key pair for the runner. Never commit the private
-   key.
-2. Copy the public key to the CORP server and install the restricted command:
+1. Generate a dedicated Ed25519 runner key. Never commit its private part.
+2. Copy the public key to CORP and run:

   ```sh
   sudo /srv/eusrr/ops/deploy/install-corp-runner-access /tmp/eusrr-gitea-runner.pub
   ```

-3. Add the private key as the `CORP_DEPLOY_SSH_KEY` repository Actions secret in
-   Gitea.
-4. Run **Deploy production** manually from the `master` branch and verify the
-   result before enabling automatic runs on pushes to `master`.
+3. Store the private key as repository Actions secret
+   `CORP_DEPLOY_SSH_KEY` in Gitea.
+4. Run **Deploy production** manually from `master` and verify the first blue
+   deployment before enabling automatic master deployments.

-The installer grants the `deploy` account no general root shell. The runner key
-is forced through `/usr/local/sbin/eusrr-deploy-ssh`, and the deployment command
-rejects every revision except the current `origin/master` SHA.
-
-PostgreSQL snapshots are stored in `/var/backups/eusrr/deploy` for 14 days. The
-last successful revision is recorded in
-`/var/lib/eusrr-deploy/last-successful-sha`, and deployment output is appended to
-`/var/log/eusrr-deploy.log`.
+The SSH key is forced through `/usr/local/sbin/eusrr-deploy-ssh`. The deploy
+command rejects every revision except the current Gitea `origin/master` SHA.
+Logs are appended to `/var/log/eusrr-deploy.log`; the active slot and successful
+SHA are recorded under `/var/lib/eusrr-deploy`.
--- a/ops/deploy/eusrr-deploy
+++ b/ops/deploy/eusrr-deploy
@ -2,19 +2,17 @@
 set -Eeuo pipefail

 readonly APP_ROOT="/srv/eusrr"
-readonly BACKEND_DIR="${APP_ROOT}/backend"
-readonly FRONTEND_DIR="${APP_ROOT}/frontend"
+readonly RELEASES_ROOT="/srv/eusrr-releases"
+readonly SLOTS_ROOT="/srv/eusrr-slots"
+readonly SHARED_ROOT="/srv/eusrr-shared"
+readonly CURRENT_LINK="/srv/eusrr-current"
 readonly DEPLOY_USER="deploy"
 readonly BACKUP_DIR="/var/backups/eusrr/deploy"
 readonly STATE_DIR="/var/lib/eusrr-deploy"
+readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot"
 readonly LOG_FILE="/var/log/eusrr-deploy.log"
 readonly LOCK_FILE="/run/lock/eusrr-deploy.lock"
-readonly -a SERVICES=(
-  gunicorn-eusrr.service
-  nextjs-eusrr.service
-  celery-eusrr-worker.service
-  celery-eusrr-beat.service
-)
+readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf"

 if [[ ${EUID} -ne 0 ]]; then
  echo "This command must run as root." >&2
@ -34,22 +32,33 @@ run_as_deploy() {
  runuser -u "${DEPLOY_USER}" -- "$@"
 }

-run_in_backend() {
+run_in_release_backend() {
  runuser -u "${DEPLOY_USER}" -- \
-    bash -c 'cd "$1"; shift; exec "$@"' _ "${BACKEND_DIR}" "$@"
+    bash -c 'cd "$1"; shift; exec "$@"' _ "${NEW_RELEASE}/backend" "$@"
+}
+
+slot_ports() {
+  case "$1" in
+    blue)  BACKEND_PORT=9001; FRONTEND_PORT=3001 ;;
+    green) BACKEND_PORT=9002; FRONTEND_PORT=3002 ;;
+    *) log "Unknown deployment slot: $1"; return 1 ;;
+  esac
 }

 if [[ ${1:-} == "--self-test" ]]; then
-  for command in git curl flock runuser pg_dump pg_restore npm; do
+  for command in git curl flock runuser pg_dump npm nginx rsync; do
    command -v "${command}" >/dev/null
  done
  test -d "${APP_ROOT}/.git"
-  test -x "${BACKEND_DIR}/.venv/bin/python"
-  run_as_deploy test -r "${BACKEND_DIR}/.env"
-  for service in "${SERVICES[@]}"; do
-    systemctl cat "${service}" >/dev/null
+  run_as_deploy test -r "${APP_ROOT}/backend/.env"
+  run_as_deploy test -r "${APP_ROOT}/frontend/.env"
+  test -r "${UPSTREAM_FILE}"
+  for unit in eusrr-backend@.service eusrr-frontend@.service \
+    eusrr-celery-worker@.service eusrr-celery-beat@.service; do
+    systemctl cat "${unit}" >/dev/null
  done
-  log "Self-test passed."
+  nginx -t
+  log "Blue-green deployment self-test passed."
  exit 0
 fi

@ -65,20 +74,37 @@ if ! flock -n 9; then
  exit 3
 fi

-OLD_SHA=""
+OLD_SLOT="$(cat "${ACTIVE_SLOT_FILE}" 2>/dev/null || printf legacy)"
+case "${OLD_SLOT}" in
+  legacy) NEW_SLOT=blue ;;
+  blue) NEW_SLOT=green ;;
+  green) NEW_SLOT=blue ;;
+  *) log "Invalid active slot state: ${OLD_SLOT}"; exit 4 ;;
+esac
+
+NEW_RELEASE="${RELEASES_ROOT}/${TARGET_SHA}"
+if [[ ${OLD_SLOT} == legacy ]]; then
+  OLD_RELEASE="${APP_ROOT}"
+else
+  OLD_RELEASE="$(readlink -f "${SLOTS_ROOT}/${OLD_SLOT}")"
+fi
+
+BACKEND_PORT=""
+FRONTEND_PORT=""
+slot_ports "${NEW_SLOT}"
+
+UPSTREAM_BACKUP="${STATE_DIR}/upstreams-before-${TARGET_SHA}"
+UPSTREAM_CHANGED=0
+OLD_STACK_STOPPED=0
+NEW_STACK_ENABLED=0
 DB_DUMP=""
-SERVICES_STOPPED=0
-MIGRATION_STARTED=0
-
-run_database_tool() {
-  local action="$1"
-  local dump_file="$2"

+run_database_dump() {
+  local dump_file="$1"
  runuser -u "${DEPLOY_USER}" -- \
-    env EUSRR_DB_ACTION="${action}" EUSRR_DB_DUMP="${dump_file}" \
-    "${BACKEND_DIR}/.venv/bin/python" - <<'PY'
+    env EUSRR_DB_DUMP="${dump_file}" \
+    "${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY'
 import os
-import sys
 from pathlib import Path

 from dotenv import dotenv_values
@ -91,66 +117,41 @@ def required(name: str) -> str:
        raise SystemExit(f"Required database setting is missing: {name}")
    return str(value)

-action = os.environ["EUSRR_DB_ACTION"]
-dump_file = os.environ["EUSRR_DB_DUMP"]
-database = required("POSTGRES_DB")
-user = required("POSTGRES_USER")
-password = required("POSTGRES_PASSWORD")
-host = str(config.get("DB_HOST") or "127.0.0.1")
-port = str(config.get("DB_PORT") or "5432")
-
-environment = os.environ.copy()
-environment["PGPASSWORD"] = password
-common = ["--host", host, "--port", port, "--username", user, "--dbname", database]
-
-if action == "dump":
-    executable = "/usr/bin/pg_dump"
-    arguments = [executable, *common, "--format=custom", "--no-owner", "--no-acl", "--file", dump_file]
-elif action == "restore":
-    executable = "/usr/bin/pg_restore"
-    arguments = [
-        executable,
-        *common,
-        "--clean",
-        "--if-exists",
-        "--no-owner",
-        "--no-acl",
-        "--exit-on-error",
-        "--single-transaction",
-        dump_file,
-    ]
-else:
-    raise SystemExit(f"Unsupported database action: {action}")
-
+executable = "/usr/bin/pg_dump"
 if not Path(executable).is_file():
    raise SystemExit(f"Database utility is not installed: {executable}")
+
+environment = os.environ.copy()
+environment["PGPASSWORD"] = required("POSTGRES_PASSWORD")
+arguments = [
+    executable,
+    "--host", str(config.get("DB_HOST") or "127.0.0.1"),
+    "--port", str(config.get("DB_PORT") or "5432"),
+    "--username", required("POSTGRES_USER"),
+    "--dbname", required("POSTGRES_DB"),
+    "--format=custom",
+    "--no-owner",
+    "--no-acl",
+    "--file", os.environ["EUSRR_DB_DUMP"],
+]
 os.execve(executable, arguments, environment)
 PY
 }

-install_dependencies_and_build() {
-  log "Installing pinned backend dependencies."
-  run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \
-    "${BACKEND_DIR}/.venv/bin/python" -m pip install --requirement "${BACKEND_DIR}/requirements.txt"
-
-  log "Installing pinned frontend dependencies."
-  run_as_deploy npm --prefix "${FRONTEND_DIR}" ci --no-audit --no-fund
-
-  log "Building the frontend."
-  run_as_deploy npm --prefix "${FRONTEND_DIR}" run build
-
-  log "Running Django system checks."
-  run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" check
-}
-
 wait_for_http() {
  local name="$1"
  local url="$2"
  local expected="$3"
+  local host_header="${4:-}"
  local attempt code
+  local -a curl_args=(--silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5)
+
+  if [[ -n ${host_header} ]]; then
+    curl_args+=(--header "Host: ${host_header}")
+  fi

  for attempt in $(seq 1 45); do
-    code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5 "${url}" || true)"
+    code="$(curl "${curl_args[@]}" "${url}" || true)"
    if [[ ${code} =~ ${expected} ]]; then
      log "Health check passed: ${name} returned ${code}."
      return 0
@ -162,91 +163,253 @@ wait_for_http() {
  return 1
 }

-start_and_verify_services() {
-  systemctl restart "${SERVICES[@]}"
-  SERVICES_STOPPED=0
-  for service in "${SERVICES[@]}"; do
-    systemctl is-active --quiet "${service}"
-  done
-  wait_for_http "frontend" "http://127.0.0.1:3000/" '^200$'
-  wait_for_http "backend" "http://127.0.0.1:9000/admin/login/" '^(200|302)$'
+write_upstreams() {
+  local backend_port="$1"
+  local frontend_port="$2"
+  local temporary_file
+  temporary_file="$(mktemp)"
+  cat >"${temporary_file}" <<EOF
+upstream eusrr_backend {
+    server 127.0.0.1:${backend_port};
+    keepalive 32;
+}
+
+upstream eusrr_frontend {
+    server 127.0.0.1:${frontend_port};
+    keepalive 32;
+}
+EOF
+  install -o root -g root -m 0644 "${temporary_file}" "${UPSTREAM_FILE}"
+  rm -f "${temporary_file}"
+  UPSTREAM_CHANGED=1
+}
+
+point_current_at() {
+  local release="$1"
+  ln -sfn "${release}" "${CURRENT_LINK}.new"
+  mv -Tf "${CURRENT_LINK}.new" "${CURRENT_LINK}"
+}
+
+start_slot_web() {
+  local slot="$1"
+  systemctl start "eusrr-backend@${slot}.service" "eusrr-frontend@${slot}.service"
+}
+
+stop_slot_stack() {
+  local slot="$1"
+  systemctl stop \
+    "eusrr-celery-beat@${slot}.service" \
+    "eusrr-celery-worker@${slot}.service" \
+    "eusrr-frontend@${slot}.service" \
+    "eusrr-backend@${slot}.service" || true
+}
+
+start_slot_celery() {
+  local slot="$1"
+  systemctl start "eusrr-celery-worker@${slot}.service" "eusrr-celery-beat@${slot}.service"
+  systemctl is-active --quiet "eusrr-celery-worker@${slot}.service"
+  systemctl is-active --quiet "eusrr-celery-beat@${slot}.service"
+}
+
+enable_slot_stack() {
+  local slot="$1"
+  systemctl enable \
+    "eusrr-backend@${slot}.service" \
+    "eusrr-frontend@${slot}.service" \
+    "eusrr-celery-worker@${slot}.service" \
+    "eusrr-celery-beat@${slot}.service"
+}
+
+disable_old_stack() {
+  if [[ ${OLD_SLOT} == legacy ]]; then
+    systemctl disable gunicorn-eusrr.service nextjs-eusrr.service \
+      celery-eusrr-worker.service celery-eusrr-beat.service || true
+  else
+    systemctl disable \
+      "eusrr-backend@${OLD_SLOT}.service" \
+      "eusrr-frontend@${OLD_SLOT}.service" \
+      "eusrr-celery-worker@${OLD_SLOT}.service" \
+      "eusrr-celery-beat@${OLD_SLOT}.service" || true
+  fi
+}
+
+enable_old_stack() {
+  if [[ ${OLD_SLOT} == legacy ]]; then
+    systemctl enable gunicorn-eusrr.service nextjs-eusrr.service \
+      celery-eusrr-worker.service celery-eusrr-beat.service || true
+  else
+    systemctl enable \
+      "eusrr-backend@${OLD_SLOT}.service" \
+      "eusrr-frontend@${OLD_SLOT}.service" \
+      "eusrr-celery-worker@${OLD_SLOT}.service" \
+      "eusrr-celery-beat@${OLD_SLOT}.service" || true
+  fi
+}
+
+stop_legacy_stack() {
+  systemctl stop \
+    celery-eusrr-beat.service celery-eusrr-worker.service \
+    nextjs-eusrr.service gunicorn-eusrr.service || true
+}
+
+start_old_stack() {
+  if [[ ${OLD_SLOT} == legacy ]]; then
+    systemctl restart gunicorn-eusrr.service nextjs-eusrr.service \
+      celery-eusrr-worker.service celery-eusrr-beat.service
+  else
+    systemctl restart "eusrr-backend@${OLD_SLOT}.service" "eusrr-frontend@${OLD_SLOT}.service" \
+      "eusrr-celery-worker@${OLD_SLOT}.service" "eusrr-celery-beat@${OLD_SLOT}.service"
+  fi
+}
+
+remove_release() {
+  local release="$1"
+  case "${release}" in
+    "${RELEASES_ROOT}"/*) ;;
+    *) log "Refusing to remove release outside ${RELEASES_ROOT}: ${release}"; return 1 ;;
+  esac
+  if [[ -e ${release} ]]; then
+    run_as_deploy git -C "${APP_ROOT}" worktree remove --force "${release}"
+  fi
 }

 rollback() {
  local original_status=$?
  trap - ERR
  set +e
-  log "Deployment failed with exit code ${original_status}; starting rollback."
+  log "Deployment failed with exit code ${original_status}; keeping the previous release active."

-  if [[ ${SERVICES_STOPPED} -eq 0 && ${MIGRATION_STARTED} -eq 1 ]]; then
-    systemctl stop "${SERVICES[@]}"
-    SERVICES_STOPPED=1
+  if [[ ${UPSTREAM_CHANGED} -eq 1 && -f ${UPSTREAM_BACKUP} ]]; then
+    install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}"
+    point_current_at "${OLD_RELEASE}"
+    nginx -t && systemctl reload nginx
  fi

-  if [[ ${MIGRATION_STARTED} -eq 1 && -n ${DB_DUMP} && -f ${DB_DUMP} ]]; then
-    log "Restoring the database snapshot."
-    run_database_tool restore "${DB_DUMP}" || log "ERROR: database restore failed; manual recovery is required."
+  stop_slot_stack "${NEW_SLOT}"
+  if [[ ${NEW_STACK_ENABLED} -eq 1 ]]; then
+    systemctl disable \
+      "eusrr-backend@${NEW_SLOT}.service" \
+      "eusrr-frontend@${NEW_SLOT}.service" \
+      "eusrr-celery-worker@${NEW_SLOT}.service" \
+      "eusrr-celery-beat@${NEW_SLOT}.service" || true
+    enable_old_stack
+  fi
+  if [[ ${OLD_STACK_STOPPED} -eq 1 ]]; then
+    start_old_stack || log "ERROR: failed to restart the previous application stack."
  fi

-  if [[ -n ${OLD_SHA} ]]; then
-    log "Restoring source revision ${OLD_SHA}."
-    run_as_deploy git -C "${APP_ROOT}" reset --hard "${OLD_SHA}"
-    install_dependencies_and_build || log "ERROR: rebuilding the previous revision failed."
-    run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput \
-      || log "ERROR: restoring static files failed."
-  fi
-
-  systemctl restart "${SERVICES[@]}" || true
-  log "Rollback finished. Inspect ${LOG_FILE} and service journals before retrying."
+  log "No database snapshot was restored because the previous release remained live and may have accepted writes."
+  log "Inspect ${LOG_FILE} and service journals before retrying."
  exit "${original_status}"
 }
 trap rollback ERR

-log "Starting deployment of ${TARGET_SHA}."
-OLD_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse HEAD)"
-
-dirty="$(run_as_deploy git -C "${APP_ROOT}" status --porcelain --untracked-files=normal)"
-unexpected_dirty="$(printf '%s\n' "${dirty}" | grep -vE '^ M frontend/next-env\.d\.ts$' || true)"
-if [[ -n ${unexpected_dirty} ]]; then
-  log "Refusing to overwrite unexpected production changes:"
-  printf '%s\n' "${unexpected_dirty}"
-  exit 4
-fi
-
-run_as_deploy git -C "${APP_ROOT}" restore --worktree frontend/next-env.d.ts 2>/dev/null || true
+log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}."
 run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \
  '+refs/heads/master:refs/remotes/origin/master'
-
 MASTER_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse refs/remotes/origin/master)"
 if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then
  log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})."
  exit 5
 fi
+if [[ ${TARGET_SHA} == "$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)" ]]; then
+  log "Revision ${TARGET_SHA} is already active."
+  exit 0
+fi

-run_as_deploy git -C "${APP_ROOT}" reset --hard "${TARGET_SHA}"
-install_dependencies_and_build
+if [[ -L ${SLOTS_ROOT}/${NEW_SLOT} ]]; then
+  stale_release="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}")"
+  if [[ ${stale_release} != "${OLD_RELEASE}" ]]; then
+    rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
+    remove_release "${stale_release}"
+  fi
+fi
+remove_release "${NEW_RELEASE}"
+run_as_deploy git -C "${APP_ROOT}" worktree prune

-log "Stopping application services for the database-safe deployment window."
-systemctl stop "${SERVICES[@]}"
-SERVICES_STOPPED=1
+available_bytes="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
+if (( available_bytes < 4294967296 )); then
+  log "At least 4 GiB of free disk space is required to build a release."
+  exit 6
+fi
+
+log "Creating isolated Git worktree ${NEW_RELEASE}."
+run_as_deploy git -C "${APP_ROOT}" worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}"
+
+run_as_deploy ln -s "${APP_ROOT}/backend/.env" "${NEW_RELEASE}/backend/.env"
+run_as_deploy ln -s "${APP_ROOT}/frontend/.env" "${NEW_RELEASE}/frontend/.env"
+run_as_deploy rm -rf "${NEW_RELEASE}/backend/media"
+run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media"
+
+log "Creating an isolated Python environment."
+run_as_deploy /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv"
+run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \
+  "${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \
+  --requirement "${NEW_RELEASE}/backend/requirements.txt"
+
+log "Installing pinned frontend dependencies and building Next.js."
+run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" ci --no-audit --no-fund
+run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" run build
+
+log "Running Django checks and collecting release static files."
+run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
+  "${NEW_RELEASE}/backend/manage.py" check
+run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
+  "${NEW_RELEASE}/backend/manage.py" collectstatic --noinput
+
+run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/"

 install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}"
 DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump"
-log "Creating PostgreSQL snapshot ${DB_DUMP}."
-run_database_tool dump "${DB_DUMP}"
+log "Creating a consistent PostgreSQL snapshot while the current release remains online."
+run_database_dump "${DB_DUMP}"
 chmod 0600 "${DB_DUMP}"

-MIGRATION_STARTED=1
-log "Applying database migrations."
-run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" migrate --noinput
+log "Planned migrations:"
+run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
+  "${NEW_RELEASE}/backend/manage.py" migrate --plan
+log "Applying backward-compatible database migrations."
+run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
+  "${NEW_RELEASE}/backend/manage.py" migrate --noinput

-log "Collecting Django static files."
-run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput
+log "Assigning release to the ${NEW_SLOT} slot and starting its web services."
+ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new"
+mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}"
+start_slot_web "${NEW_SLOT}"

-log "Starting and checking application services."
-start_and_verify_services
+wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$'
+wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$'

+cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}"
+UPSTREAM_CHANGED=1
+point_current_at "${NEW_RELEASE}"
+write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}"
+nginx -t
+systemctl reload nginx
+
+wait_for_http "public frontend" "http://127.0.0.1/" '^200$' '172.11.0.11'
+wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '172.11.0.11'
+
+log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain."
+sleep 10
+log "Switching background workers."
+if [[ ${OLD_SLOT} == legacy ]]; then
+  stop_legacy_stack
+else
+  stop_slot_stack "${OLD_SLOT}"
+fi
+OLD_STACK_STOPPED=1
+start_slot_celery "${NEW_SLOT}"
+NEW_STACK_ENABLED=1
+enable_slot_stack "${NEW_SLOT}"
+disable_old_stack
+
+find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \
+  || log "WARNING: old database snapshots could not be pruned."
+run_as_deploy find "${SHARED_ROOT}/next-static" -type f -mtime +30 -delete \
+  || log "WARNING: old Next.js assets could not be pruned."
+rm -f "${UPSTREAM_BACKUP}" || true
+printf '%s\n' "${NEW_SLOT}" >"${ACTIVE_SLOT_FILE}"
 printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha"
-find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete
 trap - ERR
-log "Deployment of ${TARGET_SHA} completed successfully."
+log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}."
--- a/ops/deploy/install-corp-runner-access
+++ b/ops/deploy/install-corp-runner-access
@ -8,28 +8,160 @@ fi

 readonly SOURCE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
 readonly PUBLIC_KEY_FILE="${1:-}"
+readonly NGINX_LINK="/etc/nginx/sites-enabled/eusrr"
+readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf"

 if [[ -z ${PUBLIC_KEY_FILE} || ! -f ${PUBLIC_KEY_FILE} ]]; then
  echo "Usage: $0 /path/to/gitea-runner.pub" >&2
  exit 2
 fi
-
 if ! id deploy >/dev/null 2>&1; then
  echo "The deploy system account does not exist." >&2
  exit 3
 fi
+if [[ ! -d ${SOURCE_DIR}/systemd ]]; then
+  echo "Systemd templates are missing from ${SOURCE_DIR}." >&2
+  exit 4
+fi

 read -r key_type key_body _ <"${PUBLIC_KEY_FILE}"
 if [[ ${key_type} != "ssh-ed25519" || -z ${key_body:-} ]]; then
  echo "Expected an Ed25519 public key." >&2
-  exit 4
+  exit 5
 fi

+nginx_config="$(readlink -f "${NGINX_LINK}")"
+nginx_backup="${nginx_config}.bak.blue-green.$(date -u +%Y%m%dT%H%M%SZ)"
+cp -a "${nginx_config}" "${nginx_backup}"
+upstream_existed=0
+upstream_backup=""
+if [[ -f ${UPSTREAM_FILE} ]]; then
+  upstream_existed=1
+  upstream_backup="${UPSTREAM_FILE}.bak.$(date -u +%Y%m%dT%H%M%SZ)"
+  cp -a "${UPSTREAM_FILE}" "${upstream_backup}"
+fi
+
+install_complete=0
+temporary_files=()
+cleanup() {
+  local status=$?
+  for file in "${temporary_files[@]:-}"; do
+    [[ -n ${file} ]] && rm -f "${file}"
+  done
+  if [[ ${install_complete} -eq 0 ]]; then
+    cp -a "${nginx_backup}" "${nginx_config}"
+    if [[ ${upstream_existed} -eq 1 ]]; then
+      cp -a "${upstream_backup}" "${UPSTREAM_FILE}"
+    else
+      rm -f "${UPSTREAM_FILE}"
+    fi
+    nginx -t >/dev/null 2>&1 && systemctl reload nginx || true
+  fi
+  exit "${status}"
+}
+trap cleanup EXIT
+
 install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy" /usr/local/sbin/eusrr-deploy
 install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy-ssh" /usr/local/sbin/eusrr-deploy-ssh
+for unit in "${SOURCE_DIR}"/systemd/*.service; do
+  install -o root -g root -m 0644 "${unit}" "/etc/systemd/system/$(basename "${unit}")"
+done
+
+install -d -o deploy -g eusrr -m 2775 /srv/eusrr-releases /srv/eusrr-slots
+install -d -o deploy -g www-data -m 0755 /srv/eusrr-shared /srv/eusrr-shared/next-static
+if [[ -d /srv/eusrr/frontend/.next/static ]]; then
+  rsync -a /srv/eusrr/frontend/.next/static/ /srv/eusrr-shared/next-static/
+fi
+chown -R deploy:www-data /srv/eusrr-shared
+find /srv/eusrr-shared -type d -exec chmod 0755 {} +
+find /srv/eusrr-shared -type f -exec chmod 0644 {} +
+
+ln -sfn /srv/eusrr /srv/eusrr-current.new
+mv -Tf /srv/eusrr-current.new /srv/eusrr-current
+
+if [[ ! -f ${UPSTREAM_FILE} ]]; then
+  cat >"${UPSTREAM_FILE}" <<'EOF'
+upstream eusrr_backend {
+    server 127.0.0.1:9000;
+    keepalive 32;
+}
+
+upstream eusrr_frontend {
+    server 127.0.0.1:3000;
+    keepalive 32;
+}
+EOF
+  chmod 0644 "${UPSTREAM_FILE}"
+fi
+
+NGINX_CONFIG="${nginx_config}" python3 <<'PY'
+import os
+import re
+from pathlib import Path
+
+path = Path(os.environ["NGINX_CONFIG"])
+text = path.read_text(encoding="utf-8")
+
+next_static = re.compile(
+    r"^    location /_next/static/ \{\n.*?^    \}",
+    flags=re.MULTILINE | re.DOTALL,
+)
+replacement = """    location /_next/static/ {
+        alias /srv/eusrr-shared/next-static/;
+        autoindex off;
+        expires 365d;
+        add_header Cache-Control \"public, immutable\";
+    }"""
+text, static_blocks = next_static.subn(replacement, text)
+if static_blocks not in (0, 2):
+    raise SystemExit(f"Expected zero or two Next.js static blocks, found {static_blocks}")
+
+text = text.replace("proxy_pass http://127.0.0.1:9000;", "proxy_pass http://eusrr_backend;")
+text = text.replace("proxy_pass http://127.0.0.1:3000;", "proxy_pass http://eusrr_frontend;")
+text = text.replace(
+    "alias /srv/eusrr/backend/staticfiles/;",
+    "alias /srv/eusrr-current/backend/staticfiles/;",
+)
+
+required = (
+    "proxy_pass http://eusrr_backend;",
+    "proxy_pass http://eusrr_frontend;",
+    "alias /srv/eusrr-shared/next-static/;",
+    "alias /srv/eusrr-current/backend/staticfiles/;",
+)
+for marker in required:
+    if marker not in text:
+        raise SystemExit(f"Nginx transformation marker is missing: {marker}")
+if "proxy_pass http://127.0.0.1:9000;" in text or "proxy_pass http://127.0.0.1:3000;" in text:
+    raise SystemExit("Legacy direct upstreams remain in Nginx configuration")
+
+path.write_text(text, encoding="utf-8")
+PY
+
+install -d -o root -g root -m 0755 /etc/eusrr/slots /var/lib/eusrr-deploy
+cat >/etc/eusrr/slots/blue.env <<'EOF'
+BACKEND_PORT=9001
+PORT=3001
+HOSTNAME=127.0.0.1
+EOF
+cat >/etc/eusrr/slots/green.env <<'EOF'
+BACKEND_PORT=9002
+PORT=3002
+HOSTNAME=127.0.0.1
+EOF
+chmod 0644 /etc/eusrr/slots/blue.env /etc/eusrr/slots/green.env
+if [[ ! -f /var/lib/eusrr-deploy/active-slot ]]; then
+  printf '%s\n' legacy >/var/lib/eusrr-deploy/active-slot
+fi
+
+install -d -o deploy -g deploy -m 0755 /var/log/celery
+install -d -o deploy -g deploy -m 0700 /var/backups/eusrr/deploy
+touch /var/log/eusrr-deploy.log
+chown root:root /var/log/eusrr-deploy.log
+chmod 0600 /var/log/eusrr-deploy.log

 sudoers_tmp="$(mktemp)"
-trap 'rm -f "${sudoers_tmp}"' EXIT
+temporary_files+=("${sudoers_tmp}")
 printf '%s\n' 'deploy ALL=(root) NOPASSWD: /usr/local/sbin/eusrr-deploy *' >"${sudoers_tmp}"
 chmod 0440 "${sudoers_tmp}"
 visudo -cf "${sudoers_tmp}" >/dev/null
@ -41,20 +173,18 @@ authorized_keys="${deploy_home}/.ssh/authorized_keys"
 touch "${authorized_keys}"
 chown deploy:deploy "${authorized_keys}"
 chmod 0600 "${authorized_keys}"
-
 filtered_keys="$(mktemp)"
+temporary_files+=("${filtered_keys}")
 grep -v ' eusrr-gitea-runner$' "${authorized_keys}" >"${filtered_keys}" || true
 printf '%s %s %s %s\n' \
  'command="/usr/local/sbin/eusrr-deploy-ssh",restrict' \
  "${key_type}" "${key_body}" 'eusrr-gitea-runner' >>"${filtered_keys}"
 install -o deploy -g deploy -m 0600 "${filtered_keys}" "${authorized_keys}"
-rm -f "${filtered_keys}"
-
-install -d -o root -g root -m 0755 /var/lib/eusrr-deploy
-install -d -o deploy -g deploy -m 0700 /var/backups/eusrr/deploy
-touch /var/log/eusrr-deploy.log
-chown root:root /var/log/eusrr-deploy.log
-chmod 0600 /var/log/eusrr-deploy.log

+systemctl daemon-reload
+nginx -t
+systemctl reload nginx
 sudo -u deploy sudo -n /usr/local/sbin/eusrr-deploy --self-test
-echo "Restricted Gitea runner access installed successfully."
+
+install_complete=1
+echo "Restricted blue-green Gitea deployment installed successfully."
--- a/ops/deploy/systemd/eusrr-backend@.service
+++ b/ops/deploy/systemd/eusrr-backend@.service
@ -0,0 +1,24 @@
+[Unit]
+Description=EUSRR backend (%i slot)
+After=network-online.target redis-server.service
+Wants=network-online.target redis-server.service
+
+[Service]
+Type=simple
+User=deploy
+Group=www-data
+WorkingDirectory=/srv/eusrr-slots/%i/backend
+EnvironmentFile=/etc/eusrr/slots/%i.env
+Environment=DJANGO_SETTINGS_MODULE=eusrr_backend.settings
+Environment=PYTHONUNBUFFERED=1
+ExecStart=/srv/eusrr-slots/%i/backend/.venv/bin/python -m gunicorn -k uvicorn.workers.UvicornWorker --bind 127.0.0.1:${BACKEND_PORT} --workers 2 --timeout 60 --graceful-timeout 30 --access-logfile - --error-logfile - --log-level info eusrr_backend.asgi:application
+Restart=always
+RestartSec=3
+KillSignal=SIGQUIT
+TimeoutStopSec=30
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=eusrr-backend-%i
+
+[Install]
+WantedBy=multi-user.target
--- a/ops/deploy/systemd/eusrr-celery-beat@.service
+++ b/ops/deploy/systemd/eusrr-celery-beat@.service
@ -0,0 +1,24 @@
+[Unit]
+Description=EUSRR Celery beat (%i slot)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=deploy
+Group=deploy
+WorkingDirectory=/srv/eusrr-slots/%i/backend
+EnvironmentFile=/srv/eusrr-slots/%i/backend/.env
+Environment=DJANGO_SETTINGS_MODULE=eusrr_backend.settings
+RuntimeDirectory=eusrr-celery-%i
+RuntimeDirectoryMode=0755
+ExecStart=/srv/eusrr-slots/%i/backend/.venv/bin/python -m celery -A eusrr_backend beat --loglevel=info --logfile=/var/log/celery/beat-%i.log --scheduler django_celery_beat.schedulers:DatabaseScheduler --pidfile=/run/eusrr-celery-%i/beat.pid
+Restart=always
+RestartSec=10
+TimeoutStopSec=30
+KillSignal=SIGTERM
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target
--- a/ops/deploy/systemd/eusrr-celery-worker@.service
+++ b/ops/deploy/systemd/eusrr-celery-worker@.service
@ -0,0 +1,22 @@
+[Unit]
+Description=EUSRR Celery worker (%i slot)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=deploy
+Group=deploy
+WorkingDirectory=/srv/eusrr-slots/%i/backend
+EnvironmentFile=/srv/eusrr-slots/%i/backend/.env
+Environment=DJANGO_SETTINGS_MODULE=eusrr_backend.settings
+ExecStart=/srv/eusrr-slots/%i/backend/.venv/bin/python -m celery -A eusrr_backend worker --loglevel=info --logfile=/var/log/celery/worker-%i.log --concurrency=4 -Q celery,notifications,commerce
+Restart=always
+RestartSec=10
+TimeoutStopSec=60
+KillSignal=SIGTERM
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target
--- a/ops/deploy/systemd/eusrr-frontend@.service
+++ b/ops/deploy/systemd/eusrr-frontend@.service
@ -0,0 +1,24 @@
+[Unit]
+Description=EUSRR frontend (%i slot)
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=simple
+User=deploy
+Group=www-data
+WorkingDirectory=/srv/eusrr-slots/%i/frontend
+EnvironmentFile=/etc/eusrr/slots/%i.env
+Environment=NODE_ENV=production
+ExecStartPre=/bin/sh -c 'cp -r /srv/eusrr-slots/%i/frontend/public /srv/eusrr-slots/%i/frontend/.next/standalone/ 2>/dev/null || true'
+ExecStartPre=/bin/sh -c 'cp -r /srv/eusrr-slots/%i/frontend/.next/static /srv/eusrr-slots/%i/frontend/.next/standalone/.next/ 2>/dev/null || true'
+ExecStart=/usr/bin/node /srv/eusrr-slots/%i/frontend/.next/standalone/server.js
+Restart=always
+RestartSec=3
+TimeoutStopSec=30
+StandardOutput=journal
+StandardError=journal
+SyslogIdentifier=eusrr-frontend-%i
+
+[Install]
+WantedBy=multi-user.target