Harden deployment cleanup, rollback and worker lifecycle

ops/deploy/README.md

@@ -46,6 +46,50 @@ command rejects every revision except the current Gitea `origin/master` SHA.
 Logs are appended to `/var/log/eusrr-deploy.log`; the active slot and successful
 SHA are recorded under `/var/lib/eusrr-deploy`.
 
+## Failure handling and disk usage
+
+The active and previous successful releases are retained. Before each attempt,
+unused Git worktrees are removed under the deployment lock. Failed builds are
+removed on error, SIGTERM, SIGINT or SIGHUP, after their processes and slot services
+have stopped. Symlink releases, mounted directories and referenced releases are
+never removed. A failed cleanup aborts the operation instead of deleting broadly.
+
+The deployment requires at least 4 GiB and 50,000 free inodes before creating a
+release. Build and database-dump process groups have a 30-minute timeout and are
+monitored every two seconds; they are stopped if free space falls below 512 MiB
+or free inodes below 50,000. This reserve is a guard, not a filesystem quota:
+other services can also consume space. Pip uses no persistent cache, and the npm
+download cache is confined to the candidate release and removed after install.
+Database snapshots are validated before receiving the final `.dump` filename;
+partial dumps are removed on failure. Existing complete dumps retain their
+14-day retention. Deployment and native Celery logs rotate through logrotate.
+
+A pending-deployment journal survives a power failure or SIGKILL. The next run
+cleans up an interrupted candidate only when the current symlink, active slot,
+successful revision and Nginx upstreams still agree. If traffic/state changes were
+interrupted, deployment stops for manual recovery and preserves the releases and
+upstream backup. Database changes are never automatically reversed.
+
+Legacy Celery services (including their umbrella unit) are stopped and disabled
+once native workers are live. Systemd conditions prevent dependency-driven
+restarts of legacy workers. The old slot must stop successfully before the new
+Celery scheduler starts.
+
+To update tooling and perform maintenance without deploying application code:
+
+```sh
+sudo /path/to/ops/deploy/install-deploy-hardening
+```
+
+For subsequent maintenance use `sudo /usr/local/sbin/eusrr-deploy --maintenance`.
+This acquires the same lock, validates live state, reconciles legacy workers and
+cleans orphan releases. It neither builds a release nor runs migrations.
+Linux integration tests run with:
+
+```sh
+python3 -m unittest discover -s ops/deploy/tests -p test_deploy_lifecycle.py -v
+```
+
 The final workflow step reports successful, failed, cancelled, and unknown
 deployment outcomes with the repository, branch, revision, actor, current
 production state, and Gitea run link. Reporting is skipped when either Telegram

ops/deploy/eusrr-deploy

@@ -13,6 +13,8 @@ readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot"
 readonly LOG_FILE="/var/log/eusrr-deploy.log"
 readonly LOCK_FILE="/run/lock/eusrr-deploy.lock"
 readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf"
+readonly PREVIOUS_RELEASE_FILE="${STATE_DIR}/previous-release"
+readonly PENDING_FILE="${STATE_DIR}/pending-deployment"
 
 if [[ ${EUID} -ne 0 ]]; then
   echo "This command must run as root." >&2
@@ -46,7 +48,7 @@ slot_ports() {
 }
 
 if [[ ${1:-} == "--self-test" ]]; then
-  for command in git curl flock runuser pg_dump npm nginx rsync; do
+  for command in git curl flock runuser pg_dump pg_restore npm nginx rsync setsid timeout findmnt; do
     command -v "${command}" >/dev/null
   done
   test -d "${APP_ROOT}/.git"
@@ -63,8 +65,11 @@ if [[ ${1:-} == "--self-test" ]]; then
 fi
 
 TARGET_SHA="${1:-}"
-if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then
-  echo "Usage: $0 <40-character lowercase commit SHA>" >&2
+MAINTENANCE=0
+if [[ ${TARGET_SHA} == --maintenance ]]; then
+  MAINTENANCE=1
+elif [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then
+  echo "Usage: $0 <40-character lowercase commit SHA> | --maintenance" >&2
   exit 2
 fi
 
@@ -98,10 +103,63 @@ UPSTREAM_CHANGED=0
 OLD_STACK_STOPPED=0
 NEW_STACK_ENABLED=0
 DB_DUMP=""
+DB_DUMP_PARTIAL=""
+PREVIOUS_RELEASE="$(cat "${PREVIOUS_RELEASE_FILE}" 2>/dev/null || true)"
+INACTIVE_RELEASE="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}" 2>/dev/null || true)"
+OLD_SUCCESSFUL_SHA="$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)"
+MUTATION_STARTED=0
+SLOT_ASSIGNED=0
+BUILD_PID=""
+WATCHDOG_PID=""
+COMMITTED=0
+
+atomic_state() {
+  printf '%s\n' "$2" >"$1.new"
+  chmod 0644 "$1.new"
+  mv -Tf "$1.new" "$1"
+}
+
+check_space() {
+  local minimum="$1" available inodes
+  available="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
+  inodes="$(df --output=iavail "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
+  if (( available < minimum || inodes < 50000 )); then
+    log "Insufficient capacity: ${available} bytes and ${inodes} inodes free; need ${minimum} bytes and 50000 inodes."
+    return 6
+  fi
+}
+
+# Build processes have their own session so cancellation also stops grandchildren.
+run_bounded_build() {
+  check_space 1073741824
+  setsid timeout --signal=TERM --kill-after=30s 30m runuser -u "${DEPLOY_USER}" -- "$@" <&0 &
+  BUILD_PID=$!
+  (
+    while kill -0 "${BUILD_PID}" 2>/dev/null; do
+      if ! check_space 536870912; then
+        log "Stopping build before it exhausts the production filesystem."
+        kill -TERM -- "-${BUILD_PID}" 2>/dev/null || true
+        sleep 5
+        kill -KILL -- "-${BUILD_PID}" 2>/dev/null || true
+        exit 6
+      fi
+      sleep 2
+    done
+  ) &
+  WATCHDOG_PID=$!
+  local status=0
+  wait "${BUILD_PID}" || status=$?
+  kill "${WATCHDOG_PID}" 2>/dev/null || true
+  wait "${WATCHDOG_PID}" 2>/dev/null || true
+  WATCHDOG_PID=""
+  BUILD_PID=""
+  (( status == 0 )) || return "${status}"
+  check_space 536870912
+}
 
 run_database_dump() {
   local dump_file="$1"
-  runuser -u "${DEPLOY_USER}" -- \
+  run_bounded_build \
     env EUSRR_DB_DUMP="${dump_file}" \
     "${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY'
 import os
@@ -201,7 +259,51 @@ stop_slot_stack() {
     "eusrr-celery-beat@${slot}.service" \
     "eusrr-celery-worker@${slot}.service" \
     "eusrr-frontend@${slot}.service" \
-    "eusrr-backend@${slot}.service" || true
+    "eusrr-backend@${slot}.service"
+  assert_slot_stopped "${slot}"
+}
+
+assert_slot_stopped() {
+  local unit state
+  for unit in backend frontend celery-worker celery-beat; do
+    state="$(systemctl show "eusrr-${unit}@$1.service" -p ActiveState --value)" || return 1
+    case "${state}" in inactive|failed) ;; *) log "Service is still ${state}: eusrr-${unit}@$1"; return 1 ;; esac
+  done
+}
+
+stop_slot_celery() {
+  local unit state
+  systemctl stop "eusrr-celery-beat@$1.service" "eusrr-celery-worker@$1.service"
+  for unit in celery-beat celery-worker; do
+    state="$(systemctl show "eusrr-${unit}@$1.service" -p ActiveState --value)" || return 1
+    case "${state}" in inactive|failed) ;; *) return 1 ;; esac
+  done
+}
+
+reconcile_legacy_workers() {
+  local active_slot="${1:-${OLD_SLOT}}"
+  [[ ${active_slot} != legacy ]] || return 0
+  systemctl is-active --quiet "eusrr-celery-worker@${active_slot}.service"
+  systemctl is-active --quiet "eusrr-celery-beat@${active_slot}.service"
+  local unit state
+  # A legacy umbrella service can start disabled worker/beat units again.
+  # A condition blocks dependency-driven starts as well as manual starts.
+  for unit in celery-eusrr.service celery-eusrr-beat.service celery-eusrr-worker.service; do
+    if [[ $(systemctl show "${unit}" -p LoadState --value) == not-found ]]; then
+      continue
+    fi
+    install -d -m 0755 "/etc/systemd/system/${unit}.d"
+    printf '%s\n' '[Unit]' 'ConditionPathExists=!/var/lib/eusrr-deploy/native-workers' \
+      >"/etc/systemd/system/${unit}.d/90-native-workers.conf"
+  done
+  touch "${STATE_DIR}/native-workers"
+  systemctl daemon-reload
+  for unit in celery-eusrr.service celery-eusrr-beat.service celery-eusrr-worker.service; do
+    [[ $(systemctl show "${unit}" -p LoadState --value) != not-found ]] || continue
+    systemctl disable --now "${unit}"
+    state="$(systemctl show "${unit}" -p ActiveState --value)"
+    case "${state}" in inactive|failed) ;; *) return 1 ;; esac
+  done
 }
 
 start_slot_celery() {
@@ -223,33 +325,41 @@ enable_slot_stack() {
 disable_old_stack() {
   if [[ ${OLD_SLOT} == legacy ]]; then
     systemctl disable gunicorn-eusrr.service nextjs-eusrr.service \
-      celery-eusrr-worker.service celery-eusrr-beat.service || true
+      celery-eusrr-worker.service celery-eusrr-beat.service
   else
     systemctl disable \
       "eusrr-backend@${OLD_SLOT}.service" \
       "eusrr-frontend@${OLD_SLOT}.service" \
       "eusrr-celery-worker@${OLD_SLOT}.service" \
-      "eusrr-celery-beat@${OLD_SLOT}.service" || true
+      "eusrr-celery-beat@${OLD_SLOT}.service"
   fi
 }
 
 enable_old_stack() {
   if [[ ${OLD_SLOT} == legacy ]]; then
     systemctl enable gunicorn-eusrr.service nextjs-eusrr.service \
-      celery-eusrr-worker.service celery-eusrr-beat.service || true
+      celery-eusrr-worker.service celery-eusrr-beat.service
   else
     systemctl enable \
       "eusrr-backend@${OLD_SLOT}.service" \
       "eusrr-frontend@${OLD_SLOT}.service" \
       "eusrr-celery-worker@${OLD_SLOT}.service" \
-      "eusrr-celery-beat@${OLD_SLOT}.service" || true
+      "eusrr-celery-beat@${OLD_SLOT}.service"
   fi
 }
 
 stop_legacy_stack() {
+  if [[ $(systemctl show celery-eusrr.service -p LoadState --value) != not-found ]]; then
+    systemctl stop celery-eusrr.service
+  fi
   systemctl stop \
     celery-eusrr-beat.service celery-eusrr-worker.service \
-    nextjs-eusrr.service gunicorn-eusrr.service || true
+    nextjs-eusrr.service gunicorn-eusrr.service
+  local unit state
+  for unit in celery-eusrr-beat.service celery-eusrr-worker.service nextjs-eusrr.service gunicorn-eusrr.service; do
+    state="$(systemctl show "${unit}" -p ActiveState --value)"
+    case "${state}" in inactive|failed) ;; *) return 1 ;; esac
+  done
 }
 
 start_old_stack() {
@@ -264,45 +374,184 @@ start_old_stack() {
 
 remove_release() {
   local release="$1"
-  case "${release}" in
-    "${RELEASES_ROOT}"/*) ;;
-    *) log "Refusing to remove release outside ${RELEASES_ROOT}: ${release}"; return 1 ;;
-  esac
+  local name="${release##*/}" protected resolved mount
+  [[ ${name} =~ ^[0-9a-f]{40}$ && ${release} == "${RELEASES_ROOT}/${name}" ]] || {
+    log "Refusing non-release path: ${release}"; return 1;
+  }
+  [[ ! -L ${release} ]] || { log "Refusing symlink release: ${release}"; return 1; }
+  resolved="$(readlink -m "${release}")"
+  [[ ${resolved} == "${release}" ]] || return 1
+  for protected in "${OLD_RELEASE}" "${PREVIOUS_RELEASE}" \
+    "$(readlink -f "${CURRENT_LINK}")" \
+    "$(readlink -f "${SLOTS_ROOT}/blue" 2>/dev/null || true)" \
+    "$(readlink -f "${SLOTS_ROOT}/green" 2>/dev/null || true)"; do
+    [[ ${release} != "${protected}" ]] || { log "Protected release: ${release}"; return 1; }
+  done
   if [[ -e ${release} ]]; then
+    # Git must never recurse into a bind mount containing shared data.
+    while IFS= read -r mount; do
+      [[ ${mount} != "${release}" && ${mount} != "${release}/"* ]] || return 1
+    done < <(findmnt -rn -o TARGET)
+    log "Removing unused release ${release}."
     run_as_deploy git -C "${APP_ROOT}" worktree remove --force "${release}"
   fi
 }
 
+prune_orphan_releases() {
+  local release
+  for release in "${RELEASES_ROOT}"/*; do
+    [[ -d ${release} && ! -L ${release} ]] || continue
+    [[ ${release##*/} =~ ^[0-9a-f]{40}$ ]] || continue
+    if [[ ${release} == "${OLD_RELEASE}" || ${release} == "${PREVIOUS_RELEASE}" ||
+          ${release} == "$(readlink -f "${CURRENT_LINK}")" ||
+          ${release} == "$(readlink -f "${SLOTS_ROOT}/blue" 2>/dev/null || true)" ||
+          ${release} == "$(readlink -f "${SLOTS_ROOT}/green" 2>/dev/null || true)" ]]; then
+      continue
+    fi
+    remove_release "${release}"
+  done
+  run_as_deploy git -C "${APP_ROOT}" worktree prune
+}
+
+validate_live_state() {
+  [[ -d ${OLD_RELEASE} && $(readlink -f "${CURRENT_LINK}") == "${OLD_RELEASE}" ]] || {
+    log "Active slot and current release disagree; refusing cleanup."; return 1;
+  }
+  if [[ ${OLD_SLOT} != legacy ]]; then
+    [[ ${OLD_RELEASE} == "${RELEASES_ROOT}/${OLD_SUCCESSFUL_SHA}" ]] || return 1
+    local backend frontend
+    case "${OLD_SLOT}" in blue) backend=9001; frontend=3001 ;; green) backend=9002; frontend=3002 ;; esac
+    grep -Eq "server 127[.]0[.]0[.]1:${backend};" "${UPSTREAM_FILE}"
+    grep -Eq "server 127[.]0[.]0[.]1:${frontend};" "${UPSTREAM_FILE}"
+    systemctl is-active --quiet "eusrr-backend@${OLD_SLOT}.service"
+    systemctl is-active --quiet "eusrr-frontend@${OLD_SLOT}.service"
+  fi
+}
+
+prune_expired_dumps() {
+  # Run on failed attempts too. A partial dump is never offered as a backup.
+  find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete
+  find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump.partial' -delete
+}
+
 rollback() {
   local original_status=$?
-  trap - ERR
+  trap - EXIT HUP INT TERM
+  [[ ${COMMITTED} -eq 0 && ${MUTATION_STARTED} -eq 1 ]] || exit "${original_status}"
+  (( original_status != 0 )) || original_status=1
   set +e
-  log "Deployment failed with exit code ${original_status}; keeping the previous release active."
-
-  if [[ ${UPSTREAM_CHANGED} -eq 1 && -f ${UPSTREAM_BACKUP} ]]; then
-    install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}"
-    point_current_at "${OLD_RELEASE}"
-    nginx -t && systemctl reload nginx
+  log "Deployment failed with exit code ${original_status}; restoring the previous release."
+  if [[ -n ${WATCHDOG_PID} ]]; then
+    kill "${WATCHDOG_PID}" 2>/dev/null
+    wait "${WATCHDOG_PID}" 2>/dev/null
+  fi
+  if [[ -n ${BUILD_PID} ]]; then
+    kill -TERM -- "-${BUILD_PID}" 2>/dev/null
+    sleep 2
+    kill -KILL -- "-${BUILD_PID}" 2>/dev/null
+    wait "${BUILD_PID}" 2>/dev/null
+  fi
+  local recovered=1
+  if [[ ${UPSTREAM_CHANGED} -eq 1 ]]; then
+    # Keep candidate HTTP services alive until Nginx has returned to the old
+    # stack. Only stop its scheduler/workers before restarting the old workers.
+    stop_slot_celery "${NEW_SLOT}" || recovered=0
+  else
+    stop_slot_stack "${NEW_SLOT}" || recovered=0
   fi
-
-  stop_slot_stack "${NEW_SLOT}"
   if [[ ${NEW_STACK_ENABLED} -eq 1 ]]; then
     systemctl disable \
       "eusrr-backend@${NEW_SLOT}.service" \
       "eusrr-frontend@${NEW_SLOT}.service" \
       "eusrr-celery-worker@${NEW_SLOT}.service" \
-      "eusrr-celery-beat@${NEW_SLOT}.service" || true
-    enable_old_stack
+      "eusrr-celery-beat@${NEW_SLOT}.service" || recovered=0
+    enable_old_stack || recovered=0
   fi
-  if [[ ${OLD_STACK_STOPPED} -eq 1 ]]; then
-    start_old_stack || log "ERROR: failed to restart the previous application stack."
+  if [[ ${OLD_STACK_STOPPED} -eq 1 && ${recovered} -eq 1 ]]; then
+    [[ ${OLD_SLOT} != legacy ]] || rm -f "${STATE_DIR}/native-workers"
+    start_old_stack || recovered=0
+  fi
+  if [[ ${UPSTREAM_CHANGED} -eq 1 ]]; then
+    if [[ -f ${UPSTREAM_BACKUP} && ${recovered} -eq 1 ]]; then
+      install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}" || recovered=0
+      point_current_at "${OLD_RELEASE}" || recovered=0
+      nginx -t && systemctl reload nginx || recovered=0
+      if [[ ${recovered} -eq 1 ]]; then
+        stop_slot_stack "${NEW_SLOT}" || recovered=0
+      fi
+    else
+      recovered=0
+    fi
+  fi
+  if [[ ${recovered} -eq 1 ]]; then
+    atomic_state "${ACTIVE_SLOT_FILE}" "${OLD_SLOT}" || recovered=0
+    atomic_state "${STATE_DIR}/last-successful-sha" "${OLD_SUCCESSFUL_SHA}" || recovered=0
+    atomic_state "${PREVIOUS_RELEASE_FILE}" "${PREVIOUS_RELEASE}" || recovered=0
+  fi
+  if [[ ${recovered} -eq 1 ]]; then
+    if [[ ${SLOT_ASSIGNED} -eq 1 ]]; then
+      rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
+      if [[ -n ${INACTIVE_RELEASE} && -d ${INACTIVE_RELEASE} ]]; then
+        ln -sfn "${INACTIVE_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}"
+      fi
+    fi
+    remove_release "${NEW_RELEASE}" || recovered=0
+  fi
+  [[ -z ${DB_DUMP_PARTIAL} ]] || rm -f "${DB_DUMP_PARTIAL}"
+  if [[ ${recovered} -eq 1 ]]; then
+    rm -f "${PENDING_FILE}" "${UPSTREAM_BACKUP}"
+    log "Previous release restored; failed build removed."
+  else
+    log "ERROR: recovery was incomplete; release and recovery journal retained. Manual inspection required."
   fi
 
   log "No database snapshot was restored because the previous release remained live and may have accepted writes."
   log "Inspect ${LOG_FILE} and service journals before retrying."
   exit "${original_status}"
 }
-trap rollback ERR
+
+validate_live_state
+# A journal from the switching phase may describe Nginx's in-memory state,
+# which cannot be inferred from its configuration file after an interrupted reload.
+if [[ -f ${PENDING_FILE} ]]; then
+  read -r pending_phase pending_sha <"${PENDING_FILE}"
+  [[ ${pending_sha} =~ ^[0-9a-f]{40}$ ]] || exit 4
+  case "${pending_phase}" in
+    building) ;;
+    switching)
+      if [[ ${pending_sha} != "${OLD_SUCCESSFUL_SHA}" ]]; then
+        log "Interrupted traffic switch: inspect services and Nginx before clearing ${PENDING_FILE}."
+        exit 4
+      fi ;;
+    *) exit 4 ;;
+  esac
+fi
+# Preserve the previous successful slot through the build of the next release.
+if [[ -n ${INACTIVE_RELEASE} && -d ${INACTIVE_RELEASE} && ! -f ${PENDING_FILE} ]]; then
+  PREVIOUS_RELEASE="${INACTIVE_RELEASE}"
+  atomic_state "${PREVIOUS_RELEASE_FILE}" "${PREVIOUS_RELEASE}"
+fi
+reconcile_legacy_workers
+stop_slot_stack "${NEW_SLOT}"
+if [[ -f ${PENDING_FILE} ]]; then
+  # SIGKILL/power failure before traffic switched: the live-state checks above
+  # prove production still uses the last committed release. Restore standby.
+  if [[ ${INACTIVE_RELEASE} == "${RELEASES_ROOT}/${pending_sha}" ]]; then
+    rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
+    if [[ -n ${PREVIOUS_RELEASE} && -d ${PREVIOUS_RELEASE} ]]; then
+      ln -sfn "${PREVIOUS_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}"
+    fi
+    INACTIVE_RELEASE="${PREVIOUS_RELEASE}"
+  fi
+  rm -f "${PENDING_FILE}"
+fi
+prune_orphan_releases
+prune_expired_dumps
+if [[ ${MAINTENANCE} -eq 1 ]]; then
+  log "Maintenance completed; active and previous releases preserved."
+  df -h "${RELEASES_ROOT}"
+  exit 0
+fi
 
 log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}."
 run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \
@@ -317,21 +566,14 @@ if [[ ${TARGET_SHA} == "$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null ||
   exit 0
 fi
 
-if [[ -L ${SLOTS_ROOT}/${NEW_SLOT} ]]; then
-  stale_release="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}")"
-  if [[ ${stale_release} != "${OLD_RELEASE}" ]]; then
-    rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
-    remove_release "${stale_release}"
-  fi
-fi
 remove_release "${NEW_RELEASE}"
-run_as_deploy git -C "${APP_ROOT}" worktree prune
-
-available_bytes="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
-if (( available_bytes < 4294967296 )); then
-  log "At least 4 GiB of free disk space is required to build a release."
-  exit 6
-fi
+check_space 4294967296
+atomic_state "${PENDING_FILE}" "building ${TARGET_SHA}"
+MUTATION_STARTED=1
+trap rollback EXIT
+trap 'exit 129' HUP
+trap 'exit 130' INT
+trap 'exit 143' TERM
 
 log "Creating isolated Git worktree ${NEW_RELEASE}."
 run_as_deploy git -C "${APP_ROOT}" worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}"
@@ -342,19 +584,20 @@ run_as_deploy rm -rf "${NEW_RELEASE}/backend/media"
 run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media"
 
 log "Creating an isolated Python environment."
-run_as_deploy /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv"
-run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \
+run_bounded_build /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv"
+run_bounded_build env PIP_DISABLE_PIP_VERSION_CHECK=1 \
   "${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \
-  --requirement "${NEW_RELEASE}/backend/requirements.txt"
+  --no-cache-dir --requirement "${NEW_RELEASE}/backend/requirements.txt"
 
 log "Installing pinned frontend dependencies, including Linux native modules, and building Next.js."
-run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" ci \
+run_bounded_build npm --cache "${NEW_RELEASE}/.npm-cache" --prefix "${NEW_RELEASE}/frontend" ci \
   --include=optional --no-audit --no-fund
+run_as_deploy rm -rf -- "${NEW_RELEASE}/.npm-cache"
 run_as_deploy test -f \
   "${NEW_RELEASE}/frontend/node_modules/lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node"
 run_as_deploy test -d \
   "${NEW_RELEASE}/frontend/node_modules/@next/swc-linux-x64-gnu"
-run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" run build
+run_bounded_build npm --prefix "${NEW_RELEASE}/frontend" run build
 
 log "Running Django checks and collecting release static files."
 run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
@@ -363,12 +606,18 @@ run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
   "${NEW_RELEASE}/backend/manage.py" collectstatic --noinput
 
 run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/"
+check_space 1073741824
 
 install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}"
 DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump"
+DB_DUMP_PARTIAL="${DB_DUMP}.partial"
 log "Creating a consistent PostgreSQL snapshot while the current release remains online."
-run_database_dump "${DB_DUMP}"
+run_database_dump "${DB_DUMP_PARTIAL}"
+pg_restore --list "${DB_DUMP_PARTIAL}" >/dev/null
+mv "${DB_DUMP_PARTIAL}" "${DB_DUMP}"
+DB_DUMP_PARTIAL=""
 chmod 0600 "${DB_DUMP}"
+check_space 536870912
 
 log "Planned migrations:"
 run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
@@ -378,6 +627,7 @@ run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
   "${NEW_RELEASE}/backend/manage.py" migrate --noinput
 
 log "Assigning release to the ${NEW_SLOT} slot and starting its web services."
+SLOT_ASSIGNED=1
 ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new"
 mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}"
 start_slot_web "${NEW_SLOT}"
@@ -386,6 +636,7 @@ wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$
 wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$'
 
 cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}"
+atomic_state "${PENDING_FILE}" "switching ${TARGET_SHA}"
 UPSTREAM_CHANGED=1
 point_current_at "${NEW_RELEASE}"
 write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}"
@@ -398,23 +649,44 @@ wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '17
 log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain."
 sleep 10
 log "Switching background workers."
+OLD_STACK_STOPPED=1
 if [[ ${OLD_SLOT} == legacy ]]; then
   stop_legacy_stack
 else
   stop_slot_stack "${OLD_SLOT}"
 fi
-OLD_STACK_STOPPED=1
 start_slot_celery "${NEW_SLOT}"
+reconcile_legacy_workers "${NEW_SLOT}"
 NEW_STACK_ENABLED=1
 enable_slot_stack "${NEW_SLOT}"
 disable_old_stack
 
 find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \
   || log "WARNING: old database snapshots could not be pruned."
-run_as_deploy find "${SHARED_ROOT}/next-static" -type f -mtime +30 -delete \
-  || log "WARNING: old Next.js assets could not be pruned."
-rm -f "${UPSTREAM_BACKUP}" || true
-printf '%s\n' "${NEW_SLOT}" >"${ACTIVE_SLOT_FILE}"
-printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha"
-trap - ERR
+# Keep files referenced by either retained release regardless of their mtime.
+run_as_deploy python3 - "${SHARED_ROOT}/next-static" "${NEW_RELEASE}" "${OLD_RELEASE}" <<'PY'
+import sys
+import time
+from pathlib import Path
+
+shared = Path(sys.argv[1])
+referenced = set()
+for release in sys.argv[2:]:
+    root = Path(release) / "frontend/.next/static"
+    referenced.update(p.relative_to(root) for p in root.rglob("*") if p.is_file())
+for path in shared.rglob("*"):
+    if (path.is_file() and not path.is_symlink()
+            and path.relative_to(shared) not in referenced
+            and path.stat().st_mtime < time.time() - 30 * 86400):
+        path.unlink()
+PY
+atomic_state "${ACTIVE_SLOT_FILE}" "${NEW_SLOT}"
+atomic_state "${STATE_DIR}/last-successful-sha" "${TARGET_SHA}"
+atomic_state "${PREVIOUS_RELEASE_FILE}" "${OLD_RELEASE}"
+COMMITTED=1
+rm -f "${UPSTREAM_BACKUP}" "${PENDING_FILE}" || true
+trap - EXIT HUP INT TERM
+# The old active slot is the rollback release; discard older unreferenced builds.
+PREVIOUS_RELEASE="${OLD_RELEASE}"
+prune_orphan_releases || log "WARNING: unused releases could not be pruned; next run will retry."
 log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}."

ops/deploy/install-corp-runner-access

@@ -24,6 +24,9 @@ if [[ ! -d ${SOURCE_DIR}/systemd ]]; then
   exit 4
 fi
 
+exec 9>/run/lock/eusrr-deploy.lock
+flock -n 9 || { echo "A deployment is running; retry bootstrap after it finishes." >&2; exit 3; }
+
 read -r key_type key_body _ <"${PUBLIC_KEY_FILE}"
 if [[ ${key_type} != "ssh-ed25519" || -z ${key_body:-} ]]; then
   echo "Expected an Ed25519 public key." >&2
@@ -63,6 +66,7 @@ trap cleanup EXIT
 
 install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy" /usr/local/sbin/eusrr-deploy
 install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy-ssh" /usr/local/sbin/eusrr-deploy-ssh
+install -o root -g root -m 0644 "${SOURCE_DIR}/logrotate.conf" /etc/logrotate.d/eusrr-deploy
 for unit in "${SOURCE_DIR}"/systemd/*.service; do
   install -o root -g root -m 0644 "${unit}" "/etc/systemd/system/$(basename "${unit}")"
 done
@@ -76,8 +80,12 @@ chown -R deploy:www-data /srv/eusrr-shared
 find /srv/eusrr-shared -type d -exec chmod 0755 {} +
 find /srv/eusrr-shared -type f -exec chmod 0644 {} +
 
-ln -sfn /srv/eusrr /srv/eusrr-current.new
-mv -Tf /srv/eusrr-current.new /srv/eusrr-current
+# Re-running bootstrap on an existing native slot must not point static files
+# back at the legacy checkout while Nginx still serves the native slot.
+if [[ ! -L /srv/eusrr-current ]]; then
+  ln -sfn /srv/eusrr /srv/eusrr-current.new
+  mv -Tf /srv/eusrr-current.new /srv/eusrr-current
+fi
 
 if [[ ! -f ${UPSTREAM_FILE} ]]; then
   cat >"${UPSTREAM_FILE}" <<'EOF'

ops/deploy/install-deploy-hardening

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Update deployment tooling and clean up without building/deploying application code.
+set -Eeuo pipefail
+[[ ${EUID} -eq 0 ]] || { echo 'Run this installer with sudo.' >&2; exit 1; }
+SOURCE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+bash -n "${SOURCE_DIR}/eusrr-deploy"
+# SCP can leave the source group-writable. Validate the exact protected copy
+# that will be installed; logrotate may return zero even when it ignores a file.
+validation_dir="$(mktemp -d /tmp/eusrr-logrotate-check.XXXXXXXX)"
+cleanup_validation() {
+  rm -f -- "${validation_dir}/logrotate.conf" "${validation_dir}/validation.log"
+  rmdir -- "${validation_dir}"
+}
+trap cleanup_validation EXIT
+install -o root -g root -m 0644 "${SOURCE_DIR}/logrotate.conf" "${validation_dir}/logrotate.conf"
+if ! LC_ALL=C logrotate --debug --state /dev/null "${validation_dir}/logrotate.conf" \
+    >"${validation_dir}/validation.log" 2>&1 ||
+    grep -Eq '^error:|Ignoring .* because' "${validation_dir}/validation.log"; then
+  cat "${validation_dir}/validation.log" >&2
+  exit 1
+fi
+exec 9>/run/lock/eusrr-deploy.lock
+flock -n 9 || { echo 'A deployment is running. Retry after it finishes.' >&2; exit 3; }
+backup="/var/lib/eusrr-deploy/tooling-backup-$(date -u +%Y%m%dT%H%M%SZ)"
+install -d -o root -g root -m 0700 "${backup}"
+cp -a /usr/local/sbin/eusrr-deploy "${backup}/eusrr-deploy"
+install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy" /usr/local/sbin/eusrr-deploy.new
+mv -Tf /usr/local/sbin/eusrr-deploy.new /usr/local/sbin/eusrr-deploy
+install -o root -g root -m 0644 "${validation_dir}/logrotate.conf" /etc/logrotate.d/eusrr-deploy
+/usr/local/sbin/eusrr-deploy --self-test
+flock -u 9
+exec 9>&-
+/usr/local/sbin/eusrr-deploy --maintenance
+systemctl is-active eusrr-backend@"$(cat /var/lib/eusrr-deploy/active-slot)".service \
+  eusrr-frontend@"$(cat /var/lib/eusrr-deploy/active-slot)".service \
+  eusrr-celery-worker@"$(cat /var/lib/eusrr-deploy/active-slot)".service \
+  eusrr-celery-beat@"$(cat /var/lib/eusrr-deploy/active-slot)".service
+printf 'Deployment tooling updated. Application was not redeployed. Script backup: %s\n' "${backup}"

ops/deploy/logrotate.conf

@@ -0,0 +1,11 @@
+/var/log/eusrr-deploy.log /var/log/celery/beat-blue.log /var/log/celery/beat-green.log /var/log/celery/worker-blue.log /var/log/celery/worker-green.log {
+    daily
+    maxsize 10M
+    rotate 5
+    compress
+    delaycompress
+    missingok
+    notifempty
+    copytruncate
+    su root root
+}

ops/deploy/tests/test_deploy_lifecycle.py

@@ -0,0 +1,279 @@
+"""Exercise the real deployment shell in an isolated Linux filesystem fixture.
+
+Run: python3 -m unittest discover -s ops/deploy/tests -p test_deploy_lifecycle.py
+Only absolute infrastructure paths and the root check are substituted. All
+deployment decisions, traps, cleanup, state writes and switching are real.
+"""
+import json
+import os
+from pathlib import Path
+import subprocess
+import tempfile
+import time
+import unittest
+
+
+SOURCE = Path(__file__).resolve().parents[1] / "eusrr-deploy"
+OLD, PREVIOUS, TARGET, ORPHAN = (c * 40 for c in "abcd")
+FAKE = r'''#!/usr/bin/env python3
+import json, os, pathlib, shutil, sys, time
+p = pathlib.Path
+root = p(os.environ['FIXTURE'])
+name = p(sys.argv[0]).name
+args = sys.argv[1:]
+with (root/'commands').open('a') as f:
+    f.write(name+' '+repr(args)+'\n')
+def fail(stage):
+    if os.environ.get('FAIL_STAGE') == stage:
+        sys.exit(17)
+if name == 'runuser':
+    args = args[args.index('--')+1:]
+    os.execvp(args[0], args)
+elif name == 'git':
+    if 'rev-parse' in args: print(os.environ['TARGET'])
+    elif 'remove' in args:
+        shutil.rmtree(args[-1])
+    elif 'add' in args:
+        release=p(args[-2]); (release/'backend').mkdir(parents=True)
+        (release/'frontend/.next/static').mkdir(parents=True)
+        (release/'frontend/.next/static/new.js').touch()
+    sys.exit(0)
+elif name == 'df':
+    print('Available')
+    available = os.environ.get('FREE_BYTES','20000000000')
+    if os.environ.get('FAIL_STAGE') == 'disk_during_build' and (root/'building').exists():
+        available = '100000000'
+    print('1000000' if '--output=iavail' in args else available)
+elif name == 'systemctl':
+    file=root/'services.json'; states=json.loads(file.read_text())
+    def units(): return [a for a in args[1:] if a.endswith('.service')]
+    def save(): file.write_text(json.dumps(states))
+    if args[0] == 'show':
+        prop=args[args.index('-p')+1]
+        print('loaded' if prop=='LoadState' else states.get(args[1],'inactive'))
+    elif args[0] == 'is-active':
+        sys.exit(0 if all(states.get(u)=='active' for u in units()) else 3)
+    elif args[0] in ('start','restart'):
+        for u in units(): states[u]='active'
+        save()
+        if any('celery-beat@blue' in u for u in units()): fail('new_beat')
+    elif args[0] == 'stop' or (args[0]=='disable' and '--now' in args):
+        if args[0]=='stop' and os.environ.get('FAIL_STAGE')=='stop_blue': sys.exit(1)
+        for u in units(): states[u]='inactive'
+        save()
+    elif args[0] == 'reload': fail('reload')
+    elif args[0] == 'enable' and any('@blue' in u for u in units()): fail('enable')
+elif name == 'fake-python':
+    if 'venv' in args:
+        dest=p(args[-1])/'bin'; dest.mkdir(parents=True)
+        (dest/'python').symlink_to(root/'bin/fake-python')
+    elif 'pip' in args: fail('pip')
+elif name == 'python':
+    if 'pip' in args: fail('pip')
+    elif '-' in args:
+        script=sys.stdin.read()
+        if 'os.execve' not in script: sys.exit(19)
+        p(os.environ['EUSRR_DB_DUMP']).write_bytes(b'dump')
+        fail('dump')
+    elif 'migrate' in args and '--noinput' in args: fail('migration')
+elif name == 'npm':
+    if 'ci' in args:
+        base=p(args[args.index('--prefix')+1])/'node_modules'
+        (base/'lightningcss-linux-x64-gnu').mkdir(parents=True)
+        (base/'lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node').touch()
+        (base/'@next/swc-linux-x64-gnu').mkdir(parents=True)
+    if 'build' in args:
+        fail('build')
+        if os.environ.get('FAIL_STAGE') in ('hang','disk_during_build'):
+            (root/'building').touch(); time.sleep(60)
+elif name == 'curl':
+    if os.environ.get('FAIL_STAGE')=='health' and any(':3001' in a for a in args): print('502')
+    else: print('200')
+elif name == 'sleep': time.sleep(0.02)
+elif name == 'findmnt': print('/')
+'''
+
+
+@unittest.skipUnless(os.name == "posix", "Linux shell integration tests")
+class DeploymentLifecycleTests(unittest.TestCase):
+    def setUp(self):
+        self.temp = tempfile.TemporaryDirectory(prefix="eusrr-deploy-test-")
+        self.addCleanup(self.temp.cleanup)
+        self.root = Path(self.temp.name)
+        self.bin = self.root / "bin"
+        self.bin.mkdir()
+        fake = self.bin / "fake"
+        fake.write_text(FAKE)
+        fake.chmod(0o755)
+        for name in ('runuser', 'git', 'df', 'systemctl', 'fake-python', 'npm',
+                     'curl', 'sleep', 'findmnt', 'nginx', 'rsync', 'pg_restore'):
+            (self.bin / name).symlink_to(fake)
+        text = SOURCE.read_text()
+        for prefix in ('/srv/', '/var/', '/run/', '/etc/'):
+            text = text.replace(prefix, str(self.root) + prefix)
+        text = text.replace('if [[ ${EUID} -ne 0 ]]; then', 'if false; then')
+        text = text.replace('/usr/bin/python3 -m venv', 'fake-python -m venv')
+        # Fixture Python entrypoint must preserve argv[0] for pip/dump handlers.
+        text = text.replace('install -d -o root -g root', 'install -d')
+        text = text.replace('install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}"', 'install -d')
+        text = text.replace('install -o root -g root', 'install')
+        self.script = self.root / "deploy"
+        self.script.write_text(text)
+        self.state = self.root / "var/lib/eusrr-deploy"
+        self.releases = self.root / "srv/eusrr-releases"
+        self.slots = self.root / "srv/eusrr-slots"
+        for directory in (self.state, self.releases, self.slots, self.root/'var/log',
+                          self.root/'run/lock', self.root/'srv/eusrr/backend',
+                          self.root/'srv/eusrr-shared/next-static', self.root/'etc/nginx/conf.d',
+                          self.root/'var/backups/eusrr/deploy'):
+            directory.mkdir(parents=True, exist_ok=True)
+        for sha in (OLD, PREVIOUS, ORPHAN):
+            (self.releases/sha/'frontend/.next/static').mkdir(parents=True)
+        self.slots.joinpath('green').symlink_to(self.releases/OLD)
+        self.slots.joinpath('blue').symlink_to(self.releases/PREVIOUS)
+        (self.root/'srv/eusrr-current').symlink_to(self.releases/OLD)
+        (self.state/'active-slot').write_text('green\n')
+        (self.state/'last-successful-sha').write_text(OLD+'\n')
+        (self.root/'etc/nginx/conf.d/eusrr-active-upstreams.conf').write_text(
+            'server 127.0.0.1:9002;\nserver 127.0.0.1:3002;\n')
+        self.services = self.root / 'services.json'
+        self.services.write_text(json.dumps({
+            **{f'eusrr-{unit}@green.service': 'active' for unit in
+               ('backend', 'frontend', 'celery-worker', 'celery-beat')},
+            'celery-eusrr.service': 'active',
+            'celery-eusrr-beat.service': 'active',
+            'celery-eusrr-worker.service': 'active',
+        }))
+        self.env = {**os.environ, 'FIXTURE': str(self.root), 'TARGET': TARGET,
+                    'PATH': str(self.bin) + ':' + os.environ['PATH']}
+
+    def run_deploy(self, mode=TARGET, **env):
+        result = subprocess.run(['bash', str(self.script), mode], env={**self.env, **env},
+                                capture_output=True, text=True, timeout=30)
+        self.output = result.stdout + result.stderr
+        return result
+
+    def assert_old_live(self):
+        self.assertEqual((self.root/'srv/eusrr-current').resolve(), self.releases/OLD, self.output)
+        self.assertEqual((self.state/'active-slot').read_text().strip(), 'green', self.output)
+        states = json.loads(self.services.read_text())
+        for unit in ('backend', 'frontend', 'celery-worker', 'celery-beat'):
+            self.assertEqual(states[f'eusrr-{unit}@green.service'], 'active', self.output)
+
+    def test_success_preserves_old_and_removes_older_releases(self):
+        result = self.run_deploy()
+        self.assertEqual(result.returncode, 0, self.output)
+        self.assertEqual((self.root/'srv/eusrr-current').resolve(), self.releases/TARGET)
+        self.assertTrue((self.releases/OLD).exists())
+        self.assertFalse((self.releases/PREVIOUS).exists())
+        self.assertFalse((self.releases/ORPHAN).exists())
+        self.assertFalse((self.state/'pending-deployment').exists())
+        states = json.loads(self.services.read_text())
+        self.assertEqual(states['celery-eusrr-beat.service'], 'inactive')
+        self.assertEqual(states['eusrr-celery-beat@green.service'], 'inactive')
+        self.assertEqual(states['eusrr-celery-beat@blue.service'], 'active')
+
+    def test_failures_remove_candidate_preserve_both_successful_releases(self):
+        for stage in ('pip', 'build', 'dump', 'migration', 'health', 'new_beat', 'enable'):
+            with self.subTest(stage=stage):
+                result = self.run_deploy(FAIL_STAGE=stage)
+                self.assertNotEqual(result.returncode, 0, self.output)
+                self.assert_old_live()
+                self.assertFalse((self.releases/TARGET).exists(), self.output)
+                self.assertTrue((self.releases/PREVIOUS).exists(), self.output)
+                self.assertFalse((self.state/'pending-deployment').exists(), self.output)
+                self.assertFalse(list((self.root/'var/backups/eusrr/deploy').glob('*.partial')))
+
+    def test_capacity_failure_does_not_remove_previous(self):
+        result = self.run_deploy(FREE_BYTES='4000000000')
+        self.assertEqual(result.returncode, 6, self.output)
+        self.assert_old_live()
+        self.assertTrue((self.releases/PREVIOUS).exists())
+
+    def test_watchdog_stops_build_before_disk_exhaustion(self):
+        result = self.run_deploy(FAIL_STAGE='disk_during_build')
+        self.assertNotEqual(result.returncode, 0, self.output)
+        self.assertIn('Stopping build before', self.output)
+        self.assert_old_live()
+        self.assertFalse((self.releases/TARGET).exists(), self.output)
+
+    def test_failed_nginx_recovery_retains_candidate_and_journal(self):
+        self.assertNotEqual(self.run_deploy(FAIL_STAGE='reload').returncode, 0)
+        self.assertTrue((self.releases/TARGET).exists(), self.output)
+        self.assertTrue((self.state/'pending-deployment').exists(), self.output)
+        self.assertTrue((self.state/f'upstreams-before-{TARGET}').exists(), self.output)
+        states = json.loads(self.services.read_text())
+        self.assertEqual(states['eusrr-backend@blue.service'], 'active')
+        self.assertEqual(states['eusrr-frontend@blue.service'], 'active')
+        self.assertEqual(self.run_deploy('--maintenance').returncode, 4, self.output)
+        self.assertTrue((self.releases/TARGET).exists())
+
+    def test_already_active_sha_never_removed(self):
+        self.env['TARGET'] = OLD
+        self.assertEqual(self.run_deploy(OLD).returncode, 0, self.output)
+        self.assert_old_live()
+        self.assertTrue((self.releases/OLD).exists())
+
+    def test_deploying_standby_sha_does_not_delete_rollback(self):
+        self.env['TARGET'] = PREVIOUS
+        self.assertNotEqual(self.run_deploy(PREVIOUS).returncode, 0)
+        self.assert_old_live()
+        self.assertTrue((self.releases/PREVIOUS).exists())
+
+    def test_existing_lock_prevents_any_maintenance(self):
+        import fcntl
+        with (self.root/'run/lock/eusrr-deploy.lock').open('w') as lock:
+            fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            self.assertEqual(self.run_deploy('--maintenance').returncode, 3, self.output)
+        self.assertTrue((self.releases/ORPHAN).exists())
+
+    def test_inconsistent_state_refuses_cleanup(self):
+        (self.state/'last-successful-sha').write_text(TARGET)
+        self.assertNotEqual(self.run_deploy('--maintenance').returncode, 0)
+        self.assertTrue((self.releases/ORPHAN).exists())
+
+    def test_failure_to_stop_refuses_cleanup(self):
+        self.assertNotEqual(self.run_deploy('--maintenance', FAIL_STAGE='stop_blue').returncode, 0)
+        self.assertTrue((self.releases/ORPHAN).exists())
+
+    def test_maintenance_preserves_symlink_targets(self):
+        external = self.root/'external'; external.mkdir()
+        external.joinpath('media').write_text('keep')
+        self.releases.joinpath('e'*40).symlink_to(external)
+        self.assertEqual(self.run_deploy('--maintenance').returncode, 0, self.output)
+        self.assertTrue(external.joinpath('media').exists())
+        self.assert_old_live()
+
+    def test_interrupted_build_recovered_by_maintenance(self):
+        (self.state/'pending-deployment').write_text('building '+TARGET+'\n')
+        (self.state/'previous-release').write_text(str(self.releases/PREVIOUS))
+        (self.releases/TARGET).mkdir()
+        self.slots.joinpath('blue').unlink()
+        self.slots.joinpath('blue').symlink_to(self.releases/TARGET)
+        self.assertEqual(self.run_deploy('--maintenance').returncode, 0, self.output)
+        self.assert_old_live()
+        self.assertFalse((self.releases/TARGET).exists())
+        self.assertEqual(self.slots.joinpath('blue').resolve(), self.releases/PREVIOUS)
+
+    def test_signal_cleans_build_process_and_candidate(self):
+        with (self.root/'output').open('w') as output:
+            process = subprocess.Popen(['bash', str(self.script), TARGET],
+                env={**self.env, 'FAIL_STAGE': 'hang'}, stdout=output, stderr=output)
+            try:
+                deadline = time.monotonic() + 15
+                while not (self.root/'building').exists() and time.monotonic() < deadline:
+                    if process.poll() is not None: break
+                    time.sleep(.05)
+                self.assertTrue((self.root/'building').exists(), (self.root/'output').read_text())
+                process.terminate()
+                process.wait(timeout=15)
+            finally:
+                if process.poll() is None: process.kill(); process.wait()
+        self.output = (self.root/'output').read_text()
+        self.assertEqual(process.returncode, 143, self.output)
+        self.assert_old_live()
+        self.assertFalse((self.releases/TARGET).exists(), self.output)
+
+
+if __name__ == '__main__':
+    unittest.main()