#!/usr/bin/env bash
set -Eeuo pipefail

readonly APP_ROOT="/srv/eusrr"
readonly RELEASES_ROOT="/srv/eusrr-releases"
readonly SLOTS_ROOT="/srv/eusrr-slots"
readonly SHARED_ROOT="/srv/eusrr-shared"
readonly CURRENT_LINK="/srv/eusrr-current"
readonly DEPLOY_USER="deploy"
readonly BACKUP_DIR="/var/backups/eusrr/deploy"
readonly STATE_DIR="/var/lib/eusrr-deploy"
readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot"
readonly LOG_FILE="/var/log/eusrr-deploy.log"
readonly LOCK_FILE="/run/lock/eusrr-deploy.lock"
readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf"
readonly PREVIOUS_RELEASE_FILE="${STATE_DIR}/previous-release"
readonly PENDING_FILE="${STATE_DIR}/pending-deployment"
readonly GITEA_ORIGIN="ssh://git@172.11.0.14/Robotail/Corp.git"
readonly GITEA_KNOWN_HOSTS="/etc/eusrr/gitea_known_hosts"

if [[ ${EUID} -ne 0 ]]; then
  echo "This command must run as root." >&2
  exit 1
fi

install -d -o root -g root -m 0755 "${STATE_DIR}"
touch "${LOG_FILE}"
chmod 0600 "${LOG_FILE}"
exec > >(tee -a "${LOG_FILE}") 2>&1

log() {
  printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*"
}

run_as_deploy() {
  setpriv --no-new-privs runuser -u "${DEPLOY_USER}" -- "$@"
}

check_node_runtime() {
  run_as_deploy node -e 'const [major, minor] = process.versions.node.split(".").map(Number); if (!((major === 22 && minor >= 13) || major === 24)) { console.error("Production requires Node.js 22.13+ (22.x) or 24.x LTS."); process.exit(1); }'
}

run_git() {
  # Repository hooks/config and per-user SSH aliases must not silently replace
  # the production source. The read-only deploy key and pinned host key are used
  # only for the explicitly selected Gitea instance.
  run_as_deploy env GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \
    GIT_SSH_COMMAND="/usr/bin/ssh -F /dev/null -i /home/deploy/.ssh/gitea_ed25519 -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${GITEA_KNOWN_HOSTS}" \
    git -c core.hooksPath=/dev/null -c protocol.allow=never -c protocol.ssh.allow=always \
    -C "${APP_ROOT}" "$@"
}

run_in_release_backend() {
  run_as_deploy \
    bash -c 'cd "$1"; shift; exec "$@"' _ "${NEW_RELEASE}/backend" "$@"
}

slot_ports() {
  case "$1" in
    blue)  BACKEND_PORT=9001; FRONTEND_PORT=3001 ;;
    green) BACKEND_PORT=9002; FRONTEND_PORT=3002 ;;
    *) log "Unknown deployment slot: $1"; return 1 ;;
  esac
}

if [[ ${1:-} == "--self-test" ]]; then
  for command in git curl flock runuser pg_dump pg_restore npm nginx rsync setsid timeout findmnt setpriv; do
    command -v "${command}" >/dev/null
  done
  if ! command -v magick >/dev/null && ! command -v convert >/dev/null; then
    log "ImageMagick is required for safe oversized-image processing."
    exit 1
  fi
  test -d "${APP_ROOT}/.git"
  run_as_deploy test -r "${APP_ROOT}/backend/.env"
  run_as_deploy test -r "${APP_ROOT}/frontend/.env"
  test -r "${UPSTREAM_FILE}"
  test -r /usr/local/libexec/eusrr-deploy-healthcheck
  run_as_deploy test -r "${GITEA_KNOWN_HOSTS}"
  check_node_runtime
  [[ $(stat -c '%u:%a' "${SLOTS_ROOT}") == 0:755 ]]
  for unit in eusrr-backend@.service eusrr-frontend@.service \
    eusrr-celery-worker@.service eusrr-celery-beat@.service; do
    systemctl cat "${unit}" >/dev/null
  done
  nginx -t
  log "Blue-green deployment self-test passed."
  exit 0
fi

TARGET_SHA="${1:-}"
MAINTENANCE=0
if [[ ${TARGET_SHA} == --maintenance ]]; then
  MAINTENANCE=1
elif [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then
  echo "Usage: $0 <40-character lowercase commit SHA> | --maintenance" >&2
  exit 2
fi

exec 9>"${LOCK_FILE}"
if ! flock -n 9; then
  log "Another deployment is already running."
  exit 3
fi

OLD_SLOT="$(cat "${ACTIVE_SLOT_FILE}" 2>/dev/null || printf legacy)"
case "${OLD_SLOT}" in
  legacy) NEW_SLOT=blue ;;
  blue) NEW_SLOT=green ;;
  green) NEW_SLOT=blue ;;
  *) log "Invalid active slot state: ${OLD_SLOT}"; exit 4 ;;
esac

NEW_RELEASE="${RELEASES_ROOT}/${TARGET_SHA}"
if [[ ${OLD_SLOT} == legacy ]]; then
  OLD_RELEASE="${APP_ROOT}"
else
  OLD_RELEASE="$(readlink -f "${SLOTS_ROOT}/${OLD_SLOT}")"
fi

BACKEND_PORT=""
FRONTEND_PORT=""
slot_ports "${NEW_SLOT}"

UPSTREAM_BACKUP="${STATE_DIR}/upstreams-before-${TARGET_SHA}"
UPSTREAM_CHANGED=0
OLD_STACK_STOPPED=0
NEW_STACK_ENABLED=0
DB_DUMP=""
DB_DUMP_PARTIAL=""
PREVIOUS_RELEASE="$(cat "${PREVIOUS_RELEASE_FILE}" 2>/dev/null || true)"
INACTIVE_RELEASE="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}" 2>/dev/null || true)"
OLD_SUCCESSFUL_SHA="$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)"
MUTATION_STARTED=0
SLOT_ASSIGNED=0
BUILD_PID=""
WATCHDOG_PID=""
COMMITTED=0

atomic_state() {
  printf '%s\n' "$2" >"$1.new"
  chmod 0644 "$1.new"
  mv -Tf "$1.new" "$1"
}

check_space() {
  local minimum="$1" available inodes
  available="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
  inodes="$(df --output=iavail "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
  if (( available < minimum || inodes < 50000 )); then
    log "Insufficient capacity: ${available} bytes and ${inodes} inodes free; need ${minimum} bytes and 50000 inodes."
    return 6
  fi
}

# Build processes have their own session so cancellation also stops grandchildren.
run_bounded_build() {
  check_space 1073741824 || return $?
  setsid timeout --signal=TERM --kill-after=30s 30m setpriv --no-new-privs \
    runuser -u "${DEPLOY_USER}" -- "$@" <&0 &
  BUILD_PID=$!
  (
    while kill -0 "${BUILD_PID}" 2>/dev/null; do
      if ! check_space 536870912; then
        log "Stopping build before it exhausts the production filesystem."
        kill -TERM -- "-${BUILD_PID}" 2>/dev/null || true
        sleep 5
        kill -KILL -- "-${BUILD_PID}" 2>/dev/null || true
        exit 6
      fi
      sleep 2
    done
  ) &
  WATCHDOG_PID=$!
  local status=0
  wait "${BUILD_PID}" || status=$?
  kill "${WATCHDOG_PID}" 2>/dev/null || true
  wait "${WATCHDOG_PID}" 2>/dev/null || true
  WATCHDOG_PID=""
  BUILD_PID=""
  (( status == 0 )) || return "${status}"
  check_space 536870912
}

install_frontend_dependencies() {
  # All attempts share one build timeout and disk watchdog. Retry only reported
  # network failures, never dependency/engine/postinstall errors. Cache remains
  # inside the candidate and is removed on success or failed-release cleanup.
  run_bounded_build bash -c '
    set -euo pipefail
    release="$1"
    output="${release}/.npm-install-attempt.log"
    for attempt in 1 2 3 4 5; do
      printf "Frontend dependency installation attempt %s/5.\n" "$attempt"
      if npm --cache "${release}/.npm-cache" --prefix "${release}/frontend" ci \
          --include=optional --no-audit --no-fund --engine-strict --prefer-offline \
          --fetch-retries=5 --fetch-retry-factor=2 \
          --fetch-retry-mintimeout=10000 --fetch-retry-maxtimeout=30000 \
          --fetch-timeout=120000 >"${output}" 2>&1; then
        cat "${output}"
        rm -f -- "${output}"
        exit 0
      else
        status=$?
        cat "${output}"
      fi
      if [[ "$attempt" == 5 ]] || ! grep -Eq \
          "npm (ERR!|error) code (ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|ECONNREFUSED|E502|E503|E504)([[:space:]]|$)" "${output}"; then
        exit "$status"
      fi
      printf "Transient npm network failure; retrying the same lockfile.\n"
      sleep "$((attempt * 15))"
    done
  ' _ "${NEW_RELEASE}"
}

run_database_dump() {
  local dump_file="$1"
  run_bounded_build \
    env EUSRR_DB_DUMP="${dump_file}" \
    "${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY'
import os
from pathlib import Path

from dotenv import dotenv_values

config = dotenv_values("/srv/eusrr/backend/.env")

def required(name: str) -> str:
    value = config.get(name)
    if not value:
        raise SystemExit(f"Required database setting is missing: {name}")
    return str(value)

executable = "/usr/bin/pg_dump"
if not Path(executable).is_file():
    raise SystemExit(f"Database utility is not installed: {executable}")

environment = os.environ.copy()
environment["PGPASSWORD"] = required("POSTGRES_PASSWORD")
arguments = [
    executable,
    "--host", str(config.get("DB_HOST") or "127.0.0.1"),
    "--port", str(config.get("DB_PORT") or "5432"),
    "--username", required("POSTGRES_USER"),
    "--dbname", required("POSTGRES_DB"),
    "--format=custom",
    "--no-owner",
    "--no-acl",
    "--file", os.environ["EUSRR_DB_DUMP"],
]
os.execve(executable, arguments, environment)
PY
}

wait_for_http() {
  local name="$1"
  local url="$2"
  local expected="$3"
  local host_header="${4:-}"
  local attempt code
  # These URLs are exclusively loopback. Match the trusted TLS terminator's
  # header so SECURE_SSL_REDIRECT can stay enabled in production.
  local -a curl_args=(--silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5
    --header 'X-Forwarded-Proto: https')

  if [[ -n ${host_header} ]]; then
    curl_args+=(--header "Host: ${host_header}")
  fi

  for attempt in $(seq 1 45); do
    code="$(curl "${curl_args[@]}" "${url}" || true)"
    if [[ ${code} =~ ${expected} ]]; then
      log "Health check passed: ${name} returned ${code}."
      return 0
    fi
    sleep 2
  done

  log "Health check failed: ${name} returned ${code:-no response}."
  return 1
}

write_upstreams() {
  local backend_port="$1"
  local frontend_port="$2"
  local temporary_file
  temporary_file="$(mktemp)"
  cat >"${temporary_file}" <<EOF
upstream eusrr_backend {
    server 127.0.0.1:${backend_port};
    keepalive 32;
}

upstream eusrr_frontend {
    server 127.0.0.1:${frontend_port};
    keepalive 32;
}
EOF
  install -o root -g root -m 0644 "${temporary_file}" "${UPSTREAM_FILE}"
  rm -f "${temporary_file}"
  UPSTREAM_CHANGED=1
}

point_current_at() {
  local release="$1"
  ln -sfn "${release}" "${CURRENT_LINK}.new"
  mv -Tf "${CURRENT_LINK}.new" "${CURRENT_LINK}"
}

start_slot_web() {
  local slot="$1"
  systemctl start "eusrr-backend@${slot}.service" "eusrr-frontend@${slot}.service"
}

stop_slot_stack() {
  local slot="$1"
  systemctl stop \
    "eusrr-celery-beat@${slot}.service" \
    "eusrr-celery-worker@${slot}.service" \
    "eusrr-frontend@${slot}.service" \
    "eusrr-backend@${slot}.service"
  assert_slot_stopped "${slot}"
}

assert_slot_stopped() {
  local unit state
  for unit in backend frontend celery-worker celery-beat; do
    state="$(systemctl show "eusrr-${unit}@$1.service" -p ActiveState --value)" || return 1
    case "${state}" in inactive|failed) ;; *) log "Service is still ${state}: eusrr-${unit}@$1"; return 1 ;; esac
  done
}

stop_slot_celery() {
  local unit state
  systemctl stop "eusrr-celery-beat@$1.service" "eusrr-celery-worker@$1.service"
  for unit in celery-beat celery-worker; do
    state="$(systemctl show "eusrr-${unit}@$1.service" -p ActiveState --value)" || return 1
    case "${state}" in inactive|failed) ;; *) return 1 ;; esac
  done
}

reconcile_legacy_workers() {
  local active_slot="${1:-${OLD_SLOT}}"
  [[ ${active_slot} != legacy ]] || return 0
  systemctl is-active --quiet "eusrr-celery-worker@${active_slot}.service"
  systemctl is-active --quiet "eusrr-celery-beat@${active_slot}.service"
  local unit state
  # A legacy umbrella service can start disabled worker/beat units again.
  # A condition blocks dependency-driven starts as well as manual starts.
  for unit in celery-eusrr.service celery-eusrr-beat.service celery-eusrr-worker.service; do
    if [[ $(systemctl show "${unit}" -p LoadState --value) == not-found ]]; then
      continue
    fi
    install -d -m 0755 "/etc/systemd/system/${unit}.d"
    printf '%s\n' '[Unit]' 'ConditionPathExists=!/var/lib/eusrr-deploy/native-workers' \
      >"/etc/systemd/system/${unit}.d/90-native-workers.conf"
  done
  touch "${STATE_DIR}/native-workers"
  systemctl daemon-reload
  for unit in celery-eusrr.service celery-eusrr-beat.service celery-eusrr-worker.service; do
    [[ $(systemctl show "${unit}" -p LoadState --value) != not-found ]] || continue
    systemctl disable --now "${unit}"
    state="$(systemctl show "${unit}" -p ActiveState --value)"
    case "${state}" in inactive|failed) ;; *) return 1 ;; esac
  done
}

start_slot_celery() {
  local slot="$1"
  systemctl start "eusrr-celery-worker@${slot}.service" "eusrr-celery-beat@${slot}.service"
  systemctl is-active --quiet "eusrr-celery-worker@${slot}.service"
  systemctl is-active --quiet "eusrr-celery-beat@${slot}.service"
}

enable_slot_stack() {
  local slot="$1"
  systemctl enable \
    "eusrr-backend@${slot}.service" \
    "eusrr-frontend@${slot}.service" \
    "eusrr-celery-worker@${slot}.service" \
    "eusrr-celery-beat@${slot}.service"
}

disable_old_stack() {
  if [[ ${OLD_SLOT} == legacy ]]; then
    systemctl disable gunicorn-eusrr.service nextjs-eusrr.service \
      celery-eusrr-worker.service celery-eusrr-beat.service
  else
    systemctl disable \
      "eusrr-backend@${OLD_SLOT}.service" \
      "eusrr-frontend@${OLD_SLOT}.service" \
      "eusrr-celery-worker@${OLD_SLOT}.service" \
      "eusrr-celery-beat@${OLD_SLOT}.service"
  fi
}

enable_old_stack() {
  if [[ ${OLD_SLOT} == legacy ]]; then
    systemctl enable gunicorn-eusrr.service nextjs-eusrr.service \
      celery-eusrr-worker.service celery-eusrr-beat.service
  else
    systemctl enable \
      "eusrr-backend@${OLD_SLOT}.service" \
      "eusrr-frontend@${OLD_SLOT}.service" \
      "eusrr-celery-worker@${OLD_SLOT}.service" \
      "eusrr-celery-beat@${OLD_SLOT}.service"
  fi
}

stop_legacy_stack() {
  if [[ $(systemctl show celery-eusrr.service -p LoadState --value) != not-found ]]; then
    systemctl stop celery-eusrr.service
  fi
  systemctl stop \
    celery-eusrr-beat.service celery-eusrr-worker.service \
    nextjs-eusrr.service gunicorn-eusrr.service
  local unit state
  for unit in celery-eusrr-beat.service celery-eusrr-worker.service nextjs-eusrr.service gunicorn-eusrr.service; do
    state="$(systemctl show "${unit}" -p ActiveState --value)"
    case "${state}" in inactive|failed) ;; *) return 1 ;; esac
  done
}

start_old_stack() {
  if [[ ${OLD_SLOT} == legacy ]]; then
    systemctl restart gunicorn-eusrr.service nextjs-eusrr.service \
      celery-eusrr-worker.service celery-eusrr-beat.service
  else
    systemctl restart "eusrr-backend@${OLD_SLOT}.service" "eusrr-frontend@${OLD_SLOT}.service" \
      "eusrr-celery-worker@${OLD_SLOT}.service" "eusrr-celery-beat@${OLD_SLOT}.service"
  fi
}

remove_release() {
  local release="$1"
  local name="${release##*/}" protected resolved mount
  [[ ${name} =~ ^[0-9a-f]{40}$ && ${release} == "${RELEASES_ROOT}/${name}" ]] || {
    log "Refusing non-release path: ${release}"; return 1;
  }
  [[ ! -L ${release} ]] || { log "Refusing symlink release: ${release}"; return 1; }
  resolved="$(readlink -m "${release}")"
  [[ ${resolved} == "${release}" ]] || return 1
  for protected in "${OLD_RELEASE}" "${PREVIOUS_RELEASE}" \
    "$(readlink -f "${CURRENT_LINK}")" \
    "$(readlink -f "${SLOTS_ROOT}/blue" 2>/dev/null || true)" \
    "$(readlink -f "${SLOTS_ROOT}/green" 2>/dev/null || true)"; do
    [[ ${release} != "${protected}" ]] || { log "Protected release: ${release}"; return 1; }
  done
  if [[ -e ${release} ]]; then
    # Git must never recurse into a bind mount containing shared data.
    while IFS= read -r mount; do
      [[ ${mount} != "${release}" && ${mount} != "${release}/"* ]] || return 1
    done < <(findmnt -rn -o TARGET)
    log "Removing unused release ${release}."
    run_git worktree remove --force "${release}"
  fi
}

prune_orphan_releases() {
  local release
  for release in "${RELEASES_ROOT}"/*; do
    [[ -d ${release} && ! -L ${release} ]] || continue
    [[ ${release##*/} =~ ^[0-9a-f]{40}$ ]] || continue
    if [[ ${release} == "${OLD_RELEASE}" || ${release} == "${PREVIOUS_RELEASE}" ||
          ${release} == "$(readlink -f "${CURRENT_LINK}")" ||
          ${release} == "$(readlink -f "${SLOTS_ROOT}/blue" 2>/dev/null || true)" ||
          ${release} == "$(readlink -f "${SLOTS_ROOT}/green" 2>/dev/null || true)" ]]; then
      continue
    fi
    remove_release "${release}"
  done
  run_git worktree prune
}

validate_live_state() {
  local slot_link slot_target
  # These links are control-plane input to root. Application code must not be
  # able to redirect a slot (or make cleanup treat shared data as a release).
  [[ ! -L ${SLOTS_ROOT} && $(stat -c '%u:%a' "${SLOTS_ROOT}") == 0:755 ]] || {
    log "Slot directory must be root-owned mode 0755; run the tooling installer."; return 1;
  }
  for slot_link in "${SLOTS_ROOT}/blue" "${SLOTS_ROOT}/green"; do
    [[ -e ${slot_link} || -L ${slot_link} ]] || continue
    [[ -L ${slot_link} ]] || { log "Slot is not a symlink: ${slot_link}"; return 1; }
    slot_target="$(readlink -f "${slot_link}")"
    [[ ${slot_target} =~ ^${RELEASES_ROOT}/[0-9a-f]{40}$ && -d ${slot_target} && ! -L ${slot_target} ]] || {
      log "Slot points outside a valid release: ${slot_link}"; return 1;
    }
  done
  [[ -d ${OLD_RELEASE} && $(readlink -f "${CURRENT_LINK}") == "${OLD_RELEASE}" ]] || {
    log "Active slot and current release disagree; refusing cleanup."; return 1;
  }
  if [[ ${OLD_SLOT} != legacy ]]; then
    [[ ${OLD_RELEASE} == "${RELEASES_ROOT}/${OLD_SUCCESSFUL_SHA}" ]] || return 1
    local backend frontend
    case "${OLD_SLOT}" in blue) backend=9001; frontend=3001 ;; green) backend=9002; frontend=3002 ;; esac
    grep -Eq "server 127[.]0[.]0[.]1:${backend};" "${UPSTREAM_FILE}"
    grep -Eq "server 127[.]0[.]0[.]1:${frontend};" "${UPSTREAM_FILE}"
    systemctl is-active --quiet "eusrr-backend@${OLD_SLOT}.service"
    systemctl is-active --quiet "eusrr-frontend@${OLD_SLOT}.service"
  fi
}

prune_expired_dumps() {
  # Run on failed attempts too. A partial dump is never offered as a backup.
  find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete
  find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump.partial' -delete
}

rollback() {
  local original_status=$?
  trap - EXIT HUP INT TERM
  [[ ${COMMITTED} -eq 0 && ${MUTATION_STARTED} -eq 1 ]] || exit "${original_status}"
  (( original_status != 0 )) || original_status=1
  set +e
  log "Deployment failed with exit code ${original_status}; restoring the previous release."
  if [[ -n ${WATCHDOG_PID} ]]; then
    kill "${WATCHDOG_PID}" 2>/dev/null
    wait "${WATCHDOG_PID}" 2>/dev/null
  fi
  if [[ -n ${BUILD_PID} ]]; then
    kill -TERM -- "-${BUILD_PID}" 2>/dev/null
    sleep 2
    kill -KILL -- "-${BUILD_PID}" 2>/dev/null
    wait "${BUILD_PID}" 2>/dev/null
  fi
  local recovered=1
  if [[ ${UPSTREAM_CHANGED} -eq 1 ]]; then
    # Keep candidate HTTP services alive until Nginx has returned to the old
    # stack. Only stop its scheduler/workers before restarting the old workers.
    stop_slot_celery "${NEW_SLOT}" || recovered=0
  else
    stop_slot_stack "${NEW_SLOT}" || recovered=0
  fi
  if [[ ${NEW_STACK_ENABLED} -eq 1 ]]; then
    systemctl disable \
      "eusrr-backend@${NEW_SLOT}.service" \
      "eusrr-frontend@${NEW_SLOT}.service" \
      "eusrr-celery-worker@${NEW_SLOT}.service" \
      "eusrr-celery-beat@${NEW_SLOT}.service" || recovered=0
    enable_old_stack || recovered=0
  fi
  if [[ ${OLD_STACK_STOPPED} -eq 1 && ${recovered} -eq 1 ]]; then
    [[ ${OLD_SLOT} != legacy ]] || rm -f "${STATE_DIR}/native-workers"
    start_old_stack || recovered=0
  fi
  if [[ ${UPSTREAM_CHANGED} -eq 1 ]]; then
    if [[ -f ${UPSTREAM_BACKUP} && ${recovered} -eq 1 ]]; then
      install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}" || recovered=0
      point_current_at "${OLD_RELEASE}" || recovered=0
      nginx -t && systemctl reload nginx || recovered=0
      if [[ ${recovered} -eq 1 ]]; then
        stop_slot_stack "${NEW_SLOT}" || recovered=0
      fi
    else
      recovered=0
    fi
  fi
  if [[ ${recovered} -eq 1 ]]; then
    atomic_state "${ACTIVE_SLOT_FILE}" "${OLD_SLOT}" || recovered=0
    atomic_state "${STATE_DIR}/last-successful-sha" "${OLD_SUCCESSFUL_SHA}" || recovered=0
    atomic_state "${PREVIOUS_RELEASE_FILE}" "${PREVIOUS_RELEASE}" || recovered=0
  fi
  if [[ ${recovered} -eq 1 ]]; then
    if [[ ${SLOT_ASSIGNED} -eq 1 ]]; then
      rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
      if [[ -n ${INACTIVE_RELEASE} && -d ${INACTIVE_RELEASE} ]]; then
        ln -sfn "${INACTIVE_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}"
      fi
    fi
    remove_release "${NEW_RELEASE}" || recovered=0
  fi
  [[ -z ${DB_DUMP_PARTIAL} ]] || rm -f "${DB_DUMP_PARTIAL}"
  if [[ ${recovered} -eq 1 ]]; then
    rm -f "${PENDING_FILE}" "${UPSTREAM_BACKUP}"
    log "Previous release restored; failed build removed."
  else
    log "ERROR: recovery was incomplete; release and recovery journal retained. Manual inspection required."
  fi

  log "No database snapshot was restored because the previous release remained live and may have accepted writes."
  log "Inspect ${LOG_FILE} and service journals before retrying."
  exit "${original_status}"
}

validate_live_state
# A journal from the switching phase may describe Nginx's in-memory state,
# which cannot be inferred from its configuration file after an interrupted reload.
if [[ -f ${PENDING_FILE} ]]; then
  read -r pending_phase pending_sha <"${PENDING_FILE}"
  [[ ${pending_sha} =~ ^[0-9a-f]{40}$ ]] || exit 4
  case "${pending_phase}" in
    building) ;;
    switching)
      if [[ ${pending_sha} != "${OLD_SUCCESSFUL_SHA}" ]]; then
        log "Interrupted traffic switch: inspect services and Nginx before clearing ${PENDING_FILE}."
        exit 4
      fi ;;
    *) exit 4 ;;
  esac
fi
# Preserve the previous successful slot through the build of the next release.
if [[ -n ${INACTIVE_RELEASE} && -d ${INACTIVE_RELEASE} && ! -f ${PENDING_FILE} ]]; then
  PREVIOUS_RELEASE="${INACTIVE_RELEASE}"
  atomic_state "${PREVIOUS_RELEASE_FILE}" "${PREVIOUS_RELEASE}"
fi
reconcile_legacy_workers
stop_slot_stack "${NEW_SLOT}"
if [[ -f ${PENDING_FILE} ]]; then
  # SIGKILL/power failure before traffic switched: the live-state checks above
  # prove production still uses the last committed release. Restore standby.
  if [[ ${INACTIVE_RELEASE} == "${RELEASES_ROOT}/${pending_sha}" ]]; then
    rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
    if [[ -n ${PREVIOUS_RELEASE} && -d ${PREVIOUS_RELEASE} ]]; then
      ln -sfn "${PREVIOUS_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}"
    fi
    INACTIVE_RELEASE="${PREVIOUS_RELEASE}"
  fi
  rm -f "${PENDING_FILE}"
fi
prune_orphan_releases
prune_expired_dumps
if [[ ${MAINTENANCE} -eq 1 ]]; then
  log "Maintenance completed; active and previous releases preserved."
  df -h "${RELEASES_ROOT}"
  exit 0
fi

log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}."
check_node_runtime
[[ $(run_git config --get remote.origin.url) == "${GITEA_ORIGIN}" ]] || {
  log "Repository origin is not the approved Gitea repository; refusing deployment."; exit 5;
}
if run_git config --get-regexp '^url\..*\.insteadof$' >/dev/null; then
  log "Git URL rewrites are not allowed for production deployment."; exit 5
fi
run_git fetch --prune --no-tags "${GITEA_ORIGIN}" \
  '+refs/heads/master:refs/remotes/origin/master'
MASTER_SHA="$(run_git rev-parse refs/remotes/origin/master)"
if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then
  log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})."
  exit 5
fi
if [[ ${TARGET_SHA} == "$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)" ]]; then
  log "Revision ${TARGET_SHA} is already active."
  exit 0
fi

remove_release "${NEW_RELEASE}"
check_space 4294967296
atomic_state "${PENDING_FILE}" "building ${TARGET_SHA}"
MUTATION_STARTED=1
trap rollback EXIT
trap 'exit 129' HUP
trap 'exit 130' INT
trap 'exit 143' TERM

log "Creating isolated Git worktree ${NEW_RELEASE}."
run_git worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}"

run_as_deploy ln -s "${APP_ROOT}/backend/.env" "${NEW_RELEASE}/backend/.env"
run_as_deploy ln -s "${APP_ROOT}/frontend/.env" "${NEW_RELEASE}/frontend/.env"
run_as_deploy rm -rf "${NEW_RELEASE}/backend/media"
run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media"

log "Creating an isolated Python environment."
run_bounded_build /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv"
run_bounded_build env PIP_DISABLE_PIP_VERSION_CHECK=1 \
  "${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \
  --no-cache-dir --timeout=60 --retries=5 --only-binary=:all: --require-hashes \
  --requirement "${NEW_RELEASE}/ops/deploy/pip-bootstrap.txt"
run_bounded_build env PIP_DISABLE_PIP_VERSION_CHECK=1 \
  "${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \
  --no-cache-dir --timeout=60 --retries=5 --resume-retries=5 \
  --requirement "${NEW_RELEASE}/backend/requirements.txt"
run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" -m pip check
run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" -c 'import shutil; assert shutil.which("magick") or shutil.which("convert"), "ImageMagick is required for safe oversized-image processing"'
run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" \
  /usr/local/libexec/eusrr-deploy-healthcheck

log "Installing pinned frontend dependencies, including Linux native modules, and building Next.js."
install_frontend_dependencies
run_as_deploy rm -rf -- "${NEW_RELEASE}/.npm-cache"
run_as_deploy test -f \
  "${NEW_RELEASE}/frontend/node_modules/lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node"
run_as_deploy test -d \
  "${NEW_RELEASE}/frontend/node_modules/@next/swc-linux-x64-gnu"
run_bounded_build npm --prefix "${NEW_RELEASE}/frontend" run build

log "Running Django checks and collecting release static files."
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
  "${NEW_RELEASE}/backend/manage.py" check
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
  "${NEW_RELEASE}/backend/manage.py" collectstatic --noinput

run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/"
check_space 1073741824

install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}"
DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump"
DB_DUMP_PARTIAL="${DB_DUMP}.partial"
log "Creating a consistent PostgreSQL snapshot while the current release remains online."
run_database_dump "${DB_DUMP_PARTIAL}"
pg_restore --list "${DB_DUMP_PARTIAL}" >/dev/null
mv "${DB_DUMP_PARTIAL}" "${DB_DUMP}"
DB_DUMP_PARTIAL=""
chmod 0600 "${DB_DUMP}"
check_space 536870912

log "Planned migrations:"
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
  "${NEW_RELEASE}/backend/manage.py" migrate --plan
log "Applying backward-compatible database migrations."
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
  "${NEW_RELEASE}/backend/manage.py" migrate --noinput

log "Assigning release to the ${NEW_SLOT} slot and starting its web services."
SLOT_ASSIGNED=1
ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new"
mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}"
start_slot_web "${NEW_SLOT}"

wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$'
wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$'
run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" \
  /usr/local/libexec/eusrr-deploy-healthcheck --app-port "${BACKEND_PORT}"

cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}"
atomic_state "${PENDING_FILE}" "switching ${TARGET_SHA}"
UPSTREAM_CHANGED=1
point_current_at "${NEW_RELEASE}"
write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}"
nginx -t
systemctl reload nginx

wait_for_http "public frontend" "http://127.0.0.1/" '^200$' '172.11.0.11'
wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '172.11.0.11'

log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain."
sleep 10
log "Switching background workers."
OLD_STACK_STOPPED=1
if [[ ${OLD_SLOT} == legacy ]]; then
  stop_legacy_stack
else
  stop_slot_stack "${OLD_SLOT}"
fi
start_slot_celery "${NEW_SLOT}"
reconcile_legacy_workers "${NEW_SLOT}"
NEW_STACK_ENABLED=1
enable_slot_stack "${NEW_SLOT}"
disable_old_stack

find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \
  || log "WARNING: old database snapshots could not be pruned."
# Keep files referenced by either retained release regardless of their mtime.
run_as_deploy python3 - "${SHARED_ROOT}/next-static" "${NEW_RELEASE}" "${OLD_RELEASE}" <<'PY'
import sys
import time
from pathlib import Path

shared = Path(sys.argv[1])
referenced = set()
for release in sys.argv[2:]:
    root = Path(release) / "frontend/.next/static"
    referenced.update(p.relative_to(root) for p in root.rglob("*") if p.is_file())
for path in shared.rglob("*"):
    if (path.is_file() and not path.is_symlink()
            and path.relative_to(shared) not in referenced
            and path.stat().st_mtime < time.time() - 30 * 86400):
        path.unlink()
PY
atomic_state "${ACTIVE_SLOT_FILE}" "${NEW_SLOT}"
atomic_state "${STATE_DIR}/last-successful-sha" "${TARGET_SHA}"
atomic_state "${PREVIOUS_RELEASE_FILE}" "${OLD_RELEASE}"
COMMITTED=1
rm -f "${UPSTREAM_BACKUP}" "${PENDING_FILE}" || true
trap - EXIT HUP INT TERM
# The old active slot is the rollback release; discard older unreferenced builds.
PREVIOUS_RELEASE="${OLD_RELEASE}"
prune_orphan_releases || log "WARNING: unused releases could not be pruned; next run will retry."
log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}."