Corp/ops/deploy/eusrr-deploy

#!/usr/bin/env bash
set -Eeuo pipefail

readonly APP_ROOT="/srv/eusrr"
readonly RELEASES_ROOT="/srv/eusrr-releases"
readonly SLOTS_ROOT="/srv/eusrr-slots"
readonly SHARED_ROOT="/srv/eusrr-shared"
readonly CURRENT_LINK="/srv/eusrr-current"
readonly DEPLOY_USER="deploy"
readonly BACKUP_DIR="/var/backups/eusrr/deploy"
readonly STATE_DIR="/var/lib/eusrr-deploy"
readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot"
readonly LOG_FILE="/var/log/eusrr-deploy.log"
readonly LOCK_FILE="/run/lock/eusrr-deploy.lock"
readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf"

if [[ ${EUID} -ne 0 ]]; then
  echo "This command must run as root." >&2
  exit 1
fi

install -d -o root -g root -m 0755 "${STATE_DIR}"
touch "${LOG_FILE}"
chmod 0600 "${LOG_FILE}"
exec > >(tee -a "${LOG_FILE}") 2>&1

log() {
  printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*"
}

run_as_deploy() {
  runuser -u "${DEPLOY_USER}" -- "$@"
}

run_in_release_backend() {
  runuser -u "${DEPLOY_USER}" -- \
    bash -c 'cd "$1"; shift; exec "$@"' _ "${NEW_RELEASE}/backend" "$@"
}

slot_ports() {
  case "$1" in
    blue)  BACKEND_PORT=9001; FRONTEND_PORT=3001 ;;
    green) BACKEND_PORT=9002; FRONTEND_PORT=3002 ;;
    *) log "Unknown deployment slot: $1"; return 1 ;;
  esac
}

if [[ ${1:-} == "--self-test" ]]; then
  for command in git curl flock runuser pg_dump npm nginx rsync; do
    command -v "${command}" >/dev/null
  done
  test -d "${APP_ROOT}/.git"
  run_as_deploy test -r "${APP_ROOT}/backend/.env"
  run_as_deploy test -r "${APP_ROOT}/frontend/.env"
  test -r "${UPSTREAM_FILE}"
  for unit in eusrr-backend@.service eusrr-frontend@.service \
    eusrr-celery-worker@.service eusrr-celery-beat@.service; do
    systemctl cat "${unit}" >/dev/null
  done
  nginx -t
  log "Blue-green deployment self-test passed."
  exit 0
fi

TARGET_SHA="${1:-}"
if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then
  echo "Usage: $0 <40-character lowercase commit SHA>" >&2
  exit 2
fi

exec 9>"${LOCK_FILE}"
if ! flock -n 9; then
  log "Another deployment is already running."
  exit 3
fi

OLD_SLOT="$(cat "${ACTIVE_SLOT_FILE}" 2>/dev/null || printf legacy)"
case "${OLD_SLOT}" in
  legacy) NEW_SLOT=blue ;;
  blue) NEW_SLOT=green ;;
  green) NEW_SLOT=blue ;;
  *) log "Invalid active slot state: ${OLD_SLOT}"; exit 4 ;;
esac

NEW_RELEASE="${RELEASES_ROOT}/${TARGET_SHA}"
if [[ ${OLD_SLOT} == legacy ]]; then
  OLD_RELEASE="${APP_ROOT}"
else
  OLD_RELEASE="$(readlink -f "${SLOTS_ROOT}/${OLD_SLOT}")"
fi

BACKEND_PORT=""
FRONTEND_PORT=""
slot_ports "${NEW_SLOT}"

UPSTREAM_BACKUP="${STATE_DIR}/upstreams-before-${TARGET_SHA}"
UPSTREAM_CHANGED=0
OLD_STACK_STOPPED=0
NEW_STACK_ENABLED=0
DB_DUMP=""

run_database_dump() {
  local dump_file="$1"
  runuser -u "${DEPLOY_USER}" -- \
    env EUSRR_DB_DUMP="${dump_file}" \
    "${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY'
import os
from pathlib import Path

from dotenv import dotenv_values

config = dotenv_values("/srv/eusrr/backend/.env")

def required(name: str) -> str:
    value = config.get(name)
    if not value:
        raise SystemExit(f"Required database setting is missing: {name}")
    return str(value)

executable = "/usr/bin/pg_dump"
if not Path(executable).is_file():
    raise SystemExit(f"Database utility is not installed: {executable}")

environment = os.environ.copy()
environment["PGPASSWORD"] = required("POSTGRES_PASSWORD")
arguments = [
    executable,
    "--host", str(config.get("DB_HOST") or "127.0.0.1"),
    "--port", str(config.get("DB_PORT") or "5432"),
    "--username", required("POSTGRES_USER"),
    "--dbname", required("POSTGRES_DB"),
    "--format=custom",
    "--no-owner",
    "--no-acl",
    "--file", os.environ["EUSRR_DB_DUMP"],
]
os.execve(executable, arguments, environment)
PY
}

wait_for_http() {
  local name="$1"
  local url="$2"
  local expected="$3"
  local host_header="${4:-}"
  local attempt code
  local -a curl_args=(--silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5)

  if [[ -n ${host_header} ]]; then
    curl_args+=(--header "Host: ${host_header}")
  fi

  for attempt in $(seq 1 45); do
    code="$(curl "${curl_args[@]}" "${url}" || true)"
    if [[ ${code} =~ ${expected} ]]; then
      log "Health check passed: ${name} returned ${code}."
      return 0
    fi
    sleep 2
  done

  log "Health check failed: ${name} returned ${code:-no response}."
  return 1
}

write_upstreams() {
  local backend_port="$1"
  local frontend_port="$2"
  local temporary_file
  temporary_file="$(mktemp)"
  cat >"${temporary_file}" <<EOF
upstream eusrr_backend {
    server 127.0.0.1:${backend_port};
    keepalive 32;
}

upstream eusrr_frontend {
    server 127.0.0.1:${frontend_port};
    keepalive 32;
}
EOF
  install -o root -g root -m 0644 "${temporary_file}" "${UPSTREAM_FILE}"
  rm -f "${temporary_file}"
  UPSTREAM_CHANGED=1
}

point_current_at() {
  local release="$1"
  ln -sfn "${release}" "${CURRENT_LINK}.new"
  mv -Tf "${CURRENT_LINK}.new" "${CURRENT_LINK}"
}

start_slot_web() {
  local slot="$1"
  systemctl start "eusrr-backend@${slot}.service" "eusrr-frontend@${slot}.service"
}

stop_slot_stack() {
  local slot="$1"
  systemctl stop \
    "eusrr-celery-beat@${slot}.service" \
    "eusrr-celery-worker@${slot}.service" \
    "eusrr-frontend@${slot}.service" \
    "eusrr-backend@${slot}.service" || true
}

start_slot_celery() {
  local slot="$1"
  systemctl start "eusrr-celery-worker@${slot}.service" "eusrr-celery-beat@${slot}.service"
  systemctl is-active --quiet "eusrr-celery-worker@${slot}.service"
  systemctl is-active --quiet "eusrr-celery-beat@${slot}.service"
}

enable_slot_stack() {
  local slot="$1"
  systemctl enable \
    "eusrr-backend@${slot}.service" \
    "eusrr-frontend@${slot}.service" \
    "eusrr-celery-worker@${slot}.service" \
    "eusrr-celery-beat@${slot}.service"
}

disable_old_stack() {
  if [[ ${OLD_SLOT} == legacy ]]; then
    systemctl disable gunicorn-eusrr.service nextjs-eusrr.service \
      celery-eusrr-worker.service celery-eusrr-beat.service || true
  else
    systemctl disable \
      "eusrr-backend@${OLD_SLOT}.service" \
      "eusrr-frontend@${OLD_SLOT}.service" \
      "eusrr-celery-worker@${OLD_SLOT}.service" \
      "eusrr-celery-beat@${OLD_SLOT}.service" || true
  fi
}

enable_old_stack() {
  if [[ ${OLD_SLOT} == legacy ]]; then
    systemctl enable gunicorn-eusrr.service nextjs-eusrr.service \
      celery-eusrr-worker.service celery-eusrr-beat.service || true
  else
    systemctl enable \
      "eusrr-backend@${OLD_SLOT}.service" \
      "eusrr-frontend@${OLD_SLOT}.service" \
      "eusrr-celery-worker@${OLD_SLOT}.service" \
      "eusrr-celery-beat@${OLD_SLOT}.service" || true
  fi
}

stop_legacy_stack() {
  systemctl stop \
    celery-eusrr-beat.service celery-eusrr-worker.service \
    nextjs-eusrr.service gunicorn-eusrr.service || true
}

start_old_stack() {
  if [[ ${OLD_SLOT} == legacy ]]; then
    systemctl restart gunicorn-eusrr.service nextjs-eusrr.service \
      celery-eusrr-worker.service celery-eusrr-beat.service
  else
    systemctl restart "eusrr-backend@${OLD_SLOT}.service" "eusrr-frontend@${OLD_SLOT}.service" \
      "eusrr-celery-worker@${OLD_SLOT}.service" "eusrr-celery-beat@${OLD_SLOT}.service"
  fi
}

remove_release() {
  local release="$1"
  case "${release}" in
    "${RELEASES_ROOT}"/*) ;;
    *) log "Refusing to remove release outside ${RELEASES_ROOT}: ${release}"; return 1 ;;
  esac
  if [[ -e ${release} ]]; then
    run_as_deploy git -C "${APP_ROOT}" worktree remove --force "${release}"
  fi
}

rollback() {
  local original_status=$?
  trap - ERR
  set +e
  log "Deployment failed with exit code ${original_status}; keeping the previous release active."

  if [[ ${UPSTREAM_CHANGED} -eq 1 && -f ${UPSTREAM_BACKUP} ]]; then
    install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}"
    point_current_at "${OLD_RELEASE}"
    nginx -t && systemctl reload nginx
  fi

  stop_slot_stack "${NEW_SLOT}"
  if [[ ${NEW_STACK_ENABLED} -eq 1 ]]; then
    systemctl disable \
      "eusrr-backend@${NEW_SLOT}.service" \
      "eusrr-frontend@${NEW_SLOT}.service" \
      "eusrr-celery-worker@${NEW_SLOT}.service" \
      "eusrr-celery-beat@${NEW_SLOT}.service" || true
    enable_old_stack
  fi
  if [[ ${OLD_STACK_STOPPED} -eq 1 ]]; then
    start_old_stack || log "ERROR: failed to restart the previous application stack."
  fi

  log "No database snapshot was restored because the previous release remained live and may have accepted writes."
  log "Inspect ${LOG_FILE} and service journals before retrying."
  exit "${original_status}"
}
trap rollback ERR

log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}."
run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \
  '+refs/heads/master:refs/remotes/origin/master'
MASTER_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse refs/remotes/origin/master)"
if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then
  log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})."
  exit 5
fi
if [[ ${TARGET_SHA} == "$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)" ]]; then
  log "Revision ${TARGET_SHA} is already active."
  exit 0
fi

if [[ -L ${SLOTS_ROOT}/${NEW_SLOT} ]]; then
  stale_release="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}")"
  if [[ ${stale_release} != "${OLD_RELEASE}" ]]; then
    rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
    remove_release "${stale_release}"
  fi
fi
remove_release "${NEW_RELEASE}"
run_as_deploy git -C "${APP_ROOT}" worktree prune

available_bytes="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
if (( available_bytes < 4294967296 )); then
  log "At least 4 GiB of free disk space is required to build a release."
  exit 6
fi

log "Creating isolated Git worktree ${NEW_RELEASE}."
run_as_deploy git -C "${APP_ROOT}" worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}"

run_as_deploy ln -s "${APP_ROOT}/backend/.env" "${NEW_RELEASE}/backend/.env"
run_as_deploy ln -s "${APP_ROOT}/frontend/.env" "${NEW_RELEASE}/frontend/.env"
run_as_deploy rm -rf "${NEW_RELEASE}/backend/media"
run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media"

log "Creating an isolated Python environment."
run_as_deploy /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv"
run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \
  "${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \
  --requirement "${NEW_RELEASE}/backend/requirements.txt"

log "Installing pinned frontend dependencies and building Next.js."
run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" ci --no-audit --no-fund
run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" run build

log "Running Django checks and collecting release static files."
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
  "${NEW_RELEASE}/backend/manage.py" check
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
  "${NEW_RELEASE}/backend/manage.py" collectstatic --noinput

run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/"

install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}"
DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump"
log "Creating a consistent PostgreSQL snapshot while the current release remains online."
run_database_dump "${DB_DUMP}"
chmod 0600 "${DB_DUMP}"

log "Planned migrations:"
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
  "${NEW_RELEASE}/backend/manage.py" migrate --plan
log "Applying backward-compatible database migrations."
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
  "${NEW_RELEASE}/backend/manage.py" migrate --noinput

log "Assigning release to the ${NEW_SLOT} slot and starting its web services."
ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new"
mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}"
start_slot_web "${NEW_SLOT}"

wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$'
wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$'

cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}"
UPSTREAM_CHANGED=1
point_current_at "${NEW_RELEASE}"
write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}"
nginx -t
systemctl reload nginx

wait_for_http "public frontend" "http://127.0.0.1/" '^200$' '172.11.0.11'
wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '172.11.0.11'

log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain."
sleep 10
log "Switching background workers."
if [[ ${OLD_SLOT} == legacy ]]; then
  stop_legacy_stack
else
  stop_slot_stack "${OLD_SLOT}"
fi
OLD_STACK_STOPPED=1
start_slot_celery "${NEW_SLOT}"
NEW_STACK_ENABLED=1
enable_slot_stack "${NEW_SLOT}"
disable_old_stack

find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \
  || log "WARNING: old database snapshots could not be pruned."
run_as_deploy find "${SHARED_ROOT}/next-static" -type f -mtime +30 -delete \
  || log "WARNING: old Next.js assets could not be pruned."
rm -f "${UPSTREAM_BACKUP}" || true
printf '%s\n' "${NEW_SLOT}" >"${ACTIVE_SLOT_FILE}"
printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha"
trap - ERR
log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}."