Corp/ops/deploy/eusrr-deploy

#!/usr/bin/env bash
set -Eeuo pipefail

readonly APP_ROOT="/srv/eusrr"
readonly BACKEND_DIR="${APP_ROOT}/backend"
readonly FRONTEND_DIR="${APP_ROOT}/frontend"
readonly DEPLOY_USER="deploy"
readonly BACKUP_DIR="/var/backups/eusrr/deploy"
readonly STATE_DIR="/var/lib/eusrr-deploy"
readonly LOG_FILE="/var/log/eusrr-deploy.log"
readonly LOCK_FILE="/run/lock/eusrr-deploy.lock"
readonly -a SERVICES=(
  gunicorn-eusrr.service
  nextjs-eusrr.service
  celery-eusrr-worker.service
  celery-eusrr-beat.service
)

if [[ ${EUID} -ne 0 ]]; then
  echo "This command must run as root." >&2
  exit 1
fi

install -d -o root -g root -m 0755 "${STATE_DIR}"
touch "${LOG_FILE}"
chmod 0600 "${LOG_FILE}"
exec > >(tee -a "${LOG_FILE}") 2>&1

log() {
  printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*"
}

run_as_deploy() {
  runuser -u "${DEPLOY_USER}" -- "$@"
}

run_in_backend() {
  runuser -u "${DEPLOY_USER}" -- \
    bash -c 'cd "$1"; shift; exec "$@"' _ "${BACKEND_DIR}" "$@"
}

if [[ ${1:-} == "--self-test" ]]; then
  for command in git curl flock runuser pg_dump pg_restore npm; do
    command -v "${command}" >/dev/null
  done
  test -d "${APP_ROOT}/.git"
  test -x "${BACKEND_DIR}/.venv/bin/python"
  run_as_deploy test -r "${BACKEND_DIR}/.env"
  for service in "${SERVICES[@]}"; do
    systemctl cat "${service}" >/dev/null
  done
  log "Self-test passed."
  exit 0
fi

TARGET_SHA="${1:-}"
if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then
  echo "Usage: $0 <40-character lowercase commit SHA>" >&2
  exit 2
fi

exec 9>"${LOCK_FILE}"
if ! flock -n 9; then
  log "Another deployment is already running."
  exit 3
fi

OLD_SHA=""
DB_DUMP=""
SERVICES_STOPPED=0
MIGRATION_STARTED=0

run_database_tool() {
  local action="$1"
  local dump_file="$2"

  runuser -u "${DEPLOY_USER}" -- \
    env EUSRR_DB_ACTION="${action}" EUSRR_DB_DUMP="${dump_file}" \
    "${BACKEND_DIR}/.venv/bin/python" - <<'PY'
import os
import sys
from pathlib import Path

from dotenv import dotenv_values

config = dotenv_values("/srv/eusrr/backend/.env")

def required(name: str) -> str:
    value = config.get(name)
    if not value:
        raise SystemExit(f"Required database setting is missing: {name}")
    return str(value)

action = os.environ["EUSRR_DB_ACTION"]
dump_file = os.environ["EUSRR_DB_DUMP"]
database = required("POSTGRES_DB")
user = required("POSTGRES_USER")
password = required("POSTGRES_PASSWORD")
host = str(config.get("DB_HOST") or "127.0.0.1")
port = str(config.get("DB_PORT") or "5432")

environment = os.environ.copy()
environment["PGPASSWORD"] = password
common = ["--host", host, "--port", port, "--username", user, "--dbname", database]

if action == "dump":
    executable = "/usr/bin/pg_dump"
    arguments = [executable, *common, "--format=custom", "--no-owner", "--no-acl", "--file", dump_file]
elif action == "restore":
    executable = "/usr/bin/pg_restore"
    arguments = [
        executable,
        *common,
        "--clean",
        "--if-exists",
        "--no-owner",
        "--no-acl",
        "--exit-on-error",
        "--single-transaction",
        dump_file,
    ]
else:
    raise SystemExit(f"Unsupported database action: {action}")

if not Path(executable).is_file():
    raise SystemExit(f"Database utility is not installed: {executable}")
os.execve(executable, arguments, environment)
PY
}

install_dependencies_and_build() {
  log "Installing pinned backend dependencies."
  run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \
    "${BACKEND_DIR}/.venv/bin/python" -m pip install --requirement "${BACKEND_DIR}/requirements.txt"

  log "Installing pinned frontend dependencies."
  run_as_deploy npm --prefix "${FRONTEND_DIR}" ci --no-audit --no-fund

  log "Building the frontend."
  run_as_deploy npm --prefix "${FRONTEND_DIR}" run build

  log "Running Django system checks."
  run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" check
}

wait_for_http() {
  local name="$1"
  local url="$2"
  local expected="$3"
  local attempt code

  for attempt in $(seq 1 45); do
    code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5 "${url}" || true)"
    if [[ ${code} =~ ${expected} ]]; then
      log "Health check passed: ${name} returned ${code}."
      return 0
    fi
    sleep 2
  done

  log "Health check failed: ${name} returned ${code:-no response}."
  return 1
}

start_and_verify_services() {
  systemctl restart "${SERVICES[@]}"
  SERVICES_STOPPED=0
  for service in "${SERVICES[@]}"; do
    systemctl is-active --quiet "${service}"
  done
  wait_for_http "frontend" "http://127.0.0.1:3000/" '^200$'
  wait_for_http "backend" "http://127.0.0.1:9000/admin/login/" '^(200|302)$'
}

rollback() {
  local original_status=$?
  trap - ERR
  set +e
  log "Deployment failed with exit code ${original_status}; starting rollback."

  if [[ ${SERVICES_STOPPED} -eq 0 && ${MIGRATION_STARTED} -eq 1 ]]; then
    systemctl stop "${SERVICES[@]}"
    SERVICES_STOPPED=1
  fi

  if [[ ${MIGRATION_STARTED} -eq 1 && -n ${DB_DUMP} && -f ${DB_DUMP} ]]; then
    log "Restoring the database snapshot."
    run_database_tool restore "${DB_DUMP}" || log "ERROR: database restore failed; manual recovery is required."
  fi

  if [[ -n ${OLD_SHA} ]]; then
    log "Restoring source revision ${OLD_SHA}."
    run_as_deploy git -C "${APP_ROOT}" reset --hard "${OLD_SHA}"
    install_dependencies_and_build || log "ERROR: rebuilding the previous revision failed."
    run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput \
      || log "ERROR: restoring static files failed."
  fi

  systemctl restart "${SERVICES[@]}" || true
  log "Rollback finished. Inspect ${LOG_FILE} and service journals before retrying."
  exit "${original_status}"
}
trap rollback ERR

log "Starting deployment of ${TARGET_SHA}."
OLD_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse HEAD)"

dirty="$(run_as_deploy git -C "${APP_ROOT}" status --porcelain --untracked-files=normal)"
unexpected_dirty="$(printf '%s\n' "${dirty}" | grep -vE '^ M frontend/next-env\.d\.ts$' || true)"
if [[ -n ${unexpected_dirty} ]]; then
  log "Refusing to overwrite unexpected production changes:"
  printf '%s\n' "${unexpected_dirty}"
  exit 4
fi

run_as_deploy git -C "${APP_ROOT}" restore --worktree frontend/next-env.d.ts 2>/dev/null || true
run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \
  '+refs/heads/master:refs/remotes/origin/master'

MASTER_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse refs/remotes/origin/master)"
if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then
  log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})."
  exit 5
fi

run_as_deploy git -C "${APP_ROOT}" reset --hard "${TARGET_SHA}"
install_dependencies_and_build

log "Stopping application services for the database-safe deployment window."
systemctl stop "${SERVICES[@]}"
SERVICES_STOPPED=1

install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}"
DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump"
log "Creating PostgreSQL snapshot ${DB_DUMP}."
run_database_tool dump "${DB_DUMP}"
chmod 0600 "${DB_DUMP}"

MIGRATION_STARTED=1
log "Applying database migrations."
run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" migrate --noinput

log "Collecting Django static files."
run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput

log "Starting and checking application services."
start_and_verify_services

printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha"
find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete
trap - ERR
log "Deployment of ${TARGET_SHA} completed successfully."