#!/usr/bin/env bash set -Eeuo pipefail readonly APP_ROOT="/srv/eusrr" readonly BACKEND_DIR="${APP_ROOT}/backend" readonly FRONTEND_DIR="${APP_ROOT}/frontend" readonly DEPLOY_USER="deploy" readonly BACKUP_DIR="/var/backups/eusrr/deploy" readonly STATE_DIR="/var/lib/eusrr-deploy" readonly LOG_FILE="/var/log/eusrr-deploy.log" readonly LOCK_FILE="/run/lock/eusrr-deploy.lock" readonly -a SERVICES=( gunicorn-eusrr.service nextjs-eusrr.service celery-eusrr-worker.service celery-eusrr-beat.service ) if [[ ${EUID} -ne 0 ]]; then echo "This command must run as root." >&2 exit 1 fi install -d -o root -g root -m 0755 "${STATE_DIR}" touch "${LOG_FILE}" chmod 0600 "${LOG_FILE}" exec > >(tee -a "${LOG_FILE}") 2>&1 log() { printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*" } run_as_deploy() { runuser -u "${DEPLOY_USER}" -- "$@" } run_in_backend() { runuser -u "${DEPLOY_USER}" -- \ bash -c 'cd "$1"; shift; exec "$@"' _ "${BACKEND_DIR}" "$@" } if [[ ${1:-} == "--self-test" ]]; then for command in git curl flock runuser pg_dump pg_restore npm; do command -v "${command}" >/dev/null done test -d "${APP_ROOT}/.git" test -x "${BACKEND_DIR}/.venv/bin/python" run_as_deploy test -r "${BACKEND_DIR}/.env" for service in "${SERVICES[@]}"; do systemctl cat "${service}" >/dev/null done log "Self-test passed." exit 0 fi TARGET_SHA="${1:-}" if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then echo "Usage: $0 <40-character lowercase commit SHA>" >&2 exit 2 fi exec 9>"${LOCK_FILE}" if ! flock -n 9; then log "Another deployment is already running." exit 3 fi OLD_SHA="" DB_DUMP="" SERVICES_STOPPED=0 MIGRATION_STARTED=0 run_database_tool() { local action="$1" local dump_file="$2" runuser -u "${DEPLOY_USER}" -- \ env EUSRR_DB_ACTION="${action}" EUSRR_DB_DUMP="${dump_file}" \ "${BACKEND_DIR}/.venv/bin/python" - <<'PY' import os import sys from pathlib import Path from dotenv import dotenv_values config = dotenv_values("/srv/eusrr/backend/.env") def required(name: str) -> str: value = config.get(name) if not value: raise SystemExit(f"Required database setting is missing: {name}") return str(value) action = os.environ["EUSRR_DB_ACTION"] dump_file = os.environ["EUSRR_DB_DUMP"] database = required("POSTGRES_DB") user = required("POSTGRES_USER") password = required("POSTGRES_PASSWORD") host = str(config.get("DB_HOST") or "127.0.0.1") port = str(config.get("DB_PORT") or "5432") environment = os.environ.copy() environment["PGPASSWORD"] = password common = ["--host", host, "--port", port, "--username", user, "--dbname", database] if action == "dump": executable = "/usr/bin/pg_dump" arguments = [executable, *common, "--format=custom", "--no-owner", "--no-acl", "--file", dump_file] elif action == "restore": executable = "/usr/bin/pg_restore" arguments = [ executable, *common, "--clean", "--if-exists", "--no-owner", "--no-acl", "--exit-on-error", "--single-transaction", dump_file, ] else: raise SystemExit(f"Unsupported database action: {action}") if not Path(executable).is_file(): raise SystemExit(f"Database utility is not installed: {executable}") os.execve(executable, arguments, environment) PY } install_dependencies_and_build() { log "Installing pinned backend dependencies." run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \ "${BACKEND_DIR}/.venv/bin/python" -m pip install --requirement "${BACKEND_DIR}/requirements.txt" log "Installing pinned frontend dependencies." run_as_deploy npm --prefix "${FRONTEND_DIR}" ci --no-audit --no-fund log "Building the frontend." run_as_deploy npm --prefix "${FRONTEND_DIR}" run build log "Running Django system checks." run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" check } wait_for_http() { local name="$1" local url="$2" local expected="$3" local attempt code for attempt in $(seq 1 45); do code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5 "${url}" || true)" if [[ ${code} =~ ${expected} ]]; then log "Health check passed: ${name} returned ${code}." return 0 fi sleep 2 done log "Health check failed: ${name} returned ${code:-no response}." return 1 } start_and_verify_services() { systemctl restart "${SERVICES[@]}" SERVICES_STOPPED=0 for service in "${SERVICES[@]}"; do systemctl is-active --quiet "${service}" done wait_for_http "frontend" "http://127.0.0.1:3000/" '^200$' wait_for_http "backend" "http://127.0.0.1:9000/admin/login/" '^(200|302)$' } rollback() { local original_status=$? trap - ERR set +e log "Deployment failed with exit code ${original_status}; starting rollback." if [[ ${SERVICES_STOPPED} -eq 0 && ${MIGRATION_STARTED} -eq 1 ]]; then systemctl stop "${SERVICES[@]}" SERVICES_STOPPED=1 fi if [[ ${MIGRATION_STARTED} -eq 1 && -n ${DB_DUMP} && -f ${DB_DUMP} ]]; then log "Restoring the database snapshot." run_database_tool restore "${DB_DUMP}" || log "ERROR: database restore failed; manual recovery is required." fi if [[ -n ${OLD_SHA} ]]; then log "Restoring source revision ${OLD_SHA}." run_as_deploy git -C "${APP_ROOT}" reset --hard "${OLD_SHA}" install_dependencies_and_build || log "ERROR: rebuilding the previous revision failed." run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput \ || log "ERROR: restoring static files failed." fi systemctl restart "${SERVICES[@]}" || true log "Rollback finished. Inspect ${LOG_FILE} and service journals before retrying." exit "${original_status}" } trap rollback ERR log "Starting deployment of ${TARGET_SHA}." OLD_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse HEAD)" dirty="$(run_as_deploy git -C "${APP_ROOT}" status --porcelain --untracked-files=normal)" unexpected_dirty="$(printf '%s\n' "${dirty}" | grep -vE '^ M frontend/next-env\.d\.ts$' || true)" if [[ -n ${unexpected_dirty} ]]; then log "Refusing to overwrite unexpected production changes:" printf '%s\n' "${unexpected_dirty}" exit 4 fi run_as_deploy git -C "${APP_ROOT}" restore --worktree frontend/next-env.d.ts 2>/dev/null || true run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \ '+refs/heads/master:refs/remotes/origin/master' MASTER_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse refs/remotes/origin/master)" if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})." exit 5 fi run_as_deploy git -C "${APP_ROOT}" reset --hard "${TARGET_SHA}" install_dependencies_and_build log "Stopping application services for the database-safe deployment window." systemctl stop "${SERVICES[@]}" SERVICES_STOPPED=1 install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}" DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump" log "Creating PostgreSQL snapshot ${DB_DUMP}." run_database_tool dump "${DB_DUMP}" chmod 0600 "${DB_DUMP}" MIGRATION_STARTED=1 log "Applying database migrations." run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" migrate --noinput log "Collecting Django static files." run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput log "Starting and checking application services." start_and_verify_services printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha" find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete trap - ERR log "Deployment of ${TARGET_SHA} completed successfully."