#!/usr/bin/env bash set -Eeuo pipefail readonly APP_ROOT="/srv/eusrr" readonly RELEASES_ROOT="/srv/eusrr-releases" readonly SLOTS_ROOT="/srv/eusrr-slots" readonly SHARED_ROOT="/srv/eusrr-shared" readonly CURRENT_LINK="/srv/eusrr-current" readonly DEPLOY_USER="deploy" readonly BACKUP_DIR="/var/backups/eusrr/deploy" readonly STATE_DIR="/var/lib/eusrr-deploy" readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot" readonly LOG_FILE="/var/log/eusrr-deploy.log" readonly LOCK_FILE="/run/lock/eusrr-deploy.lock" readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf" if [[ ${EUID} -ne 0 ]]; then echo "This command must run as root." >&2 exit 1 fi install -d -o root -g root -m 0755 "${STATE_DIR}" touch "${LOG_FILE}" chmod 0600 "${LOG_FILE}" exec > >(tee -a "${LOG_FILE}") 2>&1 log() { printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*" } run_as_deploy() { runuser -u "${DEPLOY_USER}" -- "$@" } run_in_release_backend() { runuser -u "${DEPLOY_USER}" -- \ bash -c 'cd "$1"; shift; exec "$@"' _ "${NEW_RELEASE}/backend" "$@" } slot_ports() { case "$1" in blue) BACKEND_PORT=9001; FRONTEND_PORT=3001 ;; green) BACKEND_PORT=9002; FRONTEND_PORT=3002 ;; *) log "Unknown deployment slot: $1"; return 1 ;; esac } if [[ ${1:-} == "--self-test" ]]; then for command in git curl flock runuser pg_dump npm nginx rsync; do command -v "${command}" >/dev/null done test -d "${APP_ROOT}/.git" run_as_deploy test -r "${APP_ROOT}/backend/.env" run_as_deploy test -r "${APP_ROOT}/frontend/.env" test -r "${UPSTREAM_FILE}" for unit in eusrr-backend@.service eusrr-frontend@.service \ eusrr-celery-worker@.service eusrr-celery-beat@.service; do systemctl cat "${unit}" >/dev/null done nginx -t log "Blue-green deployment self-test passed." exit 0 fi TARGET_SHA="${1:-}" if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then echo "Usage: $0 <40-character lowercase commit SHA>" >&2 exit 2 fi exec 9>"${LOCK_FILE}" if ! flock -n 9; then log "Another deployment is already running." exit 3 fi OLD_SLOT="$(cat "${ACTIVE_SLOT_FILE}" 2>/dev/null || printf legacy)" case "${OLD_SLOT}" in legacy) NEW_SLOT=blue ;; blue) NEW_SLOT=green ;; green) NEW_SLOT=blue ;; *) log "Invalid active slot state: ${OLD_SLOT}"; exit 4 ;; esac NEW_RELEASE="${RELEASES_ROOT}/${TARGET_SHA}" if [[ ${OLD_SLOT} == legacy ]]; then OLD_RELEASE="${APP_ROOT}" else OLD_RELEASE="$(readlink -f "${SLOTS_ROOT}/${OLD_SLOT}")" fi BACKEND_PORT="" FRONTEND_PORT="" slot_ports "${NEW_SLOT}" UPSTREAM_BACKUP="${STATE_DIR}/upstreams-before-${TARGET_SHA}" UPSTREAM_CHANGED=0 OLD_STACK_STOPPED=0 NEW_STACK_ENABLED=0 DB_DUMP="" run_database_dump() { local dump_file="$1" runuser -u "${DEPLOY_USER}" -- \ env EUSRR_DB_DUMP="${dump_file}" \ "${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY' import os from pathlib import Path from dotenv import dotenv_values config = dotenv_values("/srv/eusrr/backend/.env") def required(name: str) -> str: value = config.get(name) if not value: raise SystemExit(f"Required database setting is missing: {name}") return str(value) executable = "/usr/bin/pg_dump" if not Path(executable).is_file(): raise SystemExit(f"Database utility is not installed: {executable}") environment = os.environ.copy() environment["PGPASSWORD"] = required("POSTGRES_PASSWORD") arguments = [ executable, "--host", str(config.get("DB_HOST") or "127.0.0.1"), "--port", str(config.get("DB_PORT") or "5432"), "--username", required("POSTGRES_USER"), "--dbname", required("POSTGRES_DB"), "--format=custom", "--no-owner", "--no-acl", "--file", os.environ["EUSRR_DB_DUMP"], ] os.execve(executable, arguments, environment) PY } wait_for_http() { local name="$1" local url="$2" local expected="$3" local host_header="${4:-}" local attempt code local -a curl_args=(--silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5) if [[ -n ${host_header} ]]; then curl_args+=(--header "Host: ${host_header}") fi for attempt in $(seq 1 45); do code="$(curl "${curl_args[@]}" "${url}" || true)" if [[ ${code} =~ ${expected} ]]; then log "Health check passed: ${name} returned ${code}." return 0 fi sleep 2 done log "Health check failed: ${name} returned ${code:-no response}." return 1 } write_upstreams() { local backend_port="$1" local frontend_port="$2" local temporary_file temporary_file="$(mktemp)" cat >"${temporary_file}" </dev/null || true)" ]]; then log "Revision ${TARGET_SHA} is already active." exit 0 fi if [[ -L ${SLOTS_ROOT}/${NEW_SLOT} ]]; then stale_release="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}")" if [[ ${stale_release} != "${OLD_RELEASE}" ]]; then rm -f "${SLOTS_ROOT}/${NEW_SLOT}" remove_release "${stale_release}" fi fi remove_release "${NEW_RELEASE}" run_as_deploy git -C "${APP_ROOT}" worktree prune available_bytes="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')" if (( available_bytes < 4294967296 )); then log "At least 4 GiB of free disk space is required to build a release." exit 6 fi log "Creating isolated Git worktree ${NEW_RELEASE}." run_as_deploy git -C "${APP_ROOT}" worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}" run_as_deploy ln -s "${APP_ROOT}/backend/.env" "${NEW_RELEASE}/backend/.env" run_as_deploy ln -s "${APP_ROOT}/frontend/.env" "${NEW_RELEASE}/frontend/.env" run_as_deploy rm -rf "${NEW_RELEASE}/backend/media" run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media" log "Creating an isolated Python environment." run_as_deploy /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv" run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \ "${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \ --requirement "${NEW_RELEASE}/backend/requirements.txt" log "Installing pinned frontend dependencies and building Next.js." run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" ci --no-audit --no-fund run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" run build log "Running Django checks and collecting release static files." run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ "${NEW_RELEASE}/backend/manage.py" check run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ "${NEW_RELEASE}/backend/manage.py" collectstatic --noinput run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/" install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}" DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump" log "Creating a consistent PostgreSQL snapshot while the current release remains online." run_database_dump "${DB_DUMP}" chmod 0600 "${DB_DUMP}" log "Planned migrations:" run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ "${NEW_RELEASE}/backend/manage.py" migrate --plan log "Applying backward-compatible database migrations." run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ "${NEW_RELEASE}/backend/manage.py" migrate --noinput log "Assigning release to the ${NEW_SLOT} slot and starting its web services." ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new" mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}" start_slot_web "${NEW_SLOT}" wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$' wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$' cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}" UPSTREAM_CHANGED=1 point_current_at "${NEW_RELEASE}" write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}" nginx -t systemctl reload nginx wait_for_http "public frontend" "http://127.0.0.1/" '^200$' '172.11.0.11' wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '172.11.0.11' log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain." sleep 10 log "Switching background workers." if [[ ${OLD_SLOT} == legacy ]]; then stop_legacy_stack else stop_slot_stack "${OLD_SLOT}" fi OLD_STACK_STOPPED=1 start_slot_celery "${NEW_SLOT}" NEW_STACK_ENABLED=1 enable_slot_stack "${NEW_SLOT}" disable_old_stack find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \ || log "WARNING: old database snapshots could not be pruned." run_as_deploy find "${SHARED_ROOT}/next-static" -type f -mtime +30 -delete \ || log "WARNING: old Next.js assets could not be pruned." rm -f "${UPSTREAM_BACKUP}" || true printf '%s\n' "${NEW_SLOT}" >"${ACTIVE_SLOT_FILE}" printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha" trap - ERR log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}."