diff --git a/.gitattributes b/.gitattributes index dd6b4c87..4c244ca6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,2 +1,3 @@ ops/deploy/* text eol=lf +ops/deploy/systemd/*.service text eol=lf .gitea/workflows/*.yml text eol=lf diff --git a/ops/deploy/README.md b/ops/deploy/README.md index eeef42f6..bb63f6ea 100644 --- a/ops/deploy/README.md +++ b/ops/deploy/README.md @@ -1,30 +1,44 @@ -# CORP production deployment +# CORP blue-green production deployment -Production is deployed from the current `master` commit in the Gitea repository -`Robotail/Corp`. The Gitea Actions runner only opens a restricted SSH connection; -all validation, backup, build, migration, restart, health checks, and rollback are -performed on the CORP server. +Production is deployed from the current `master` commit in Gitea repository +`Robotail/Corp`. The Gitea runner only opens a restricted SSH connection. Build, +backup, migrations, health checks, and traffic switching run on the CORP server. + +Two native slots are used instead of containers: + +- `blue`: backend `9001`, frontend `3001`; +- `green`: backend `9002`, frontend `3002`; +- releases: `/srv/eusrr-releases/`; +- slot links: `/srv/eusrr-slots/{blue,green}`; +- active release link: `/srv/eusrr-current`. + +The inactive release is built and checked while the active release continues to +serve users. Nginx then reloads an atomic upstream change. Existing WebSocket +connections may reconnect when the old backend is drained, but HTTP traffic does +not require a maintenance window. Old and new Next.js static assets are retained +in `/srv/eusrr-shared/next-static` so already-open pages keep working. + +Database migrations run before traffic switches and therefore must be backward +compatible with the currently active application. Do not combine destructive +schema removal with the release that stops using the schema. PostgreSQL snapshots +are stored in `/var/backups/eusrr/deploy` for 14 days, but they are not restored +automatically while the old release is accepting writes. ## Bootstrap -1. Generate a dedicated Ed25519 key pair for the runner. Never commit the private - key. -2. Copy the public key to the CORP server and install the restricted command: +1. Generate a dedicated Ed25519 runner key. Never commit its private part. +2. Copy the public key to CORP and run: ```sh sudo /srv/eusrr/ops/deploy/install-corp-runner-access /tmp/eusrr-gitea-runner.pub ``` -3. Add the private key as the `CORP_DEPLOY_SSH_KEY` repository Actions secret in - Gitea. -4. Run **Deploy production** manually from the `master` branch and verify the - result before enabling automatic runs on pushes to `master`. +3. Store the private key as repository Actions secret + `CORP_DEPLOY_SSH_KEY` in Gitea. +4. Run **Deploy production** manually from `master` and verify the first blue + deployment before enabling automatic master deployments. -The installer grants the `deploy` account no general root shell. The runner key -is forced through `/usr/local/sbin/eusrr-deploy-ssh`, and the deployment command -rejects every revision except the current `origin/master` SHA. - -PostgreSQL snapshots are stored in `/var/backups/eusrr/deploy` for 14 days. The -last successful revision is recorded in -`/var/lib/eusrr-deploy/last-successful-sha`, and deployment output is appended to -`/var/log/eusrr-deploy.log`. +The SSH key is forced through `/usr/local/sbin/eusrr-deploy-ssh`. The deploy +command rejects every revision except the current Gitea `origin/master` SHA. +Logs are appended to `/var/log/eusrr-deploy.log`; the active slot and successful +SHA are recorded under `/var/lib/eusrr-deploy`. diff --git a/ops/deploy/eusrr-deploy b/ops/deploy/eusrr-deploy index 6e10254e..57e1eae7 100755 --- a/ops/deploy/eusrr-deploy +++ b/ops/deploy/eusrr-deploy @@ -2,19 +2,17 @@ set -Eeuo pipefail readonly APP_ROOT="/srv/eusrr" -readonly BACKEND_DIR="${APP_ROOT}/backend" -readonly FRONTEND_DIR="${APP_ROOT}/frontend" +readonly RELEASES_ROOT="/srv/eusrr-releases" +readonly SLOTS_ROOT="/srv/eusrr-slots" +readonly SHARED_ROOT="/srv/eusrr-shared" +readonly CURRENT_LINK="/srv/eusrr-current" readonly DEPLOY_USER="deploy" readonly BACKUP_DIR="/var/backups/eusrr/deploy" readonly STATE_DIR="/var/lib/eusrr-deploy" +readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot" readonly LOG_FILE="/var/log/eusrr-deploy.log" readonly LOCK_FILE="/run/lock/eusrr-deploy.lock" -readonly -a SERVICES=( - gunicorn-eusrr.service - nextjs-eusrr.service - celery-eusrr-worker.service - celery-eusrr-beat.service -) +readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf" if [[ ${EUID} -ne 0 ]]; then echo "This command must run as root." >&2 @@ -34,22 +32,33 @@ run_as_deploy() { runuser -u "${DEPLOY_USER}" -- "$@" } -run_in_backend() { +run_in_release_backend() { runuser -u "${DEPLOY_USER}" -- \ - bash -c 'cd "$1"; shift; exec "$@"' _ "${BACKEND_DIR}" "$@" + bash -c 'cd "$1"; shift; exec "$@"' _ "${NEW_RELEASE}/backend" "$@" +} + +slot_ports() { + case "$1" in + blue) BACKEND_PORT=9001; FRONTEND_PORT=3001 ;; + green) BACKEND_PORT=9002; FRONTEND_PORT=3002 ;; + *) log "Unknown deployment slot: $1"; return 1 ;; + esac } if [[ ${1:-} == "--self-test" ]]; then - for command in git curl flock runuser pg_dump pg_restore npm; do + for command in git curl flock runuser pg_dump npm nginx rsync; do command -v "${command}" >/dev/null done test -d "${APP_ROOT}/.git" - test -x "${BACKEND_DIR}/.venv/bin/python" - run_as_deploy test -r "${BACKEND_DIR}/.env" - for service in "${SERVICES[@]}"; do - systemctl cat "${service}" >/dev/null + run_as_deploy test -r "${APP_ROOT}/backend/.env" + run_as_deploy test -r "${APP_ROOT}/frontend/.env" + test -r "${UPSTREAM_FILE}" + for unit in eusrr-backend@.service eusrr-frontend@.service \ + eusrr-celery-worker@.service eusrr-celery-beat@.service; do + systemctl cat "${unit}" >/dev/null done - log "Self-test passed." + nginx -t + log "Blue-green deployment self-test passed." exit 0 fi @@ -65,20 +74,37 @@ if ! flock -n 9; then exit 3 fi -OLD_SHA="" +OLD_SLOT="$(cat "${ACTIVE_SLOT_FILE}" 2>/dev/null || printf legacy)" +case "${OLD_SLOT}" in + legacy) NEW_SLOT=blue ;; + blue) NEW_SLOT=green ;; + green) NEW_SLOT=blue ;; + *) log "Invalid active slot state: ${OLD_SLOT}"; exit 4 ;; +esac + +NEW_RELEASE="${RELEASES_ROOT}/${TARGET_SHA}" +if [[ ${OLD_SLOT} == legacy ]]; then + OLD_RELEASE="${APP_ROOT}" +else + OLD_RELEASE="$(readlink -f "${SLOTS_ROOT}/${OLD_SLOT}")" +fi + +BACKEND_PORT="" +FRONTEND_PORT="" +slot_ports "${NEW_SLOT}" + +UPSTREAM_BACKUP="${STATE_DIR}/upstreams-before-${TARGET_SHA}" +UPSTREAM_CHANGED=0 +OLD_STACK_STOPPED=0 +NEW_STACK_ENABLED=0 DB_DUMP="" -SERVICES_STOPPED=0 -MIGRATION_STARTED=0 - -run_database_tool() { - local action="$1" - local dump_file="$2" +run_database_dump() { + local dump_file="$1" runuser -u "${DEPLOY_USER}" -- \ - env EUSRR_DB_ACTION="${action}" EUSRR_DB_DUMP="${dump_file}" \ - "${BACKEND_DIR}/.venv/bin/python" - <<'PY' + env EUSRR_DB_DUMP="${dump_file}" \ + "${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY' import os -import sys from pathlib import Path from dotenv import dotenv_values @@ -91,66 +117,41 @@ def required(name: str) -> str: raise SystemExit(f"Required database setting is missing: {name}") return str(value) -action = os.environ["EUSRR_DB_ACTION"] -dump_file = os.environ["EUSRR_DB_DUMP"] -database = required("POSTGRES_DB") -user = required("POSTGRES_USER") -password = required("POSTGRES_PASSWORD") -host = str(config.get("DB_HOST") or "127.0.0.1") -port = str(config.get("DB_PORT") or "5432") - -environment = os.environ.copy() -environment["PGPASSWORD"] = password -common = ["--host", host, "--port", port, "--username", user, "--dbname", database] - -if action == "dump": - executable = "/usr/bin/pg_dump" - arguments = [executable, *common, "--format=custom", "--no-owner", "--no-acl", "--file", dump_file] -elif action == "restore": - executable = "/usr/bin/pg_restore" - arguments = [ - executable, - *common, - "--clean", - "--if-exists", - "--no-owner", - "--no-acl", - "--exit-on-error", - "--single-transaction", - dump_file, - ] -else: - raise SystemExit(f"Unsupported database action: {action}") - +executable = "/usr/bin/pg_dump" if not Path(executable).is_file(): raise SystemExit(f"Database utility is not installed: {executable}") + +environment = os.environ.copy() +environment["PGPASSWORD"] = required("POSTGRES_PASSWORD") +arguments = [ + executable, + "--host", str(config.get("DB_HOST") or "127.0.0.1"), + "--port", str(config.get("DB_PORT") or "5432"), + "--username", required("POSTGRES_USER"), + "--dbname", required("POSTGRES_DB"), + "--format=custom", + "--no-owner", + "--no-acl", + "--file", os.environ["EUSRR_DB_DUMP"], +] os.execve(executable, arguments, environment) PY } -install_dependencies_and_build() { - log "Installing pinned backend dependencies." - run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \ - "${BACKEND_DIR}/.venv/bin/python" -m pip install --requirement "${BACKEND_DIR}/requirements.txt" - - log "Installing pinned frontend dependencies." - run_as_deploy npm --prefix "${FRONTEND_DIR}" ci --no-audit --no-fund - - log "Building the frontend." - run_as_deploy npm --prefix "${FRONTEND_DIR}" run build - - log "Running Django system checks." - run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" check -} - wait_for_http() { local name="$1" local url="$2" local expected="$3" + local host_header="${4:-}" local attempt code + local -a curl_args=(--silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5) + + if [[ -n ${host_header} ]]; then + curl_args+=(--header "Host: ${host_header}") + fi for attempt in $(seq 1 45); do - code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5 "${url}" || true)" + code="$(curl "${curl_args[@]}" "${url}" || true)" if [[ ${code} =~ ${expected} ]]; then log "Health check passed: ${name} returned ${code}." return 0 @@ -162,91 +163,253 @@ wait_for_http() { return 1 } -start_and_verify_services() { - systemctl restart "${SERVICES[@]}" - SERVICES_STOPPED=0 - for service in "${SERVICES[@]}"; do - systemctl is-active --quiet "${service}" - done - wait_for_http "frontend" "http://127.0.0.1:3000/" '^200$' - wait_for_http "backend" "http://127.0.0.1:9000/admin/login/" '^(200|302)$' +write_upstreams() { + local backend_port="$1" + local frontend_port="$2" + local temporary_file + temporary_file="$(mktemp)" + cat >"${temporary_file}" </dev/null || true +log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}." run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \ '+refs/heads/master:refs/remotes/origin/master' - MASTER_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse refs/remotes/origin/master)" if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})." exit 5 fi +if [[ ${TARGET_SHA} == "$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)" ]]; then + log "Revision ${TARGET_SHA} is already active." + exit 0 +fi -run_as_deploy git -C "${APP_ROOT}" reset --hard "${TARGET_SHA}" -install_dependencies_and_build +if [[ -L ${SLOTS_ROOT}/${NEW_SLOT} ]]; then + stale_release="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}")" + if [[ ${stale_release} != "${OLD_RELEASE}" ]]; then + rm -f "${SLOTS_ROOT}/${NEW_SLOT}" + remove_release "${stale_release}" + fi +fi +remove_release "${NEW_RELEASE}" +run_as_deploy git -C "${APP_ROOT}" worktree prune -log "Stopping application services for the database-safe deployment window." -systemctl stop "${SERVICES[@]}" -SERVICES_STOPPED=1 +available_bytes="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')" +if (( available_bytes < 4294967296 )); then + log "At least 4 GiB of free disk space is required to build a release." + exit 6 +fi + +log "Creating isolated Git worktree ${NEW_RELEASE}." +run_as_deploy git -C "${APP_ROOT}" worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}" + +run_as_deploy ln -s "${APP_ROOT}/backend/.env" "${NEW_RELEASE}/backend/.env" +run_as_deploy ln -s "${APP_ROOT}/frontend/.env" "${NEW_RELEASE}/frontend/.env" +run_as_deploy rm -rf "${NEW_RELEASE}/backend/media" +run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media" + +log "Creating an isolated Python environment." +run_as_deploy /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv" +run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \ + "${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \ + --requirement "${NEW_RELEASE}/backend/requirements.txt" + +log "Installing pinned frontend dependencies and building Next.js." +run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" ci --no-audit --no-fund +run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" run build + +log "Running Django checks and collecting release static files." +run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ + "${NEW_RELEASE}/backend/manage.py" check +run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ + "${NEW_RELEASE}/backend/manage.py" collectstatic --noinput + +run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/" install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}" DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump" -log "Creating PostgreSQL snapshot ${DB_DUMP}." -run_database_tool dump "${DB_DUMP}" +log "Creating a consistent PostgreSQL snapshot while the current release remains online." +run_database_dump "${DB_DUMP}" chmod 0600 "${DB_DUMP}" -MIGRATION_STARTED=1 -log "Applying database migrations." -run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" migrate --noinput +log "Planned migrations:" +run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ + "${NEW_RELEASE}/backend/manage.py" migrate --plan +log "Applying backward-compatible database migrations." +run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ + "${NEW_RELEASE}/backend/manage.py" migrate --noinput -log "Collecting Django static files." -run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput +log "Assigning release to the ${NEW_SLOT} slot and starting its web services." +ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new" +mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}" +start_slot_web "${NEW_SLOT}" -log "Starting and checking application services." -start_and_verify_services +wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$' +wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$' +cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}" +UPSTREAM_CHANGED=1 +point_current_at "${NEW_RELEASE}" +write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}" +nginx -t +systemctl reload nginx + +wait_for_http "public frontend" "http://127.0.0.1/" '^200$' '172.11.0.11' +wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '172.11.0.11' + +log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain." +sleep 10 +log "Switching background workers." +if [[ ${OLD_SLOT} == legacy ]]; then + stop_legacy_stack +else + stop_slot_stack "${OLD_SLOT}" +fi +OLD_STACK_STOPPED=1 +start_slot_celery "${NEW_SLOT}" +NEW_STACK_ENABLED=1 +enable_slot_stack "${NEW_SLOT}" +disable_old_stack + +find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \ + || log "WARNING: old database snapshots could not be pruned." +run_as_deploy find "${SHARED_ROOT}/next-static" -type f -mtime +30 -delete \ + || log "WARNING: old Next.js assets could not be pruned." +rm -f "${UPSTREAM_BACKUP}" || true +printf '%s\n' "${NEW_SLOT}" >"${ACTIVE_SLOT_FILE}" printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha" -find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete trap - ERR -log "Deployment of ${TARGET_SHA} completed successfully." +log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}." diff --git a/ops/deploy/install-corp-runner-access b/ops/deploy/install-corp-runner-access index f06ca0c6..d8b2e59e 100755 --- a/ops/deploy/install-corp-runner-access +++ b/ops/deploy/install-corp-runner-access @@ -8,28 +8,160 @@ fi readonly SOURCE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" readonly PUBLIC_KEY_FILE="${1:-}" +readonly NGINX_LINK="/etc/nginx/sites-enabled/eusrr" +readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf" if [[ -z ${PUBLIC_KEY_FILE} || ! -f ${PUBLIC_KEY_FILE} ]]; then echo "Usage: $0 /path/to/gitea-runner.pub" >&2 exit 2 fi - if ! id deploy >/dev/null 2>&1; then echo "The deploy system account does not exist." >&2 exit 3 fi +if [[ ! -d ${SOURCE_DIR}/systemd ]]; then + echo "Systemd templates are missing from ${SOURCE_DIR}." >&2 + exit 4 +fi read -r key_type key_body _ <"${PUBLIC_KEY_FILE}" if [[ ${key_type} != "ssh-ed25519" || -z ${key_body:-} ]]; then echo "Expected an Ed25519 public key." >&2 - exit 4 + exit 5 fi +nginx_config="$(readlink -f "${NGINX_LINK}")" +nginx_backup="${nginx_config}.bak.blue-green.$(date -u +%Y%m%dT%H%M%SZ)" +cp -a "${nginx_config}" "${nginx_backup}" +upstream_existed=0 +upstream_backup="" +if [[ -f ${UPSTREAM_FILE} ]]; then + upstream_existed=1 + upstream_backup="${UPSTREAM_FILE}.bak.$(date -u +%Y%m%dT%H%M%SZ)" + cp -a "${UPSTREAM_FILE}" "${upstream_backup}" +fi + +install_complete=0 +temporary_files=() +cleanup() { + local status=$? + for file in "${temporary_files[@]:-}"; do + [[ -n ${file} ]] && rm -f "${file}" + done + if [[ ${install_complete} -eq 0 ]]; then + cp -a "${nginx_backup}" "${nginx_config}" + if [[ ${upstream_existed} -eq 1 ]]; then + cp -a "${upstream_backup}" "${UPSTREAM_FILE}" + else + rm -f "${UPSTREAM_FILE}" + fi + nginx -t >/dev/null 2>&1 && systemctl reload nginx || true + fi + exit "${status}" +} +trap cleanup EXIT + install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy" /usr/local/sbin/eusrr-deploy install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy-ssh" /usr/local/sbin/eusrr-deploy-ssh +for unit in "${SOURCE_DIR}"/systemd/*.service; do + install -o root -g root -m 0644 "${unit}" "/etc/systemd/system/$(basename "${unit}")" +done + +install -d -o deploy -g eusrr -m 2775 /srv/eusrr-releases /srv/eusrr-slots +install -d -o deploy -g www-data -m 0755 /srv/eusrr-shared /srv/eusrr-shared/next-static +if [[ -d /srv/eusrr/frontend/.next/static ]]; then + rsync -a /srv/eusrr/frontend/.next/static/ /srv/eusrr-shared/next-static/ +fi +chown -R deploy:www-data /srv/eusrr-shared +find /srv/eusrr-shared -type d -exec chmod 0755 {} + +find /srv/eusrr-shared -type f -exec chmod 0644 {} + + +ln -sfn /srv/eusrr /srv/eusrr-current.new +mv -Tf /srv/eusrr-current.new /srv/eusrr-current + +if [[ ! -f ${UPSTREAM_FILE} ]]; then + cat >"${UPSTREAM_FILE}" <<'EOF' +upstream eusrr_backend { + server 127.0.0.1:9000; + keepalive 32; +} + +upstream eusrr_frontend { + server 127.0.0.1:3000; + keepalive 32; +} +EOF + chmod 0644 "${UPSTREAM_FILE}" +fi + +NGINX_CONFIG="${nginx_config}" python3 <<'PY' +import os +import re +from pathlib import Path + +path = Path(os.environ["NGINX_CONFIG"]) +text = path.read_text(encoding="utf-8") + +next_static = re.compile( + r"^ location /_next/static/ \{\n.*?^ \}", + flags=re.MULTILINE | re.DOTALL, +) +replacement = """ location /_next/static/ { + alias /srv/eusrr-shared/next-static/; + autoindex off; + expires 365d; + add_header Cache-Control \"public, immutable\"; + }""" +text, static_blocks = next_static.subn(replacement, text) +if static_blocks not in (0, 2): + raise SystemExit(f"Expected zero or two Next.js static blocks, found {static_blocks}") + +text = text.replace("proxy_pass http://127.0.0.1:9000;", "proxy_pass http://eusrr_backend;") +text = text.replace("proxy_pass http://127.0.0.1:3000;", "proxy_pass http://eusrr_frontend;") +text = text.replace( + "alias /srv/eusrr/backend/staticfiles/;", + "alias /srv/eusrr-current/backend/staticfiles/;", +) + +required = ( + "proxy_pass http://eusrr_backend;", + "proxy_pass http://eusrr_frontend;", + "alias /srv/eusrr-shared/next-static/;", + "alias /srv/eusrr-current/backend/staticfiles/;", +) +for marker in required: + if marker not in text: + raise SystemExit(f"Nginx transformation marker is missing: {marker}") +if "proxy_pass http://127.0.0.1:9000;" in text or "proxy_pass http://127.0.0.1:3000;" in text: + raise SystemExit("Legacy direct upstreams remain in Nginx configuration") + +path.write_text(text, encoding="utf-8") +PY + +install -d -o root -g root -m 0755 /etc/eusrr/slots /var/lib/eusrr-deploy +cat >/etc/eusrr/slots/blue.env <<'EOF' +BACKEND_PORT=9001 +PORT=3001 +HOSTNAME=127.0.0.1 +EOF +cat >/etc/eusrr/slots/green.env <<'EOF' +BACKEND_PORT=9002 +PORT=3002 +HOSTNAME=127.0.0.1 +EOF +chmod 0644 /etc/eusrr/slots/blue.env /etc/eusrr/slots/green.env +if [[ ! -f /var/lib/eusrr-deploy/active-slot ]]; then + printf '%s\n' legacy >/var/lib/eusrr-deploy/active-slot +fi + +install -d -o deploy -g deploy -m 0755 /var/log/celery +install -d -o deploy -g deploy -m 0700 /var/backups/eusrr/deploy +touch /var/log/eusrr-deploy.log +chown root:root /var/log/eusrr-deploy.log +chmod 0600 /var/log/eusrr-deploy.log sudoers_tmp="$(mktemp)" -trap 'rm -f "${sudoers_tmp}"' EXIT +temporary_files+=("${sudoers_tmp}") printf '%s\n' 'deploy ALL=(root) NOPASSWD: /usr/local/sbin/eusrr-deploy *' >"${sudoers_tmp}" chmod 0440 "${sudoers_tmp}" visudo -cf "${sudoers_tmp}" >/dev/null @@ -41,20 +173,18 @@ authorized_keys="${deploy_home}/.ssh/authorized_keys" touch "${authorized_keys}" chown deploy:deploy "${authorized_keys}" chmod 0600 "${authorized_keys}" - filtered_keys="$(mktemp)" +temporary_files+=("${filtered_keys}") grep -v ' eusrr-gitea-runner$' "${authorized_keys}" >"${filtered_keys}" || true printf '%s %s %s %s\n' \ 'command="/usr/local/sbin/eusrr-deploy-ssh",restrict' \ "${key_type}" "${key_body}" 'eusrr-gitea-runner' >>"${filtered_keys}" install -o deploy -g deploy -m 0600 "${filtered_keys}" "${authorized_keys}" -rm -f "${filtered_keys}" - -install -d -o root -g root -m 0755 /var/lib/eusrr-deploy -install -d -o deploy -g deploy -m 0700 /var/backups/eusrr/deploy -touch /var/log/eusrr-deploy.log -chown root:root /var/log/eusrr-deploy.log -chmod 0600 /var/log/eusrr-deploy.log +systemctl daemon-reload +nginx -t +systemctl reload nginx sudo -u deploy sudo -n /usr/local/sbin/eusrr-deploy --self-test -echo "Restricted Gitea runner access installed successfully." + +install_complete=1 +echo "Restricted blue-green Gitea deployment installed successfully." diff --git a/ops/deploy/systemd/eusrr-backend@.service b/ops/deploy/systemd/eusrr-backend@.service new file mode 100644 index 00000000..d6c889f0 --- /dev/null +++ b/ops/deploy/systemd/eusrr-backend@.service @@ -0,0 +1,24 @@ +[Unit] +Description=EUSRR backend (%i slot) +After=network-online.target redis-server.service +Wants=network-online.target redis-server.service + +[Service] +Type=simple +User=deploy +Group=www-data +WorkingDirectory=/srv/eusrr-slots/%i/backend +EnvironmentFile=/etc/eusrr/slots/%i.env +Environment=DJANGO_SETTINGS_MODULE=eusrr_backend.settings +Environment=PYTHONUNBUFFERED=1 +ExecStart=/srv/eusrr-slots/%i/backend/.venv/bin/python -m gunicorn -k uvicorn.workers.UvicornWorker --bind 127.0.0.1:${BACKEND_PORT} --workers 2 --timeout 60 --graceful-timeout 30 --access-logfile - --error-logfile - --log-level info eusrr_backend.asgi:application +Restart=always +RestartSec=3 +KillSignal=SIGQUIT +TimeoutStopSec=30 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=eusrr-backend-%i + +[Install] +WantedBy=multi-user.target diff --git a/ops/deploy/systemd/eusrr-celery-beat@.service b/ops/deploy/systemd/eusrr-celery-beat@.service new file mode 100644 index 00000000..e08e205f --- /dev/null +++ b/ops/deploy/systemd/eusrr-celery-beat@.service @@ -0,0 +1,24 @@ +[Unit] +Description=EUSRR Celery beat (%i slot) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=deploy +Group=deploy +WorkingDirectory=/srv/eusrr-slots/%i/backend +EnvironmentFile=/srv/eusrr-slots/%i/backend/.env +Environment=DJANGO_SETTINGS_MODULE=eusrr_backend.settings +RuntimeDirectory=eusrr-celery-%i +RuntimeDirectoryMode=0755 +ExecStart=/srv/eusrr-slots/%i/backend/.venv/bin/python -m celery -A eusrr_backend beat --loglevel=info --logfile=/var/log/celery/beat-%i.log --scheduler django_celery_beat.schedulers:DatabaseScheduler --pidfile=/run/eusrr-celery-%i/beat.pid +Restart=always +RestartSec=10 +TimeoutStopSec=30 +KillSignal=SIGTERM +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ops/deploy/systemd/eusrr-celery-worker@.service b/ops/deploy/systemd/eusrr-celery-worker@.service new file mode 100644 index 00000000..dcc70a96 --- /dev/null +++ b/ops/deploy/systemd/eusrr-celery-worker@.service @@ -0,0 +1,22 @@ +[Unit] +Description=EUSRR Celery worker (%i slot) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=deploy +Group=deploy +WorkingDirectory=/srv/eusrr-slots/%i/backend +EnvironmentFile=/srv/eusrr-slots/%i/backend/.env +Environment=DJANGO_SETTINGS_MODULE=eusrr_backend.settings +ExecStart=/srv/eusrr-slots/%i/backend/.venv/bin/python -m celery -A eusrr_backend worker --loglevel=info --logfile=/var/log/celery/worker-%i.log --concurrency=4 -Q celery,notifications,commerce +Restart=always +RestartSec=10 +TimeoutStopSec=60 +KillSignal=SIGTERM +StandardOutput=journal +StandardError=journal + +[Install] +WantedBy=multi-user.target diff --git a/ops/deploy/systemd/eusrr-frontend@.service b/ops/deploy/systemd/eusrr-frontend@.service new file mode 100644 index 00000000..83eb597e --- /dev/null +++ b/ops/deploy/systemd/eusrr-frontend@.service @@ -0,0 +1,24 @@ +[Unit] +Description=EUSRR frontend (%i slot) +After=network-online.target +Wants=network-online.target + +[Service] +Type=simple +User=deploy +Group=www-data +WorkingDirectory=/srv/eusrr-slots/%i/frontend +EnvironmentFile=/etc/eusrr/slots/%i.env +Environment=NODE_ENV=production +ExecStartPre=/bin/sh -c 'cp -r /srv/eusrr-slots/%i/frontend/public /srv/eusrr-slots/%i/frontend/.next/standalone/ 2>/dev/null || true' +ExecStartPre=/bin/sh -c 'cp -r /srv/eusrr-slots/%i/frontend/.next/static /srv/eusrr-slots/%i/frontend/.next/standalone/.next/ 2>/dev/null || true' +ExecStart=/usr/bin/node /srv/eusrr-slots/%i/frontend/.next/standalone/server.js +Restart=always +RestartSec=3 +TimeoutStopSec=30 +StandardOutput=journal +StandardError=journal +SyslogIdentifier=eusrr-frontend-%i + +[Install] +WantedBy=multi-user.target