diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..dd6b4c87 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +ops/deploy/* text eol=lf +.gitea/workflows/*.yml text eol=lf diff --git a/.gitea/workflows/deploy-production.yml b/.gitea/workflows/deploy-production.yml new file mode 100644 index 00000000..62f45c2e --- /dev/null +++ b/.gitea/workflows/deploy-production.yml @@ -0,0 +1,34 @@ +name: Deploy production + +on: + workflow_dispatch: + +concurrency: + group: production + cancel-in-progress: false + +jobs: + deploy: + runs-on: deploy + timeout-minutes: 30 + steps: + - name: Deploy current Gitea revision to CORP + shell: sh + env: + CORP_DEPLOY_SSH_KEY: ${{ secrets.CORP_DEPLOY_SSH_KEY }} + TARGET_SHA: ${{ gitea.sha }} + run: | + set -eu + work_dir="$(mktemp -d)" + trap 'rm -rf "${work_dir}"' EXIT HUP INT TERM + umask 077 + printf '%s\n' "${CORP_DEPLOY_SSH_KEY}" > "${work_dir}/key" + printf '%s\n' '172.11.0.11 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJsNn4tOkIiZgZCWCumPi/OTuB7kSUkFd8u7iJnIFTwx' > "${work_dir}/known_hosts" + ssh \ + -i "${work_dir}/key" \ + -o BatchMode=yes \ + -o IdentitiesOnly=yes \ + -o StrictHostKeyChecking=yes \ + -o UserKnownHostsFile="${work_dir}/known_hosts" \ + deploy@172.11.0.11 \ + "deploy ${TARGET_SHA}" diff --git a/frontend/next-env.d.ts b/frontend/next-env.d.ts deleted file mode 100644 index a419cbe4..00000000 --- a/frontend/next-env.d.ts +++ /dev/null @@ -1,7 +0,0 @@ -/// -/// -import "./.next/dev/types/routes.d.ts"; -import "./.next/dev/types/root-params.d.ts"; - -// NOTE: This file should not be edited -// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. diff --git a/ops/deploy/README.md b/ops/deploy/README.md new file mode 100644 index 00000000..eeef42f6 --- /dev/null +++ b/ops/deploy/README.md @@ -0,0 +1,30 @@ +# CORP production deployment + +Production is deployed from the current `master` commit in the Gitea repository +`Robotail/Corp`. The Gitea Actions runner only opens a restricted SSH connection; +all validation, backup, build, migration, restart, health checks, and rollback are +performed on the CORP server. + +## Bootstrap + +1. Generate a dedicated Ed25519 key pair for the runner. Never commit the private + key. +2. Copy the public key to the CORP server and install the restricted command: + + ```sh + sudo /srv/eusrr/ops/deploy/install-corp-runner-access /tmp/eusrr-gitea-runner.pub + ``` + +3. Add the private key as the `CORP_DEPLOY_SSH_KEY` repository Actions secret in + Gitea. +4. Run **Deploy production** manually from the `master` branch and verify the + result before enabling automatic runs on pushes to `master`. + +The installer grants the `deploy` account no general root shell. The runner key +is forced through `/usr/local/sbin/eusrr-deploy-ssh`, and the deployment command +rejects every revision except the current `origin/master` SHA. + +PostgreSQL snapshots are stored in `/var/backups/eusrr/deploy` for 14 days. The +last successful revision is recorded in +`/var/lib/eusrr-deploy/last-successful-sha`, and deployment output is appended to +`/var/log/eusrr-deploy.log`. diff --git a/ops/deploy/eusrr-deploy b/ops/deploy/eusrr-deploy new file mode 100755 index 00000000..6e10254e --- /dev/null +++ b/ops/deploy/eusrr-deploy @@ -0,0 +1,252 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly APP_ROOT="/srv/eusrr" +readonly BACKEND_DIR="${APP_ROOT}/backend" +readonly FRONTEND_DIR="${APP_ROOT}/frontend" +readonly DEPLOY_USER="deploy" +readonly BACKUP_DIR="/var/backups/eusrr/deploy" +readonly STATE_DIR="/var/lib/eusrr-deploy" +readonly LOG_FILE="/var/log/eusrr-deploy.log" +readonly LOCK_FILE="/run/lock/eusrr-deploy.lock" +readonly -a SERVICES=( + gunicorn-eusrr.service + nextjs-eusrr.service + celery-eusrr-worker.service + celery-eusrr-beat.service +) + +if [[ ${EUID} -ne 0 ]]; then + echo "This command must run as root." >&2 + exit 1 +fi + +install -d -o root -g root -m 0755 "${STATE_DIR}" +touch "${LOG_FILE}" +chmod 0600 "${LOG_FILE}" +exec > >(tee -a "${LOG_FILE}") 2>&1 + +log() { + printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*" +} + +run_as_deploy() { + runuser -u "${DEPLOY_USER}" -- "$@" +} + +run_in_backend() { + runuser -u "${DEPLOY_USER}" -- \ + bash -c 'cd "$1"; shift; exec "$@"' _ "${BACKEND_DIR}" "$@" +} + +if [[ ${1:-} == "--self-test" ]]; then + for command in git curl flock runuser pg_dump pg_restore npm; do + command -v "${command}" >/dev/null + done + test -d "${APP_ROOT}/.git" + test -x "${BACKEND_DIR}/.venv/bin/python" + run_as_deploy test -r "${BACKEND_DIR}/.env" + for service in "${SERVICES[@]}"; do + systemctl cat "${service}" >/dev/null + done + log "Self-test passed." + exit 0 +fi + +TARGET_SHA="${1:-}" +if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then + echo "Usage: $0 <40-character lowercase commit SHA>" >&2 + exit 2 +fi + +exec 9>"${LOCK_FILE}" +if ! flock -n 9; then + log "Another deployment is already running." + exit 3 +fi + +OLD_SHA="" +DB_DUMP="" +SERVICES_STOPPED=0 +MIGRATION_STARTED=0 + +run_database_tool() { + local action="$1" + local dump_file="$2" + + runuser -u "${DEPLOY_USER}" -- \ + env EUSRR_DB_ACTION="${action}" EUSRR_DB_DUMP="${dump_file}" \ + "${BACKEND_DIR}/.venv/bin/python" - <<'PY' +import os +import sys +from pathlib import Path + +from dotenv import dotenv_values + +config = dotenv_values("/srv/eusrr/backend/.env") + +def required(name: str) -> str: + value = config.get(name) + if not value: + raise SystemExit(f"Required database setting is missing: {name}") + return str(value) + +action = os.environ["EUSRR_DB_ACTION"] +dump_file = os.environ["EUSRR_DB_DUMP"] +database = required("POSTGRES_DB") +user = required("POSTGRES_USER") +password = required("POSTGRES_PASSWORD") +host = str(config.get("DB_HOST") or "127.0.0.1") +port = str(config.get("DB_PORT") or "5432") + +environment = os.environ.copy() +environment["PGPASSWORD"] = password +common = ["--host", host, "--port", port, "--username", user, "--dbname", database] + +if action == "dump": + executable = "/usr/bin/pg_dump" + arguments = [executable, *common, "--format=custom", "--no-owner", "--no-acl", "--file", dump_file] +elif action == "restore": + executable = "/usr/bin/pg_restore" + arguments = [ + executable, + *common, + "--clean", + "--if-exists", + "--no-owner", + "--no-acl", + "--exit-on-error", + "--single-transaction", + dump_file, + ] +else: + raise SystemExit(f"Unsupported database action: {action}") + +if not Path(executable).is_file(): + raise SystemExit(f"Database utility is not installed: {executable}") +os.execve(executable, arguments, environment) +PY +} + +install_dependencies_and_build() { + log "Installing pinned backend dependencies." + run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \ + "${BACKEND_DIR}/.venv/bin/python" -m pip install --requirement "${BACKEND_DIR}/requirements.txt" + + log "Installing pinned frontend dependencies." + run_as_deploy npm --prefix "${FRONTEND_DIR}" ci --no-audit --no-fund + + log "Building the frontend." + run_as_deploy npm --prefix "${FRONTEND_DIR}" run build + + log "Running Django system checks." + run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" check +} + +wait_for_http() { + local name="$1" + local url="$2" + local expected="$3" + local attempt code + + for attempt in $(seq 1 45); do + code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5 "${url}" || true)" + if [[ ${code} =~ ${expected} ]]; then + log "Health check passed: ${name} returned ${code}." + return 0 + fi + sleep 2 + done + + log "Health check failed: ${name} returned ${code:-no response}." + return 1 +} + +start_and_verify_services() { + systemctl restart "${SERVICES[@]}" + SERVICES_STOPPED=0 + for service in "${SERVICES[@]}"; do + systemctl is-active --quiet "${service}" + done + wait_for_http "frontend" "http://127.0.0.1:3000/" '^200$' + wait_for_http "backend" "http://127.0.0.1:9000/admin/login/" '^(200|302)$' +} + +rollback() { + local original_status=$? + trap - ERR + set +e + log "Deployment failed with exit code ${original_status}; starting rollback." + + if [[ ${SERVICES_STOPPED} -eq 0 && ${MIGRATION_STARTED} -eq 1 ]]; then + systemctl stop "${SERVICES[@]}" + SERVICES_STOPPED=1 + fi + + if [[ ${MIGRATION_STARTED} -eq 1 && -n ${DB_DUMP} && -f ${DB_DUMP} ]]; then + log "Restoring the database snapshot." + run_database_tool restore "${DB_DUMP}" || log "ERROR: database restore failed; manual recovery is required." + fi + + if [[ -n ${OLD_SHA} ]]; then + log "Restoring source revision ${OLD_SHA}." + run_as_deploy git -C "${APP_ROOT}" reset --hard "${OLD_SHA}" + install_dependencies_and_build || log "ERROR: rebuilding the previous revision failed." + run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput \ + || log "ERROR: restoring static files failed." + fi + + systemctl restart "${SERVICES[@]}" || true + log "Rollback finished. Inspect ${LOG_FILE} and service journals before retrying." + exit "${original_status}" +} +trap rollback ERR + +log "Starting deployment of ${TARGET_SHA}." +OLD_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse HEAD)" + +dirty="$(run_as_deploy git -C "${APP_ROOT}" status --porcelain --untracked-files=normal)" +unexpected_dirty="$(printf '%s\n' "${dirty}" | grep -vE '^ M frontend/next-env\.d\.ts$' || true)" +if [[ -n ${unexpected_dirty} ]]; then + log "Refusing to overwrite unexpected production changes:" + printf '%s\n' "${unexpected_dirty}" + exit 4 +fi + +run_as_deploy git -C "${APP_ROOT}" restore --worktree frontend/next-env.d.ts 2>/dev/null || true +run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \ + '+refs/heads/master:refs/remotes/origin/master' + +MASTER_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse refs/remotes/origin/master)" +if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then + log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})." + exit 5 +fi + +run_as_deploy git -C "${APP_ROOT}" reset --hard "${TARGET_SHA}" +install_dependencies_and_build + +log "Stopping application services for the database-safe deployment window." +systemctl stop "${SERVICES[@]}" +SERVICES_STOPPED=1 + +install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}" +DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump" +log "Creating PostgreSQL snapshot ${DB_DUMP}." +run_database_tool dump "${DB_DUMP}" +chmod 0600 "${DB_DUMP}" + +MIGRATION_STARTED=1 +log "Applying database migrations." +run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" migrate --noinput + +log "Collecting Django static files." +run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput + +log "Starting and checking application services." +start_and_verify_services + +printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha" +find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete +trap - ERR +log "Deployment of ${TARGET_SHA} completed successfully." diff --git a/ops/deploy/eusrr-deploy-ssh b/ops/deploy/eusrr-deploy-ssh new file mode 100755 index 00000000..e4b31d58 --- /dev/null +++ b/ops/deploy/eusrr-deploy-ssh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +readonly requested_command="${SSH_ORIGINAL_COMMAND:-}" + +if [[ ${requested_command} =~ ^deploy[[:space:]]([0-9a-f]{40})$ ]]; then + exec sudo -n /usr/local/sbin/eusrr-deploy "${BASH_REMATCH[1]}" +fi + +echo "Only 'deploy <40-character lowercase commit SHA>' is allowed." >&2 +exit 2 diff --git a/ops/deploy/install-corp-runner-access b/ops/deploy/install-corp-runner-access new file mode 100755 index 00000000..f06ca0c6 --- /dev/null +++ b/ops/deploy/install-corp-runner-access @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +if [[ ${EUID} -ne 0 ]]; then + echo "Run this installer with sudo." >&2 + exit 1 +fi + +readonly SOURCE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +readonly PUBLIC_KEY_FILE="${1:-}" + +if [[ -z ${PUBLIC_KEY_FILE} || ! -f ${PUBLIC_KEY_FILE} ]]; then + echo "Usage: $0 /path/to/gitea-runner.pub" >&2 + exit 2 +fi + +if ! id deploy >/dev/null 2>&1; then + echo "The deploy system account does not exist." >&2 + exit 3 +fi + +read -r key_type key_body _ <"${PUBLIC_KEY_FILE}" +if [[ ${key_type} != "ssh-ed25519" || -z ${key_body:-} ]]; then + echo "Expected an Ed25519 public key." >&2 + exit 4 +fi + +install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy" /usr/local/sbin/eusrr-deploy +install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy-ssh" /usr/local/sbin/eusrr-deploy-ssh + +sudoers_tmp="$(mktemp)" +trap 'rm -f "${sudoers_tmp}"' EXIT +printf '%s\n' 'deploy ALL=(root) NOPASSWD: /usr/local/sbin/eusrr-deploy *' >"${sudoers_tmp}" +chmod 0440 "${sudoers_tmp}" +visudo -cf "${sudoers_tmp}" >/dev/null +install -o root -g root -m 0440 "${sudoers_tmp}" /etc/sudoers.d/eusrr-deploy + +deploy_home="$(getent passwd deploy | cut -d: -f6)" +install -d -o deploy -g deploy -m 0700 "${deploy_home}/.ssh" +authorized_keys="${deploy_home}/.ssh/authorized_keys" +touch "${authorized_keys}" +chown deploy:deploy "${authorized_keys}" +chmod 0600 "${authorized_keys}" + +filtered_keys="$(mktemp)" +grep -v ' eusrr-gitea-runner$' "${authorized_keys}" >"${filtered_keys}" || true +printf '%s %s %s %s\n' \ + 'command="/usr/local/sbin/eusrr-deploy-ssh",restrict' \ + "${key_type}" "${key_body}" 'eusrr-gitea-runner' >>"${filtered_keys}" +install -o deploy -g deploy -m 0600 "${filtered_keys}" "${authorized_keys}" +rm -f "${filtered_keys}" + +install -d -o root -g root -m 0755 /var/lib/eusrr-deploy +install -d -o deploy -g deploy -m 0700 /var/backups/eusrr/deploy +touch /var/log/eusrr-deploy.log +chown root:root /var/log/eusrr-deploy.log +chmod 0600 /var/log/eusrr-deploy.log + +sudo -u deploy sudo -n /usr/local/sbin/eusrr-deploy --self-test +echo "Restricted Gitea runner access installed successfully."