Merge Gitea deployment automation

2026-09-04 15:30:22 +03:00 · 2026-09-04 15:30:22 +03:00 · 93cf86803c
commit 93cf86803c
parent 287951e237 6bb9811cd6
7 changed files with 389 additions and 7 deletions
--- a/.gitattributes
+++ b/.gitattributes
@ -0,0 +1,2 @@
+ops/deploy/* text eol=lf
+.gitea/workflows/*.yml text eol=lf
--- a/.gitea/workflows/deploy-production.yml
+++ b/.gitea/workflows/deploy-production.yml
@ -0,0 +1,34 @@
+name: Deploy production
+
+on:
+  workflow_dispatch:
+
+concurrency:
+  group: production
+  cancel-in-progress: false
+
+jobs:
+  deploy:
+    runs-on: deploy
+    timeout-minutes: 30
+    steps:
+      - name: Deploy current Gitea revision to CORP
+        shell: sh
+        env:
+          CORP_DEPLOY_SSH_KEY: ${{ secrets.CORP_DEPLOY_SSH_KEY }}
+          TARGET_SHA: ${{ gitea.sha }}
+        run: |
+          set -eu
+          work_dir="$(mktemp -d)"
+          trap 'rm -rf "${work_dir}"' EXIT HUP INT TERM
+          umask 077
+          printf '%s\n' "${CORP_DEPLOY_SSH_KEY}" > "${work_dir}/key"
+          printf '%s\n' '172.11.0.11 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJsNn4tOkIiZgZCWCumPi/OTuB7kSUkFd8u7iJnIFTwx' > "${work_dir}/known_hosts"
+          ssh \
+            -i "${work_dir}/key" \
+            -o BatchMode=yes \
+            -o IdentitiesOnly=yes \
+            -o StrictHostKeyChecking=yes \
+            -o UserKnownHostsFile="${work_dir}/known_hosts" \
+            deploy@172.11.0.11 \
+            "deploy ${TARGET_SHA}"
--- a/frontend/next-env.d.ts
+++ b/frontend/next-env.d.ts
@ -1,7 +0,0 @@
-/// <reference types="next" />
-/// <reference types="next/image-types/global" />
-import "./.next/dev/types/routes.d.ts";
-import "./.next/dev/types/root-params.d.ts";
-
-// NOTE: This file should not be edited
-// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.
--- a/ops/deploy/README.md
+++ b/ops/deploy/README.md
@ -0,0 +1,30 @@
+# CORP production deployment
+
+Production is deployed from the current `master` commit in the Gitea repository
+`Robotail/Corp`. The Gitea Actions runner only opens a restricted SSH connection;
+all validation, backup, build, migration, restart, health checks, and rollback are
+performed on the CORP server.
+
+## Bootstrap
+
+1. Generate a dedicated Ed25519 key pair for the runner. Never commit the private
+   key.
+2. Copy the public key to the CORP server and install the restricted command:
+
+   ```sh
+   sudo /srv/eusrr/ops/deploy/install-corp-runner-access /tmp/eusrr-gitea-runner.pub
+   ```
+
+3. Add the private key as the `CORP_DEPLOY_SSH_KEY` repository Actions secret in
+   Gitea.
+4. Run **Deploy production** manually from the `master` branch and verify the
+   result before enabling automatic runs on pushes to `master`.
+
+The installer grants the `deploy` account no general root shell. The runner key
+is forced through `/usr/local/sbin/eusrr-deploy-ssh`, and the deployment command
+rejects every revision except the current `origin/master` SHA.
+
+PostgreSQL snapshots are stored in `/var/backups/eusrr/deploy` for 14 days. The
+last successful revision is recorded in
+`/var/lib/eusrr-deploy/last-successful-sha`, and deployment output is appended to
+`/var/log/eusrr-deploy.log`.
--- a/ops/deploy/eusrr-deploy
+++ b/ops/deploy/eusrr-deploy
@ -0,0 +1,252 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+readonly APP_ROOT="/srv/eusrr"
+readonly BACKEND_DIR="${APP_ROOT}/backend"
+readonly FRONTEND_DIR="${APP_ROOT}/frontend"
+readonly DEPLOY_USER="deploy"
+readonly BACKUP_DIR="/var/backups/eusrr/deploy"
+readonly STATE_DIR="/var/lib/eusrr-deploy"
+readonly LOG_FILE="/var/log/eusrr-deploy.log"
+readonly LOCK_FILE="/run/lock/eusrr-deploy.lock"
+readonly -a SERVICES=(
+  gunicorn-eusrr.service
+  nextjs-eusrr.service
+  celery-eusrr-worker.service
+  celery-eusrr-beat.service
+)
+
+if [[ ${EUID} -ne 0 ]]; then
+  echo "This command must run as root." >&2
+  exit 1
+fi
+
+install -d -o root -g root -m 0755 "${STATE_DIR}"
+touch "${LOG_FILE}"
+chmod 0600 "${LOG_FILE}"
+exec > >(tee -a "${LOG_FILE}") 2>&1
+
+log() {
+  printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*"
+}
+
+run_as_deploy() {
+  runuser -u "${DEPLOY_USER}" -- "$@"
+}
+
+run_in_backend() {
+  runuser -u "${DEPLOY_USER}" -- \
+    bash -c 'cd "$1"; shift; exec "$@"' _ "${BACKEND_DIR}" "$@"
+}
+
+if [[ ${1:-} == "--self-test" ]]; then
+  for command in git curl flock runuser pg_dump pg_restore npm; do
+    command -v "${command}" >/dev/null
+  done
+  test -d "${APP_ROOT}/.git"
+  test -x "${BACKEND_DIR}/.venv/bin/python"
+  run_as_deploy test -r "${BACKEND_DIR}/.env"
+  for service in "${SERVICES[@]}"; do
+    systemctl cat "${service}" >/dev/null
+  done
+  log "Self-test passed."
+  exit 0
+fi
+
+TARGET_SHA="${1:-}"
+if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then
+  echo "Usage: $0 <40-character lowercase commit SHA>" >&2
+  exit 2
+fi
+
+exec 9>"${LOCK_FILE}"
+if ! flock -n 9; then
+  log "Another deployment is already running."
+  exit 3
+fi
+
+OLD_SHA=""
+DB_DUMP=""
+SERVICES_STOPPED=0
+MIGRATION_STARTED=0
+
+run_database_tool() {
+  local action="$1"
+  local dump_file="$2"
+
+  runuser -u "${DEPLOY_USER}" -- \
+    env EUSRR_DB_ACTION="${action}" EUSRR_DB_DUMP="${dump_file}" \
+    "${BACKEND_DIR}/.venv/bin/python" - <<'PY'
+import os
+import sys
+from pathlib import Path
+
+from dotenv import dotenv_values
+
+config = dotenv_values("/srv/eusrr/backend/.env")
+
+def required(name: str) -> str:
+    value = config.get(name)
+    if not value:
+        raise SystemExit(f"Required database setting is missing: {name}")
+    return str(value)
+
+action = os.environ["EUSRR_DB_ACTION"]
+dump_file = os.environ["EUSRR_DB_DUMP"]
+database = required("POSTGRES_DB")
+user = required("POSTGRES_USER")
+password = required("POSTGRES_PASSWORD")
+host = str(config.get("DB_HOST") or "127.0.0.1")
+port = str(config.get("DB_PORT") or "5432")
+
+environment = os.environ.copy()
+environment["PGPASSWORD"] = password
+common = ["--host", host, "--port", port, "--username", user, "--dbname", database]
+
+if action == "dump":
+    executable = "/usr/bin/pg_dump"
+    arguments = [executable, *common, "--format=custom", "--no-owner", "--no-acl", "--file", dump_file]
+elif action == "restore":
+    executable = "/usr/bin/pg_restore"
+    arguments = [
+        executable,
+        *common,
+        "--clean",
+        "--if-exists",
+        "--no-owner",
+        "--no-acl",
+        "--exit-on-error",
+        "--single-transaction",
+        dump_file,
+    ]
+else:
+    raise SystemExit(f"Unsupported database action: {action}")
+
+if not Path(executable).is_file():
+    raise SystemExit(f"Database utility is not installed: {executable}")
+os.execve(executable, arguments, environment)
+PY
+}
+
+install_dependencies_and_build() {
+  log "Installing pinned backend dependencies."
+  run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \
+    "${BACKEND_DIR}/.venv/bin/python" -m pip install --requirement "${BACKEND_DIR}/requirements.txt"
+
+  log "Installing pinned frontend dependencies."
+  run_as_deploy npm --prefix "${FRONTEND_DIR}" ci --no-audit --no-fund
+
+  log "Building the frontend."
+  run_as_deploy npm --prefix "${FRONTEND_DIR}" run build
+
+  log "Running Django system checks."
+  run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" check
+}
+
+wait_for_http() {
+  local name="$1"
+  local url="$2"
+  local expected="$3"
+  local attempt code
+
+  for attempt in $(seq 1 45); do
+    code="$(curl --silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5 "${url}" || true)"
+    if [[ ${code} =~ ${expected} ]]; then
+      log "Health check passed: ${name} returned ${code}."
+      return 0
+    fi
+    sleep 2
+  done
+
+  log "Health check failed: ${name} returned ${code:-no response}."
+  return 1
+}
+
+start_and_verify_services() {
+  systemctl restart "${SERVICES[@]}"
+  SERVICES_STOPPED=0
+  for service in "${SERVICES[@]}"; do
+    systemctl is-active --quiet "${service}"
+  done
+  wait_for_http "frontend" "http://127.0.0.1:3000/" '^200$'
+  wait_for_http "backend" "http://127.0.0.1:9000/admin/login/" '^(200|302)$'
+}
+
+rollback() {
+  local original_status=$?
+  trap - ERR
+  set +e
+  log "Deployment failed with exit code ${original_status}; starting rollback."
+
+  if [[ ${SERVICES_STOPPED} -eq 0 && ${MIGRATION_STARTED} -eq 1 ]]; then
+    systemctl stop "${SERVICES[@]}"
+    SERVICES_STOPPED=1
+  fi
+
+  if [[ ${MIGRATION_STARTED} -eq 1 && -n ${DB_DUMP} && -f ${DB_DUMP} ]]; then
+    log "Restoring the database snapshot."
+    run_database_tool restore "${DB_DUMP}" || log "ERROR: database restore failed; manual recovery is required."
+  fi
+
+  if [[ -n ${OLD_SHA} ]]; then
+    log "Restoring source revision ${OLD_SHA}."
+    run_as_deploy git -C "${APP_ROOT}" reset --hard "${OLD_SHA}"
+    install_dependencies_and_build || log "ERROR: rebuilding the previous revision failed."
+    run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput \
+      || log "ERROR: restoring static files failed."
+  fi
+
+  systemctl restart "${SERVICES[@]}" || true
+  log "Rollback finished. Inspect ${LOG_FILE} and service journals before retrying."
+  exit "${original_status}"
+}
+trap rollback ERR
+
+log "Starting deployment of ${TARGET_SHA}."
+OLD_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse HEAD)"
+
+dirty="$(run_as_deploy git -C "${APP_ROOT}" status --porcelain --untracked-files=normal)"
+unexpected_dirty="$(printf '%s\n' "${dirty}" | grep -vE '^ M frontend/next-env\.d\.ts$' || true)"
+if [[ -n ${unexpected_dirty} ]]; then
+  log "Refusing to overwrite unexpected production changes:"
+  printf '%s\n' "${unexpected_dirty}"
+  exit 4
+fi
+
+run_as_deploy git -C "${APP_ROOT}" restore --worktree frontend/next-env.d.ts 2>/dev/null || true
+run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \
+  '+refs/heads/master:refs/remotes/origin/master'
+
+MASTER_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse refs/remotes/origin/master)"
+if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then
+  log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})."
+  exit 5
+fi
+
+run_as_deploy git -C "${APP_ROOT}" reset --hard "${TARGET_SHA}"
+install_dependencies_and_build
+
+log "Stopping application services for the database-safe deployment window."
+systemctl stop "${SERVICES[@]}"
+SERVICES_STOPPED=1
+
+install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}"
+DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump"
+log "Creating PostgreSQL snapshot ${DB_DUMP}."
+run_database_tool dump "${DB_DUMP}"
+chmod 0600 "${DB_DUMP}"
+
+MIGRATION_STARTED=1
+log "Applying database migrations."
+run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" migrate --noinput
+
+log "Collecting Django static files."
+run_in_backend "${BACKEND_DIR}/.venv/bin/python" "${BACKEND_DIR}/manage.py" collectstatic --noinput
+
+log "Starting and checking application services."
+start_and_verify_services
+
+printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha"
+find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete
+trap - ERR
+log "Deployment of ${TARGET_SHA} completed successfully."
--- a/ops/deploy/eusrr-deploy-ssh
+++ b/ops/deploy/eusrr-deploy-ssh
@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+readonly requested_command="${SSH_ORIGINAL_COMMAND:-}"
+
+if [[ ${requested_command} =~ ^deploy[[:space:]]([0-9a-f]{40})$ ]]; then
+  exec sudo -n /usr/local/sbin/eusrr-deploy "${BASH_REMATCH[1]}"
+fi
+
+echo "Only 'deploy <40-character lowercase commit SHA>' is allowed." >&2
+exit 2
--- a/ops/deploy/install-corp-runner-access
+++ b/ops/deploy/install-corp-runner-access
@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+if [[ ${EUID} -ne 0 ]]; then
+  echo "Run this installer with sudo." >&2
+  exit 1
+fi
+
+readonly SOURCE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+readonly PUBLIC_KEY_FILE="${1:-}"
+
+if [[ -z ${PUBLIC_KEY_FILE} || ! -f ${PUBLIC_KEY_FILE} ]]; then
+  echo "Usage: $0 /path/to/gitea-runner.pub" >&2
+  exit 2
+fi
+
+if ! id deploy >/dev/null 2>&1; then
+  echo "The deploy system account does not exist." >&2
+  exit 3
+fi
+
+read -r key_type key_body _ <"${PUBLIC_KEY_FILE}"
+if [[ ${key_type} != "ssh-ed25519" || -z ${key_body:-} ]]; then
+  echo "Expected an Ed25519 public key." >&2
+  exit 4
+fi
+
+install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy" /usr/local/sbin/eusrr-deploy
+install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy-ssh" /usr/local/sbin/eusrr-deploy-ssh
+
+sudoers_tmp="$(mktemp)"
+trap 'rm -f "${sudoers_tmp}"' EXIT
+printf '%s\n' 'deploy ALL=(root) NOPASSWD: /usr/local/sbin/eusrr-deploy *' >"${sudoers_tmp}"
+chmod 0440 "${sudoers_tmp}"
+visudo -cf "${sudoers_tmp}" >/dev/null
+install -o root -g root -m 0440 "${sudoers_tmp}" /etc/sudoers.d/eusrr-deploy
+
+deploy_home="$(getent passwd deploy | cut -d: -f6)"
+install -d -o deploy -g deploy -m 0700 "${deploy_home}/.ssh"
+authorized_keys="${deploy_home}/.ssh/authorized_keys"
+touch "${authorized_keys}"
+chown deploy:deploy "${authorized_keys}"
+chmod 0600 "${authorized_keys}"
+
+filtered_keys="$(mktemp)"
+grep -v ' eusrr-gitea-runner$' "${authorized_keys}" >"${filtered_keys}" || true
+printf '%s %s %s %s\n' \
+  'command="/usr/local/sbin/eusrr-deploy-ssh",restrict' \
+  "${key_type}" "${key_body}" 'eusrr-gitea-runner' >>"${filtered_keys}"
+install -o deploy -g deploy -m 0600 "${filtered_keys}" "${authorized_keys}"
+rm -f "${filtered_keys}"
+
+install -d -o root -g root -m 0755 /var/lib/eusrr-deploy
+install -d -o deploy -g deploy -m 0700 /var/backups/eusrr/deploy
+touch /var/log/eusrr-deploy.log
+chown root:root /var/log/eusrr-deploy.log
+chmod 0600 /var/log/eusrr-deploy.log
+
+sudo -u deploy sudo -n /usr/local/sbin/eusrr-deploy --self-test
+echo "Restricted Gitea runner access installed successfully."