421 lines
13 KiB
Bash
Executable File
421 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
readonly APP_ROOT="/srv/eusrr"
|
|
readonly RELEASES_ROOT="/srv/eusrr-releases"
|
|
readonly SLOTS_ROOT="/srv/eusrr-slots"
|
|
readonly SHARED_ROOT="/srv/eusrr-shared"
|
|
readonly CURRENT_LINK="/srv/eusrr-current"
|
|
readonly DEPLOY_USER="deploy"
|
|
readonly BACKUP_DIR="/var/backups/eusrr/deploy"
|
|
readonly STATE_DIR="/var/lib/eusrr-deploy"
|
|
readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot"
|
|
readonly LOG_FILE="/var/log/eusrr-deploy.log"
|
|
readonly LOCK_FILE="/run/lock/eusrr-deploy.lock"
|
|
readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf"
|
|
|
|
if [[ ${EUID} -ne 0 ]]; then
|
|
echo "This command must run as root." >&2
|
|
exit 1
|
|
fi
|
|
|
|
install -d -o root -g root -m 0755 "${STATE_DIR}"
|
|
touch "${LOG_FILE}"
|
|
chmod 0600 "${LOG_FILE}"
|
|
exec > >(tee -a "${LOG_FILE}") 2>&1
|
|
|
|
log() {
|
|
printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*"
|
|
}
|
|
|
|
run_as_deploy() {
|
|
runuser -u "${DEPLOY_USER}" -- "$@"
|
|
}
|
|
|
|
run_in_release_backend() {
|
|
runuser -u "${DEPLOY_USER}" -- \
|
|
bash -c 'cd "$1"; shift; exec "$@"' _ "${NEW_RELEASE}/backend" "$@"
|
|
}
|
|
|
|
slot_ports() {
|
|
case "$1" in
|
|
blue) BACKEND_PORT=9001; FRONTEND_PORT=3001 ;;
|
|
green) BACKEND_PORT=9002; FRONTEND_PORT=3002 ;;
|
|
*) log "Unknown deployment slot: $1"; return 1 ;;
|
|
esac
|
|
}
|
|
|
|
if [[ ${1:-} == "--self-test" ]]; then
|
|
for command in git curl flock runuser pg_dump npm nginx rsync; do
|
|
command -v "${command}" >/dev/null
|
|
done
|
|
test -d "${APP_ROOT}/.git"
|
|
run_as_deploy test -r "${APP_ROOT}/backend/.env"
|
|
run_as_deploy test -r "${APP_ROOT}/frontend/.env"
|
|
test -r "${UPSTREAM_FILE}"
|
|
for unit in eusrr-backend@.service eusrr-frontend@.service \
|
|
eusrr-celery-worker@.service eusrr-celery-beat@.service; do
|
|
systemctl cat "${unit}" >/dev/null
|
|
done
|
|
nginx -t
|
|
log "Blue-green deployment self-test passed."
|
|
exit 0
|
|
fi
|
|
|
|
TARGET_SHA="${1:-}"
|
|
if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then
|
|
echo "Usage: $0 <40-character lowercase commit SHA>" >&2
|
|
exit 2
|
|
fi
|
|
|
|
exec 9>"${LOCK_FILE}"
|
|
if ! flock -n 9; then
|
|
log "Another deployment is already running."
|
|
exit 3
|
|
fi
|
|
|
|
OLD_SLOT="$(cat "${ACTIVE_SLOT_FILE}" 2>/dev/null || printf legacy)"
|
|
case "${OLD_SLOT}" in
|
|
legacy) NEW_SLOT=blue ;;
|
|
blue) NEW_SLOT=green ;;
|
|
green) NEW_SLOT=blue ;;
|
|
*) log "Invalid active slot state: ${OLD_SLOT}"; exit 4 ;;
|
|
esac
|
|
|
|
NEW_RELEASE="${RELEASES_ROOT}/${TARGET_SHA}"
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
OLD_RELEASE="${APP_ROOT}"
|
|
else
|
|
OLD_RELEASE="$(readlink -f "${SLOTS_ROOT}/${OLD_SLOT}")"
|
|
fi
|
|
|
|
BACKEND_PORT=""
|
|
FRONTEND_PORT=""
|
|
slot_ports "${NEW_SLOT}"
|
|
|
|
UPSTREAM_BACKUP="${STATE_DIR}/upstreams-before-${TARGET_SHA}"
|
|
UPSTREAM_CHANGED=0
|
|
OLD_STACK_STOPPED=0
|
|
NEW_STACK_ENABLED=0
|
|
DB_DUMP=""
|
|
|
|
run_database_dump() {
|
|
local dump_file="$1"
|
|
runuser -u "${DEPLOY_USER}" -- \
|
|
env EUSRR_DB_DUMP="${dump_file}" \
|
|
"${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY'
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from dotenv import dotenv_values
|
|
|
|
config = dotenv_values("/srv/eusrr/backend/.env")
|
|
|
|
def required(name: str) -> str:
|
|
value = config.get(name)
|
|
if not value:
|
|
raise SystemExit(f"Required database setting is missing: {name}")
|
|
return str(value)
|
|
|
|
executable = "/usr/bin/pg_dump"
|
|
if not Path(executable).is_file():
|
|
raise SystemExit(f"Database utility is not installed: {executable}")
|
|
|
|
environment = os.environ.copy()
|
|
environment["PGPASSWORD"] = required("POSTGRES_PASSWORD")
|
|
arguments = [
|
|
executable,
|
|
"--host", str(config.get("DB_HOST") or "127.0.0.1"),
|
|
"--port", str(config.get("DB_PORT") or "5432"),
|
|
"--username", required("POSTGRES_USER"),
|
|
"--dbname", required("POSTGRES_DB"),
|
|
"--format=custom",
|
|
"--no-owner",
|
|
"--no-acl",
|
|
"--file", os.environ["EUSRR_DB_DUMP"],
|
|
]
|
|
os.execve(executable, arguments, environment)
|
|
PY
|
|
}
|
|
|
|
wait_for_http() {
|
|
local name="$1"
|
|
local url="$2"
|
|
local expected="$3"
|
|
local host_header="${4:-}"
|
|
local attempt code
|
|
local -a curl_args=(--silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5)
|
|
|
|
if [[ -n ${host_header} ]]; then
|
|
curl_args+=(--header "Host: ${host_header}")
|
|
fi
|
|
|
|
for attempt in $(seq 1 45); do
|
|
code="$(curl "${curl_args[@]}" "${url}" || true)"
|
|
if [[ ${code} =~ ${expected} ]]; then
|
|
log "Health check passed: ${name} returned ${code}."
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
log "Health check failed: ${name} returned ${code:-no response}."
|
|
return 1
|
|
}
|
|
|
|
write_upstreams() {
|
|
local backend_port="$1"
|
|
local frontend_port="$2"
|
|
local temporary_file
|
|
temporary_file="$(mktemp)"
|
|
cat >"${temporary_file}" <<EOF
|
|
upstream eusrr_backend {
|
|
server 127.0.0.1:${backend_port};
|
|
keepalive 32;
|
|
}
|
|
|
|
upstream eusrr_frontend {
|
|
server 127.0.0.1:${frontend_port};
|
|
keepalive 32;
|
|
}
|
|
EOF
|
|
install -o root -g root -m 0644 "${temporary_file}" "${UPSTREAM_FILE}"
|
|
rm -f "${temporary_file}"
|
|
UPSTREAM_CHANGED=1
|
|
}
|
|
|
|
point_current_at() {
|
|
local release="$1"
|
|
ln -sfn "${release}" "${CURRENT_LINK}.new"
|
|
mv -Tf "${CURRENT_LINK}.new" "${CURRENT_LINK}"
|
|
}
|
|
|
|
start_slot_web() {
|
|
local slot="$1"
|
|
systemctl start "eusrr-backend@${slot}.service" "eusrr-frontend@${slot}.service"
|
|
}
|
|
|
|
stop_slot_stack() {
|
|
local slot="$1"
|
|
systemctl stop \
|
|
"eusrr-celery-beat@${slot}.service" \
|
|
"eusrr-celery-worker@${slot}.service" \
|
|
"eusrr-frontend@${slot}.service" \
|
|
"eusrr-backend@${slot}.service" || true
|
|
}
|
|
|
|
start_slot_celery() {
|
|
local slot="$1"
|
|
systemctl start "eusrr-celery-worker@${slot}.service" "eusrr-celery-beat@${slot}.service"
|
|
systemctl is-active --quiet "eusrr-celery-worker@${slot}.service"
|
|
systemctl is-active --quiet "eusrr-celery-beat@${slot}.service"
|
|
}
|
|
|
|
enable_slot_stack() {
|
|
local slot="$1"
|
|
systemctl enable \
|
|
"eusrr-backend@${slot}.service" \
|
|
"eusrr-frontend@${slot}.service" \
|
|
"eusrr-celery-worker@${slot}.service" \
|
|
"eusrr-celery-beat@${slot}.service"
|
|
}
|
|
|
|
disable_old_stack() {
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
systemctl disable gunicorn-eusrr.service nextjs-eusrr.service \
|
|
celery-eusrr-worker.service celery-eusrr-beat.service || true
|
|
else
|
|
systemctl disable \
|
|
"eusrr-backend@${OLD_SLOT}.service" \
|
|
"eusrr-frontend@${OLD_SLOT}.service" \
|
|
"eusrr-celery-worker@${OLD_SLOT}.service" \
|
|
"eusrr-celery-beat@${OLD_SLOT}.service" || true
|
|
fi
|
|
}
|
|
|
|
enable_old_stack() {
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
systemctl enable gunicorn-eusrr.service nextjs-eusrr.service \
|
|
celery-eusrr-worker.service celery-eusrr-beat.service || true
|
|
else
|
|
systemctl enable \
|
|
"eusrr-backend@${OLD_SLOT}.service" \
|
|
"eusrr-frontend@${OLD_SLOT}.service" \
|
|
"eusrr-celery-worker@${OLD_SLOT}.service" \
|
|
"eusrr-celery-beat@${OLD_SLOT}.service" || true
|
|
fi
|
|
}
|
|
|
|
stop_legacy_stack() {
|
|
systemctl stop \
|
|
celery-eusrr-beat.service celery-eusrr-worker.service \
|
|
nextjs-eusrr.service gunicorn-eusrr.service || true
|
|
}
|
|
|
|
start_old_stack() {
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
systemctl restart gunicorn-eusrr.service nextjs-eusrr.service \
|
|
celery-eusrr-worker.service celery-eusrr-beat.service
|
|
else
|
|
systemctl restart "eusrr-backend@${OLD_SLOT}.service" "eusrr-frontend@${OLD_SLOT}.service" \
|
|
"eusrr-celery-worker@${OLD_SLOT}.service" "eusrr-celery-beat@${OLD_SLOT}.service"
|
|
fi
|
|
}
|
|
|
|
remove_release() {
|
|
local release="$1"
|
|
case "${release}" in
|
|
"${RELEASES_ROOT}"/*) ;;
|
|
*) log "Refusing to remove release outside ${RELEASES_ROOT}: ${release}"; return 1 ;;
|
|
esac
|
|
if [[ -e ${release} ]]; then
|
|
run_as_deploy git -C "${APP_ROOT}" worktree remove --force "${release}"
|
|
fi
|
|
}
|
|
|
|
rollback() {
|
|
local original_status=$?
|
|
trap - ERR
|
|
set +e
|
|
log "Deployment failed with exit code ${original_status}; keeping the previous release active."
|
|
|
|
if [[ ${UPSTREAM_CHANGED} -eq 1 && -f ${UPSTREAM_BACKUP} ]]; then
|
|
install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}"
|
|
point_current_at "${OLD_RELEASE}"
|
|
nginx -t && systemctl reload nginx
|
|
fi
|
|
|
|
stop_slot_stack "${NEW_SLOT}"
|
|
if [[ ${NEW_STACK_ENABLED} -eq 1 ]]; then
|
|
systemctl disable \
|
|
"eusrr-backend@${NEW_SLOT}.service" \
|
|
"eusrr-frontend@${NEW_SLOT}.service" \
|
|
"eusrr-celery-worker@${NEW_SLOT}.service" \
|
|
"eusrr-celery-beat@${NEW_SLOT}.service" || true
|
|
enable_old_stack
|
|
fi
|
|
if [[ ${OLD_STACK_STOPPED} -eq 1 ]]; then
|
|
start_old_stack || log "ERROR: failed to restart the previous application stack."
|
|
fi
|
|
|
|
log "No database snapshot was restored because the previous release remained live and may have accepted writes."
|
|
log "Inspect ${LOG_FILE} and service journals before retrying."
|
|
exit "${original_status}"
|
|
}
|
|
trap rollback ERR
|
|
|
|
log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}."
|
|
run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \
|
|
'+refs/heads/master:refs/remotes/origin/master'
|
|
MASTER_SHA="$(run_as_deploy git -C "${APP_ROOT}" rev-parse refs/remotes/origin/master)"
|
|
if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then
|
|
log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})."
|
|
exit 5
|
|
fi
|
|
if [[ ${TARGET_SHA} == "$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)" ]]; then
|
|
log "Revision ${TARGET_SHA} is already active."
|
|
exit 0
|
|
fi
|
|
|
|
if [[ -L ${SLOTS_ROOT}/${NEW_SLOT} ]]; then
|
|
stale_release="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}")"
|
|
if [[ ${stale_release} != "${OLD_RELEASE}" ]]; then
|
|
rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
|
|
remove_release "${stale_release}"
|
|
fi
|
|
fi
|
|
remove_release "${NEW_RELEASE}"
|
|
run_as_deploy git -C "${APP_ROOT}" worktree prune
|
|
|
|
available_bytes="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
|
|
if (( available_bytes < 4294967296 )); then
|
|
log "At least 4 GiB of free disk space is required to build a release."
|
|
exit 6
|
|
fi
|
|
|
|
log "Creating isolated Git worktree ${NEW_RELEASE}."
|
|
run_as_deploy git -C "${APP_ROOT}" worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}"
|
|
|
|
run_as_deploy ln -s "${APP_ROOT}/backend/.env" "${NEW_RELEASE}/backend/.env"
|
|
run_as_deploy ln -s "${APP_ROOT}/frontend/.env" "${NEW_RELEASE}/frontend/.env"
|
|
run_as_deploy rm -rf "${NEW_RELEASE}/backend/media"
|
|
run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media"
|
|
|
|
log "Creating an isolated Python environment."
|
|
run_as_deploy /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv"
|
|
run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
|
"${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \
|
|
--requirement "${NEW_RELEASE}/backend/requirements.txt"
|
|
|
|
log "Installing pinned frontend dependencies, including Linux native modules, and building Next.js."
|
|
run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" ci \
|
|
--include=optional --no-audit --no-fund
|
|
run_as_deploy test -f \
|
|
"${NEW_RELEASE}/frontend/node_modules/lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node"
|
|
run_as_deploy test -d \
|
|
"${NEW_RELEASE}/frontend/node_modules/@next/swc-linux-x64-gnu"
|
|
run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" run build
|
|
|
|
log "Running Django checks and collecting release static files."
|
|
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
"${NEW_RELEASE}/backend/manage.py" check
|
|
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
"${NEW_RELEASE}/backend/manage.py" collectstatic --noinput
|
|
|
|
run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/"
|
|
|
|
install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}"
|
|
DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump"
|
|
log "Creating a consistent PostgreSQL snapshot while the current release remains online."
|
|
run_database_dump "${DB_DUMP}"
|
|
chmod 0600 "${DB_DUMP}"
|
|
|
|
log "Planned migrations:"
|
|
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
"${NEW_RELEASE}/backend/manage.py" migrate --plan
|
|
log "Applying backward-compatible database migrations."
|
|
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
"${NEW_RELEASE}/backend/manage.py" migrate --noinput
|
|
|
|
log "Assigning release to the ${NEW_SLOT} slot and starting its web services."
|
|
ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new"
|
|
mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}"
|
|
start_slot_web "${NEW_SLOT}"
|
|
|
|
wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$'
|
|
wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$'
|
|
|
|
cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}"
|
|
UPSTREAM_CHANGED=1
|
|
point_current_at "${NEW_RELEASE}"
|
|
write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}"
|
|
nginx -t
|
|
systemctl reload nginx
|
|
|
|
wait_for_http "public frontend" "http://127.0.0.1/" '^200$' '172.11.0.11'
|
|
wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '172.11.0.11'
|
|
|
|
log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain."
|
|
sleep 10
|
|
log "Switching background workers."
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
stop_legacy_stack
|
|
else
|
|
stop_slot_stack "${OLD_SLOT}"
|
|
fi
|
|
OLD_STACK_STOPPED=1
|
|
start_slot_celery "${NEW_SLOT}"
|
|
NEW_STACK_ENABLED=1
|
|
enable_slot_stack "${NEW_SLOT}"
|
|
disable_old_stack
|
|
|
|
find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \
|
|
|| log "WARNING: old database snapshots could not be pruned."
|
|
run_as_deploy find "${SHARED_ROOT}/next-static" -type f -mtime +30 -delete \
|
|
|| log "WARNING: old Next.js assets could not be pruned."
|
|
rm -f "${UPSTREAM_BACKUP}" || true
|
|
printf '%s\n' "${NEW_SLOT}" >"${ACTIVE_SLOT_FILE}"
|
|
printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha"
|
|
trap - ERR
|
|
log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}."
|