779 lines
28 KiB
Bash
Executable File
779 lines
28 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
|
|
readonly APP_ROOT="/srv/eusrr"
|
|
readonly RELEASES_ROOT="/srv/eusrr-releases"
|
|
readonly SLOTS_ROOT="/srv/eusrr-slots"
|
|
readonly SHARED_ROOT="/srv/eusrr-shared"
|
|
readonly CURRENT_LINK="/srv/eusrr-current"
|
|
readonly DEPLOY_USER="deploy"
|
|
readonly BACKUP_DIR="/var/backups/eusrr/deploy"
|
|
readonly STATE_DIR="/var/lib/eusrr-deploy"
|
|
readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot"
|
|
readonly LOG_FILE="/var/log/eusrr-deploy.log"
|
|
readonly LOCK_FILE="/run/lock/eusrr-deploy.lock"
|
|
readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf"
|
|
readonly PREVIOUS_RELEASE_FILE="${STATE_DIR}/previous-release"
|
|
readonly PENDING_FILE="${STATE_DIR}/pending-deployment"
|
|
readonly GITEA_ORIGIN="ssh://git@172.11.0.14/Robotail/Corp.git"
|
|
readonly GITEA_KNOWN_HOSTS="/etc/eusrr/gitea_known_hosts"
|
|
|
|
if [[ ${EUID} -ne 0 ]]; then
|
|
echo "This command must run as root." >&2
|
|
exit 1
|
|
fi
|
|
|
|
install -d -o root -g root -m 0755 "${STATE_DIR}"
|
|
touch "${LOG_FILE}"
|
|
chmod 0600 "${LOG_FILE}"
|
|
exec > >(tee -a "${LOG_FILE}") 2>&1
|
|
|
|
log() {
|
|
printf '[%s] %s\n' "$(date --iso-8601=seconds)" "$*"
|
|
}
|
|
|
|
run_as_deploy() {
|
|
setpriv --no-new-privs runuser -u "${DEPLOY_USER}" -- "$@"
|
|
}
|
|
|
|
check_node_runtime() {
|
|
run_as_deploy node -e 'const [major, minor] = process.versions.node.split(".").map(Number); if (!((major === 22 && minor >= 13) || major === 24)) { console.error("Production requires Node.js 22.13+ (22.x) or 24.x LTS."); process.exit(1); }'
|
|
}
|
|
|
|
run_git() {
|
|
# Repository hooks/config and per-user SSH aliases must not silently replace
|
|
# the production source. The read-only deploy key and pinned host key are used
|
|
# only for the explicitly selected Gitea instance.
|
|
run_as_deploy env GIT_CONFIG_NOSYSTEM=1 GIT_CONFIG_GLOBAL=/dev/null \
|
|
GIT_SSH_COMMAND="/usr/bin/ssh -F /dev/null -i /home/deploy/.ssh/gitea_ed25519 -o BatchMode=yes -o IdentitiesOnly=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=${GITEA_KNOWN_HOSTS}" \
|
|
git -c core.hooksPath=/dev/null -c protocol.allow=never -c protocol.ssh.allow=always \
|
|
-C "${APP_ROOT}" "$@"
|
|
}
|
|
|
|
run_in_release_backend() {
|
|
run_as_deploy \
|
|
bash -c 'cd "$1"; shift; exec "$@"' _ "${NEW_RELEASE}/backend" "$@"
|
|
}
|
|
|
|
slot_ports() {
|
|
case "$1" in
|
|
blue) BACKEND_PORT=9001; FRONTEND_PORT=3001 ;;
|
|
green) BACKEND_PORT=9002; FRONTEND_PORT=3002 ;;
|
|
*) log "Unknown deployment slot: $1"; return 1 ;;
|
|
esac
|
|
}
|
|
|
|
if [[ ${1:-} == "--self-test" ]]; then
|
|
for command in git curl flock runuser pg_dump pg_restore npm nginx rsync setsid timeout findmnt setpriv; do
|
|
command -v "${command}" >/dev/null
|
|
done
|
|
if ! command -v magick >/dev/null && ! command -v convert >/dev/null; then
|
|
log "ImageMagick is required for safe oversized-image processing."
|
|
exit 1
|
|
fi
|
|
test -d "${APP_ROOT}/.git"
|
|
run_as_deploy test -r "${APP_ROOT}/backend/.env"
|
|
run_as_deploy test -r "${APP_ROOT}/frontend/.env"
|
|
test -r "${UPSTREAM_FILE}"
|
|
test -r /usr/local/libexec/eusrr-deploy-healthcheck
|
|
run_as_deploy test -r "${GITEA_KNOWN_HOSTS}"
|
|
check_node_runtime
|
|
[[ $(stat -c '%u:%a' "${SLOTS_ROOT}") == 0:755 ]]
|
|
for unit in eusrr-backend@.service eusrr-frontend@.service \
|
|
eusrr-celery-worker@.service eusrr-celery-beat@.service; do
|
|
systemctl cat "${unit}" >/dev/null
|
|
done
|
|
nginx -t
|
|
log "Blue-green deployment self-test passed."
|
|
exit 0
|
|
fi
|
|
|
|
TARGET_SHA="${1:-}"
|
|
MAINTENANCE=0
|
|
if [[ ${TARGET_SHA} == --maintenance ]]; then
|
|
MAINTENANCE=1
|
|
elif [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then
|
|
echo "Usage: $0 <40-character lowercase commit SHA> | --maintenance" >&2
|
|
exit 2
|
|
fi
|
|
|
|
exec 9>"${LOCK_FILE}"
|
|
if ! flock -n 9; then
|
|
log "Another deployment is already running."
|
|
exit 3
|
|
fi
|
|
|
|
OLD_SLOT="$(cat "${ACTIVE_SLOT_FILE}" 2>/dev/null || printf legacy)"
|
|
case "${OLD_SLOT}" in
|
|
legacy) NEW_SLOT=blue ;;
|
|
blue) NEW_SLOT=green ;;
|
|
green) NEW_SLOT=blue ;;
|
|
*) log "Invalid active slot state: ${OLD_SLOT}"; exit 4 ;;
|
|
esac
|
|
|
|
NEW_RELEASE="${RELEASES_ROOT}/${TARGET_SHA}"
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
OLD_RELEASE="${APP_ROOT}"
|
|
else
|
|
OLD_RELEASE="$(readlink -f "${SLOTS_ROOT}/${OLD_SLOT}")"
|
|
fi
|
|
|
|
BACKEND_PORT=""
|
|
FRONTEND_PORT=""
|
|
slot_ports "${NEW_SLOT}"
|
|
|
|
UPSTREAM_BACKUP="${STATE_DIR}/upstreams-before-${TARGET_SHA}"
|
|
UPSTREAM_CHANGED=0
|
|
OLD_STACK_STOPPED=0
|
|
NEW_STACK_ENABLED=0
|
|
DB_DUMP=""
|
|
DB_DUMP_PARTIAL=""
|
|
PREVIOUS_RELEASE="$(cat "${PREVIOUS_RELEASE_FILE}" 2>/dev/null || true)"
|
|
INACTIVE_RELEASE="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}" 2>/dev/null || true)"
|
|
OLD_SUCCESSFUL_SHA="$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)"
|
|
MUTATION_STARTED=0
|
|
SLOT_ASSIGNED=0
|
|
BUILD_PID=""
|
|
WATCHDOG_PID=""
|
|
COMMITTED=0
|
|
|
|
atomic_state() {
|
|
printf '%s\n' "$2" >"$1.new"
|
|
chmod 0644 "$1.new"
|
|
mv -Tf "$1.new" "$1"
|
|
}
|
|
|
|
check_space() {
|
|
local minimum="$1" available inodes
|
|
available="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
|
|
inodes="$(df --output=iavail "${RELEASES_ROOT}" | tail -1 | tr -d ' ')"
|
|
if (( available < minimum || inodes < 50000 )); then
|
|
log "Insufficient capacity: ${available} bytes and ${inodes} inodes free; need ${minimum} bytes and 50000 inodes."
|
|
return 6
|
|
fi
|
|
}
|
|
|
|
# Build processes have their own session so cancellation also stops grandchildren.
|
|
run_bounded_build() {
|
|
check_space 1073741824 || return $?
|
|
setsid timeout --signal=TERM --kill-after=30s 30m setpriv --no-new-privs \
|
|
runuser -u "${DEPLOY_USER}" -- "$@" <&0 &
|
|
BUILD_PID=$!
|
|
(
|
|
while kill -0 "${BUILD_PID}" 2>/dev/null; do
|
|
if ! check_space 536870912; then
|
|
log "Stopping build before it exhausts the production filesystem."
|
|
kill -TERM -- "-${BUILD_PID}" 2>/dev/null || true
|
|
sleep 5
|
|
kill -KILL -- "-${BUILD_PID}" 2>/dev/null || true
|
|
exit 6
|
|
fi
|
|
sleep 2
|
|
done
|
|
) &
|
|
WATCHDOG_PID=$!
|
|
local status=0
|
|
wait "${BUILD_PID}" || status=$?
|
|
kill "${WATCHDOG_PID}" 2>/dev/null || true
|
|
wait "${WATCHDOG_PID}" 2>/dev/null || true
|
|
WATCHDOG_PID=""
|
|
BUILD_PID=""
|
|
(( status == 0 )) || return "${status}"
|
|
check_space 536870912
|
|
}
|
|
|
|
install_frontend_dependencies() {
|
|
# All attempts share one build timeout and disk watchdog. Retry only reported
|
|
# network failures, never dependency/engine/postinstall errors. Cache remains
|
|
# inside the candidate and is removed on success or failed-release cleanup.
|
|
run_bounded_build bash -c '
|
|
set -euo pipefail
|
|
release="$1"
|
|
output="${release}/.npm-install-attempt.log"
|
|
for attempt in 1 2 3; do
|
|
printf "Frontend dependency installation attempt %s/3.\n" "$attempt"
|
|
if npm --cache "${release}/.npm-cache" --prefix "${release}/frontend" ci \
|
|
--include=optional --no-audit --no-fund --engine-strict \
|
|
--fetch-retries=3 --fetch-retry-factor=2 \
|
|
--fetch-retry-mintimeout=10000 --fetch-retry-maxtimeout=30000 \
|
|
--fetch-timeout=120000 >"${output}" 2>&1; then
|
|
cat "${output}"
|
|
rm -f -- "${output}"
|
|
exit 0
|
|
else
|
|
status=$?
|
|
cat "${output}"
|
|
fi
|
|
if [[ "$attempt" == 3 ]] || ! grep -Eq \
|
|
"npm (ERR!|error) code (ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|ECONNREFUSED|E502|E503|E504)([[:space:]]|$)" "${output}"; then
|
|
exit "$status"
|
|
fi
|
|
printf "Transient npm network failure; retrying the same lockfile.\n"
|
|
sleep "$((attempt * 10))"
|
|
done
|
|
' _ "${NEW_RELEASE}"
|
|
}
|
|
|
|
run_database_dump() {
|
|
local dump_file="$1"
|
|
run_bounded_build \
|
|
env EUSRR_DB_DUMP="${dump_file}" \
|
|
"${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY'
|
|
import os
|
|
from pathlib import Path
|
|
|
|
from dotenv import dotenv_values
|
|
|
|
config = dotenv_values("/srv/eusrr/backend/.env")
|
|
|
|
def required(name: str) -> str:
|
|
value = config.get(name)
|
|
if not value:
|
|
raise SystemExit(f"Required database setting is missing: {name}")
|
|
return str(value)
|
|
|
|
executable = "/usr/bin/pg_dump"
|
|
if not Path(executable).is_file():
|
|
raise SystemExit(f"Database utility is not installed: {executable}")
|
|
|
|
environment = os.environ.copy()
|
|
environment["PGPASSWORD"] = required("POSTGRES_PASSWORD")
|
|
arguments = [
|
|
executable,
|
|
"--host", str(config.get("DB_HOST") or "127.0.0.1"),
|
|
"--port", str(config.get("DB_PORT") or "5432"),
|
|
"--username", required("POSTGRES_USER"),
|
|
"--dbname", required("POSTGRES_DB"),
|
|
"--format=custom",
|
|
"--no-owner",
|
|
"--no-acl",
|
|
"--file", os.environ["EUSRR_DB_DUMP"],
|
|
]
|
|
os.execve(executable, arguments, environment)
|
|
PY
|
|
}
|
|
|
|
wait_for_http() {
|
|
local name="$1"
|
|
local url="$2"
|
|
local expected="$3"
|
|
local host_header="${4:-}"
|
|
local attempt code
|
|
# These URLs are exclusively loopback. Match the trusted TLS terminator's
|
|
# header so SECURE_SSL_REDIRECT can stay enabled in production.
|
|
local -a curl_args=(--silent --show-error --output /dev/null --write-out '%{http_code}' --max-time 5
|
|
--header 'X-Forwarded-Proto: https')
|
|
|
|
if [[ -n ${host_header} ]]; then
|
|
curl_args+=(--header "Host: ${host_header}")
|
|
fi
|
|
|
|
for attempt in $(seq 1 45); do
|
|
code="$(curl "${curl_args[@]}" "${url}" || true)"
|
|
if [[ ${code} =~ ${expected} ]]; then
|
|
log "Health check passed: ${name} returned ${code}."
|
|
return 0
|
|
fi
|
|
sleep 2
|
|
done
|
|
|
|
log "Health check failed: ${name} returned ${code:-no response}."
|
|
return 1
|
|
}
|
|
|
|
write_upstreams() {
|
|
local backend_port="$1"
|
|
local frontend_port="$2"
|
|
local temporary_file
|
|
temporary_file="$(mktemp)"
|
|
cat >"${temporary_file}" <<EOF
|
|
upstream eusrr_backend {
|
|
server 127.0.0.1:${backend_port};
|
|
keepalive 32;
|
|
}
|
|
|
|
upstream eusrr_frontend {
|
|
server 127.0.0.1:${frontend_port};
|
|
keepalive 32;
|
|
}
|
|
EOF
|
|
install -o root -g root -m 0644 "${temporary_file}" "${UPSTREAM_FILE}"
|
|
rm -f "${temporary_file}"
|
|
UPSTREAM_CHANGED=1
|
|
}
|
|
|
|
point_current_at() {
|
|
local release="$1"
|
|
ln -sfn "${release}" "${CURRENT_LINK}.new"
|
|
mv -Tf "${CURRENT_LINK}.new" "${CURRENT_LINK}"
|
|
}
|
|
|
|
start_slot_web() {
|
|
local slot="$1"
|
|
systemctl start "eusrr-backend@${slot}.service" "eusrr-frontend@${slot}.service"
|
|
}
|
|
|
|
stop_slot_stack() {
|
|
local slot="$1"
|
|
systemctl stop \
|
|
"eusrr-celery-beat@${slot}.service" \
|
|
"eusrr-celery-worker@${slot}.service" \
|
|
"eusrr-frontend@${slot}.service" \
|
|
"eusrr-backend@${slot}.service"
|
|
assert_slot_stopped "${slot}"
|
|
}
|
|
|
|
assert_slot_stopped() {
|
|
local unit state
|
|
for unit in backend frontend celery-worker celery-beat; do
|
|
state="$(systemctl show "eusrr-${unit}@$1.service" -p ActiveState --value)" || return 1
|
|
case "${state}" in inactive|failed) ;; *) log "Service is still ${state}: eusrr-${unit}@$1"; return 1 ;; esac
|
|
done
|
|
}
|
|
|
|
stop_slot_celery() {
|
|
local unit state
|
|
systemctl stop "eusrr-celery-beat@$1.service" "eusrr-celery-worker@$1.service"
|
|
for unit in celery-beat celery-worker; do
|
|
state="$(systemctl show "eusrr-${unit}@$1.service" -p ActiveState --value)" || return 1
|
|
case "${state}" in inactive|failed) ;; *) return 1 ;; esac
|
|
done
|
|
}
|
|
|
|
reconcile_legacy_workers() {
|
|
local active_slot="${1:-${OLD_SLOT}}"
|
|
[[ ${active_slot} != legacy ]] || return 0
|
|
systemctl is-active --quiet "eusrr-celery-worker@${active_slot}.service"
|
|
systemctl is-active --quiet "eusrr-celery-beat@${active_slot}.service"
|
|
local unit state
|
|
# A legacy umbrella service can start disabled worker/beat units again.
|
|
# A condition blocks dependency-driven starts as well as manual starts.
|
|
for unit in celery-eusrr.service celery-eusrr-beat.service celery-eusrr-worker.service; do
|
|
if [[ $(systemctl show "${unit}" -p LoadState --value) == not-found ]]; then
|
|
continue
|
|
fi
|
|
install -d -m 0755 "/etc/systemd/system/${unit}.d"
|
|
printf '%s\n' '[Unit]' 'ConditionPathExists=!/var/lib/eusrr-deploy/native-workers' \
|
|
>"/etc/systemd/system/${unit}.d/90-native-workers.conf"
|
|
done
|
|
touch "${STATE_DIR}/native-workers"
|
|
systemctl daemon-reload
|
|
for unit in celery-eusrr.service celery-eusrr-beat.service celery-eusrr-worker.service; do
|
|
[[ $(systemctl show "${unit}" -p LoadState --value) != not-found ]] || continue
|
|
systemctl disable --now "${unit}"
|
|
state="$(systemctl show "${unit}" -p ActiveState --value)"
|
|
case "${state}" in inactive|failed) ;; *) return 1 ;; esac
|
|
done
|
|
}
|
|
|
|
start_slot_celery() {
|
|
local slot="$1"
|
|
systemctl start "eusrr-celery-worker@${slot}.service" "eusrr-celery-beat@${slot}.service"
|
|
systemctl is-active --quiet "eusrr-celery-worker@${slot}.service"
|
|
systemctl is-active --quiet "eusrr-celery-beat@${slot}.service"
|
|
}
|
|
|
|
enable_slot_stack() {
|
|
local slot="$1"
|
|
systemctl enable \
|
|
"eusrr-backend@${slot}.service" \
|
|
"eusrr-frontend@${slot}.service" \
|
|
"eusrr-celery-worker@${slot}.service" \
|
|
"eusrr-celery-beat@${slot}.service"
|
|
}
|
|
|
|
disable_old_stack() {
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
systemctl disable gunicorn-eusrr.service nextjs-eusrr.service \
|
|
celery-eusrr-worker.service celery-eusrr-beat.service
|
|
else
|
|
systemctl disable \
|
|
"eusrr-backend@${OLD_SLOT}.service" \
|
|
"eusrr-frontend@${OLD_SLOT}.service" \
|
|
"eusrr-celery-worker@${OLD_SLOT}.service" \
|
|
"eusrr-celery-beat@${OLD_SLOT}.service"
|
|
fi
|
|
}
|
|
|
|
enable_old_stack() {
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
systemctl enable gunicorn-eusrr.service nextjs-eusrr.service \
|
|
celery-eusrr-worker.service celery-eusrr-beat.service
|
|
else
|
|
systemctl enable \
|
|
"eusrr-backend@${OLD_SLOT}.service" \
|
|
"eusrr-frontend@${OLD_SLOT}.service" \
|
|
"eusrr-celery-worker@${OLD_SLOT}.service" \
|
|
"eusrr-celery-beat@${OLD_SLOT}.service"
|
|
fi
|
|
}
|
|
|
|
stop_legacy_stack() {
|
|
if [[ $(systemctl show celery-eusrr.service -p LoadState --value) != not-found ]]; then
|
|
systemctl stop celery-eusrr.service
|
|
fi
|
|
systemctl stop \
|
|
celery-eusrr-beat.service celery-eusrr-worker.service \
|
|
nextjs-eusrr.service gunicorn-eusrr.service
|
|
local unit state
|
|
for unit in celery-eusrr-beat.service celery-eusrr-worker.service nextjs-eusrr.service gunicorn-eusrr.service; do
|
|
state="$(systemctl show "${unit}" -p ActiveState --value)"
|
|
case "${state}" in inactive|failed) ;; *) return 1 ;; esac
|
|
done
|
|
}
|
|
|
|
start_old_stack() {
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
systemctl restart gunicorn-eusrr.service nextjs-eusrr.service \
|
|
celery-eusrr-worker.service celery-eusrr-beat.service
|
|
else
|
|
systemctl restart "eusrr-backend@${OLD_SLOT}.service" "eusrr-frontend@${OLD_SLOT}.service" \
|
|
"eusrr-celery-worker@${OLD_SLOT}.service" "eusrr-celery-beat@${OLD_SLOT}.service"
|
|
fi
|
|
}
|
|
|
|
remove_release() {
|
|
local release="$1"
|
|
local name="${release##*/}" protected resolved mount
|
|
[[ ${name} =~ ^[0-9a-f]{40}$ && ${release} == "${RELEASES_ROOT}/${name}" ]] || {
|
|
log "Refusing non-release path: ${release}"; return 1;
|
|
}
|
|
[[ ! -L ${release} ]] || { log "Refusing symlink release: ${release}"; return 1; }
|
|
resolved="$(readlink -m "${release}")"
|
|
[[ ${resolved} == "${release}" ]] || return 1
|
|
for protected in "${OLD_RELEASE}" "${PREVIOUS_RELEASE}" \
|
|
"$(readlink -f "${CURRENT_LINK}")" \
|
|
"$(readlink -f "${SLOTS_ROOT}/blue" 2>/dev/null || true)" \
|
|
"$(readlink -f "${SLOTS_ROOT}/green" 2>/dev/null || true)"; do
|
|
[[ ${release} != "${protected}" ]] || { log "Protected release: ${release}"; return 1; }
|
|
done
|
|
if [[ -e ${release} ]]; then
|
|
# Git must never recurse into a bind mount containing shared data.
|
|
while IFS= read -r mount; do
|
|
[[ ${mount} != "${release}" && ${mount} != "${release}/"* ]] || return 1
|
|
done < <(findmnt -rn -o TARGET)
|
|
log "Removing unused release ${release}."
|
|
run_git worktree remove --force "${release}"
|
|
fi
|
|
}
|
|
|
|
prune_orphan_releases() {
|
|
local release
|
|
for release in "${RELEASES_ROOT}"/*; do
|
|
[[ -d ${release} && ! -L ${release} ]] || continue
|
|
[[ ${release##*/} =~ ^[0-9a-f]{40}$ ]] || continue
|
|
if [[ ${release} == "${OLD_RELEASE}" || ${release} == "${PREVIOUS_RELEASE}" ||
|
|
${release} == "$(readlink -f "${CURRENT_LINK}")" ||
|
|
${release} == "$(readlink -f "${SLOTS_ROOT}/blue" 2>/dev/null || true)" ||
|
|
${release} == "$(readlink -f "${SLOTS_ROOT}/green" 2>/dev/null || true)" ]]; then
|
|
continue
|
|
fi
|
|
remove_release "${release}"
|
|
done
|
|
run_git worktree prune
|
|
}
|
|
|
|
validate_live_state() {
|
|
local slot_link slot_target
|
|
# These links are control-plane input to root. Application code must not be
|
|
# able to redirect a slot (or make cleanup treat shared data as a release).
|
|
[[ ! -L ${SLOTS_ROOT} && $(stat -c '%u:%a' "${SLOTS_ROOT}") == 0:755 ]] || {
|
|
log "Slot directory must be root-owned mode 0755; run the tooling installer."; return 1;
|
|
}
|
|
for slot_link in "${SLOTS_ROOT}/blue" "${SLOTS_ROOT}/green"; do
|
|
[[ -e ${slot_link} || -L ${slot_link} ]] || continue
|
|
[[ -L ${slot_link} ]] || { log "Slot is not a symlink: ${slot_link}"; return 1; }
|
|
slot_target="$(readlink -f "${slot_link}")"
|
|
[[ ${slot_target} =~ ^${RELEASES_ROOT}/[0-9a-f]{40}$ && -d ${slot_target} && ! -L ${slot_target} ]] || {
|
|
log "Slot points outside a valid release: ${slot_link}"; return 1;
|
|
}
|
|
done
|
|
[[ -d ${OLD_RELEASE} && $(readlink -f "${CURRENT_LINK}") == "${OLD_RELEASE}" ]] || {
|
|
log "Active slot and current release disagree; refusing cleanup."; return 1;
|
|
}
|
|
if [[ ${OLD_SLOT} != legacy ]]; then
|
|
[[ ${OLD_RELEASE} == "${RELEASES_ROOT}/${OLD_SUCCESSFUL_SHA}" ]] || return 1
|
|
local backend frontend
|
|
case "${OLD_SLOT}" in blue) backend=9001; frontend=3001 ;; green) backend=9002; frontend=3002 ;; esac
|
|
grep -Eq "server 127[.]0[.]0[.]1:${backend};" "${UPSTREAM_FILE}"
|
|
grep -Eq "server 127[.]0[.]0[.]1:${frontend};" "${UPSTREAM_FILE}"
|
|
systemctl is-active --quiet "eusrr-backend@${OLD_SLOT}.service"
|
|
systemctl is-active --quiet "eusrr-frontend@${OLD_SLOT}.service"
|
|
fi
|
|
}
|
|
|
|
prune_expired_dumps() {
|
|
# Run on failed attempts too. A partial dump is never offered as a backup.
|
|
find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete
|
|
find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump.partial' -delete
|
|
}
|
|
|
|
rollback() {
|
|
local original_status=$?
|
|
trap - EXIT HUP INT TERM
|
|
[[ ${COMMITTED} -eq 0 && ${MUTATION_STARTED} -eq 1 ]] || exit "${original_status}"
|
|
(( original_status != 0 )) || original_status=1
|
|
set +e
|
|
log "Deployment failed with exit code ${original_status}; restoring the previous release."
|
|
if [[ -n ${WATCHDOG_PID} ]]; then
|
|
kill "${WATCHDOG_PID}" 2>/dev/null
|
|
wait "${WATCHDOG_PID}" 2>/dev/null
|
|
fi
|
|
if [[ -n ${BUILD_PID} ]]; then
|
|
kill -TERM -- "-${BUILD_PID}" 2>/dev/null
|
|
sleep 2
|
|
kill -KILL -- "-${BUILD_PID}" 2>/dev/null
|
|
wait "${BUILD_PID}" 2>/dev/null
|
|
fi
|
|
local recovered=1
|
|
if [[ ${UPSTREAM_CHANGED} -eq 1 ]]; then
|
|
# Keep candidate HTTP services alive until Nginx has returned to the old
|
|
# stack. Only stop its scheduler/workers before restarting the old workers.
|
|
stop_slot_celery "${NEW_SLOT}" || recovered=0
|
|
else
|
|
stop_slot_stack "${NEW_SLOT}" || recovered=0
|
|
fi
|
|
if [[ ${NEW_STACK_ENABLED} -eq 1 ]]; then
|
|
systemctl disable \
|
|
"eusrr-backend@${NEW_SLOT}.service" \
|
|
"eusrr-frontend@${NEW_SLOT}.service" \
|
|
"eusrr-celery-worker@${NEW_SLOT}.service" \
|
|
"eusrr-celery-beat@${NEW_SLOT}.service" || recovered=0
|
|
enable_old_stack || recovered=0
|
|
fi
|
|
if [[ ${OLD_STACK_STOPPED} -eq 1 && ${recovered} -eq 1 ]]; then
|
|
[[ ${OLD_SLOT} != legacy ]] || rm -f "${STATE_DIR}/native-workers"
|
|
start_old_stack || recovered=0
|
|
fi
|
|
if [[ ${UPSTREAM_CHANGED} -eq 1 ]]; then
|
|
if [[ -f ${UPSTREAM_BACKUP} && ${recovered} -eq 1 ]]; then
|
|
install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}" || recovered=0
|
|
point_current_at "${OLD_RELEASE}" || recovered=0
|
|
nginx -t && systemctl reload nginx || recovered=0
|
|
if [[ ${recovered} -eq 1 ]]; then
|
|
stop_slot_stack "${NEW_SLOT}" || recovered=0
|
|
fi
|
|
else
|
|
recovered=0
|
|
fi
|
|
fi
|
|
if [[ ${recovered} -eq 1 ]]; then
|
|
atomic_state "${ACTIVE_SLOT_FILE}" "${OLD_SLOT}" || recovered=0
|
|
atomic_state "${STATE_DIR}/last-successful-sha" "${OLD_SUCCESSFUL_SHA}" || recovered=0
|
|
atomic_state "${PREVIOUS_RELEASE_FILE}" "${PREVIOUS_RELEASE}" || recovered=0
|
|
fi
|
|
if [[ ${recovered} -eq 1 ]]; then
|
|
if [[ ${SLOT_ASSIGNED} -eq 1 ]]; then
|
|
rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
|
|
if [[ -n ${INACTIVE_RELEASE} && -d ${INACTIVE_RELEASE} ]]; then
|
|
ln -sfn "${INACTIVE_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}"
|
|
fi
|
|
fi
|
|
remove_release "${NEW_RELEASE}" || recovered=0
|
|
fi
|
|
[[ -z ${DB_DUMP_PARTIAL} ]] || rm -f "${DB_DUMP_PARTIAL}"
|
|
if [[ ${recovered} -eq 1 ]]; then
|
|
rm -f "${PENDING_FILE}" "${UPSTREAM_BACKUP}"
|
|
log "Previous release restored; failed build removed."
|
|
else
|
|
log "ERROR: recovery was incomplete; release and recovery journal retained. Manual inspection required."
|
|
fi
|
|
|
|
log "No database snapshot was restored because the previous release remained live and may have accepted writes."
|
|
log "Inspect ${LOG_FILE} and service journals before retrying."
|
|
exit "${original_status}"
|
|
}
|
|
|
|
validate_live_state
|
|
# A journal from the switching phase may describe Nginx's in-memory state,
|
|
# which cannot be inferred from its configuration file after an interrupted reload.
|
|
if [[ -f ${PENDING_FILE} ]]; then
|
|
read -r pending_phase pending_sha <"${PENDING_FILE}"
|
|
[[ ${pending_sha} =~ ^[0-9a-f]{40}$ ]] || exit 4
|
|
case "${pending_phase}" in
|
|
building) ;;
|
|
switching)
|
|
if [[ ${pending_sha} != "${OLD_SUCCESSFUL_SHA}" ]]; then
|
|
log "Interrupted traffic switch: inspect services and Nginx before clearing ${PENDING_FILE}."
|
|
exit 4
|
|
fi ;;
|
|
*) exit 4 ;;
|
|
esac
|
|
fi
|
|
# Preserve the previous successful slot through the build of the next release.
|
|
if [[ -n ${INACTIVE_RELEASE} && -d ${INACTIVE_RELEASE} && ! -f ${PENDING_FILE} ]]; then
|
|
PREVIOUS_RELEASE="${INACTIVE_RELEASE}"
|
|
atomic_state "${PREVIOUS_RELEASE_FILE}" "${PREVIOUS_RELEASE}"
|
|
fi
|
|
reconcile_legacy_workers
|
|
stop_slot_stack "${NEW_SLOT}"
|
|
if [[ -f ${PENDING_FILE} ]]; then
|
|
# SIGKILL/power failure before traffic switched: the live-state checks above
|
|
# prove production still uses the last committed release. Restore standby.
|
|
if [[ ${INACTIVE_RELEASE} == "${RELEASES_ROOT}/${pending_sha}" ]]; then
|
|
rm -f "${SLOTS_ROOT}/${NEW_SLOT}"
|
|
if [[ -n ${PREVIOUS_RELEASE} && -d ${PREVIOUS_RELEASE} ]]; then
|
|
ln -sfn "${PREVIOUS_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}"
|
|
fi
|
|
INACTIVE_RELEASE="${PREVIOUS_RELEASE}"
|
|
fi
|
|
rm -f "${PENDING_FILE}"
|
|
fi
|
|
prune_orphan_releases
|
|
prune_expired_dumps
|
|
if [[ ${MAINTENANCE} -eq 1 ]]; then
|
|
log "Maintenance completed; active and previous releases preserved."
|
|
df -h "${RELEASES_ROOT}"
|
|
exit 0
|
|
fi
|
|
|
|
log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}."
|
|
check_node_runtime
|
|
[[ $(run_git config --get remote.origin.url) == "${GITEA_ORIGIN}" ]] || {
|
|
log "Repository origin is not the approved Gitea repository; refusing deployment."; exit 5;
|
|
}
|
|
if run_git config --get-regexp '^url\..*\.insteadof$' >/dev/null; then
|
|
log "Git URL rewrites are not allowed for production deployment."; exit 5
|
|
fi
|
|
run_git fetch --prune --no-tags "${GITEA_ORIGIN}" \
|
|
'+refs/heads/master:refs/remotes/origin/master'
|
|
MASTER_SHA="$(run_git rev-parse refs/remotes/origin/master)"
|
|
if [[ ${TARGET_SHA} != "${MASTER_SHA}" ]]; then
|
|
log "Refusing deployment: requested SHA is not the current Gitea master (${MASTER_SHA})."
|
|
exit 5
|
|
fi
|
|
if [[ ${TARGET_SHA} == "$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)" ]]; then
|
|
log "Revision ${TARGET_SHA} is already active."
|
|
exit 0
|
|
fi
|
|
|
|
remove_release "${NEW_RELEASE}"
|
|
check_space 4294967296
|
|
atomic_state "${PENDING_FILE}" "building ${TARGET_SHA}"
|
|
MUTATION_STARTED=1
|
|
trap rollback EXIT
|
|
trap 'exit 129' HUP
|
|
trap 'exit 130' INT
|
|
trap 'exit 143' TERM
|
|
|
|
log "Creating isolated Git worktree ${NEW_RELEASE}."
|
|
run_git worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}"
|
|
|
|
run_as_deploy ln -s "${APP_ROOT}/backend/.env" "${NEW_RELEASE}/backend/.env"
|
|
run_as_deploy ln -s "${APP_ROOT}/frontend/.env" "${NEW_RELEASE}/frontend/.env"
|
|
run_as_deploy rm -rf "${NEW_RELEASE}/backend/media"
|
|
run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media"
|
|
|
|
log "Creating an isolated Python environment."
|
|
run_bounded_build /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv"
|
|
run_bounded_build env PIP_DISABLE_PIP_VERSION_CHECK=1 \
|
|
"${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \
|
|
--no-cache-dir --requirement "${NEW_RELEASE}/backend/requirements.txt"
|
|
run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" -m pip check
|
|
run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" -c 'import shutil; assert shutil.which("magick") or shutil.which("convert"), "ImageMagick is required for safe oversized-image processing"'
|
|
run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
/usr/local/libexec/eusrr-deploy-healthcheck
|
|
|
|
log "Installing pinned frontend dependencies, including Linux native modules, and building Next.js."
|
|
install_frontend_dependencies
|
|
run_as_deploy rm -rf -- "${NEW_RELEASE}/.npm-cache"
|
|
run_as_deploy test -f \
|
|
"${NEW_RELEASE}/frontend/node_modules/lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node"
|
|
run_as_deploy test -d \
|
|
"${NEW_RELEASE}/frontend/node_modules/@next/swc-linux-x64-gnu"
|
|
run_bounded_build npm --prefix "${NEW_RELEASE}/frontend" run build
|
|
|
|
log "Running Django checks and collecting release static files."
|
|
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
"${NEW_RELEASE}/backend/manage.py" check
|
|
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
"${NEW_RELEASE}/backend/manage.py" collectstatic --noinput
|
|
|
|
run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/"
|
|
check_space 1073741824
|
|
|
|
install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}"
|
|
DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump"
|
|
DB_DUMP_PARTIAL="${DB_DUMP}.partial"
|
|
log "Creating a consistent PostgreSQL snapshot while the current release remains online."
|
|
run_database_dump "${DB_DUMP_PARTIAL}"
|
|
pg_restore --list "${DB_DUMP_PARTIAL}" >/dev/null
|
|
mv "${DB_DUMP_PARTIAL}" "${DB_DUMP}"
|
|
DB_DUMP_PARTIAL=""
|
|
chmod 0600 "${DB_DUMP}"
|
|
check_space 536870912
|
|
|
|
log "Planned migrations:"
|
|
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
"${NEW_RELEASE}/backend/manage.py" migrate --plan
|
|
log "Applying backward-compatible database migrations."
|
|
run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
"${NEW_RELEASE}/backend/manage.py" migrate --noinput
|
|
|
|
log "Assigning release to the ${NEW_SLOT} slot and starting its web services."
|
|
SLOT_ASSIGNED=1
|
|
ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new"
|
|
mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}"
|
|
start_slot_web "${NEW_SLOT}"
|
|
|
|
wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$'
|
|
wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$'
|
|
run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" \
|
|
/usr/local/libexec/eusrr-deploy-healthcheck --app-port "${BACKEND_PORT}"
|
|
|
|
cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}"
|
|
atomic_state "${PENDING_FILE}" "switching ${TARGET_SHA}"
|
|
UPSTREAM_CHANGED=1
|
|
point_current_at "${NEW_RELEASE}"
|
|
write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}"
|
|
nginx -t
|
|
systemctl reload nginx
|
|
|
|
wait_for_http "public frontend" "http://127.0.0.1/" '^200$' '172.11.0.11'
|
|
wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '172.11.0.11'
|
|
|
|
log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain."
|
|
sleep 10
|
|
log "Switching background workers."
|
|
OLD_STACK_STOPPED=1
|
|
if [[ ${OLD_SLOT} == legacy ]]; then
|
|
stop_legacy_stack
|
|
else
|
|
stop_slot_stack "${OLD_SLOT}"
|
|
fi
|
|
start_slot_celery "${NEW_SLOT}"
|
|
reconcile_legacy_workers "${NEW_SLOT}"
|
|
NEW_STACK_ENABLED=1
|
|
enable_slot_stack "${NEW_SLOT}"
|
|
disable_old_stack
|
|
|
|
find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \
|
|
|| log "WARNING: old database snapshots could not be pruned."
|
|
# Keep files referenced by either retained release regardless of their mtime.
|
|
run_as_deploy python3 - "${SHARED_ROOT}/next-static" "${NEW_RELEASE}" "${OLD_RELEASE}" <<'PY'
|
|
import sys
|
|
import time
|
|
from pathlib import Path
|
|
|
|
shared = Path(sys.argv[1])
|
|
referenced = set()
|
|
for release in sys.argv[2:]:
|
|
root = Path(release) / "frontend/.next/static"
|
|
referenced.update(p.relative_to(root) for p in root.rglob("*") if p.is_file())
|
|
for path in shared.rglob("*"):
|
|
if (path.is_file() and not path.is_symlink()
|
|
and path.relative_to(shared) not in referenced
|
|
and path.stat().st_mtime < time.time() - 30 * 86400):
|
|
path.unlink()
|
|
PY
|
|
atomic_state "${ACTIVE_SLOT_FILE}" "${NEW_SLOT}"
|
|
atomic_state "${STATE_DIR}/last-successful-sha" "${TARGET_SHA}"
|
|
atomic_state "${PREVIOUS_RELEASE_FILE}" "${OLD_RELEASE}"
|
|
COMMITTED=1
|
|
rm -f "${UPSTREAM_BACKUP}" "${PENDING_FILE}" || true
|
|
trap - EXIT HUP INT TERM
|
|
# The old active slot is the rollback release; discard older unreferenced builds.
|
|
PREVIOUS_RELEASE="${OLD_RELEASE}"
|
|
prune_orphan_releases || log "WARNING: unused releases could not be pruned; next run will retry."
|
|
log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}."
|