From 96fe59d8426bcb31396e397c5d25d9f193b34889 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=98=D0=B3=D0=BE=D1=80=D1=8C=20=D0=9D=D0=B0=D0=B4=D0=B5?= =?UTF-8?q?=D0=B8=D0=BD?= Date: Thu, 10 Sep 2026 13:55:35 +0300 Subject: [PATCH] Harden deployment cleanup, rollback and worker lifecycle --- ops/deploy/README.md | 44 +++ ops/deploy/eusrr-deploy | 382 ++++++++++++++++++---- ops/deploy/install-corp-runner-access | 12 +- ops/deploy/install-deploy-hardening | 38 +++ ops/deploy/logrotate.conf | 11 + ops/deploy/tests/test_deploy_lifecycle.py | 279 ++++++++++++++++ 6 files changed, 709 insertions(+), 57 deletions(-) create mode 100755 ops/deploy/install-deploy-hardening create mode 100644 ops/deploy/logrotate.conf create mode 100644 ops/deploy/tests/test_deploy_lifecycle.py diff --git a/ops/deploy/README.md b/ops/deploy/README.md index 65e0e55a..290425b7 100644 --- a/ops/deploy/README.md +++ b/ops/deploy/README.md @@ -46,6 +46,50 @@ command rejects every revision except the current Gitea `origin/master` SHA. Logs are appended to `/var/log/eusrr-deploy.log`; the active slot and successful SHA are recorded under `/var/lib/eusrr-deploy`. +## Failure handling and disk usage + +The active and previous successful releases are retained. Before each attempt, +unused Git worktrees are removed under the deployment lock. Failed builds are +removed on error, SIGTERM, SIGINT or SIGHUP, after their processes and slot services +have stopped. Symlink releases, mounted directories and referenced releases are +never removed. A failed cleanup aborts the operation instead of deleting broadly. + +The deployment requires at least 4 GiB and 50,000 free inodes before creating a +release. Build and database-dump process groups have a 30-minute timeout and are +monitored every two seconds; they are stopped if free space falls below 512 MiB +or free inodes below 50,000. This reserve is a guard, not a filesystem quota: +other services can also consume space. Pip uses no persistent cache, and the npm +download cache is confined to the candidate release and removed after install. +Database snapshots are validated before receiving the final `.dump` filename; +partial dumps are removed on failure. Existing complete dumps retain their +14-day retention. Deployment and native Celery logs rotate through logrotate. + +A pending-deployment journal survives a power failure or SIGKILL. The next run +cleans up an interrupted candidate only when the current symlink, active slot, +successful revision and Nginx upstreams still agree. If traffic/state changes were +interrupted, deployment stops for manual recovery and preserves the releases and +upstream backup. Database changes are never automatically reversed. + +Legacy Celery services (including their umbrella unit) are stopped and disabled +once native workers are live. Systemd conditions prevent dependency-driven +restarts of legacy workers. The old slot must stop successfully before the new +Celery scheduler starts. + +To update tooling and perform maintenance without deploying application code: + +```sh +sudo /path/to/ops/deploy/install-deploy-hardening +``` + +For subsequent maintenance use `sudo /usr/local/sbin/eusrr-deploy --maintenance`. +This acquires the same lock, validates live state, reconciles legacy workers and +cleans orphan releases. It neither builds a release nor runs migrations. +Linux integration tests run with: + +```sh +python3 -m unittest discover -s ops/deploy/tests -p test_deploy_lifecycle.py -v +``` + The final workflow step reports successful, failed, cancelled, and unknown deployment outcomes with the repository, branch, revision, actor, current production state, and Gitea run link. Reporting is skipped when either Telegram diff --git a/ops/deploy/eusrr-deploy b/ops/deploy/eusrr-deploy index f9bfbd37..9108219e 100755 --- a/ops/deploy/eusrr-deploy +++ b/ops/deploy/eusrr-deploy @@ -13,6 +13,8 @@ readonly ACTIVE_SLOT_FILE="${STATE_DIR}/active-slot" readonly LOG_FILE="/var/log/eusrr-deploy.log" readonly LOCK_FILE="/run/lock/eusrr-deploy.lock" readonly UPSTREAM_FILE="/etc/nginx/conf.d/eusrr-active-upstreams.conf" +readonly PREVIOUS_RELEASE_FILE="${STATE_DIR}/previous-release" +readonly PENDING_FILE="${STATE_DIR}/pending-deployment" if [[ ${EUID} -ne 0 ]]; then echo "This command must run as root." >&2 @@ -46,7 +48,7 @@ slot_ports() { } if [[ ${1:-} == "--self-test" ]]; then - for command in git curl flock runuser pg_dump npm nginx rsync; do + for command in git curl flock runuser pg_dump pg_restore npm nginx rsync setsid timeout findmnt; do command -v "${command}" >/dev/null done test -d "${APP_ROOT}/.git" @@ -63,8 +65,11 @@ if [[ ${1:-} == "--self-test" ]]; then fi TARGET_SHA="${1:-}" -if [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then - echo "Usage: $0 <40-character lowercase commit SHA>" >&2 +MAINTENANCE=0 +if [[ ${TARGET_SHA} == --maintenance ]]; then + MAINTENANCE=1 +elif [[ ! ${TARGET_SHA} =~ ^[0-9a-f]{40}$ ]]; then + echo "Usage: $0 <40-character lowercase commit SHA> | --maintenance" >&2 exit 2 fi @@ -98,10 +103,63 @@ UPSTREAM_CHANGED=0 OLD_STACK_STOPPED=0 NEW_STACK_ENABLED=0 DB_DUMP="" +DB_DUMP_PARTIAL="" +PREVIOUS_RELEASE="$(cat "${PREVIOUS_RELEASE_FILE}" 2>/dev/null || true)" +INACTIVE_RELEASE="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}" 2>/dev/null || true)" +OLD_SUCCESSFUL_SHA="$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || true)" +MUTATION_STARTED=0 +SLOT_ASSIGNED=0 +BUILD_PID="" +WATCHDOG_PID="" +COMMITTED=0 + +atomic_state() { + printf '%s\n' "$2" >"$1.new" + chmod 0644 "$1.new" + mv -Tf "$1.new" "$1" +} + +check_space() { + local minimum="$1" available inodes + available="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')" + inodes="$(df --output=iavail "${RELEASES_ROOT}" | tail -1 | tr -d ' ')" + if (( available < minimum || inodes < 50000 )); then + log "Insufficient capacity: ${available} bytes and ${inodes} inodes free; need ${minimum} bytes and 50000 inodes." + return 6 + fi +} + +# Build processes have their own session so cancellation also stops grandchildren. +run_bounded_build() { + check_space 1073741824 + setsid timeout --signal=TERM --kill-after=30s 30m runuser -u "${DEPLOY_USER}" -- "$@" <&0 & + BUILD_PID=$! + ( + while kill -0 "${BUILD_PID}" 2>/dev/null; do + if ! check_space 536870912; then + log "Stopping build before it exhausts the production filesystem." + kill -TERM -- "-${BUILD_PID}" 2>/dev/null || true + sleep 5 + kill -KILL -- "-${BUILD_PID}" 2>/dev/null || true + exit 6 + fi + sleep 2 + done + ) & + WATCHDOG_PID=$! + local status=0 + wait "${BUILD_PID}" || status=$? + kill "${WATCHDOG_PID}" 2>/dev/null || true + wait "${WATCHDOG_PID}" 2>/dev/null || true + WATCHDOG_PID="" + BUILD_PID="" + (( status == 0 )) || return "${status}" + check_space 536870912 +} run_database_dump() { local dump_file="$1" - runuser -u "${DEPLOY_USER}" -- \ + run_bounded_build \ env EUSRR_DB_DUMP="${dump_file}" \ "${NEW_RELEASE}/backend/.venv/bin/python" - <<'PY' import os @@ -201,7 +259,51 @@ stop_slot_stack() { "eusrr-celery-beat@${slot}.service" \ "eusrr-celery-worker@${slot}.service" \ "eusrr-frontend@${slot}.service" \ - "eusrr-backend@${slot}.service" || true + "eusrr-backend@${slot}.service" + assert_slot_stopped "${slot}" +} + +assert_slot_stopped() { + local unit state + for unit in backend frontend celery-worker celery-beat; do + state="$(systemctl show "eusrr-${unit}@$1.service" -p ActiveState --value)" || return 1 + case "${state}" in inactive|failed) ;; *) log "Service is still ${state}: eusrr-${unit}@$1"; return 1 ;; esac + done +} + +stop_slot_celery() { + local unit state + systemctl stop "eusrr-celery-beat@$1.service" "eusrr-celery-worker@$1.service" + for unit in celery-beat celery-worker; do + state="$(systemctl show "eusrr-${unit}@$1.service" -p ActiveState --value)" || return 1 + case "${state}" in inactive|failed) ;; *) return 1 ;; esac + done +} + +reconcile_legacy_workers() { + local active_slot="${1:-${OLD_SLOT}}" + [[ ${active_slot} != legacy ]] || return 0 + systemctl is-active --quiet "eusrr-celery-worker@${active_slot}.service" + systemctl is-active --quiet "eusrr-celery-beat@${active_slot}.service" + local unit state + # A legacy umbrella service can start disabled worker/beat units again. + # A condition blocks dependency-driven starts as well as manual starts. + for unit in celery-eusrr.service celery-eusrr-beat.service celery-eusrr-worker.service; do + if [[ $(systemctl show "${unit}" -p LoadState --value) == not-found ]]; then + continue + fi + install -d -m 0755 "/etc/systemd/system/${unit}.d" + printf '%s\n' '[Unit]' 'ConditionPathExists=!/var/lib/eusrr-deploy/native-workers' \ + >"/etc/systemd/system/${unit}.d/90-native-workers.conf" + done + touch "${STATE_DIR}/native-workers" + systemctl daemon-reload + for unit in celery-eusrr.service celery-eusrr-beat.service celery-eusrr-worker.service; do + [[ $(systemctl show "${unit}" -p LoadState --value) != not-found ]] || continue + systemctl disable --now "${unit}" + state="$(systemctl show "${unit}" -p ActiveState --value)" + case "${state}" in inactive|failed) ;; *) return 1 ;; esac + done } start_slot_celery() { @@ -223,33 +325,41 @@ enable_slot_stack() { disable_old_stack() { if [[ ${OLD_SLOT} == legacy ]]; then systemctl disable gunicorn-eusrr.service nextjs-eusrr.service \ - celery-eusrr-worker.service celery-eusrr-beat.service || true + celery-eusrr-worker.service celery-eusrr-beat.service else systemctl disable \ "eusrr-backend@${OLD_SLOT}.service" \ "eusrr-frontend@${OLD_SLOT}.service" \ "eusrr-celery-worker@${OLD_SLOT}.service" \ - "eusrr-celery-beat@${OLD_SLOT}.service" || true + "eusrr-celery-beat@${OLD_SLOT}.service" fi } enable_old_stack() { if [[ ${OLD_SLOT} == legacy ]]; then systemctl enable gunicorn-eusrr.service nextjs-eusrr.service \ - celery-eusrr-worker.service celery-eusrr-beat.service || true + celery-eusrr-worker.service celery-eusrr-beat.service else systemctl enable \ "eusrr-backend@${OLD_SLOT}.service" \ "eusrr-frontend@${OLD_SLOT}.service" \ "eusrr-celery-worker@${OLD_SLOT}.service" \ - "eusrr-celery-beat@${OLD_SLOT}.service" || true + "eusrr-celery-beat@${OLD_SLOT}.service" fi } stop_legacy_stack() { + if [[ $(systemctl show celery-eusrr.service -p LoadState --value) != not-found ]]; then + systemctl stop celery-eusrr.service + fi systemctl stop \ celery-eusrr-beat.service celery-eusrr-worker.service \ - nextjs-eusrr.service gunicorn-eusrr.service || true + nextjs-eusrr.service gunicorn-eusrr.service + local unit state + for unit in celery-eusrr-beat.service celery-eusrr-worker.service nextjs-eusrr.service gunicorn-eusrr.service; do + state="$(systemctl show "${unit}" -p ActiveState --value)" + case "${state}" in inactive|failed) ;; *) return 1 ;; esac + done } start_old_stack() { @@ -264,45 +374,184 @@ start_old_stack() { remove_release() { local release="$1" - case "${release}" in - "${RELEASES_ROOT}"/*) ;; - *) log "Refusing to remove release outside ${RELEASES_ROOT}: ${release}"; return 1 ;; - esac + local name="${release##*/}" protected resolved mount + [[ ${name} =~ ^[0-9a-f]{40}$ && ${release} == "${RELEASES_ROOT}/${name}" ]] || { + log "Refusing non-release path: ${release}"; return 1; + } + [[ ! -L ${release} ]] || { log "Refusing symlink release: ${release}"; return 1; } + resolved="$(readlink -m "${release}")" + [[ ${resolved} == "${release}" ]] || return 1 + for protected in "${OLD_RELEASE}" "${PREVIOUS_RELEASE}" \ + "$(readlink -f "${CURRENT_LINK}")" \ + "$(readlink -f "${SLOTS_ROOT}/blue" 2>/dev/null || true)" \ + "$(readlink -f "${SLOTS_ROOT}/green" 2>/dev/null || true)"; do + [[ ${release} != "${protected}" ]] || { log "Protected release: ${release}"; return 1; } + done if [[ -e ${release} ]]; then + # Git must never recurse into a bind mount containing shared data. + while IFS= read -r mount; do + [[ ${mount} != "${release}" && ${mount} != "${release}/"* ]] || return 1 + done < <(findmnt -rn -o TARGET) + log "Removing unused release ${release}." run_as_deploy git -C "${APP_ROOT}" worktree remove --force "${release}" fi } +prune_orphan_releases() { + local release + for release in "${RELEASES_ROOT}"/*; do + [[ -d ${release} && ! -L ${release} ]] || continue + [[ ${release##*/} =~ ^[0-9a-f]{40}$ ]] || continue + if [[ ${release} == "${OLD_RELEASE}" || ${release} == "${PREVIOUS_RELEASE}" || + ${release} == "$(readlink -f "${CURRENT_LINK}")" || + ${release} == "$(readlink -f "${SLOTS_ROOT}/blue" 2>/dev/null || true)" || + ${release} == "$(readlink -f "${SLOTS_ROOT}/green" 2>/dev/null || true)" ]]; then + continue + fi + remove_release "${release}" + done + run_as_deploy git -C "${APP_ROOT}" worktree prune +} + +validate_live_state() { + [[ -d ${OLD_RELEASE} && $(readlink -f "${CURRENT_LINK}") == "${OLD_RELEASE}" ]] || { + log "Active slot and current release disagree; refusing cleanup."; return 1; + } + if [[ ${OLD_SLOT} != legacy ]]; then + [[ ${OLD_RELEASE} == "${RELEASES_ROOT}/${OLD_SUCCESSFUL_SHA}" ]] || return 1 + local backend frontend + case "${OLD_SLOT}" in blue) backend=9001; frontend=3001 ;; green) backend=9002; frontend=3002 ;; esac + grep -Eq "server 127[.]0[.]0[.]1:${backend};" "${UPSTREAM_FILE}" + grep -Eq "server 127[.]0[.]0[.]1:${frontend};" "${UPSTREAM_FILE}" + systemctl is-active --quiet "eusrr-backend@${OLD_SLOT}.service" + systemctl is-active --quiet "eusrr-frontend@${OLD_SLOT}.service" + fi +} + +prune_expired_dumps() { + # Run on failed attempts too. A partial dump is never offered as a backup. + find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete + find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump.partial' -delete +} + rollback() { local original_status=$? - trap - ERR + trap - EXIT HUP INT TERM + [[ ${COMMITTED} -eq 0 && ${MUTATION_STARTED} -eq 1 ]] || exit "${original_status}" + (( original_status != 0 )) || original_status=1 set +e - log "Deployment failed with exit code ${original_status}; keeping the previous release active." - - if [[ ${UPSTREAM_CHANGED} -eq 1 && -f ${UPSTREAM_BACKUP} ]]; then - install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}" - point_current_at "${OLD_RELEASE}" - nginx -t && systemctl reload nginx + log "Deployment failed with exit code ${original_status}; restoring the previous release." + if [[ -n ${WATCHDOG_PID} ]]; then + kill "${WATCHDOG_PID}" 2>/dev/null + wait "${WATCHDOG_PID}" 2>/dev/null + fi + if [[ -n ${BUILD_PID} ]]; then + kill -TERM -- "-${BUILD_PID}" 2>/dev/null + sleep 2 + kill -KILL -- "-${BUILD_PID}" 2>/dev/null + wait "${BUILD_PID}" 2>/dev/null + fi + local recovered=1 + if [[ ${UPSTREAM_CHANGED} -eq 1 ]]; then + # Keep candidate HTTP services alive until Nginx has returned to the old + # stack. Only stop its scheduler/workers before restarting the old workers. + stop_slot_celery "${NEW_SLOT}" || recovered=0 + else + stop_slot_stack "${NEW_SLOT}" || recovered=0 fi - - stop_slot_stack "${NEW_SLOT}" if [[ ${NEW_STACK_ENABLED} -eq 1 ]]; then systemctl disable \ "eusrr-backend@${NEW_SLOT}.service" \ "eusrr-frontend@${NEW_SLOT}.service" \ "eusrr-celery-worker@${NEW_SLOT}.service" \ - "eusrr-celery-beat@${NEW_SLOT}.service" || true - enable_old_stack + "eusrr-celery-beat@${NEW_SLOT}.service" || recovered=0 + enable_old_stack || recovered=0 fi - if [[ ${OLD_STACK_STOPPED} -eq 1 ]]; then - start_old_stack || log "ERROR: failed to restart the previous application stack." + if [[ ${OLD_STACK_STOPPED} -eq 1 && ${recovered} -eq 1 ]]; then + [[ ${OLD_SLOT} != legacy ]] || rm -f "${STATE_DIR}/native-workers" + start_old_stack || recovered=0 + fi + if [[ ${UPSTREAM_CHANGED} -eq 1 ]]; then + if [[ -f ${UPSTREAM_BACKUP} && ${recovered} -eq 1 ]]; then + install -o root -g root -m 0644 "${UPSTREAM_BACKUP}" "${UPSTREAM_FILE}" || recovered=0 + point_current_at "${OLD_RELEASE}" || recovered=0 + nginx -t && systemctl reload nginx || recovered=0 + if [[ ${recovered} -eq 1 ]]; then + stop_slot_stack "${NEW_SLOT}" || recovered=0 + fi + else + recovered=0 + fi + fi + if [[ ${recovered} -eq 1 ]]; then + atomic_state "${ACTIVE_SLOT_FILE}" "${OLD_SLOT}" || recovered=0 + atomic_state "${STATE_DIR}/last-successful-sha" "${OLD_SUCCESSFUL_SHA}" || recovered=0 + atomic_state "${PREVIOUS_RELEASE_FILE}" "${PREVIOUS_RELEASE}" || recovered=0 + fi + if [[ ${recovered} -eq 1 ]]; then + if [[ ${SLOT_ASSIGNED} -eq 1 ]]; then + rm -f "${SLOTS_ROOT}/${NEW_SLOT}" + if [[ -n ${INACTIVE_RELEASE} && -d ${INACTIVE_RELEASE} ]]; then + ln -sfn "${INACTIVE_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}" + fi + fi + remove_release "${NEW_RELEASE}" || recovered=0 + fi + [[ -z ${DB_DUMP_PARTIAL} ]] || rm -f "${DB_DUMP_PARTIAL}" + if [[ ${recovered} -eq 1 ]]; then + rm -f "${PENDING_FILE}" "${UPSTREAM_BACKUP}" + log "Previous release restored; failed build removed." + else + log "ERROR: recovery was incomplete; release and recovery journal retained. Manual inspection required." fi log "No database snapshot was restored because the previous release remained live and may have accepted writes." log "Inspect ${LOG_FILE} and service journals before retrying." exit "${original_status}" } -trap rollback ERR + +validate_live_state +# A journal from the switching phase may describe Nginx's in-memory state, +# which cannot be inferred from its configuration file after an interrupted reload. +if [[ -f ${PENDING_FILE} ]]; then + read -r pending_phase pending_sha <"${PENDING_FILE}" + [[ ${pending_sha} =~ ^[0-9a-f]{40}$ ]] || exit 4 + case "${pending_phase}" in + building) ;; + switching) + if [[ ${pending_sha} != "${OLD_SUCCESSFUL_SHA}" ]]; then + log "Interrupted traffic switch: inspect services and Nginx before clearing ${PENDING_FILE}." + exit 4 + fi ;; + *) exit 4 ;; + esac +fi +# Preserve the previous successful slot through the build of the next release. +if [[ -n ${INACTIVE_RELEASE} && -d ${INACTIVE_RELEASE} && ! -f ${PENDING_FILE} ]]; then + PREVIOUS_RELEASE="${INACTIVE_RELEASE}" + atomic_state "${PREVIOUS_RELEASE_FILE}" "${PREVIOUS_RELEASE}" +fi +reconcile_legacy_workers +stop_slot_stack "${NEW_SLOT}" +if [[ -f ${PENDING_FILE} ]]; then + # SIGKILL/power failure before traffic switched: the live-state checks above + # prove production still uses the last committed release. Restore standby. + if [[ ${INACTIVE_RELEASE} == "${RELEASES_ROOT}/${pending_sha}" ]]; then + rm -f "${SLOTS_ROOT}/${NEW_SLOT}" + if [[ -n ${PREVIOUS_RELEASE} && -d ${PREVIOUS_RELEASE} ]]; then + ln -sfn "${PREVIOUS_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}" + fi + INACTIVE_RELEASE="${PREVIOUS_RELEASE}" + fi + rm -f "${PENDING_FILE}" +fi +prune_orphan_releases +prune_expired_dumps +if [[ ${MAINTENANCE} -eq 1 ]]; then + log "Maintenance completed; active and previous releases preserved." + df -h "${RELEASES_ROOT}" + exit 0 +fi log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}." run_as_deploy git -C "${APP_ROOT}" fetch --prune origin \ @@ -317,21 +566,14 @@ if [[ ${TARGET_SHA} == "$(cat "${STATE_DIR}/last-successful-sha" 2>/dev/null || exit 0 fi -if [[ -L ${SLOTS_ROOT}/${NEW_SLOT} ]]; then - stale_release="$(readlink -f "${SLOTS_ROOT}/${NEW_SLOT}")" - if [[ ${stale_release} != "${OLD_RELEASE}" ]]; then - rm -f "${SLOTS_ROOT}/${NEW_SLOT}" - remove_release "${stale_release}" - fi -fi remove_release "${NEW_RELEASE}" -run_as_deploy git -C "${APP_ROOT}" worktree prune - -available_bytes="$(df --output=avail -B1 "${RELEASES_ROOT}" | tail -1 | tr -d ' ')" -if (( available_bytes < 4294967296 )); then - log "At least 4 GiB of free disk space is required to build a release." - exit 6 -fi +check_space 4294967296 +atomic_state "${PENDING_FILE}" "building ${TARGET_SHA}" +MUTATION_STARTED=1 +trap rollback EXIT +trap 'exit 129' HUP +trap 'exit 130' INT +trap 'exit 143' TERM log "Creating isolated Git worktree ${NEW_RELEASE}." run_as_deploy git -C "${APP_ROOT}" worktree add --detach "${NEW_RELEASE}" "${TARGET_SHA}" @@ -342,19 +584,20 @@ run_as_deploy rm -rf "${NEW_RELEASE}/backend/media" run_as_deploy ln -s "${APP_ROOT}/backend/media" "${NEW_RELEASE}/backend/media" log "Creating an isolated Python environment." -run_as_deploy /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv" -run_as_deploy env PIP_DISABLE_PIP_VERSION_CHECK=1 \ +run_bounded_build /usr/bin/python3 -m venv "${NEW_RELEASE}/backend/.venv" +run_bounded_build env PIP_DISABLE_PIP_VERSION_CHECK=1 \ "${NEW_RELEASE}/backend/.venv/bin/python" -m pip install \ - --requirement "${NEW_RELEASE}/backend/requirements.txt" + --no-cache-dir --requirement "${NEW_RELEASE}/backend/requirements.txt" log "Installing pinned frontend dependencies, including Linux native modules, and building Next.js." -run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" ci \ +run_bounded_build npm --cache "${NEW_RELEASE}/.npm-cache" --prefix "${NEW_RELEASE}/frontend" ci \ --include=optional --no-audit --no-fund +run_as_deploy rm -rf -- "${NEW_RELEASE}/.npm-cache" run_as_deploy test -f \ "${NEW_RELEASE}/frontend/node_modules/lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node" run_as_deploy test -d \ "${NEW_RELEASE}/frontend/node_modules/@next/swc-linux-x64-gnu" -run_as_deploy npm --prefix "${NEW_RELEASE}/frontend" run build +run_bounded_build npm --prefix "${NEW_RELEASE}/frontend" run build log "Running Django checks and collecting release static files." run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ @@ -363,12 +606,18 @@ run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ "${NEW_RELEASE}/backend/manage.py" collectstatic --noinput run_as_deploy rsync -a "${NEW_RELEASE}/frontend/.next/static/" "${SHARED_ROOT}/next-static/" +check_space 1073741824 install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}" -m 0700 "${BACKUP_DIR}" DB_DUMP="${BACKUP_DIR}/postgres-before-${TARGET_SHA}-$(date -u +%Y%m%dT%H%M%SZ).dump" +DB_DUMP_PARTIAL="${DB_DUMP}.partial" log "Creating a consistent PostgreSQL snapshot while the current release remains online." -run_database_dump "${DB_DUMP}" +run_database_dump "${DB_DUMP_PARTIAL}" +pg_restore --list "${DB_DUMP_PARTIAL}" >/dev/null +mv "${DB_DUMP_PARTIAL}" "${DB_DUMP}" +DB_DUMP_PARTIAL="" chmod 0600 "${DB_DUMP}" +check_space 536870912 log "Planned migrations:" run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ @@ -378,6 +627,7 @@ run_in_release_backend "${NEW_RELEASE}/backend/.venv/bin/python" \ "${NEW_RELEASE}/backend/manage.py" migrate --noinput log "Assigning release to the ${NEW_SLOT} slot and starting its web services." +SLOT_ASSIGNED=1 ln -sfn "${NEW_RELEASE}" "${SLOTS_ROOT}/${NEW_SLOT}.new" mv -Tf "${SLOTS_ROOT}/${NEW_SLOT}.new" "${SLOTS_ROOT}/${NEW_SLOT}" start_slot_web "${NEW_SLOT}" @@ -386,6 +636,7 @@ wait_for_http "${NEW_SLOT} frontend" "http://127.0.0.1:${FRONTEND_PORT}/" '^200$ wait_for_http "${NEW_SLOT} backend" "http://127.0.0.1:${BACKEND_PORT}/admin/login/" '^(200|302)$' cp -a "${UPSTREAM_FILE}" "${UPSTREAM_BACKUP}" +atomic_state "${PENDING_FILE}" "switching ${TARGET_SHA}" UPSTREAM_CHANGED=1 point_current_at "${NEW_RELEASE}" write_upstreams "${BACKEND_PORT}" "${FRONTEND_PORT}" @@ -398,23 +649,44 @@ wait_for_http "public backend" "http://127.0.0.1/admin/login/" '^(200|302)$' '17 log "Traffic is on ${NEW_SLOT}; allowing in-flight HTTP requests to drain." sleep 10 log "Switching background workers." +OLD_STACK_STOPPED=1 if [[ ${OLD_SLOT} == legacy ]]; then stop_legacy_stack else stop_slot_stack "${OLD_SLOT}" fi -OLD_STACK_STOPPED=1 start_slot_celery "${NEW_SLOT}" +reconcile_legacy_workers "${NEW_SLOT}" NEW_STACK_ENABLED=1 enable_slot_stack "${NEW_SLOT}" disable_old_stack find "${BACKUP_DIR}" -mindepth 1 -maxdepth 1 -type f -name 'postgres-before-*.dump' -mtime +14 -delete \ || log "WARNING: old database snapshots could not be pruned." -run_as_deploy find "${SHARED_ROOT}/next-static" -type f -mtime +30 -delete \ - || log "WARNING: old Next.js assets could not be pruned." -rm -f "${UPSTREAM_BACKUP}" || true -printf '%s\n' "${NEW_SLOT}" >"${ACTIVE_SLOT_FILE}" -printf '%s\n' "${TARGET_SHA}" >"${STATE_DIR}/last-successful-sha" -trap - ERR +# Keep files referenced by either retained release regardless of their mtime. +run_as_deploy python3 - "${SHARED_ROOT}/next-static" "${NEW_RELEASE}" "${OLD_RELEASE}" <<'PY' +import sys +import time +from pathlib import Path + +shared = Path(sys.argv[1]) +referenced = set() +for release in sys.argv[2:]: + root = Path(release) / "frontend/.next/static" + referenced.update(p.relative_to(root) for p in root.rglob("*") if p.is_file()) +for path in shared.rglob("*"): + if (path.is_file() and not path.is_symlink() + and path.relative_to(shared) not in referenced + and path.stat().st_mtime < time.time() - 30 * 86400): + path.unlink() +PY +atomic_state "${ACTIVE_SLOT_FILE}" "${NEW_SLOT}" +atomic_state "${STATE_DIR}/last-successful-sha" "${TARGET_SHA}" +atomic_state "${PREVIOUS_RELEASE_FILE}" "${OLD_RELEASE}" +COMMITTED=1 +rm -f "${UPSTREAM_BACKUP}" "${PENDING_FILE}" || true +trap - EXIT HUP INT TERM +# The old active slot is the rollback release; discard older unreferenced builds. +PREVIOUS_RELEASE="${OLD_RELEASE}" +prune_orphan_releases || log "WARNING: unused releases could not be pruned; next run will retry." log "Blue-green deployment of ${TARGET_SHA} completed successfully on ${NEW_SLOT}." diff --git a/ops/deploy/install-corp-runner-access b/ops/deploy/install-corp-runner-access index d8b2e59e..ba813539 100755 --- a/ops/deploy/install-corp-runner-access +++ b/ops/deploy/install-corp-runner-access @@ -24,6 +24,9 @@ if [[ ! -d ${SOURCE_DIR}/systemd ]]; then exit 4 fi +exec 9>/run/lock/eusrr-deploy.lock +flock -n 9 || { echo "A deployment is running; retry bootstrap after it finishes." >&2; exit 3; } + read -r key_type key_body _ <"${PUBLIC_KEY_FILE}" if [[ ${key_type} != "ssh-ed25519" || -z ${key_body:-} ]]; then echo "Expected an Ed25519 public key." >&2 @@ -63,6 +66,7 @@ trap cleanup EXIT install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy" /usr/local/sbin/eusrr-deploy install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy-ssh" /usr/local/sbin/eusrr-deploy-ssh +install -o root -g root -m 0644 "${SOURCE_DIR}/logrotate.conf" /etc/logrotate.d/eusrr-deploy for unit in "${SOURCE_DIR}"/systemd/*.service; do install -o root -g root -m 0644 "${unit}" "/etc/systemd/system/$(basename "${unit}")" done @@ -76,8 +80,12 @@ chown -R deploy:www-data /srv/eusrr-shared find /srv/eusrr-shared -type d -exec chmod 0755 {} + find /srv/eusrr-shared -type f -exec chmod 0644 {} + -ln -sfn /srv/eusrr /srv/eusrr-current.new -mv -Tf /srv/eusrr-current.new /srv/eusrr-current +# Re-running bootstrap on an existing native slot must not point static files +# back at the legacy checkout while Nginx still serves the native slot. +if [[ ! -L /srv/eusrr-current ]]; then + ln -sfn /srv/eusrr /srv/eusrr-current.new + mv -Tf /srv/eusrr-current.new /srv/eusrr-current +fi if [[ ! -f ${UPSTREAM_FILE} ]]; then cat >"${UPSTREAM_FILE}" <<'EOF' diff --git a/ops/deploy/install-deploy-hardening b/ops/deploy/install-deploy-hardening new file mode 100755 index 00000000..7ab8c378 --- /dev/null +++ b/ops/deploy/install-deploy-hardening @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Update deployment tooling and clean up without building/deploying application code. +set -Eeuo pipefail +[[ ${EUID} -eq 0 ]] || { echo 'Run this installer with sudo.' >&2; exit 1; } +SOURCE_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)" +bash -n "${SOURCE_DIR}/eusrr-deploy" +# SCP can leave the source group-writable. Validate the exact protected copy +# that will be installed; logrotate may return zero even when it ignores a file. +validation_dir="$(mktemp -d /tmp/eusrr-logrotate-check.XXXXXXXX)" +cleanup_validation() { + rm -f -- "${validation_dir}/logrotate.conf" "${validation_dir}/validation.log" + rmdir -- "${validation_dir}" +} +trap cleanup_validation EXIT +install -o root -g root -m 0644 "${SOURCE_DIR}/logrotate.conf" "${validation_dir}/logrotate.conf" +if ! LC_ALL=C logrotate --debug --state /dev/null "${validation_dir}/logrotate.conf" \ + >"${validation_dir}/validation.log" 2>&1 || + grep -Eq '^error:|Ignoring .* because' "${validation_dir}/validation.log"; then + cat "${validation_dir}/validation.log" >&2 + exit 1 +fi +exec 9>/run/lock/eusrr-deploy.lock +flock -n 9 || { echo 'A deployment is running. Retry after it finishes.' >&2; exit 3; } +backup="/var/lib/eusrr-deploy/tooling-backup-$(date -u +%Y%m%dT%H%M%SZ)" +install -d -o root -g root -m 0700 "${backup}" +cp -a /usr/local/sbin/eusrr-deploy "${backup}/eusrr-deploy" +install -o root -g root -m 0755 "${SOURCE_DIR}/eusrr-deploy" /usr/local/sbin/eusrr-deploy.new +mv -Tf /usr/local/sbin/eusrr-deploy.new /usr/local/sbin/eusrr-deploy +install -o root -g root -m 0644 "${validation_dir}/logrotate.conf" /etc/logrotate.d/eusrr-deploy +/usr/local/sbin/eusrr-deploy --self-test +flock -u 9 +exec 9>&- +/usr/local/sbin/eusrr-deploy --maintenance +systemctl is-active eusrr-backend@"$(cat /var/lib/eusrr-deploy/active-slot)".service \ + eusrr-frontend@"$(cat /var/lib/eusrr-deploy/active-slot)".service \ + eusrr-celery-worker@"$(cat /var/lib/eusrr-deploy/active-slot)".service \ + eusrr-celery-beat@"$(cat /var/lib/eusrr-deploy/active-slot)".service +printf 'Deployment tooling updated. Application was not redeployed. Script backup: %s\n' "${backup}" diff --git a/ops/deploy/logrotate.conf b/ops/deploy/logrotate.conf new file mode 100644 index 00000000..8a3e8817 --- /dev/null +++ b/ops/deploy/logrotate.conf @@ -0,0 +1,11 @@ +/var/log/eusrr-deploy.log /var/log/celery/beat-blue.log /var/log/celery/beat-green.log /var/log/celery/worker-blue.log /var/log/celery/worker-green.log { + daily + maxsize 10M + rotate 5 + compress + delaycompress + missingok + notifempty + copytruncate + su root root +} diff --git a/ops/deploy/tests/test_deploy_lifecycle.py b/ops/deploy/tests/test_deploy_lifecycle.py new file mode 100644 index 00000000..f7f595f3 --- /dev/null +++ b/ops/deploy/tests/test_deploy_lifecycle.py @@ -0,0 +1,279 @@ +"""Exercise the real deployment shell in an isolated Linux filesystem fixture. + +Run: python3 -m unittest discover -s ops/deploy/tests -p test_deploy_lifecycle.py +Only absolute infrastructure paths and the root check are substituted. All +deployment decisions, traps, cleanup, state writes and switching are real. +""" +import json +import os +from pathlib import Path +import subprocess +import tempfile +import time +import unittest + + +SOURCE = Path(__file__).resolve().parents[1] / "eusrr-deploy" +OLD, PREVIOUS, TARGET, ORPHAN = (c * 40 for c in "abcd") +FAKE = r'''#!/usr/bin/env python3 +import json, os, pathlib, shutil, sys, time +p = pathlib.Path +root = p(os.environ['FIXTURE']) +name = p(sys.argv[0]).name +args = sys.argv[1:] +with (root/'commands').open('a') as f: + f.write(name+' '+repr(args)+'\n') +def fail(stage): + if os.environ.get('FAIL_STAGE') == stage: + sys.exit(17) +if name == 'runuser': + args = args[args.index('--')+1:] + os.execvp(args[0], args) +elif name == 'git': + if 'rev-parse' in args: print(os.environ['TARGET']) + elif 'remove' in args: + shutil.rmtree(args[-1]) + elif 'add' in args: + release=p(args[-2]); (release/'backend').mkdir(parents=True) + (release/'frontend/.next/static').mkdir(parents=True) + (release/'frontend/.next/static/new.js').touch() + sys.exit(0) +elif name == 'df': + print('Available') + available = os.environ.get('FREE_BYTES','20000000000') + if os.environ.get('FAIL_STAGE') == 'disk_during_build' and (root/'building').exists(): + available = '100000000' + print('1000000' if '--output=iavail' in args else available) +elif name == 'systemctl': + file=root/'services.json'; states=json.loads(file.read_text()) + def units(): return [a for a in args[1:] if a.endswith('.service')] + def save(): file.write_text(json.dumps(states)) + if args[0] == 'show': + prop=args[args.index('-p')+1] + print('loaded' if prop=='LoadState' else states.get(args[1],'inactive')) + elif args[0] == 'is-active': + sys.exit(0 if all(states.get(u)=='active' for u in units()) else 3) + elif args[0] in ('start','restart'): + for u in units(): states[u]='active' + save() + if any('celery-beat@blue' in u for u in units()): fail('new_beat') + elif args[0] == 'stop' or (args[0]=='disable' and '--now' in args): + if args[0]=='stop' and os.environ.get('FAIL_STAGE')=='stop_blue': sys.exit(1) + for u in units(): states[u]='inactive' + save() + elif args[0] == 'reload': fail('reload') + elif args[0] == 'enable' and any('@blue' in u for u in units()): fail('enable') +elif name == 'fake-python': + if 'venv' in args: + dest=p(args[-1])/'bin'; dest.mkdir(parents=True) + (dest/'python').symlink_to(root/'bin/fake-python') + elif 'pip' in args: fail('pip') +elif name == 'python': + if 'pip' in args: fail('pip') + elif '-' in args: + script=sys.stdin.read() + if 'os.execve' not in script: sys.exit(19) + p(os.environ['EUSRR_DB_DUMP']).write_bytes(b'dump') + fail('dump') + elif 'migrate' in args and '--noinput' in args: fail('migration') +elif name == 'npm': + if 'ci' in args: + base=p(args[args.index('--prefix')+1])/'node_modules' + (base/'lightningcss-linux-x64-gnu').mkdir(parents=True) + (base/'lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node').touch() + (base/'@next/swc-linux-x64-gnu').mkdir(parents=True) + if 'build' in args: + fail('build') + if os.environ.get('FAIL_STAGE') in ('hang','disk_during_build'): + (root/'building').touch(); time.sleep(60) +elif name == 'curl': + if os.environ.get('FAIL_STAGE')=='health' and any(':3001' in a for a in args): print('502') + else: print('200') +elif name == 'sleep': time.sleep(0.02) +elif name == 'findmnt': print('/') +''' + + +@unittest.skipUnless(os.name == "posix", "Linux shell integration tests") +class DeploymentLifecycleTests(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory(prefix="eusrr-deploy-test-") + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + self.bin = self.root / "bin" + self.bin.mkdir() + fake = self.bin / "fake" + fake.write_text(FAKE) + fake.chmod(0o755) + for name in ('runuser', 'git', 'df', 'systemctl', 'fake-python', 'npm', + 'curl', 'sleep', 'findmnt', 'nginx', 'rsync', 'pg_restore'): + (self.bin / name).symlink_to(fake) + text = SOURCE.read_text() + for prefix in ('/srv/', '/var/', '/run/', '/etc/'): + text = text.replace(prefix, str(self.root) + prefix) + text = text.replace('if [[ ${EUID} -ne 0 ]]; then', 'if false; then') + text = text.replace('/usr/bin/python3 -m venv', 'fake-python -m venv') + # Fixture Python entrypoint must preserve argv[0] for pip/dump handlers. + text = text.replace('install -d -o root -g root', 'install -d') + text = text.replace('install -d -o "${DEPLOY_USER}" -g "${DEPLOY_USER}"', 'install -d') + text = text.replace('install -o root -g root', 'install') + self.script = self.root / "deploy" + self.script.write_text(text) + self.state = self.root / "var/lib/eusrr-deploy" + self.releases = self.root / "srv/eusrr-releases" + self.slots = self.root / "srv/eusrr-slots" + for directory in (self.state, self.releases, self.slots, self.root/'var/log', + self.root/'run/lock', self.root/'srv/eusrr/backend', + self.root/'srv/eusrr-shared/next-static', self.root/'etc/nginx/conf.d', + self.root/'var/backups/eusrr/deploy'): + directory.mkdir(parents=True, exist_ok=True) + for sha in (OLD, PREVIOUS, ORPHAN): + (self.releases/sha/'frontend/.next/static').mkdir(parents=True) + self.slots.joinpath('green').symlink_to(self.releases/OLD) + self.slots.joinpath('blue').symlink_to(self.releases/PREVIOUS) + (self.root/'srv/eusrr-current').symlink_to(self.releases/OLD) + (self.state/'active-slot').write_text('green\n') + (self.state/'last-successful-sha').write_text(OLD+'\n') + (self.root/'etc/nginx/conf.d/eusrr-active-upstreams.conf').write_text( + 'server 127.0.0.1:9002;\nserver 127.0.0.1:3002;\n') + self.services = self.root / 'services.json' + self.services.write_text(json.dumps({ + **{f'eusrr-{unit}@green.service': 'active' for unit in + ('backend', 'frontend', 'celery-worker', 'celery-beat')}, + 'celery-eusrr.service': 'active', + 'celery-eusrr-beat.service': 'active', + 'celery-eusrr-worker.service': 'active', + })) + self.env = {**os.environ, 'FIXTURE': str(self.root), 'TARGET': TARGET, + 'PATH': str(self.bin) + ':' + os.environ['PATH']} + + def run_deploy(self, mode=TARGET, **env): + result = subprocess.run(['bash', str(self.script), mode], env={**self.env, **env}, + capture_output=True, text=True, timeout=30) + self.output = result.stdout + result.stderr + return result + + def assert_old_live(self): + self.assertEqual((self.root/'srv/eusrr-current').resolve(), self.releases/OLD, self.output) + self.assertEqual((self.state/'active-slot').read_text().strip(), 'green', self.output) + states = json.loads(self.services.read_text()) + for unit in ('backend', 'frontend', 'celery-worker', 'celery-beat'): + self.assertEqual(states[f'eusrr-{unit}@green.service'], 'active', self.output) + + def test_success_preserves_old_and_removes_older_releases(self): + result = self.run_deploy() + self.assertEqual(result.returncode, 0, self.output) + self.assertEqual((self.root/'srv/eusrr-current').resolve(), self.releases/TARGET) + self.assertTrue((self.releases/OLD).exists()) + self.assertFalse((self.releases/PREVIOUS).exists()) + self.assertFalse((self.releases/ORPHAN).exists()) + self.assertFalse((self.state/'pending-deployment').exists()) + states = json.loads(self.services.read_text()) + self.assertEqual(states['celery-eusrr-beat.service'], 'inactive') + self.assertEqual(states['eusrr-celery-beat@green.service'], 'inactive') + self.assertEqual(states['eusrr-celery-beat@blue.service'], 'active') + + def test_failures_remove_candidate_preserve_both_successful_releases(self): + for stage in ('pip', 'build', 'dump', 'migration', 'health', 'new_beat', 'enable'): + with self.subTest(stage=stage): + result = self.run_deploy(FAIL_STAGE=stage) + self.assertNotEqual(result.returncode, 0, self.output) + self.assert_old_live() + self.assertFalse((self.releases/TARGET).exists(), self.output) + self.assertTrue((self.releases/PREVIOUS).exists(), self.output) + self.assertFalse((self.state/'pending-deployment').exists(), self.output) + self.assertFalse(list((self.root/'var/backups/eusrr/deploy').glob('*.partial'))) + + def test_capacity_failure_does_not_remove_previous(self): + result = self.run_deploy(FREE_BYTES='4000000000') + self.assertEqual(result.returncode, 6, self.output) + self.assert_old_live() + self.assertTrue((self.releases/PREVIOUS).exists()) + + def test_watchdog_stops_build_before_disk_exhaustion(self): + result = self.run_deploy(FAIL_STAGE='disk_during_build') + self.assertNotEqual(result.returncode, 0, self.output) + self.assertIn('Stopping build before', self.output) + self.assert_old_live() + self.assertFalse((self.releases/TARGET).exists(), self.output) + + def test_failed_nginx_recovery_retains_candidate_and_journal(self): + self.assertNotEqual(self.run_deploy(FAIL_STAGE='reload').returncode, 0) + self.assertTrue((self.releases/TARGET).exists(), self.output) + self.assertTrue((self.state/'pending-deployment').exists(), self.output) + self.assertTrue((self.state/f'upstreams-before-{TARGET}').exists(), self.output) + states = json.loads(self.services.read_text()) + self.assertEqual(states['eusrr-backend@blue.service'], 'active') + self.assertEqual(states['eusrr-frontend@blue.service'], 'active') + self.assertEqual(self.run_deploy('--maintenance').returncode, 4, self.output) + self.assertTrue((self.releases/TARGET).exists()) + + def test_already_active_sha_never_removed(self): + self.env['TARGET'] = OLD + self.assertEqual(self.run_deploy(OLD).returncode, 0, self.output) + self.assert_old_live() + self.assertTrue((self.releases/OLD).exists()) + + def test_deploying_standby_sha_does_not_delete_rollback(self): + self.env['TARGET'] = PREVIOUS + self.assertNotEqual(self.run_deploy(PREVIOUS).returncode, 0) + self.assert_old_live() + self.assertTrue((self.releases/PREVIOUS).exists()) + + def test_existing_lock_prevents_any_maintenance(self): + import fcntl + with (self.root/'run/lock/eusrr-deploy.lock').open('w') as lock: + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + self.assertEqual(self.run_deploy('--maintenance').returncode, 3, self.output) + self.assertTrue((self.releases/ORPHAN).exists()) + + def test_inconsistent_state_refuses_cleanup(self): + (self.state/'last-successful-sha').write_text(TARGET) + self.assertNotEqual(self.run_deploy('--maintenance').returncode, 0) + self.assertTrue((self.releases/ORPHAN).exists()) + + def test_failure_to_stop_refuses_cleanup(self): + self.assertNotEqual(self.run_deploy('--maintenance', FAIL_STAGE='stop_blue').returncode, 0) + self.assertTrue((self.releases/ORPHAN).exists()) + + def test_maintenance_preserves_symlink_targets(self): + external = self.root/'external'; external.mkdir() + external.joinpath('media').write_text('keep') + self.releases.joinpath('e'*40).symlink_to(external) + self.assertEqual(self.run_deploy('--maintenance').returncode, 0, self.output) + self.assertTrue(external.joinpath('media').exists()) + self.assert_old_live() + + def test_interrupted_build_recovered_by_maintenance(self): + (self.state/'pending-deployment').write_text('building '+TARGET+'\n') + (self.state/'previous-release').write_text(str(self.releases/PREVIOUS)) + (self.releases/TARGET).mkdir() + self.slots.joinpath('blue').unlink() + self.slots.joinpath('blue').symlink_to(self.releases/TARGET) + self.assertEqual(self.run_deploy('--maintenance').returncode, 0, self.output) + self.assert_old_live() + self.assertFalse((self.releases/TARGET).exists()) + self.assertEqual(self.slots.joinpath('blue').resolve(), self.releases/PREVIOUS) + + def test_signal_cleans_build_process_and_candidate(self): + with (self.root/'output').open('w') as output: + process = subprocess.Popen(['bash', str(self.script), TARGET], + env={**self.env, 'FAIL_STAGE': 'hang'}, stdout=output, stderr=output) + try: + deadline = time.monotonic() + 15 + while not (self.root/'building').exists() and time.monotonic() < deadline: + if process.poll() is not None: break + time.sleep(.05) + self.assertTrue((self.root/'building').exists(), (self.root/'output').read_text()) + process.terminate() + process.wait(timeout=15) + finally: + if process.poll() is None: process.kill(); process.wait() + self.output = (self.root/'output').read_text() + self.assertEqual(process.returncode, 143, self.output) + self.assert_old_live() + self.assertFalse((self.releases/TARGET).exists(), self.output) + + +if __name__ == '__main__': + unittest.main()