From c9d7e101859e418df1ca5cfa0f1a323a303aafac Mon Sep 17 00:00:00 2001 From: lizerk Date: Thu, 10 Sep 2026 21:09:38 +0300 Subject: [PATCH] Retry transient npm installs more resiliently --- ops/deploy/README.md | 5 +++-- ops/deploy/eusrr-deploy | 12 ++++++------ ops/deploy/tests/test_deploy_lifecycle.py | 3 ++- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/ops/deploy/README.md b/ops/deploy/README.md index fc53344c..b33bf015 100644 --- a/ops/deploy/README.md +++ b/ops/deploy/README.md @@ -63,8 +63,9 @@ never removed. A failed cleanup aborts the operation instead of deleting broadly Node.js must be 22.13+ on the 22.x LTS line, or 24.x LTS; this is checked before creating a candidate. Frontend `npm ci` uses strict engine validation and the -committed lockfile. Network-only failures may retry up to three installation -attempts under one 30-minute timeout and the same disk watchdog. Dependency, +committed lockfile. Network-only failures may retry up to five installation +attempts under one 30-minute timeout and the same disk watchdog. Completed +downloads are preferred from the candidate-local cache between attempts. Dependency, engine and postinstall failures are not retried. The per-candidate npm cache is removed after installation and with any failed candidate. diff --git a/ops/deploy/eusrr-deploy b/ops/deploy/eusrr-deploy index e19e6401..22c55f53 100755 --- a/ops/deploy/eusrr-deploy +++ b/ops/deploy/eusrr-deploy @@ -190,11 +190,11 @@ install_frontend_dependencies() { set -euo pipefail release="$1" output="${release}/.npm-install-attempt.log" - for attempt in 1 2 3; do - printf "Frontend dependency installation attempt %s/3.\n" "$attempt" + for attempt in 1 2 3 4 5; do + printf "Frontend dependency installation attempt %s/5.\n" "$attempt" if npm --cache "${release}/.npm-cache" --prefix "${release}/frontend" ci \ - --include=optional --no-audit --no-fund --engine-strict \ - --fetch-retries=3 --fetch-retry-factor=2 \ + --include=optional --no-audit --no-fund --engine-strict --prefer-offline \ + --fetch-retries=5 --fetch-retry-factor=2 \ --fetch-retry-mintimeout=10000 --fetch-retry-maxtimeout=30000 \ --fetch-timeout=120000 >"${output}" 2>&1; then cat "${output}" @@ -204,12 +204,12 @@ install_frontend_dependencies() { status=$? cat "${output}" fi - if [[ "$attempt" == 3 ]] || ! grep -Eq \ + if [[ "$attempt" == 5 ]] || ! grep -Eq \ "npm (ERR!|error) code (ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|ECONNREFUSED|E502|E503|E504)([[:space:]]|$)" "${output}"; then exit "$status" fi printf "Transient npm network failure; retrying the same lockfile.\n" - sleep "$((attempt * 10))" + sleep "$((attempt * 15))" done ' _ "${NEW_RELEASE}" } diff --git a/ops/deploy/tests/test_deploy_lifecycle.py b/ops/deploy/tests/test_deploy_lifecycle.py index 7ce9b5a0..8aba8451 100644 --- a/ops/deploy/tests/test_deploy_lifecycle.py +++ b/ops/deploy/tests/test_deploy_lifecycle.py @@ -223,7 +223,7 @@ class DeploymentLifecycleTests(unittest.TestCase): def test_persistent_npm_network_failure_is_bounded_and_cleans_candidate(self): result = self.run_deploy(FAIL_STAGE='npm_network_always') self.assertEqual(result.returncode, 152, self.output) - self.assertEqual((self.root/'npm_attempts').read_text(), '3') + self.assertEqual((self.root/'npm_attempts').read_text(), '5') self.assert_old_live() self.assertFalse((self.releases/TARGET).exists()) @@ -247,6 +247,7 @@ class DeploymentLifecycleTests(unittest.TestCase): self.assertIn('pip-bootstrap.txt', commands) self.assertIn('--require-hashes', commands) self.assertIn('--resume-retries=5', commands) + self.assertIn('--prefer-offline', commands) self.assertNotIn('--trusted-host', commands) self.assertNotIn('--no-require-hashes', commands) bootstrap = (SOURCE.parent/'pip-bootstrap.txt').read_text()