Retry transient npm installs more resiliently

ops/deploy/README.md

@@ -63,8 +63,9 @@ never removed. A failed cleanup aborts the operation instead of deleting broadly
 
 Node.js must be 22.13+ on the 22.x LTS line, or 24.x LTS; this is checked before
 creating a candidate. Frontend `npm ci` uses strict engine validation and the
-committed lockfile. Network-only failures may retry up to three installation
-attempts under one 30-minute timeout and the same disk watchdog. Dependency,
+committed lockfile. Network-only failures may retry up to five installation
+attempts under one 30-minute timeout and the same disk watchdog. Completed
+downloads are preferred from the candidate-local cache between attempts. Dependency,
 engine and postinstall failures are not retried. The per-candidate npm cache is
 removed after installation and with any failed candidate.
 

ops/deploy/eusrr-deploy

@@ -190,11 +190,11 @@ install_frontend_dependencies() {
     set -euo pipefail
     release="$1"
     output="${release}/.npm-install-attempt.log"
-    for attempt in 1 2 3; do
-      printf "Frontend dependency installation attempt %s/3.\n" "$attempt"
+    for attempt in 1 2 3 4 5; do
+      printf "Frontend dependency installation attempt %s/5.\n" "$attempt"
       if npm --cache "${release}/.npm-cache" --prefix "${release}/frontend" ci \
-          --include=optional --no-audit --no-fund --engine-strict \
-          --fetch-retries=3 --fetch-retry-factor=2 \
+          --include=optional --no-audit --no-fund --engine-strict --prefer-offline \
+          --fetch-retries=5 --fetch-retry-factor=2 \
           --fetch-retry-mintimeout=10000 --fetch-retry-maxtimeout=30000 \
           --fetch-timeout=120000 >"${output}" 2>&1; then
         cat "${output}"
@@ -204,12 +204,12 @@ install_frontend_dependencies() {
         status=$?
         cat "${output}"
       fi
-      if [[ "$attempt" == 3 ]] || ! grep -Eq \
+      if [[ "$attempt" == 5 ]] || ! grep -Eq \
           "npm (ERR!|error) code (ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|ECONNREFUSED|E502|E503|E504)([[:space:]]|$)" "${output}"; then
         exit "$status"
       fi
       printf "Transient npm network failure; retrying the same lockfile.\n"
-      sleep "$((attempt * 10))"
+      sleep "$((attempt * 15))"
     done
   ' _ "${NEW_RELEASE}"
 }

ops/deploy/tests/test_deploy_lifecycle.py

@@ -223,7 +223,7 @@ class DeploymentLifecycleTests(unittest.TestCase):
     def test_persistent_npm_network_failure_is_bounded_and_cleans_candidate(self):
         result = self.run_deploy(FAIL_STAGE='npm_network_always')
         self.assertEqual(result.returncode, 152, self.output)
-        self.assertEqual((self.root/'npm_attempts').read_text(), '3')
+        self.assertEqual((self.root/'npm_attempts').read_text(), '5')
         self.assert_old_live()
         self.assertFalse((self.releases/TARGET).exists())
 
@@ -247,6 +247,7 @@ class DeploymentLifecycleTests(unittest.TestCase):
         self.assertIn('pip-bootstrap.txt', commands)
         self.assertIn('--require-hashes', commands)
         self.assertIn('--resume-retries=5', commands)
+        self.assertIn('--prefer-offline', commands)
         self.assertNotIn('--trusted-host', commands)
         self.assertNotIn('--no-require-hashes', commands)
         bootstrap = (SOURCE.parent/'pip-bootstrap.txt').read_text()