Require supported Node LTS and retry transient npm fetch failures safely
Some checks failed
Deploy production / deploy (push) Failing after 6m38s
Some checks failed
Deploy production / deploy (push) Failing after 6m38s
This commit is contained in:
parent
e3c677dda0
commit
7ed495ce64
@ -64,6 +64,13 @@ Runner остаётся **глобальным** по решению владе
|
||||
forced command и актуальным SHA master; сервер Gitea проверяется по закреплённому
|
||||
SSH host key. URL источника и Git hooks нельзя незаметно подменить конфигом.
|
||||
|
||||
UID 10001 имеет настоящую запись пользователя `runner` в производном образе;
|
||||
перед заменой контейнера отдельно проверяется OpenSSH без сети и ключей.
|
||||
Для frontend требуется Node.js 22.13+ (22.x) или 24.x LTS, проверка выполняется
|
||||
до создания кандидата. `npm ci` использует строгую проверку engines и повторяет
|
||||
только сетевые сбои: максимум три попытки под общим таймаутом и контролем диска.
|
||||
Ошибки зависимостей и postinstall не маскируются повторными попытками.
|
||||
|
||||
Остановка Gunicorn использует SIGTERM и достаточный таймаут мягкого завершения.
|
||||
Код 143 Next.js при штатной остановке не считается аварией. Telegram-транспорт
|
||||
и его fallback-механизмы сохранены; ссылка использует номер запуска Gitea.
|
||||
|
||||
3
frontend/package-lock.json
generated
3
frontend/package-lock.json
generated
@ -8,6 +8,9 @@
|
||||
"name": "corp.front",
|
||||
"version": "0.1.0",
|
||||
"hasInstallScript": true,
|
||||
"engines": {
|
||||
"node": ">=22.13.0 <23 || >=24 <25"
|
||||
},
|
||||
"dependencies": {
|
||||
"@dnd-kit/core": "^6.3.1",
|
||||
"@dnd-kit/sortable": "^10.0.0",
|
||||
|
||||
@ -3,6 +3,9 @@
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"engines": {
|
||||
"node": ">=22.13.0 <23 || >=24 <25"
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "next dev --webpack",
|
||||
"dev:turbo": "next dev --turbo",
|
||||
|
||||
@ -61,6 +61,13 @@ removed on error, SIGTERM, SIGINT or SIGHUP, after their processes and slot serv
|
||||
have stopped. Symlink releases, mounted directories and referenced releases are
|
||||
never removed. A failed cleanup aborts the operation instead of deleting broadly.
|
||||
|
||||
Node.js must be 22.13+ on the 22.x LTS line, or 24.x LTS; this is checked before
|
||||
creating a candidate. Frontend `npm ci` uses strict engine validation and the
|
||||
committed lockfile. Network-only failures may retry up to three installation
|
||||
attempts under one 30-minute timeout and the same disk watchdog. Dependency,
|
||||
engine and postinstall failures are not retried. The per-candidate npm cache is
|
||||
removed after installation and with any failed candidate.
|
||||
|
||||
The deployment requires at least 4 GiB and 50,000 free inodes before creating a
|
||||
release. Build and database-dump process groups have a 30-minute timeout and are
|
||||
monitored every two seconds; they are stopped if free space falls below 512 MiB
|
||||
|
||||
@ -36,6 +36,10 @@ run_as_deploy() {
|
||||
setpriv --no-new-privs runuser -u "${DEPLOY_USER}" -- "$@"
|
||||
}
|
||||
|
||||
check_node_runtime() {
|
||||
run_as_deploy node -e 'const [major, minor] = process.versions.node.split(".").map(Number); if (!((major === 22 && minor >= 13) || major === 24)) { console.error("Production requires Node.js 22.13+ (22.x) or 24.x LTS."); process.exit(1); }'
|
||||
}
|
||||
|
||||
run_git() {
|
||||
# Repository hooks/config and per-user SSH aliases must not silently replace
|
||||
# the production source. The read-only deploy key and pinned host key are used
|
||||
@ -73,6 +77,7 @@ if [[ ${1:-} == "--self-test" ]]; then
|
||||
test -r "${UPSTREAM_FILE}"
|
||||
test -r /usr/local/libexec/eusrr-deploy-healthcheck
|
||||
run_as_deploy test -r "${GITEA_KNOWN_HOSTS}"
|
||||
check_node_runtime
|
||||
[[ $(stat -c '%u:%a' "${SLOTS_ROOT}") == 0:755 ]]
|
||||
for unit in eusrr-backend@.service eusrr-frontend@.service \
|
||||
eusrr-celery-worker@.service eusrr-celery-beat@.service; do
|
||||
@ -150,7 +155,7 @@ check_space() {
|
||||
|
||||
# Build processes have their own session so cancellation also stops grandchildren.
|
||||
run_bounded_build() {
|
||||
check_space 1073741824
|
||||
check_space 1073741824 || return $?
|
||||
setsid timeout --signal=TERM --kill-after=30s 30m setpriv --no-new-privs \
|
||||
runuser -u "${DEPLOY_USER}" -- "$@" <&0 &
|
||||
BUILD_PID=$!
|
||||
@ -177,6 +182,38 @@ run_bounded_build() {
|
||||
check_space 536870912
|
||||
}
|
||||
|
||||
install_frontend_dependencies() {
|
||||
# All attempts share one build timeout and disk watchdog. Retry only reported
|
||||
# network failures, never dependency/engine/postinstall errors. Cache remains
|
||||
# inside the candidate and is removed on success or failed-release cleanup.
|
||||
run_bounded_build bash -c '
|
||||
set -euo pipefail
|
||||
release="$1"
|
||||
output="${release}/.npm-install-attempt.log"
|
||||
for attempt in 1 2 3; do
|
||||
printf "Frontend dependency installation attempt %s/3.\n" "$attempt"
|
||||
if npm --cache "${release}/.npm-cache" --prefix "${release}/frontend" ci \
|
||||
--include=optional --no-audit --no-fund --engine-strict \
|
||||
--fetch-retries=3 --fetch-retry-factor=2 \
|
||||
--fetch-retry-mintimeout=10000 --fetch-retry-maxtimeout=30000 \
|
||||
--fetch-timeout=120000 >"${output}" 2>&1; then
|
||||
cat "${output}"
|
||||
rm -f -- "${output}"
|
||||
exit 0
|
||||
else
|
||||
status=$?
|
||||
cat "${output}"
|
||||
fi
|
||||
if [[ "$attempt" == 3 ]] || ! grep -Eq \
|
||||
"npm (ERR!|error) code (ECONNRESET|ETIMEDOUT|EAI_AGAIN|ENETUNREACH|EHOSTUNREACH|ECONNREFUSED|E502|E503|E504)([[:space:]]|$)" "${output}"; then
|
||||
exit "$status"
|
||||
fi
|
||||
printf "Transient npm network failure; retrying the same lockfile.\n"
|
||||
sleep "$((attempt * 10))"
|
||||
done
|
||||
' _ "${NEW_RELEASE}"
|
||||
}
|
||||
|
||||
run_database_dump() {
|
||||
local dump_file="$1"
|
||||
run_bounded_build \
|
||||
@ -591,6 +628,7 @@ if [[ ${MAINTENANCE} -eq 1 ]]; then
|
||||
fi
|
||||
|
||||
log "Preparing ${TARGET_SHA} in the ${NEW_SLOT} slot; current slot is ${OLD_SLOT}."
|
||||
check_node_runtime
|
||||
[[ $(run_git config --get remote.origin.url) == "${GITEA_ORIGIN}" ]] || {
|
||||
log "Repository origin is not the approved Gitea repository; refusing deployment."; exit 5;
|
||||
}
|
||||
@ -637,8 +675,7 @@ run_bounded_build "${NEW_RELEASE}/backend/.venv/bin/python" \
|
||||
/usr/local/libexec/eusrr-deploy-healthcheck
|
||||
|
||||
log "Installing pinned frontend dependencies, including Linux native modules, and building Next.js."
|
||||
run_bounded_build npm --cache "${NEW_RELEASE}/.npm-cache" --prefix "${NEW_RELEASE}/frontend" ci \
|
||||
--include=optional --no-audit --no-fund
|
||||
install_frontend_dependencies
|
||||
run_as_deploy rm -rf -- "${NEW_RELEASE}/.npm-cache"
|
||||
run_as_deploy test -f \
|
||||
"${NEW_RELEASE}/frontend/node_modules/lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node"
|
||||
|
||||
@ -85,6 +85,16 @@ elif name == 'python':
|
||||
elif 'migrate' in args and '--noinput' in args: fail('migration')
|
||||
elif name == 'npm':
|
||||
if 'ci' in args:
|
||||
count_file=root/'npm_attempts'
|
||||
count=int(count_file.read_text())+1 if count_file.exists() else 1
|
||||
count_file.write_text(str(count))
|
||||
stage=os.environ.get('FAIL_STAGE')
|
||||
if stage=='npm_network_always' or (stage=='npm_network_once' and count==1):
|
||||
print('npm error code ECONNRESET', file=sys.stderr)
|
||||
sys.exit(152)
|
||||
if stage=='npm_dependency':
|
||||
print('npm error code ERESOLVE', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
base=p(args[args.index('--prefix')+1])/'node_modules'
|
||||
(base/'lightningcss-linux-x64-gnu').mkdir(parents=True)
|
||||
(base/'lightningcss-linux-x64-gnu/lightningcss.linux-x64-gnu.node').touch()
|
||||
@ -93,6 +103,7 @@ elif name == 'npm':
|
||||
fail('build')
|
||||
if os.environ.get('FAIL_STAGE') in ('hang','disk_during_build'):
|
||||
(root/'building').touch(); time.sleep(60)
|
||||
elif name == 'node': fail('node_runtime')
|
||||
elif name == 'curl':
|
||||
if os.environ.get('FAIL_STAGE')=='health' and any(':3001' in a for a in args): print('502')
|
||||
else: print('200')
|
||||
@ -112,7 +123,7 @@ class DeploymentLifecycleTests(unittest.TestCase):
|
||||
fake = self.bin / "fake"
|
||||
fake.write_text(FAKE)
|
||||
fake.chmod(0o755)
|
||||
for name in ('runuser', 'git', 'df', 'systemctl', 'fake-python', 'npm',
|
||||
for name in ('runuser', 'git', 'df', 'systemctl', 'fake-python', 'npm', 'node',
|
||||
'curl', 'sleep', 'findmnt', 'nginx', 'rsync', 'pg_restore'):
|
||||
(self.bin / name).symlink_to(fake)
|
||||
text = SOURCE.read_text()
|
||||
@ -198,6 +209,33 @@ class DeploymentLifecycleTests(unittest.TestCase):
|
||||
self.assert_old_live()
|
||||
self.assertTrue((self.releases/PREVIOUS).exists())
|
||||
|
||||
def test_transient_npm_network_failure_retries_same_candidate(self):
|
||||
result = self.run_deploy(FAIL_STAGE='npm_network_once')
|
||||
self.assertEqual(result.returncode, 0, self.output)
|
||||
self.assertEqual((self.root/'npm_attempts').read_text(), '2')
|
||||
self.assertIn('retrying the same lockfile', self.output)
|
||||
self.assertFalse((self.releases/TARGET/'.npm-install-attempt.log').exists())
|
||||
|
||||
def test_persistent_npm_network_failure_is_bounded_and_cleans_candidate(self):
|
||||
result = self.run_deploy(FAIL_STAGE='npm_network_always')
|
||||
self.assertEqual(result.returncode, 152, self.output)
|
||||
self.assertEqual((self.root/'npm_attempts').read_text(), '3')
|
||||
self.assert_old_live()
|
||||
self.assertFalse((self.releases/TARGET).exists())
|
||||
|
||||
def test_npm_dependency_errors_are_not_retried(self):
|
||||
result = self.run_deploy(FAIL_STAGE='npm_dependency')
|
||||
self.assertEqual(result.returncode, 1, self.output)
|
||||
self.assertEqual((self.root/'npm_attempts').read_text(), '1')
|
||||
self.assert_old_live()
|
||||
|
||||
def test_unsupported_node_rejected_before_creating_candidate(self):
|
||||
result = self.run_deploy(FAIL_STAGE='node_runtime')
|
||||
self.assertEqual(result.returncode, 17, self.output)
|
||||
self.assert_old_live()
|
||||
self.assertFalse((self.releases/TARGET).exists())
|
||||
self.assertFalse((self.root/'npm_attempts').exists())
|
||||
|
||||
def test_source_override_is_rejected_without_touching_live_release(self):
|
||||
for stage in ('origin', 'url_rewrite'):
|
||||
with self.subTest(stage=stage):
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user