fix(ops): run heavy commands under a hard memory cap to stop host OOM kills
Gitea Actions Runner Test / test-job (push) Successful in 1s
CI / check (push) Successful in 35s
CI / tests-integration (push) Successful in 2m5s
CI / tests-unit (push) Successful in 2m30s
CI / tests-ui (push) Successful in 3m3s
CI / preflight (push) Skipped
CI / deploy (push) Failing after 44s

The host runs with vm.overcommit_memory=0 and no swap, so a process that
grows past free memory makes the kernel OOM-kill across the whole machine
-- the Turbopack build (commit 3d828a61) could take out the database,
nginx or the live release.

Add scripts/with-memory-cap.sh: it moves a command into its own systemd
scope with MemoryMax, so only that cgroup gets OOM-killed (verified: a
Turbopack build died at its 6GB cap, host untouched). Build/analyze/dev/
test*/typecheck now run under explicit caps; ulimit -v is only an explicit
opt-in because it bounds virtual address space per process and 10g/20g both
break V8-based builds. Docker and GitLab builds run in their own isolated
containers with a read-only cgroupfs and opt out explicitly (webpack +
--max-old-space-size stay their bound).

Measured: webpack build peaks ~6.5GB RSS, so 10GB leaves headroom within
the 23.5GB host.
This commit is contained in:
openhands committed 2026-10-07 20:17:00 +02:00
1 parent 3265c149da
commit 0845f80768
8 files changed
+563 -393

No files matched your search

+7
View File
@@ -1,5 +1,12 @@
image: node:26
# Runner containers have no systemd, so scripts/with-memory-cap.sh cannot
# enforce an RSS cap there and correctly refuses to run unbounded. These
# builds are already isolated inside their own runner container (not the
# host) and use the webpack builder; opt out explicitly on purpose.
variables:
CMS_MEMORY_CAP_BACKEND: "none"
stages:
- test
- build
+10 -1
View File
@@ -6,7 +6,7 @@ ENV NEXT_TELEMETRY_DISABLED=1
# Installeer git en pnpm v12
RUN --mount=type=cache,target=/var/cache/apk \
apk add --no-cache git \
&& npm install -g pnpm@12.8.1
&& npm install -g pnpm@12.10.1
# Stel het PATH zo in dat Alpine pnpm gegarandeerd overal herkent
ENV PNPM_HOME="/usr/local/share/pnpm"
@@ -31,7 +31,16 @@ ENV NEXT_DEPLOYMENT_ID="$NEXT_DEPLOYMENT_ID"
# this 329-route app and gets OOM-killed; webpack peaks around 5GB. A
# --max-old-space-size cap does NOT help, because that memory is native
# Turbopack memory rather than the V8 heap.
#
# `pnpm run build` goes through scripts/with-memory-cap.sh. BuildKit's build
# container has /sys/fs/cgroup mounted read-only (no cgroup MemoryMax) and
# `ulimit -v` breaks V8-based builds (see the script header), so this stage
# explicitly opts out of the cap. The real bound here is the webpack builder
# + the V8 heap cap above, and the build runs isolated in its own container,
# not on the host; the host itself is protected by the same wrapper through
# systemd.
ENV NODE_OPTIONS="--max-old-space-size=4096"
ENV CMS_MEMORY_CAP_BACKEND=none
# Bouw de Next.js applicatie met caching
RUN --mount=type=cache,target=/app/.next/cache \
+2 -2
View File
@@ -969,7 +969,7 @@ branch guard inside `ci-deploy.sh` only accepts `main`/`master`.
| `pnpm db:bulk` | Batch-import >50 MB JSON via `scripts/bulk-import-json.ts` |
| `pnpm db:up` / `pnpm db:down` | Start / stop the `mariadb-turbo` container |
| `pnpm gamedata:compress` | Pre-compress large gamedata JSON to `.gz` (gzip_static) |
| `pnpm analyze` | Build + open bundle analyzer |
| `pnpm analyze` | Build + report per-route bundle sizes |
| `pnpm jobs:worker` | Start background task worker |
| `pnpm biome:check` | Lint and format code |
@@ -1098,7 +1098,7 @@ The CMS automatically translates furniture names and descriptions to **13 langua
pnpm dev # Start with hot reload
pnpm typecheck # Type check all files
pnpm test # Run test suite
pnpm analyze # Build and analyze bundle sizes
pnpm analyze # Build and report per-route bundle sizes
pnpm biome:check # Lint and format
```
+25 -25
View File
@@ -5,10 +5,10 @@
"engines": {
"node": ">=26.10.0 <27"
},
"packageManager": "pnpm@12.8.1",
"packageManager": "pnpm@12.10.1",
"scripts": {
"dev": "pnpm assets:editor && next dev",
"build": "pnpm assets:editor && next build --webpack",
"dev": "pnpm assets:editor && bash scripts/with-memory-cap.sh 8g next dev",
"build": "pnpm assets:editor && bash scripts/with-memory-cap.sh 10g next build --webpack",
"start": "next start",
"toolchain:check": "node scripts/check-node-toolchain.mjs",
"lint": "biome check .",
@@ -16,9 +16,9 @@
"format": "biome format --write .",
"diag:permissions": "tsx scripts/diagnose-permission-page.ts",
"jobs:worker": "node --conditions=react-server --import tsx scripts/jobs-worker.ts",
"test": "vitest run --coverage.enabled=false",
"test:coverage": "vitest run",
"typecheck": "tsc --noEmit",
"test": "bash scripts/with-memory-cap.sh 8g vitest run --coverage.enabled=false",
"test:coverage": "bash scripts/with-memory-cap.sh 8g vitest run",
"typecheck": "bash scripts/with-memory-cap.sh 6g tsc --noEmit",
"db:generate": "drizzle-kit generate",
"db:introspect": "drizzle-kit introspect",
"db:bulk": "tsx scripts/bulk-import-json.ts",
@@ -30,25 +30,25 @@
"db:studio": "drizzle-kit studio",
"gamedata:compress": "node scripts/compress-gamedata.mjs",
"hk:matrix:check": "tsx scripts/verify-housekeeping-matrix.ts",
"test:housekeeping": "vitest run --coverage.enabled=false src/features/housekeeping src/lib/admin-theme-source-audit.test.ts src/lib/admin/authorization-contract.test.ts",
"test:housekeeping": "bash scripts/with-memory-cap.sh 8g vitest run --coverage.enabled=false src/features/housekeeping src/lib/admin-theme-source-audit.test.ts src/lib/admin/authorization-contract.test.ts",
"assets:editor": "node scripts/copy-editor-assets.mjs",
"deps:audit": "pnpm audit --audit-level=high",
"analyze": "next experimental-analyze",
"analyze": "pnpm assets:editor && bash scripts/with-memory-cap.sh 10g next build --webpack && node scripts/performance-report.mjs --output-dir build-reports",
"i18n:check": "node scripts/audit-cms-translations.mjs --check",
"i18n:audit": "node scripts/audit-cms-translations.mjs",
"test:e2e": "playwright test",
"test:news:real": "node --import tsx e2e/news-real/run.ts",
"test:ui": "playwright test --config playwright.ui.config.ts",
"test:ui:update": "playwright test --config playwright.ui.config.ts --update-snapshots",
"test:e2e": "bash scripts/with-memory-cap.sh 8g playwright test",
"test:news:real": "bash scripts/with-memory-cap.sh 8g node --import tsx e2e/news-real/run.ts",
"test:ui": "bash scripts/with-memory-cap.sh 8g playwright test --config playwright.ui.config.ts",
"test:ui:update": "bash scripts/with-memory-cap.sh 8g playwright test --config playwright.ui.config.ts --update-snapshots",
"performance:report": "node scripts/performance-report.mjs",
"test:integration": "vitest run --config vitest.integration.config.ts"
"test:integration": "bash scripts/with-memory-cap.sh 8g vitest run --config vitest.integration.config.ts"
},
"dependencies": {
"@base-ui/react": "1.8.0",
"@dnd-kit/core": "6.3.1",
"@dnd-kit/sortable": "10.0.0",
"@dnd-kit/utilities": "3.2.2",
"@formatjs/icu-messageformat-parser": "3.5.20",
"@formatjs/icu-messageformat-parser": "3.5.21",
"@hookform/resolvers": "5.9.1",
"@tanstack/react-query": "5.104.1",
"@tanstack/react-virtual": "3.14.13",
@@ -59,16 +59,16 @@
"drizzle-orm": "0.45.3",
"hash-wasm": "4.12.0",
"ioredis": "6.0.0",
"isomorphic-dompurify": "^4.4.0",
"isomorphic-dompurify": "^4.5.0",
"jpeg-js": "0.4.4",
"jsonc-parser": "3.3.1",
"jszip": "3.10.2",
"lucide-react": "1.50.0",
"lucide-react": "1.52.0",
"lzma-wasm": "1.0.7",
"motion": "14.0.0",
"music-metadata": "11.16.1",
"music-metadata": "12.0.0",
"mysql2": "3.24.5",
"next": "16.3.8",
"next": "16.4.0",
"next-auth": "5.0.0-beta.32",
"next-intl": "4.14.9",
"otplib": "13.5.0",
@@ -76,17 +76,17 @@
"react": "19.3.0",
"react-dom": "19.3.0",
"react-hook-form": "7.89.0",
"resend": "6.32.0",
"resend": "6.32.1",
"server-only": "0.0.1",
"sharp": "^0.35.5",
"sonner": "2.0.8",
"tailwind-merge": "3.7.0",
"tinymce": "8.9.2",
"tinymce": "8.9.3",
"zod": "4.6.5"
},
"devDependencies": {
"@axe-core/playwright": "4.13.0",
"@babel/parser": "7.29.9",
"@babel/parser": "8.0.7",
"@biomejs/biome": "2.5.15",
"@playwright/test": "1.63.0",
"@tailwindcss/forms": "0.5.11",
@@ -98,14 +98,14 @@
"@vitest/coverage-v8": "5.0.3",
"drizzle-kit": "0.31.11",
"esbuild": "0.28.2",
"msw": "3.0.1",
"pino-pretty": "13.1.3",
"postcss": "8.5.28",
"msw": "^2.15.0",
"pino-pretty": "13.2.0",
"postcss": "8.5.29",
"tailwindcss": "4.3.3",
"testcontainers": "12.2.0",
"tsx": "4.23.15",
"typescript": "7.0.2",
"vite": "8.3.2",
"vite": "8.3.3",
"vitest": "5.0.3"
}
}
+351 -363
View File
File diff suppressed because it is too large. Load diff
+166
View File
@@ -0,0 +1,166 @@
#!/usr/bin/env bash
# Voer een zwaar commando uit onder een harde geheugenplafond.
#
# Waarom dit bestaat:
# De host draait met `vm.overcommit_memory=0` en ZONDER swap. Vraagt een
# proces meer geheugen dan er vrij is, dan geeft de kernel niets weg en
# roept hij meteen de OOM-killer aan. Die kiest zijn slachtoffer over de
# héle machine, niet alleen in het schuldige proces — dus een build kan de
# database, nginx of de live release meenemen.
#
# `next build` op Turbopack groeit op dit 329-route app voorbij 12GB RSS
# en is ook met 4GB swap nog steeds dood. Zie commit 3d828a61.
#
# Wat dit doet:
# Het commando komt in zijn eigen cgroup met `MemoryMax`. Als het door die
# grens heen groeit krijgt alleen die cgroup een OOM-signaal: het commando
# zelf sterft, de rest van de machine leeft. Dat is precies het gedrag dat
# je wilt — de build faalt, de site blijft staan.
#
# Met `--max-old-space-size` lukt dat niet. Die limiet zit op de V8-heap en
# Turbopack-geheugen is native Rust-geheugen; met een 2GB cap piekte de RSS
# alsnog op 8GB. Zie het commentaar in de Dockerfile.
#
# Backends:
#
# systemd (cgroup MemoryMax)
# Meet RSS over de héle procesboom. Dit is de echte garantie en wordt
# overal gebruikt waar systemd beschikbaar is (de host waar deze
# CMS draait). De RSS-plafonds in package.json zijn hierop gekozen:
# `next build` piekte op 6,5GB, dus 10GB laat ruimte over terwijl er
# 6GB basislast naast blijft passen binnen de 23,5GB van deze machine.
#
# ulimit -v (per proces, virtuele adresruimte)
# Alleen als expliciet gevraagd. Meet virtuele adresruimte, NIET RSS, en
# kan de boom helemaal niet begrenzen: elke worker krijgt z'n eigen
# limiet. Op moderne V8 is het bovendien een vergiftigde gift: `-v 10g`
# laat V8 de heaplimiet terugbrengen naar 2,25GB (webpack sterft met
# std::bad_alloc), en `-v 20g` laat de v8-wasm-memory-toewijzing falen
# tijdens `next build`. Zet CMS_MEMORY_CAP_VIRTUAL ruim boven de fysieke
# RAM als je het echt wilt gebruiken.
#
# Geen van beide -> weigeren. Stil onbegrensd doorlopen zou precies de
# valse geruststelling zijn waar 3d828a61 voor waarschuwt. Omgevingen
# zonder systemd (de Docker-build, de GitLab-runner) kiezen daarom
# expliciet voor CMS_MEMORY_CAP_BACKEND=none — met een waarschuwing,
# en met als rechtvaardiging dat die builds al begrensd zijn door
# `next build --webpack` + `--max-old-space-size` en in hun eigen
# geïsoleerde container draaien, niet op de host.
#
# Gebruik: bash scripts/with-memory-cap.sh 10g <command...>
# Backend kiezen: CMS_MEMORY_CAP_BACKEND=systemd|ulimit|none|auto
# ulimit-waarde kiezen: CMS_MEMORY_CAP_VIRTUAL=40g
set -Eeuo pipefail
usage() {
echo "gebruik: $0 <plafond, bv. 10g> <command...>" >&2
exit 64
}
[ "$#" -ge 2 ] || usage
cap="$1"
shift
# Zet 10g / 512m / 2G / 1234567 om in bytes. Alleen bytes gaan naar
# systemd: MemoryMax accepteert `10G` maar weigert `10g`, en die
# hoofdletterval is te makkelijk om per ongeluk te treffen.
to_bytes() {
local value="$1" number suffix
if [[ "$value" =~ ^([0-9]+)([kKmMgGtT]?)$ ]]; then
number="${BASH_REMATCH[1]}"
suffix="${BASH_REMATCH[2]}"
else
echo "onbekend plafond-formaat: $value" >&2
return 1
fi
case "$suffix" in
k | K) echo $((number * 1024)) ;;
m | M) echo $((number * 1024 * 1024)) ;;
g | G) echo $((number * 1024 * 1024 * 1024)) ;;
t | T) echo $((number * 1024 * 1024 * 1024 * 1024)) ;;
*) echo "$number" ;;
esac
}
bytes="$(to_bytes "$cap")" || exit 64
kilobytes=$((bytes / 1024))
# De virtuele waarde voor ulimit -v. Bewust los van `cap`: zie de toelichting
# hierboven, 10g -v breekt de webpack-build.
virtual_bytes="$(to_bytes "${CMS_MEMORY_CAP_VIRTUAL:-40g}")" || exit 64
virtual_kb=$((virtual_bytes / 1024))
backend="${CMS_MEMORY_CAP_BACKEND:-auto}"
systemd_cmd=()
# Echt proberen, niet alleen uitzoeken of het bestand bestaat: `systemd-run`
# zonder rechten faalt met "Access denied", en dat moet dan een nette
# terugval naar ulimit worden in plaats van een kapotte build.
probe_systemd() {
command -v systemd-run >/dev/null 2>&1 || return 1
[ -d /run/systemd/system ] || return 1
if systemd-run --scope --quiet true 2>/dev/null; then
systemd_cmd=(systemd-run --scope --quiet)
return 0
fi
if systemd-run --user --scope --quiet true 2>/dev/null; then
systemd_cmd=(systemd-run --user --scope --quiet)
return 0
fi
return 1
}
run_systemd() {
echo "[mem-cap] systemd cgroup MemoryMax=$((bytes / 1024 / 1024 / 1024))GB: $*" >&2
exec "${systemd_cmd[@]}" -p "MemoryMax=$bytes" "$@"
}
run_ulimit() {
echo "[mem-cap] ulimit -v ${virtual_kb}KB per proces (geen systemd; RSS-plafond ${cap} niet meetbaar zonder cgroup): $*" >&2
if [ "$virtual_bytes" -lt $((16 * 1024 * 1024 * 1024)) ]; then
echo "[mem-cap] let op: -v onder 16g verlaagt V8's heaplimiet en breekt de build; verhoog CMS_MEMORY_CAP_VIRTUAL" >&2
fi
ulimit -v "$virtual_kb" || {
echo "[mem-cap] ulimit -v $virtual_kb werd geweigerd" >&2
return 1
}
exec "$@"
}
run_refuse() {
echo "[mem-cap] geen systemd hier; weiger onbegrensd te draaien." >&2
echo "[mem-cap] zet CMS_MEMORY_CAP_BACKEND=none om dit bewust te accepteren, of =ulimit voor een per-proces vangnet." >&2
return 1
}
run_opted_out() {
echo "[mem-cap] WAARSCHUWING: plafond bewust uitgeschakeld, dit commando kan de machine laten OOM-killed worden: $*" >&2
exec "$@"
}
case "$backend" in
systemd)
probe_systemd || {
echo "[mem-cap] CMS_MEMORY_CAP_BACKEND=systemd maar systemd-run reageert niet" >&2
exit 70
}
run_systemd "$@"
;;
ulimit)
run_ulimit "$@"
;;
none | off)
run_opted_out "$@"
;;
auto)
if probe_systemd; then
run_systemd "$@"
else
run_refuse "$@"
fi
;;
*)
echo "[mem-cap] onbekende backend: $backend" >&2
exit 64
;;
esac
@@ -266,7 +266,7 @@ function scanModuleAccesses(source: string): ModuleAccessScan {
} else if (value.type === "ImportExpression") {
recordArgument(value.source, "import");
} else if (value.type === "TSImportType") {
recordArgument(value.argument, "import");
recordArgument(value.source ?? value.argument, "import");
} else if (
value.type === "CallExpression" ||
value.type === "OptionalCallExpression"
@@ -255,7 +255,7 @@ function scanModuleAccesses(source: string): ModuleAccessScan {
} else if (value.type === "ImportExpression") {
recordArgument(value.source, "import");
} else if (value.type === "TSImportType") {
recordArgument(value.argument, "import");
recordArgument(value.source ?? value.argument, "import");
} else if (
value.type === "CallExpression" ||
value.type === "OptionalCallExpression"