From 745d0b7247ebedd7aa2ef7bf925d04aa25bc0c30 Mon Sep 17 00:00:00 2001 From: simoleo89 Date: Mon, 7 Sep 2026 22:10:13 +0200 Subject: [PATCH] feat(docker): restore failed Compose updates and retain recent releases --- README.md | 41 +++++++++++++-- scripts/docker-update.sh | 83 +++++++++++++++++++++++++++---- src/lib/docker-update.test.ts | 34 +++++++++++++ src/test/docker-update-harness.sh | 17 ++++++- 4 files changed, 159 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 31d1095f94..72c979d9e4 100644 --- a/README.md +++ b/README.md @@ -182,11 +182,42 @@ cron entry with the absolute path of **your** checkout; keep logs in its `logs` directory. Do not configure both CI and Compose updates for the same instance. The updater refuses an active `epicnext-cms-app` CI-managed container. -Logs: `logs/docker-update.log`. A failure after recreation leaves the candidate -running for diagnosis and returns nonzero; this Compose updater does not promise -automatic application or database rollback. Persistent volumes are preserved. -Before production migrations, retain your normal database backup. The existing -CI deploy continues to restore its previous container on failed verification. +Logs: `logs/docker-update.log`. After cutover, a startup, image or HTTP check +failure automatically recreates the CMS with its previous image and verifies its +local HTTP release and database health. The command still returns nonzero: a +rollback is not a successful update. Recovery uses the current Compose configuration +and `.env`; it restores application code, not configuration edits or database migrations. +The previous image must carry a valid release label. First installations have no +previous release: a failed candidate remains available for diagnosis. A failed +rollback is explicitly logged with a retained `epicnext-cms:rollback-...` recovery tag. +Public routing must be checked separately after rollback. SIGKILL or host power loss +cannot run the rollback handler. + +After success, the updater keeps the two most recent distinct releases tracked in +`logs/docker-release-history.log`. Cleanup only removes older recorded CMS image +tags, without force; images referenced by other containers (including stopped ones) +are preserved and retried on later updates. Untracked historical images, build cache, +other services and persistent volumes are not pruned. Retain the history file between +updates. Before production migrations, retain your normal database backup. + +### Shared prebuilt image audit + +The current Docker build is installation-specific. A shared registry image needs +these changes before it can safely replace local builds: + +| Configuration | Current source | Required preparation | +| --- | --- | --- | +| Avatar imager URL | `src/lib/imager.ts`, direct `NEXT_PUBLIC_IMAGER_URL` access | Supply public runtime configuration; remove the Epicnabbo-specific fallback for other installations. | +| Badge URL | Admin user badge components, `NEXT_PUBLIC_BADGE_URL` | Pass runtime configuration from the server to client components. | +| Public application URL | Diagnostics and photo helpers, `NEXT_PUBLIC_APP_URL` fallback | Use server runtime `APP_URL`; audit any generated absolute URLs. | +| Release identity | `next.config.ts`, `NEXT_PUBLIC_CMS_RELEASE` | Keep baked into the image: one commit identifies the same application binary everywhere. | +| Environment validation | `src/env.ts` requires database URL, hotel name and production auth secret | Build with isolated fixture configuration, then validate each installation at startup. Do not publish production environment files or builder images. | +| Build context | `.dockerignore` currently includes `.env`; Dockerfile copies the context into the builder | Separate build inputs from runtime secrets; inspect final image and build layers before registry publication. | + +This is a source audit, not a validated portable image. The next verification is to +run the **same image digest** for two installations with different names, domains, +imager/badge URLs and credentials, then check rendered pages and browser requests. +The registry publishing workflow is intentionally not enabled yet. ### Diagnose an update that is not visible diff --git a/scripts/docker-update.sh b/scripts/docker-update.sh index 1324411fb4..77995de4ac 100755 --- a/scripts/docker-update.sh +++ b/scripts/docker-update.sh @@ -10,7 +10,48 @@ mkdir -p "$(dirname "$LOG_FILE")" log() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG_FILE"; } die() { log "ERROR: $*"; exit 1; } migration_image="" -trap 'if [[ -n "$migration_image" ]]; then docker image rm "$migration_image" >>"$LOG_FILE" 2>&1 || true; fi' EXIT +previous_image="" +previous_release="" +rollback_tag="" +cutover=0 +probe='const r=await fetch(process.argv[1],{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.database!==true||d.release!==process.argv[2]){console.error(JSON.stringify({http:r.status,database:d.database,release:d.release,expected:process.argv[2]}));process.exit(1)}' +verify_container() { + local target="$1" release="$2" attempt + for attempt in $(seq 1 30); do + if docker exec "$target" node --input-type=module -e "$probe" "http://127.0.0.1:3002/api/health" "$release" >>"$LOG_FILE" 2>&1; then return 0; fi + sleep 3 + done + return 1 +} +finish() { + local status=$? restored + trap - EXIT + if [[ "$status" != 0 && "$cutover" = 1 ]]; then + docker compose logs --tail 100 cms >>"$LOG_FILE" 2>&1 || true + if [[ -n "$previous_image" ]]; then + log "Update failed; restoring previous image $previous_image. Database migrations are not reversed." + if CMS_RELEASE="$rollback_tag" docker compose up -d --no-deps --no-build --force-recreate cms >>"$LOG_FILE" 2>&1; then + restored="$(docker compose ps -q cms 2>>"$LOG_FILE" || true)" + if [[ -n "$restored" && "$(docker inspect --format '{{.Image}}' "$restored")" = "$previous_image" ]] && verify_container "$restored" "$previous_release"; then + log "Rollback verified locally: $previous_release. Check public routing separately." + else + log "ERROR: rollback verification failed. Inspect $LOG_FILE; previous image retained as epicnext-cms:$rollback_tag." + fi + else + log "ERROR: rollback could not recreate CMS. Previous image retained as epicnext-cms:$rollback_tag." + fi + else + log "First installation failed: no previous image exists to restore. Candidate retained for diagnosis." + fi + fi + if [[ -n "$migration_image" ]]; then docker image rm "$migration_image" >>"$LOG_FILE" 2>&1 || true; fi + # Keep the recovery tag on failure for manual recovery, including same-commit rebuilds. + if [[ ( "$status" = 0 || "$cutover" = 0 ) && -n "$rollback_tag" ]]; then docker image rm "epicnext-cms:$rollback_tag" >>"$LOG_FILE" 2>&1 || true; fi + exit "$status" +} +trap finish EXIT +trap 'exit 130' INT +trap 'exit 143' TERM trap 'log "Update failed; inspect $LOG_FILE. No volumes or local files were deleted."' ERR # An existing CI deployment is a different owner of the same host port. @@ -31,6 +72,14 @@ export CMS_RELEASE="$(git rev-parse HEAD)" [[ -f .env ]] || die "Create .env before installing or updating." docker info >/dev/null docker compose config --quiet +previous_container="$(docker compose ps -q cms)" +if [[ -n "$previous_container" ]]; then + previous_image="$(docker inspect --format '{{.Image}}' "$previous_container")" + previous_release="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$previous_image")" + [[ "$previous_release" =~ ^[0-9a-f]{40}$ ]] || die "Previous image lacks a valid release label; automatic rollback cannot be verified." + rollback_tag="rollback-$CMS_RELEASE-$$" + docker image tag "$previous_image" "epicnext-cms:$rollback_tag" +fi log "Building release $CMS_RELEASE from $DIR" # The builder contains the matching migration source and locked dependencies. # No Node/package manager installation on the host is required. @@ -42,19 +91,13 @@ revision="$(docker image inspect --format '{{index .Config.Labels "org.openconta [[ "$revision" = "$CMS_RELEASE" ]] || die "Built image has revision $revision, expected $CMS_RELEASE." docker run --rm --network host --entrypoint pnpm "$migration_image" db:migrate >>"$LOG_FILE" 2>&1 log "Build and migrations completed; recreating only the CMS service." +cutover=1 docker compose up -d --no-deps --no-build --force-recreate cms >>"$LOG_FILE" 2>&1 container="$(docker compose ps -q cms)" [[ -n "$container" ]] || die "Compose did not start the CMS container." actual_image="$(docker inspect --format '{{.Image}}' "$container")" [[ "$actual_image" = "$expected_image" ]] || die "Running image $actual_image differs from built image $expected_image." -# Verify the actual HTTP response, not an environment variable supplied at run time. -probe='const r=await fetch(process.argv[1],{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.database!==true||d.release!==process.argv[2]){console.error(JSON.stringify({http:r.status(),database:d.database,release:d.release,expected:process.argv[2]}));process.exit(1)}' -healthy=0 -for attempt in $(seq 1 30); do - if docker exec "$container" node --input-type=module -e "$probe" "http://127.0.0.1:3002/api/health" "$CMS_RELEASE" >>"$LOG_FILE" 2>&1; then healthy=1; break; fi - sleep 3 -done -[[ "$healthy" = 1 ]] || die "HTTP health/release verification failed. The candidate remains available for diagnosis; no success was recorded." +verify_container "$container" "$CMS_RELEASE" || die "HTTP health/release verification failed." if [[ -n "${CMS_PUBLIC_URL:-}" ]]; then [[ "$CMS_PUBLIC_URL" = https://* || "$CMS_PUBLIC_URL" = http://* ]] || die "CMS_PUBLIC_URL must be an HTTP(S) URL." docker exec "$container" node --input-type=module -e "$probe" "${CMS_PUBLIC_URL%/}/api/health?release=$CMS_RELEASE" "$CMS_RELEASE" >>"$LOG_FILE" 2>&1 || die "Public domain serves another release or is unhealthy. Check reverse proxy/CDN destination." @@ -63,3 +106,25 @@ else log "Public domain was not checked. Set CMS_PUBLIC_URL to verify reverse proxy/CDN routing as well." fi log "Verified release $CMS_RELEASE, image $actual_image, container $container" +cutover=0 +# Record only this checkout's successful release tags; never prune Docker globally. +history="$DIR/logs/docker-release-history.log" +mkdir -p "$DIR/logs" +touch "$history" +mapfile -t releases < <(printf '%s\n' "$CMS_RELEASE" "$previous_release"; cat "$history") +kept=() +pending=() +for release in "${releases[@]}"; do + [[ "$release" =~ ^[0-9a-f]{40}$ ]] || continue + [[ " ${kept[*]} ${pending[*]} " != *" $release "* ]] || continue + if [[ "${#kept[@]}" -lt 2 ]]; then kept+=("$release"); continue; fi + # Even stopped containers belonging to other deployments protect an image. + if users="$(docker ps -aq --filter "ancestor=epicnext-cms:$release")" && [[ -z "$users" ]] && docker image rm "epicnext-cms:$release" >>"$LOG_FILE" 2>&1; then + log "Removed superseded release tag $release" + else + pending+=("$release") + fi +done +printf '%s\n' "${kept[@]}" "${pending[@]}" > "$history.tmp" +mv "$history.tmp" "$history" +log "Keeping the two latest releases; in-use images and persistent volumes are preserved." diff --git a/src/lib/docker-update.test.ts b/src/lib/docker-update.test.ts index 648c875452..a9b46b6a25 100644 --- a/src/lib/docker-update.test.ts +++ b/src/lib/docker-update.test.ts @@ -29,6 +29,11 @@ function simulate(scenario: string) { const dir = mkdtempSync(join(tmpdir(), "cms-compose-test-")); try { mkdirSync(join(dir, "scripts")); + mkdirSync(join(dir, "logs")); + writeFileSync( + join(dir, "logs/docker-release-history.log"), + `${"b".repeat(40)}\n${"c".repeat(40)}\n`, + ); copyFileSync( resolve(root, "scripts/docker-update.sh"), join(dir, "scripts/docker-update.sh"), @@ -50,6 +55,7 @@ function simulate(scenario: string) { if (result.error) throw result.error; return { status: result.status, + restored: existsSync(join(dir, "restored")), output: result.stdout + result.stderr, calls: existsSync(join(dir, "calls")) ? readFileSync(join(dir, "calls"), "utf8") @@ -60,6 +66,32 @@ function simulate(scenario: string) { } } describe("Docker clone updates", () => { + it("removes only historical release tags beyond the current and previous", () => { + const r = simulate("success"); + expect(r.calls).toContain(`docker image rm epicnext-cms:${"c".repeat(40)}`); + expect(r.calls).not.toContain( + `docker image rm epicnext-cms:${"b".repeat(40)}`, + ); + }); + it("preserves older images used by another container", () => { + const r = simulate("retained-in-use"); + expect(r.status, r.output).toBe(0); + expect(r.calls).not.toContain( + `docker image rm epicnext-cms:${"c".repeat(40)}`, + ); + }); + it("reports an unsuccessful rollback and retains the recovery image", () => { + const r = simulate("rollback-failure"); + expect(r.status).not.toBe(0); + expect(r.output).toContain("rollback could not recreate CMS"); + expect(r.calls).not.toContain("docker image rm epicnext-cms:rollback-"); + }); + it("handles first installation failures without claiming rollback", () => { + const r = simulate("first-failure"); + expect(r.status).not.toBe(0); + expect(r.restored).toBe(false); + expect(r.output).toContain("no previous image exists"); + }); it("builds the pulled commit, migrates before recreation and verifies local/public HTTP", () => { const r = simulate("success"); expect(r.status, r.output).toBe(0); @@ -91,6 +123,8 @@ describe("Docker clone updates", () => { const r = simulate(scenario); expect(r.status, r.output).not.toBe(0); expect(r.output).not.toContain("Verified release"); + expect(r.restored).toBe(true); + expect(r.output).toContain("Rollback verified locally"); }, ); }); diff --git a/src/test/docker-update-harness.sh b/src/test/docker-update-harness.sh index 435a3e93bd..ff23a3eafe 100644 --- a/src/test/docker-update-harness.sh +++ b/src/test/docker-update-harness.sh @@ -15,15 +15,28 @@ docker() { case "$1 ${2:-}" in 'inspect --format') if [ "${@: -1}" = epicnext-cms-app ]; then [ "$SCENARIO" = ci-active ] && echo true; return 0; fi + if [ "${@: -1}" = previous123 ] || [ -f "$TEST_DIR/restored" ]; then echo sha256:old; return 0; fi if [ "$SCENARIO" = wrong-image ]; then echo sha256:old; else echo sha256:new; fi ;; 'image inspect') + if [[ "$*" = *org.opencontainers* ]] && [[ "$*" = *sha256:old* ]]; then printf 'b%.0s' {1..40}; echo; return 0; fi if [[ "$*" = *org.opencontainers* ]]; then echo "$TEST_SHA"; else echo sha256:new; fi ;; 'compose config') return 0 ;; 'compose build') [ "$SCENARIO" != build-failure ] ;; - 'compose up') [ "$SCENARIO" != recreate-failure ] ;; - 'compose ps') echo container123 ;; + 'compose up') + if [[ "$CMS_RELEASE" = rollback-* ]]; then + [ "$SCENARIO" != rollback-failure ] || return 1 + touch "$TEST_DIR/restored"; return 0 + fi + touch "$TEST_DIR/cutover" + [ "$SCENARIO" != recreate-failure ] ;; + 'compose ps') + if [ -f "$TEST_DIR/cutover" ]; then echo container123 + elif [ "$SCENARIO" != first-failure ]; then echo previous123; fi ;; 'run --rm') [ "$SCENARIO" != migration-failure ] ;; + 'ps -aq') if [ "$SCENARIO" = retained-in-use ]; then echo other-container; fi ;; 'exec container123') + if [ -f "$TEST_DIR/restored" ]; then return 0; fi + if [ "$SCENARIO" = first-failure ] || [ "$SCENARIO" = rollback-failure ]; then return 1; fi if [ "$SCENARIO" = wrong-release ]; then return 1; fi if [ "$SCENARIO" = wrong-public ] && [[ "$*" = *example.test* ]]; then return 1; fi ;; *) return 0 ;;