feat(docker): restore failed Compose updates and retain recent releases

README.md

@@ -182,11 +182,42 @@ cron entry with the absolute path of **your** checkout; keep logs in its `logs`
 directory. Do not configure both CI and Compose updates for the same instance.
 The updater refuses an active `epicnext-cms-app` CI-managed container.
 
-Logs: `logs/docker-update.log`. A failure after recreation leaves the candidate
-running for diagnosis and returns nonzero; this Compose updater does not promise
-automatic application or database rollback. Persistent volumes are preserved.
-Before production migrations, retain your normal database backup. The existing
-CI deploy continues to restore its previous container on failed verification.
+Logs: `logs/docker-update.log`. After cutover, a startup, image or HTTP check
+failure automatically recreates the CMS with its previous image and verifies its
+local HTTP release and database health. The command still returns nonzero: a
+rollback is not a successful update. Recovery uses the current Compose configuration
+and `.env`; it restores application code, not configuration edits or database migrations.
+The previous image must carry a valid release label. First installations have no
+previous release: a failed candidate remains available for diagnosis. A failed
+rollback is explicitly logged with a retained `epicnext-cms:rollback-...` recovery tag.
+Public routing must be checked separately after rollback. SIGKILL or host power loss
+cannot run the rollback handler.
+
+After success, the updater keeps the two most recent distinct releases tracked in
+`logs/docker-release-history.log`. Cleanup only removes older recorded CMS image
+tags, without force; images referenced by other containers (including stopped ones)
+are preserved and retried on later updates. Untracked historical images, build cache,
+other services and persistent volumes are not pruned. Retain the history file between
+updates. Before production migrations, retain your normal database backup.
+
+### Shared prebuilt image audit
+
+The current Docker build is installation-specific. A shared registry image needs
+these changes before it can safely replace local builds:
+
+| Configuration | Current source | Required preparation |
+| --- | --- | --- |
+| Avatar imager URL | `src/lib/imager.ts`, direct `NEXT_PUBLIC_IMAGER_URL` access | Supply public runtime configuration; remove the Epicnabbo-specific fallback for other installations. |
+| Badge URL | Admin user badge components, `NEXT_PUBLIC_BADGE_URL` | Pass runtime configuration from the server to client components. |
+| Public application URL | Diagnostics and photo helpers, `NEXT_PUBLIC_APP_URL` fallback | Use server runtime `APP_URL`; audit any generated absolute URLs. |
+| Release identity | `next.config.ts`, `NEXT_PUBLIC_CMS_RELEASE` | Keep baked into the image: one commit identifies the same application binary everywhere. |
+| Environment validation | `src/env.ts` requires database URL, hotel name and production auth secret | Build with isolated fixture configuration, then validate each installation at startup. Do not publish production environment files or builder images. |
+| Build context | `.dockerignore` currently includes `.env`; Dockerfile copies the context into the builder | Separate build inputs from runtime secrets; inspect final image and build layers before registry publication. |
+
+This is a source audit, not a validated portable image. The next verification is to
+run the **same image digest** for two installations with different names, domains,
+imager/badge URLs and credentials, then check rendered pages and browser requests.
+The registry publishing workflow is intentionally not enabled yet.
 
 ### Diagnose an update that is not visible
 

scripts/docker-update.sh

@@ -10,7 +10,48 @@ mkdir -p "$(dirname "$LOG_FILE")"
 log() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG_FILE"; }
 die() { log "ERROR: $*"; exit 1; }
 migration_image=""
-trap 'if [[ -n "$migration_image" ]]; then docker image rm "$migration_image" >>"$LOG_FILE" 2>&1 || true; fi' EXIT
+previous_image=""
+previous_release=""
+rollback_tag=""
+cutover=0
+probe='const r=await fetch(process.argv[1],{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.database!==true||d.release!==process.argv[2]){console.error(JSON.stringify({http:r.status,database:d.database,release:d.release,expected:process.argv[2]}));process.exit(1)}'
+verify_container() {
+  local target="$1" release="$2" attempt
+  for attempt in $(seq 1 30); do
+    if docker exec "$target" node --input-type=module -e "$probe" "http://127.0.0.1:3002/api/health" "$release" >>"$LOG_FILE" 2>&1; then return 0; fi
+    sleep 3
+  done
+  return 1
+}
+finish() {
+  local status=$? restored
+  trap - EXIT
+  if [[ "$status" != 0 && "$cutover" = 1 ]]; then
+    docker compose logs --tail 100 cms >>"$LOG_FILE" 2>&1 || true
+    if [[ -n "$previous_image" ]]; then
+      log "Update failed; restoring previous image $previous_image. Database migrations are not reversed."
+      if CMS_RELEASE="$rollback_tag" docker compose up -d --no-deps --no-build --force-recreate cms >>"$LOG_FILE" 2>&1; then
+        restored="$(docker compose ps -q cms 2>>"$LOG_FILE" || true)"
+        if [[ -n "$restored" && "$(docker inspect --format '{{.Image}}' "$restored")" = "$previous_image" ]] && verify_container "$restored" "$previous_release"; then
+          log "Rollback verified locally: $previous_release. Check public routing separately."
+        else
+          log "ERROR: rollback verification failed. Inspect $LOG_FILE; previous image retained as epicnext-cms:$rollback_tag."
+        fi
+      else
+        log "ERROR: rollback could not recreate CMS. Previous image retained as epicnext-cms:$rollback_tag."
+      fi
+    else
+      log "First installation failed: no previous image exists to restore. Candidate retained for diagnosis."
+    fi
+  fi
+  if [[ -n "$migration_image" ]]; then docker image rm "$migration_image" >>"$LOG_FILE" 2>&1 || true; fi
+  # Keep the recovery tag on failure for manual recovery, including same-commit rebuilds.
+  if [[ ( "$status" = 0 || "$cutover" = 0 ) && -n "$rollback_tag" ]]; then docker image rm "epicnext-cms:$rollback_tag" >>"$LOG_FILE" 2>&1 || true; fi
+  exit "$status"
+}
+trap finish EXIT
+trap 'exit 130' INT
+trap 'exit 143' TERM
 trap 'log "Update failed; inspect $LOG_FILE. No volumes or local files were deleted."' ERR
 
 # An existing CI deployment is a different owner of the same host port.
@@ -31,6 +72,14 @@ export CMS_RELEASE="$(git rev-parse HEAD)"
 [[ -f .env ]] || die "Create .env before installing or updating."
 docker info >/dev/null
 docker compose config --quiet
+previous_container="$(docker compose ps -q cms)"
+if [[ -n "$previous_container" ]]; then
+  previous_image="$(docker inspect --format '{{.Image}}' "$previous_container")"
+  previous_release="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$previous_image")"
+  [[ "$previous_release" =~ ^[0-9a-f]{40}$ ]] || die "Previous image lacks a valid release label; automatic rollback cannot be verified."
+  rollback_tag="rollback-$CMS_RELEASE-$$"
+  docker image tag "$previous_image" "epicnext-cms:$rollback_tag"
+fi
 log "Building release $CMS_RELEASE from $DIR"
 # The builder contains the matching migration source and locked dependencies.
 # No Node/package manager installation on the host is required.
@@ -42,19 +91,13 @@ revision="$(docker image inspect --format '{{index .Config.Labels "org.openconta
 [[ "$revision" = "$CMS_RELEASE" ]] || die "Built image has revision $revision, expected $CMS_RELEASE."
 docker run --rm --network host --entrypoint pnpm "$migration_image" db:migrate >>"$LOG_FILE" 2>&1
 log "Build and migrations completed; recreating only the CMS service."
+cutover=1
 docker compose up -d --no-deps --no-build --force-recreate cms >>"$LOG_FILE" 2>&1
 container="$(docker compose ps -q cms)"
 [[ -n "$container" ]] || die "Compose did not start the CMS container."
 actual_image="$(docker inspect --format '{{.Image}}' "$container")"
 [[ "$actual_image" = "$expected_image" ]] || die "Running image $actual_image differs from built image $expected_image."
-# Verify the actual HTTP response, not an environment variable supplied at run time.
-probe='const r=await fetch(process.argv[1],{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.database!==true||d.release!==process.argv[2]){console.error(JSON.stringify({http:r.status(),database:d.database,release:d.release,expected:process.argv[2]}));process.exit(1)}'
-healthy=0
-for attempt in $(seq 1 30); do
-  if docker exec "$container" node --input-type=module -e "$probe" "http://127.0.0.1:3002/api/health" "$CMS_RELEASE" >>"$LOG_FILE" 2>&1; then healthy=1; break; fi
-  sleep 3
-done
-[[ "$healthy" = 1 ]] || die "HTTP health/release verification failed. The candidate remains available for diagnosis; no success was recorded."
+verify_container "$container" "$CMS_RELEASE" || die "HTTP health/release verification failed."
 if [[ -n "${CMS_PUBLIC_URL:-}" ]]; then
   [[ "$CMS_PUBLIC_URL" = https://* || "$CMS_PUBLIC_URL" = http://* ]] || die "CMS_PUBLIC_URL must be an HTTP(S) URL."
   docker exec "$container" node --input-type=module -e "$probe" "${CMS_PUBLIC_URL%/}/api/health?release=$CMS_RELEASE" "$CMS_RELEASE" >>"$LOG_FILE" 2>&1 || die "Public domain serves another release or is unhealthy. Check reverse proxy/CDN destination."
@@ -63,3 +106,25 @@ else
   log "Public domain was not checked. Set CMS_PUBLIC_URL to verify reverse proxy/CDN routing as well."
 fi
 log "Verified release $CMS_RELEASE, image $actual_image, container $container"
+cutover=0
+# Record only this checkout's successful release tags; never prune Docker globally.
+history="$DIR/logs/docker-release-history.log"
+mkdir -p "$DIR/logs"
+touch "$history"
+mapfile -t releases < <(printf '%s\n' "$CMS_RELEASE" "$previous_release"; cat "$history")
+kept=()
+pending=()
+for release in "${releases[@]}"; do
+  [[ "$release" =~ ^[0-9a-f]{40}$ ]] || continue
+  [[ " ${kept[*]} ${pending[*]} " != *" $release "* ]] || continue
+  if [[ "${#kept[@]}" -lt 2 ]]; then kept+=("$release"); continue; fi
+  # Even stopped containers belonging to other deployments protect an image.
+  if users="$(docker ps -aq --filter "ancestor=epicnext-cms:$release")" && [[ -z "$users" ]] && docker image rm "epicnext-cms:$release" >>"$LOG_FILE" 2>&1; then
+    log "Removed superseded release tag $release"
+  else
+    pending+=("$release")
+  fi
+done
+printf '%s\n' "${kept[@]}" "${pending[@]}" > "$history.tmp"
+mv "$history.tmp" "$history"
+log "Keeping the two latest releases; in-use images and persistent volumes are preserved."

src/lib/docker-update.test.ts

@@ -29,6 +29,11 @@ function simulate(scenario: string) {
 	const dir = mkdtempSync(join(tmpdir(), "cms-compose-test-"));
 	try {
 		mkdirSync(join(dir, "scripts"));
+		mkdirSync(join(dir, "logs"));
+		writeFileSync(
+			join(dir, "logs/docker-release-history.log"),
+			`${"b".repeat(40)}\n${"c".repeat(40)}\n`,
+		);
 		copyFileSync(
 			resolve(root, "scripts/docker-update.sh"),
 			join(dir, "scripts/docker-update.sh"),
@@ -50,6 +55,7 @@ function simulate(scenario: string) {
 		if (result.error) throw result.error;
 		return {
 			status: result.status,
+			restored: existsSync(join(dir, "restored")),
 			output: result.stdout + result.stderr,
 			calls: existsSync(join(dir, "calls"))
 				? readFileSync(join(dir, "calls"), "utf8")
@@ -60,6 +66,32 @@ function simulate(scenario: string) {
 	}
 }
 describe("Docker clone updates", () => {
+	it("removes only historical release tags beyond the current and previous", () => {
+		const r = simulate("success");
+		expect(r.calls).toContain(`docker image rm epicnext-cms:${"c".repeat(40)}`);
+		expect(r.calls).not.toContain(
+			`docker image rm epicnext-cms:${"b".repeat(40)}`,
+		);
+	});
+	it("preserves older images used by another container", () => {
+		const r = simulate("retained-in-use");
+		expect(r.status, r.output).toBe(0);
+		expect(r.calls).not.toContain(
+			`docker image rm epicnext-cms:${"c".repeat(40)}`,
+		);
+	});
+	it("reports an unsuccessful rollback and retains the recovery image", () => {
+		const r = simulate("rollback-failure");
+		expect(r.status).not.toBe(0);
+		expect(r.output).toContain("rollback could not recreate CMS");
+		expect(r.calls).not.toContain("docker image rm epicnext-cms:rollback-");
+	});
+	it("handles first installation failures without claiming rollback", () => {
+		const r = simulate("first-failure");
+		expect(r.status).not.toBe(0);
+		expect(r.restored).toBe(false);
+		expect(r.output).toContain("no previous image exists");
+	});
 	it("builds the pulled commit, migrates before recreation and verifies local/public HTTP", () => {
 		const r = simulate("success");
 		expect(r.status, r.output).toBe(0);
@@ -91,6 +123,8 @@ describe("Docker clone updates", () => {
 			const r = simulate(scenario);
 			expect(r.status, r.output).not.toBe(0);
 			expect(r.output).not.toContain("Verified release");
+			expect(r.restored).toBe(true);
+			expect(r.output).toContain("Rollback verified locally");
 		},
 	);
 });

src/test/docker-update-harness.sh

@@ -15,15 +15,28 @@ docker() {
  case "$1 ${2:-}" in
   'inspect --format')
    if [ "${@: -1}" = epicnext-cms-app ]; then [ "$SCENARIO" = ci-active ] && echo true; return 0; fi
+   if [ "${@: -1}" = previous123 ] || [ -f "$TEST_DIR/restored" ]; then echo sha256:old; return 0; fi
    if [ "$SCENARIO" = wrong-image ]; then echo sha256:old; else echo sha256:new; fi ;;
   'image inspect')
+   if [[ "$*" = *org.opencontainers* ]] && [[ "$*" = *sha256:old* ]]; then printf 'b%.0s' {1..40}; echo; return 0; fi
    if [[ "$*" = *org.opencontainers* ]]; then echo "$TEST_SHA"; else echo sha256:new; fi ;;
   'compose config') return 0 ;;
   'compose build') [ "$SCENARIO" != build-failure ] ;;
-  'compose up') [ "$SCENARIO" != recreate-failure ] ;;
-  'compose ps') echo container123 ;;
+  'compose up')
+   if [[ "$CMS_RELEASE" = rollback-* ]]; then
+     [ "$SCENARIO" != rollback-failure ] || return 1
+     touch "$TEST_DIR/restored"; return 0
+   fi
+   touch "$TEST_DIR/cutover"
+   [ "$SCENARIO" != recreate-failure ] ;;
+  'compose ps')
+   if [ -f "$TEST_DIR/cutover" ]; then echo container123
+   elif [ "$SCENARIO" != first-failure ]; then echo previous123; fi ;;
   'run --rm') [ "$SCENARIO" != migration-failure ] ;;
+  'ps -aq') if [ "$SCENARIO" = retained-in-use ]; then echo other-container; fi ;;
   'exec container123')
+   if [ -f "$TEST_DIR/restored" ]; then return 0; fi
+   if [ "$SCENARIO" = first-failure ] || [ "$SCENARIO" = rollback-failure ]; then return 1; fi
    if [ "$SCENARIO" = wrong-release ]; then return 1; fi
    if [ "$SCENARIO" = wrong-public ] && [[ "$*" = *example.test* ]]; then return 1; fi ;;
   *) return 0 ;;