feat(docker): restore failed Compose updates and retain recent releases
This commit is contained in:
Simo committed
1 parent
fe1ee8a6fe
commit
745d0b7247
4 files changed
+159
-16
No files matched your search
@@ -10,7 +10,48 @@ mkdir -p "$(dirname "$LOG_FILE")"
|
||||
log() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG_FILE"; }
|
||||
die() { log "ERROR: $*"; exit 1; }
|
||||
migration_image=""
|
||||
trap 'if [[ -n "$migration_image" ]]; then docker image rm "$migration_image" >>"$LOG_FILE" 2>&1 || true; fi' EXIT
|
||||
previous_image=""
|
||||
previous_release=""
|
||||
rollback_tag=""
|
||||
cutover=0
|
||||
probe='const r=await fetch(process.argv[1],{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.database!==true||d.release!==process.argv[2]){console.error(JSON.stringify({http:r.status,database:d.database,release:d.release,expected:process.argv[2]}));process.exit(1)}'
|
||||
verify_container() {
|
||||
local target="$1" release="$2" attempt
|
||||
for attempt in $(seq 1 30); do
|
||||
if docker exec "$target" node --input-type=module -e "$probe" "http://127.0.0.1:3002/api/health" "$release" >>"$LOG_FILE" 2>&1; then return 0; fi
|
||||
sleep 3
|
||||
done
|
||||
return 1
|
||||
}
|
||||
finish() {
|
||||
local status=$? restored
|
||||
trap - EXIT
|
||||
if [[ "$status" != 0 && "$cutover" = 1 ]]; then
|
||||
docker compose logs --tail 100 cms >>"$LOG_FILE" 2>&1 || true
|
||||
if [[ -n "$previous_image" ]]; then
|
||||
log "Update failed; restoring previous image $previous_image. Database migrations are not reversed."
|
||||
if CMS_RELEASE="$rollback_tag" docker compose up -d --no-deps --no-build --force-recreate cms >>"$LOG_FILE" 2>&1; then
|
||||
restored="$(docker compose ps -q cms 2>>"$LOG_FILE" || true)"
|
||||
if [[ -n "$restored" && "$(docker inspect --format '{{.Image}}' "$restored")" = "$previous_image" ]] && verify_container "$restored" "$previous_release"; then
|
||||
log "Rollback verified locally: $previous_release. Check public routing separately."
|
||||
else
|
||||
log "ERROR: rollback verification failed. Inspect $LOG_FILE; previous image retained as epicnext-cms:$rollback_tag."
|
||||
fi
|
||||
else
|
||||
log "ERROR: rollback could not recreate CMS. Previous image retained as epicnext-cms:$rollback_tag."
|
||||
fi
|
||||
else
|
||||
log "First installation failed: no previous image exists to restore. Candidate retained for diagnosis."
|
||||
fi
|
||||
fi
|
||||
if [[ -n "$migration_image" ]]; then docker image rm "$migration_image" >>"$LOG_FILE" 2>&1 || true; fi
|
||||
# Keep the recovery tag on failure for manual recovery, including same-commit rebuilds.
|
||||
if [[ ( "$status" = 0 || "$cutover" = 0 ) && -n "$rollback_tag" ]]; then docker image rm "epicnext-cms:$rollback_tag" >>"$LOG_FILE" 2>&1 || true; fi
|
||||
exit "$status"
|
||||
}
|
||||
trap finish EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
trap 'log "Update failed; inspect $LOG_FILE. No volumes or local files were deleted."' ERR
|
||||
|
||||
# An existing CI deployment is a different owner of the same host port.
|
||||
@@ -31,6 +72,14 @@ export CMS_RELEASE="$(git rev-parse HEAD)"
|
||||
[[ -f .env ]] || die "Create .env before installing or updating."
|
||||
docker info >/dev/null
|
||||
docker compose config --quiet
|
||||
previous_container="$(docker compose ps -q cms)"
|
||||
if [[ -n "$previous_container" ]]; then
|
||||
previous_image="$(docker inspect --format '{{.Image}}' "$previous_container")"
|
||||
previous_release="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$previous_image")"
|
||||
[[ "$previous_release" =~ ^[0-9a-f]{40}$ ]] || die "Previous image lacks a valid release label; automatic rollback cannot be verified."
|
||||
rollback_tag="rollback-$CMS_RELEASE-$$"
|
||||
docker image tag "$previous_image" "epicnext-cms:$rollback_tag"
|
||||
fi
|
||||
log "Building release $CMS_RELEASE from $DIR"
|
||||
# The builder contains the matching migration source and locked dependencies.
|
||||
# No Node/package manager installation on the host is required.
|
||||
@@ -42,19 +91,13 @@ revision="$(docker image inspect --format '{{index .Config.Labels "org.openconta
|
||||
[[ "$revision" = "$CMS_RELEASE" ]] || die "Built image has revision $revision, expected $CMS_RELEASE."
|
||||
docker run --rm --network host --entrypoint pnpm "$migration_image" db:migrate >>"$LOG_FILE" 2>&1
|
||||
log "Build and migrations completed; recreating only the CMS service."
|
||||
cutover=1
|
||||
docker compose up -d --no-deps --no-build --force-recreate cms >>"$LOG_FILE" 2>&1
|
||||
container="$(docker compose ps -q cms)"
|
||||
[[ -n "$container" ]] || die "Compose did not start the CMS container."
|
||||
actual_image="$(docker inspect --format '{{.Image}}' "$container")"
|
||||
[[ "$actual_image" = "$expected_image" ]] || die "Running image $actual_image differs from built image $expected_image."
|
||||
# Verify the actual HTTP response, not an environment variable supplied at run time.
|
||||
probe='const r=await fetch(process.argv[1],{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.database!==true||d.release!==process.argv[2]){console.error(JSON.stringify({http:r.status(),database:d.database,release:d.release,expected:process.argv[2]}));process.exit(1)}'
|
||||
healthy=0
|
||||
for attempt in $(seq 1 30); do
|
||||
if docker exec "$container" node --input-type=module -e "$probe" "http://127.0.0.1:3002/api/health" "$CMS_RELEASE" >>"$LOG_FILE" 2>&1; then healthy=1; break; fi
|
||||
sleep 3
|
||||
done
|
||||
[[ "$healthy" = 1 ]] || die "HTTP health/release verification failed. The candidate remains available for diagnosis; no success was recorded."
|
||||
verify_container "$container" "$CMS_RELEASE" || die "HTTP health/release verification failed."
|
||||
if [[ -n "${CMS_PUBLIC_URL:-}" ]]; then
|
||||
[[ "$CMS_PUBLIC_URL" = https://* || "$CMS_PUBLIC_URL" = http://* ]] || die "CMS_PUBLIC_URL must be an HTTP(S) URL."
|
||||
docker exec "$container" node --input-type=module -e "$probe" "${CMS_PUBLIC_URL%/}/api/health?release=$CMS_RELEASE" "$CMS_RELEASE" >>"$LOG_FILE" 2>&1 || die "Public domain serves another release or is unhealthy. Check reverse proxy/CDN destination."
|
||||
@@ -63,3 +106,25 @@ else
|
||||
log "Public domain was not checked. Set CMS_PUBLIC_URL to verify reverse proxy/CDN routing as well."
|
||||
fi
|
||||
log "Verified release $CMS_RELEASE, image $actual_image, container $container"
|
||||
cutover=0
|
||||
# Record only this checkout's successful release tags; never prune Docker globally.
|
||||
history="$DIR/logs/docker-release-history.log"
|
||||
mkdir -p "$DIR/logs"
|
||||
touch "$history"
|
||||
mapfile -t releases < <(printf '%s\n' "$CMS_RELEASE" "$previous_release"; cat "$history")
|
||||
kept=()
|
||||
pending=()
|
||||
for release in "${releases[@]}"; do
|
||||
[[ "$release" =~ ^[0-9a-f]{40}$ ]] || continue
|
||||
[[ " ${kept[*]} ${pending[*]} " != *" $release "* ]] || continue
|
||||
if [[ "${#kept[@]}" -lt 2 ]]; then kept+=("$release"); continue; fi
|
||||
# Even stopped containers belonging to other deployments protect an image.
|
||||
if users="$(docker ps -aq --filter "ancestor=epicnext-cms:$release")" && [[ -z "$users" ]] && docker image rm "epicnext-cms:$release" >>"$LOG_FILE" 2>&1; then
|
||||
log "Removed superseded release tag $release"
|
||||
else
|
||||
pending+=("$release")
|
||||
fi
|
||||
done
|
||||
printf '%s\n' "${kept[@]}" "${pending[@]}" > "$history.tmp"
|
||||
mv "$history.tmp" "$history"
|
||||
log "Keeping the two latest releases; in-use images and persistent volumes are preserved."
|
||||
Reference in new issue
Block a user