fix(deploy): verify Docker clone updates against the served release
This commit is contained in:
Simo committed
1 parent
3bac126ace
commit
cbaa115d56
15 files changed
+321
-187
No files matched your search
+54
-108
@@ -1,119 +1,65 @@
|
||||
#!/usr/bin/env bash
|
||||
# ==============================================================================
|
||||
# docker-update.sh — Automatic daily update for the Dockerized EpicNext-CMS.
|
||||
#
|
||||
# Steps:
|
||||
# 1. Verify the working tree is clean (uncommitted changes abort).
|
||||
# 2. git pull (fast-forward only).
|
||||
# 3. Run CMS migrations on the HOST (the slim runtime container has no source).
|
||||
# 4. docker compose build (auto-detects pnpm/yarn/npm via lockfile).
|
||||
# 5. docker compose up -d && wait for a healthy container.
|
||||
# 6. Record everything in update.log.
|
||||
#
|
||||
# Exit codes: 0 ok, 1 update skipped, 2 build/deploy failed, 3 health failed.
|
||||
# ==============================================================================
|
||||
|
||||
set -uo pipefail
|
||||
|
||||
# Update a Linux Docker Compose clone from its configured Git upstream.
|
||||
set -Eeuo pipefail
|
||||
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$DIR" || exit 2
|
||||
|
||||
# Share the CI lock before pulling or touching the live application.
|
||||
cd "$DIR"
|
||||
exec 9>"$DIR/.deploy.lock"
|
||||
flock -w 1800 9 || exit 2
|
||||
|
||||
flock -w 1800 9
|
||||
LOG_FILE="${LOG_FILE:-$DIR/logs/docker-update.log}"
|
||||
PM2_APP="${PM2_APP:-next}" # host-side CMS that must stay stopped (port 3002)
|
||||
mkdir -p "$(dirname "$LOG_FILE")"
|
||||
log() { printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" | tee -a "$LOG_FILE"; }
|
||||
die() { log "ERROR: $*"; exit 1; }
|
||||
migration_image=""
|
||||
trap 'if [[ -n "$migration_image" ]]; then docker image rm "$migration_image" >>"$LOG_FILE" 2>&1 || true; fi' EXIT
|
||||
trap 'log "Update failed; inspect $LOG_FILE. No volumes or local files were deleted."' ERR
|
||||
|
||||
log() { echo "[$(date '+%Y-%m-%d %H:%M:%S')] $*" | tee -a "$LOG_FILE"; }
|
||||
die() { log "ERROR: $*"; exit "${2:-2}"; }
|
||||
|
||||
touch "$LOG_FILE"
|
||||
|
||||
log "=== Start docker-update ==="
|
||||
|
||||
# --- 0. Preflight: verify this VPS is ready (permissions, ports, deps) ---
|
||||
if ! "$DIR/scripts/docker-preflight.sh"; then
|
||||
die "preflight failed — fix issues first (see '--fix' flag)" 1
|
||||
# An existing CI deployment is a different owner of the same host port.
|
||||
if [ "$(docker inspect --format '{{.State.Running}}' epicnext-cms-app 2>/dev/null || true)" = true ]; then
|
||||
die "This host is managed by CI (epicnext-cms-app). Update through CI, not a second Compose deployment."
|
||||
fi
|
||||
log "preflight OK"
|
||||
|
||||
# --- 0b. Guard: uncommitted changes would break git pull / taint deploys ---
|
||||
if ! { git diff --quiet --exit-code && git diff --cached --quiet --exit-code; }; then
|
||||
die "working tree has uncommitted changes; commit or stash first" 1
|
||||
[[ -z "$(git status --porcelain --untracked-files=normal)" ]] || die "Working tree is not clean. Commit or stash local work first."
|
||||
git rev-parse --abbrev-ref --symbolic-full-name '@{upstream}' >/dev/null || die "Configure this branch's Git upstream before updating."
|
||||
script_before="$(git hash-object scripts/docker-update.sh)"
|
||||
git pull --ff-only >>"$LOG_FILE" 2>&1
|
||||
if [ "$script_before" != "$(git hash-object scripts/docker-update.sh)" ]; then
|
||||
log "Updater changed; restarting the newly pulled script."
|
||||
exec 9>&-
|
||||
exec bash "$DIR/scripts/docker-update.sh"
|
||||
fi
|
||||
|
||||
# --- 1. Pull latest ---
|
||||
git pull --ff-only --quiet 2>>"$LOG_FILE"
|
||||
pull_status=$?
|
||||
if [ $pull_status -ne 0 ]; then
|
||||
die "git pull failed (status $pull_status)" 1
|
||||
fi
|
||||
log "git pull OK: $(git rev-parse --short HEAD)"
|
||||
|
||||
# --- 2. Host-side migrations (idempotent; only applies CMS-owned tables) ---
|
||||
if [ -f pnpm-lock.yaml ] && command -v pnpm >/dev/null 2>&1; then
|
||||
pnpm db:migrate >>"$LOG_FILE" 2>&1 || die "db:migrate (pnpm) failed"
|
||||
elif command -v npm >/dev/null 2>&1; then
|
||||
npm run db:migrate >>"$LOG_FILE" 2>&1 || die "db:migrate (npm) failed"
|
||||
else
|
||||
die "no package manager found for migrations" 2
|
||||
fi
|
||||
log "db:migrate OK"
|
||||
|
||||
# --- 3. Node major gate (patches auto, major upgrades need review) ---
|
||||
# `node:alpine` floats within, then across, Node majors. Patches/minors are
|
||||
# safe to apply silently; a NEW major (e.g. 26 -> 27) is a breaking risk for
|
||||
# native addons / Next compatibility, so require an explicit review before it
|
||||
# goes live. Compare the major of the deployed runtime image vs the floating
|
||||
# tag; abort (not deploy) when they differ.
|
||||
deployed_major="$(docker inspect --format '{{.Config.Image}}' epicnext-cms 2>/dev/null || true)"
|
||||
# Resolve the currently-deployed Node major from its image.
|
||||
if [ -n "$deployed_major" ] && docker image inspect "$deployed_major" >/dev/null 2>&1; then
|
||||
deployed_major="$(docker run --rm --entrypoint sh "$deployed_major" -c 'node -p "process.versions.node.split(\".\")[0]"' 2>/dev/null || true)"
|
||||
fi
|
||||
float_major="$(docker run --rm --entrypoint sh node:alpine -c 'node -p "process.versions.node.split(\".\")[0]"' 2>/dev/null || true)"
|
||||
if [ -n "$deployed_major" ] && [ -n "$float_major" ] && [ "$deployed_major" != "$float_major" ]; then
|
||||
die "Node major change detected (deployed v$deployed_major, floating tag v$float_major). Major upgrades require review; update engines/Dockerfile deliberately first." 1
|
||||
fi
|
||||
log "Node major gate OK (major=${float_major:-?})"
|
||||
|
||||
# --- 4. Rebuild the image ---
|
||||
# Reset the BuildKit cache first so the build doesn't accumulate unbounded
|
||||
# layers on disk across daily rebuilds.
|
||||
docker builder prune -af --filter "until=1h" --keep-storage=0 2>>"$LOG_FILE" || true
|
||||
docker compose build >>"$LOG_FILE" 2>&1 || die "docker compose build failed" 2
|
||||
log "docker compose build OK"
|
||||
|
||||
# --- 5. Recreate the container ---
|
||||
docker compose up -d >>"$LOG_FILE" 2>&1 || die "docker compose up failed" 2
|
||||
log "docker compose up OK"
|
||||
|
||||
# --- 6. Wait for health (up to ~4 min) ---
|
||||
export CMS_RELEASE="$(git rev-parse HEAD)"
|
||||
[[ "$CMS_RELEASE" =~ ^[0-9a-f]{40}$ ]] || die "Invalid Git commit."
|
||||
[[ -f .env ]] || die "Create .env before installing or updating."
|
||||
docker info >/dev/null
|
||||
docker compose config --quiet
|
||||
log "Building release $CMS_RELEASE from $DIR"
|
||||
# The builder contains the matching migration source and locked dependencies.
|
||||
# No Node/package manager installation on the host is required.
|
||||
migration_image="epicnext-cms-migrations:$CMS_RELEASE"
|
||||
docker build --network=host --target builder --build-arg NEXT_DEPLOYMENT_ID="$CMS_RELEASE" -t "$migration_image" . >>"$LOG_FILE" 2>&1
|
||||
docker compose build --build-arg NEXT_DEPLOYMENT_ID="$CMS_RELEASE" cms >>"$LOG_FILE" 2>&1
|
||||
expected_image="$(docker image inspect --format '{{.Id}}' "epicnext-cms:$CMS_RELEASE")"
|
||||
revision="$(docker image inspect --format '{{index .Config.Labels "org.opencontainers.image.revision"}}' "$expected_image")"
|
||||
[[ "$revision" = "$CMS_RELEASE" ]] || die "Built image has revision $revision, expected $CMS_RELEASE."
|
||||
docker run --rm --network host --entrypoint pnpm "$migration_image" db:migrate >>"$LOG_FILE" 2>&1
|
||||
log "Build and migrations completed; recreating only the CMS service."
|
||||
docker compose up -d --no-deps --no-build --force-recreate cms >>"$LOG_FILE" 2>&1
|
||||
container="$(docker compose ps -q cms)"
|
||||
[[ -n "$container" ]] || die "Compose did not start the CMS container."
|
||||
actual_image="$(docker inspect --format '{{.Image}}' "$container")"
|
||||
[[ "$actual_image" = "$expected_image" ]] || die "Running image $actual_image differs from built image $expected_image."
|
||||
# Verify the actual HTTP response, not an environment variable supplied at run time.
|
||||
probe='const r=await fetch(process.argv[1],{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.database!==true||d.release!==process.argv[2]){console.error(JSON.stringify({http:r.status(),database:d.database,release:d.release,expected:process.argv[2]}));process.exit(1)}'
|
||||
healthy=0
|
||||
for i in $(seq 1 16); do
|
||||
status="$(docker inspect --format='{{.State.Health.Status}}' epicnext-cms 2>/dev/null || true)"
|
||||
case "$status" in
|
||||
healthy) healthy=1; break ;;
|
||||
unhealthy) break ;;
|
||||
esac
|
||||
sleep 15
|
||||
for attempt in $(seq 1 30); do
|
||||
if docker exec "$container" node --input-type=module -e "$probe" "http://127.0.0.1:3002/api/health" "$CMS_RELEASE" >>"$LOG_FILE" 2>&1; then healthy=1; break; fi
|
||||
sleep 3
|
||||
done
|
||||
|
||||
if [ "$healthy" -eq 1 ]; then
|
||||
log "CMS healthy after update (commit $(git rev-parse --short HEAD))"
|
||||
[[ "$healthy" = 1 ]] || die "HTTP health/release verification failed. The candidate remains available for diagnosis; no success was recorded."
|
||||
if [[ -n "${CMS_PUBLIC_URL:-}" ]]; then
|
||||
[[ "$CMS_PUBLIC_URL" = https://* || "$CMS_PUBLIC_URL" = http://* ]] || die "CMS_PUBLIC_URL must be an HTTP(S) URL."
|
||||
docker exec "$container" node --input-type=module -e "$probe" "${CMS_PUBLIC_URL%/}/api/health?release=$CMS_RELEASE" "$CMS_RELEASE" >>"$LOG_FILE" 2>&1 || die "Public domain serves another release or is unhealthy. Check reverse proxy/CDN destination."
|
||||
log "Public URL verified: $CMS_PUBLIC_URL"
|
||||
else
|
||||
log "WARNING: container not healthy (status='${status:-unknown}')"
|
||||
# Leave the container running so it can be debugged; report failure exit.
|
||||
exit 3
|
||||
log "Public domain was not checked. Set CMS_PUBLIC_URL to verify reverse proxy/CDN routing as well."
|
||||
fi
|
||||
|
||||
# --- 7. Make sure the stale host-side PM2 CMS stays stopped ---
|
||||
if command -v pm2 >/dev/null 2>&1 && pm2 jlist >/dev/null 2>&1; then
|
||||
if pm2 list 2>/dev/null | grep -q "${PM2_APP}"; then
|
||||
pm2 stop "$PM2_APP" >/dev/null 2>&1 && log "pm2 '${PM2_APP}' kept stopped (avoids port 3002 clash)"
|
||||
fi
|
||||
fi
|
||||
|
||||
log "=== docker-update finished OK ==="
|
||||
exit 0
|
||||
log "Verified release $CMS_RELEASE, image $actual_image, container $container"
|
||||
Reference in new issue
Block a user