ci: serialize deployments and restore previous release on smoke failure
This commit is contained in:
Simo committed
1 parent
85a6df162b
commit
7c1f109a9d
7 files changed
+369
-234
No files matched your search
@@ -0,0 +1,148 @@
|
||||
#!/usr/bin/env bash
|
||||
# One lock covers build, migrations, cutover, health checks and smoke tests.
|
||||
set -Eeuo pipefail
|
||||
|
||||
deploy_dir="${CMS_DEPLOY_DIR:-/var/www/atom-nexst}"
|
||||
branch="${DEPLOY_BRANCH:-main}"
|
||||
case "$branch" in main|master) ;; *) echo "Unsupported deployment branch" >&2; exit 1 ;; esac
|
||||
exec 9>"$deploy_dir/.deploy.lock"
|
||||
flock -w 1800 9
|
||||
|
||||
sha="$(git rev-parse HEAD)"
|
||||
[[ "$sha" =~ ^[0-9a-f]{40}$ ]] || { echo "Invalid commit" >&2; exit 1; }
|
||||
image="epicnext-cms:$sha"
|
||||
previous_name=""
|
||||
previous_image=""
|
||||
backup_name="epicnext-cms-rollback"
|
||||
cutover_started=0
|
||||
candidate_attempted=0
|
||||
backup_created=0
|
||||
|
||||
is_current() {
|
||||
local head
|
||||
head="$(git ls-remote --exit-code origin "refs/heads/$branch")" || return 2
|
||||
head="${head%%[[:space:]]*}"
|
||||
if [ "$head" != "$sha" ]; then
|
||||
echo "Skipping superseded commit $sha (branch now at $head)"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_current() {
|
||||
local status=0
|
||||
is_current || status=$?
|
||||
case "$status" in 0) ;; 1) exit 0 ;; *) echo "Cannot verify remote branch" >&2; exit 1 ;; esac
|
||||
}
|
||||
|
||||
healthy() {
|
||||
local attempt
|
||||
for attempt in $(seq 1 30); do
|
||||
if curl -sf --max-time 5 http://127.0.0.1:3002/api/health | grep -q '"database":true'; then return 0; fi
|
||||
sleep 3
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
finish() {
|
||||
local status=$?
|
||||
trap - EXIT
|
||||
if [ "$status" -ne 0 ] && [ "$cutover_started" -eq 1 ]; then
|
||||
echo "Deployment failed; restoring previous container" >&2
|
||||
docker logs epicnext-cms-app --tail 50 >&2 || true
|
||||
if [ "$candidate_attempted" -eq 1 ]; then docker rm -f epicnext-cms-app || true; fi
|
||||
if [ "$backup_created" -eq 1 ]; then docker rename "$backup_name" "$previous_name" || true; fi
|
||||
if [ -n "$previous_name" ]; then
|
||||
if docker start "$previous_name" && healthy; then
|
||||
echo "Rollback verified: $previous_image"
|
||||
else
|
||||
echo "ERROR: previous container could not be restored to healthy state" >&2
|
||||
fi
|
||||
else
|
||||
echo "No previous container exists; rollback is unavailable" >&2
|
||||
fi
|
||||
fi
|
||||
exit "$status"
|
||||
}
|
||||
trap finish EXIT
|
||||
trap 'exit 130' INT
|
||||
trap 'exit 143' TERM
|
||||
|
||||
check_current
|
||||
cp "$deploy_dir/.env" .env
|
||||
pnpm install --frozen-lockfile
|
||||
# Prepare browser before cutover so installation failures cannot interrupt the site.
|
||||
pnpm exec playwright install chromium
|
||||
|
||||
echo "Building $image"
|
||||
DOCKER_BUILDKIT=1 docker build --network=host --progress=plain --cache-from epicnext-cms:latest -t "$image" .
|
||||
check_current
|
||||
pnpm db:migrate
|
||||
check_current
|
||||
|
||||
# Prefer the active CI container, or the active legacy compose container.
|
||||
for name in epicnext-cms-app epicnext-cms; do
|
||||
if [ "$(docker inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)" = true ]; then
|
||||
previous_name="$name"
|
||||
break
|
||||
fi
|
||||
done
|
||||
if [ -z "$previous_name" ]; then
|
||||
for name in epicnext-cms-app epicnext-cms; do
|
||||
if docker inspect "$name" >/dev/null 2>&1; then previous_name="$name"; break; fi
|
||||
done
|
||||
fi
|
||||
if docker inspect "$backup_name" >/dev/null 2>&1; then
|
||||
echo "Unresolved rollback container exists; refusing to overwrite it" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -n "$previous_name" ]; then
|
||||
previous_image="$(docker inspect --format '{{.Image}}' "$previous_name")"
|
||||
docker tag "$previous_image" epicnext-cms:previous
|
||||
fi
|
||||
# Remove a stopped leftover CI container when the compose container is active.
|
||||
if [ "$previous_name" != epicnext-cms-app ] && docker inspect epicnext-cms-app >/dev/null 2>&1; then
|
||||
docker rm epicnext-cms-app
|
||||
fi
|
||||
|
||||
cutover_started=1
|
||||
if [ -n "$previous_name" ]; then
|
||||
docker stop "$previous_name"
|
||||
docker rename "$previous_name" "$backup_name"
|
||||
backup_created=1
|
||||
fi
|
||||
candidate_attempted=1
|
||||
(
|
||||
set -a
|
||||
# shellcheck disable=SC1091
|
||||
. "$deploy_dir/.env"
|
||||
set +a
|
||||
ENV_ARGS=()
|
||||
while IFS='=' read -r key _; do
|
||||
case "$key" in ''|'#'*|*[!A-Za-z0-9_]* ) continue ;; esac
|
||||
ENV_ARGS+=(-e "$key")
|
||||
done < "$deploy_dir/.env"
|
||||
docker run -d --name epicnext-cms-app --restart always --net=host \
|
||||
"${ENV_ARGS[@]}" \
|
||||
-v "$deploy_dir/public/nitro-assets:/app/public/nitro-assets" \
|
||||
-v "$deploy_dir/public/swf:/app/public/swf" \
|
||||
-v "$deploy_dir/storage:/app/storage" \
|
||||
-v /var/www/Gamedata:/var/www/Gamedata \
|
||||
"$image"
|
||||
)
|
||||
healthy
|
||||
PLAYWRIGHT_BASE_URL=http://127.0.0.1:3002 pnpm test:e2e
|
||||
# Publish the latest alias only after health and browser checks pass.
|
||||
docker tag "$image" epicnext-cms:latest
|
||||
cutover_started=0
|
||||
if [ "$backup_created" -eq 1 ]; then docker rm "$backup_name" || true; fi
|
||||
|
||||
echo "Deployment verified: $sha"
|
||||
# Retain the current and previous releases; do not remove arbitrary named tags.
|
||||
while IFS= read -r tag; do
|
||||
if [[ "$tag" =~ ^epicnext-cms:[0-9a-f]{40}$ ]] && [ "$tag" != "$image" ]; then
|
||||
tagged_image="$(docker image inspect --format '{{.Id}}' "$tag" 2>/dev/null || true)"
|
||||
if [ -n "$tagged_image" ] && [ "$tagged_image" != "$previous_image" ]; then docker image rm "$tag" || true; fi
|
||||
fi
|
||||
done < <(docker image ls --format '{{.Repository}}:{{.Tag}}' epicnext-cms)
|
||||
docker builder prune -af --filter "until=72h" --keep-storage=2g || true
|
||||
docker image prune -f --filter "until=168h" || true
|
||||
Reference in new issue
Block a user