From c35fc764bc7d2a820d09e5f7e69384aca6d9ee2a Mon Sep 17 00:00:00 2001 From: simoleo89 Date: Mon, 7 Sep 2026 21:28:18 +0200 Subject: [PATCH] fix(deploy): wait for the expected HTTP release and report failed probes --- scripts/ci-deploy.sh | 5 ++- scripts/verify-deployed-release.mjs | 58 +++++++++++++++++++++++++ src/lib/verify-deployed-release.test.ts | 48 ++++++++++++++++++++ 3 files changed, 109 insertions(+), 2 deletions(-) create mode 100644 scripts/verify-deployed-release.mjs create mode 100644 src/lib/verify-deployed-release.test.ts diff --git a/scripts/ci-deploy.sh b/scripts/ci-deploy.sh index 44012f8425..b77c89073b 100644 --- a/scripts/ci-deploy.sh +++ b/scripts/ci-deploy.sh @@ -49,6 +49,7 @@ finish() { if [ "$status" -ne 0 ] && [ "$cutover_started" -eq 1 ]; then echo "Deployment failed; restoring previous container" >&2 docker logs epicnext-cms-app --tail 50 >&2 || true + if command -v ss >/dev/null 2>&1; then ss -ltnp 'sport = :3002' >&2 || true; fi if [ "$candidate_attempted" -eq 1 ]; then docker rm -f epicnext-cms-app || true; fi if [ "$backup_created" -eq 1 ]; then docker rename "$backup_name" "$previous_name" || true; fi if [ -n "$previous_name" ]; then @@ -123,7 +124,7 @@ candidate_attempted=1 ENV_ARGS+=(-e "$key") done < "$deploy_dir/.env" docker run -d --name epicnext-cms-app --restart always --net=host \ - "${ENV_ARGS[@]}" \ + "${ENV_ARGS[@]}" -e PORT=3002 -e HOSTNAME=0.0.0.0 \ -v "$deploy_dir/public/nitro-assets:/app/public/nitro-assets" \ -v "$deploy_dir/public/swf:/app/public/swf" \ -v "$deploy_dir/storage:/app/storage" \ @@ -131,7 +132,7 @@ candidate_attempted=1 "$image" ) healthy -node --input-type=module -e 'const r=await fetch("http://127.0.0.1:3002/api/health",{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.release!==process.argv[1]){console.error("Release mismatch",d.release,process.argv[1]);process.exit(1)}' "$sha" +node scripts/verify-deployed-release.mjs http://127.0.0.1:3002/api/health "$sha" PLAYWRIGHT_BASE_URL=http://127.0.0.1:3002 pnpm test:e2e # Publish the latest alias only after health and browser checks pass. docker tag "$image" epicnext-cms:latest diff --git a/scripts/verify-deployed-release.mjs b/scripts/verify-deployed-release.mjs new file mode 100644 index 0000000000..b70877ea81 --- /dev/null +++ b/scripts/verify-deployed-release.mjs @@ -0,0 +1,58 @@ +import { resolve } from "node:path"; +import { setTimeout as delay } from "node:timers/promises"; +import { pathToFileURL } from "node:url"; + +export async function verifyDeployedRelease( + url, + expected, + { attempts = 30, interval = 3000, report = console.error } = {}, +) { + if (!/^[a-f0-9]{40}$/.test(expected)) + throw new Error("Expected a full Git commit SHA"); + const target = new URL(url); + if (!["http:", "https:"].includes(target.protocol)) + throw new Error("Expected an HTTP(S) health URL"); + for (let attempt = 1; attempt <= attempts; attempt++) { + let observed; + try { + target.searchParams.set("expectedRelease", expected); + target.searchParams.set("probe", String(attempt)); + const response = await fetch(target, { + cache: "no-store", + redirect: "error", + headers: { "Cache-Control": "no-cache" }, + signal: AbortSignal.timeout(5000), + }); + const body = await response.json(); + observed = { + http: response.status, + status: body.status ?? null, + database: body.database ?? null, + release: body.release ?? null, + }; + if (response.ok && body.database === true && body.release === expected) + return; + } catch (error) { + observed = { + error: error instanceof Error ? error.message : String(error), + }; + } + report(JSON.stringify({ attempt, attempts, expected, observed })); + if (attempt < attempts) await delay(interval); + } + throw new Error( + "Expected release never became healthy. Check container startup, port ownership and the HTTP destination.", + ); +} +if ( + process.argv[1] && + import.meta.url === pathToFileURL(resolve(process.argv[1])).href +) { + try { + await verifyDeployedRelease(process.argv[2], process.argv[3]); + console.log(`HTTP release verified: ${process.argv[3]}`); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 1; + } +} diff --git a/src/lib/verify-deployed-release.test.ts b/src/lib/verify-deployed-release.test.ts new file mode 100644 index 0000000000..06b74ab298 --- /dev/null +++ b/src/lib/verify-deployed-release.test.ts @@ -0,0 +1,48 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; +import { verifyDeployedRelease } from "../../scripts/verify-deployed-release.mjs"; + +const sha = "a".repeat(40); +afterEach(() => vi.unstubAllGlobals()); +describe("deployed HTTP release readiness", () => { + it("waits through an old response and startup failure until the exact release is ready", async () => { + const fetch = vi + .fn() + .mockResolvedValueOnce(Response.json({ database: true })) + .mockRejectedValueOnce(new Error("connection refused")) + .mockResolvedValueOnce(Response.json({ database: true, release: sha })); + vi.stubGlobal("fetch", fetch); + const report = vi.fn(); + await verifyDeployedRelease("http://localhost:3002/api/health", sha, { + attempts: 3, + interval: 0, + report, + }); + expect(fetch).toHaveBeenCalledTimes(3); + expect(report).toHaveBeenCalledTimes(2); + expect(JSON.parse(report.mock.calls[0][0]).observed.release).toBeNull(); + }); + it.each([ + [200, { database: true, release: "old" }], + [200, { database: true }], + [429, { status: "rate_limited" }], + [200, { database: false, release: sha }], + ])( + "rejects HTTP %s without the expected healthy release", + async (status, body) => { + vi.stubGlobal( + "fetch", + vi.fn().mockResolvedValue(Response.json(body, { status })), + ); + const report = vi.fn(); + await expect( + verifyDeployedRelease("http://localhost:3002/api/health", sha, { + attempts: 2, + interval: 0, + report, + }), + ).rejects.toThrow("never became healthy"); + expect(report).toHaveBeenCalledTimes(2); + expect(JSON.parse(report.mock.calls[0][0]).observed.http).toBe(status); + }, + ); +});