fix(deploy): wait for the expected HTTP release and report failed probes

scripts/ci-deploy.sh

@@ -49,6 +49,7 @@ finish() {
   if [ "$status" -ne 0 ] && [ "$cutover_started" -eq 1 ]; then
     echo "Deployment failed; restoring previous container" >&2
     docker logs epicnext-cms-app --tail 50 >&2 || true
+    if command -v ss >/dev/null 2>&1; then ss -ltnp 'sport = :3002' >&2 || true; fi
     if [ "$candidate_attempted" -eq 1 ]; then docker rm -f epicnext-cms-app || true; fi
     if [ "$backup_created" -eq 1 ]; then docker rename "$backup_name" "$previous_name" || true; fi
     if [ -n "$previous_name" ]; then
@@ -123,7 +124,7 @@ candidate_attempted=1
     ENV_ARGS+=(-e "$key")
   done < "$deploy_dir/.env"
   docker run -d --name epicnext-cms-app --restart always --net=host \
-    "${ENV_ARGS[@]}" \
+    "${ENV_ARGS[@]}" -e PORT=3002 -e HOSTNAME=0.0.0.0 \
     -v "$deploy_dir/public/nitro-assets:/app/public/nitro-assets" \
     -v "$deploy_dir/public/swf:/app/public/swf" \
     -v "$deploy_dir/storage:/app/storage" \
@@ -131,7 +132,7 @@ candidate_attempted=1
     "$image"
 )
 healthy
-node --input-type=module -e 'const r=await fetch("http://127.0.0.1:3002/api/health",{cache:"no-store",signal:AbortSignal.timeout(5000)});const d=await r.json();if(!r.ok||d.release!==process.argv[1]){console.error("Release mismatch",d.release,process.argv[1]);process.exit(1)}' "$sha"
+node scripts/verify-deployed-release.mjs http://127.0.0.1:3002/api/health "$sha"
 PLAYWRIGHT_BASE_URL=http://127.0.0.1:3002 pnpm test:e2e
 # Publish the latest alias only after health and browser checks pass.
 docker tag "$image" epicnext-cms:latest

scripts/verify-deployed-release.mjs

@@ -0,0 +1,58 @@
+import { resolve } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { pathToFileURL } from "node:url";
+
+export async function verifyDeployedRelease(
+	url,
+	expected,
+	{ attempts = 30, interval = 3000, report = console.error } = {},
+) {
+	if (!/^[a-f0-9]{40}$/.test(expected))
+		throw new Error("Expected a full Git commit SHA");
+	const target = new URL(url);
+	if (!["http:", "https:"].includes(target.protocol))
+		throw new Error("Expected an HTTP(S) health URL");
+	for (let attempt = 1; attempt <= attempts; attempt++) {
+		let observed;
+		try {
+			target.searchParams.set("expectedRelease", expected);
+			target.searchParams.set("probe", String(attempt));
+			const response = await fetch(target, {
+				cache: "no-store",
+				redirect: "error",
+				headers: { "Cache-Control": "no-cache" },
+				signal: AbortSignal.timeout(5000),
+			});
+			const body = await response.json();
+			observed = {
+				http: response.status,
+				status: body.status ?? null,
+				database: body.database ?? null,
+				release: body.release ?? null,
+			};
+			if (response.ok && body.database === true && body.release === expected)
+				return;
+		} catch (error) {
+			observed = {
+				error: error instanceof Error ? error.message : String(error),
+			};
+		}
+		report(JSON.stringify({ attempt, attempts, expected, observed }));
+		if (attempt < attempts) await delay(interval);
+	}
+	throw new Error(
+		"Expected release never became healthy. Check container startup, port ownership and the HTTP destination.",
+	);
+}
+if (
+	process.argv[1] &&
+	import.meta.url === pathToFileURL(resolve(process.argv[1])).href
+) {
+	try {
+		await verifyDeployedRelease(process.argv[2], process.argv[3]);
+		console.log(`HTTP release verified: ${process.argv[3]}`);
+	} catch (error) {
+		console.error(error instanceof Error ? error.message : String(error));
+		process.exitCode = 1;
+	}
+}

src/lib/verify-deployed-release.test.ts

@@ -0,0 +1,48 @@
+import { afterEach, describe, expect, it, vi } from "vitest";
+import { verifyDeployedRelease } from "../../scripts/verify-deployed-release.mjs";
+
+const sha = "a".repeat(40);
+afterEach(() => vi.unstubAllGlobals());
+describe("deployed HTTP release readiness", () => {
+	it("waits through an old response and startup failure until the exact release is ready", async () => {
+		const fetch = vi
+			.fn()
+			.mockResolvedValueOnce(Response.json({ database: true }))
+			.mockRejectedValueOnce(new Error("connection refused"))
+			.mockResolvedValueOnce(Response.json({ database: true, release: sha }));
+		vi.stubGlobal("fetch", fetch);
+		const report = vi.fn();
+		await verifyDeployedRelease("http://localhost:3002/api/health", sha, {
+			attempts: 3,
+			interval: 0,
+			report,
+		});
+		expect(fetch).toHaveBeenCalledTimes(3);
+		expect(report).toHaveBeenCalledTimes(2);
+		expect(JSON.parse(report.mock.calls[0][0]).observed.release).toBeNull();
+	});
+	it.each([
+		[200, { database: true, release: "old" }],
+		[200, { database: true }],
+		[429, { status: "rate_limited" }],
+		[200, { database: false, release: sha }],
+	])(
+		"rejects HTTP %s without the expected healthy release",
+		async (status, body) => {
+			vi.stubGlobal(
+				"fetch",
+				vi.fn().mockResolvedValue(Response.json(body, { status })),
+			);
+			const report = vi.fn();
+			await expect(
+				verifyDeployedRelease("http://localhost:3002/api/health", sha, {
+					attempts: 2,
+					interval: 0,
+					report,
+				}),
+			).rejects.toThrow("never became healthy");
+			expect(report).toHaveBeenCalledTimes(2);
+			expect(JSON.parse(report.mock.calls[0][0]).observed.http).toBe(status);
+		},
+	);
+});