ci: serialize deployments and restore previous release on smoke failure

src/lib/ci-deploy.test.ts

@@ -0,0 +1,129 @@
+import { spawnSync } from "node:child_process";
+import {
+	existsSync,
+	mkdirSync,
+	mkdtempSync,
+	readFileSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { delimiter, dirname, join, resolve } from "node:path";
+import { describe, expect, it } from "vitest";
+
+const root = process.cwd();
+const bash =
+	process.platform === "win32"
+		? ((process.env.PATH ?? "")
+				.split(delimiter)
+				.flatMap((dir) => [
+					join(dir, "bash.exe"),
+					join(dirname(dir), "bin", "bash.exe"),
+					join(dirname(dirname(dir)), "bin", "bash.exe"),
+				])
+				.find((path) => existsSync(path)) ?? "bash")
+		: "bash";
+const sha = "a".repeat(40);
+function simulate(scenario: string, previous = "epicnext-cms-app") {
+	const dir = mkdtempSync(join(tmpdir(), "cms-deploy-test-"));
+	try {
+		mkdirSync(join(dir, "production"));
+		writeFileSync(join(dir, "production", ".env"), 'HOTEL_NAME="Test Hotel"\n');
+		if (previous) writeFileSync(join(dir, previous), "old\n");
+		const result = spawnSync(bash, [resolve(root, "scripts/ci-deploy.sh")], {
+			cwd: dir,
+			encoding: "utf8",
+			timeout: 15000,
+			env: {
+				...process.env,
+				BASH_ENV: resolve(root, "src/test/ci-deploy-harness.sh"),
+				TEST_DIR: dir.replaceAll("\\", "/"),
+				CMS_DEPLOY_DIR: join(dir, "production").replaceAll("\\", "/"),
+				TEST_SHA: sha,
+				SCENARIO: scenario,
+				DEPLOY_BRANCH: "main",
+			},
+		});
+		if (result.error) throw result.error;
+		return {
+			status: result.status,
+			output: result.stdout + result.stderr,
+			calls: existsSync(join(dir, "calls"))
+				? readFileSync(join(dir, "calls"), "utf8")
+				: "",
+			app: existsSync(join(dir, "epicnext-cms-app"))
+				? readFileSync(join(dir, "epicnext-cms-app"), "utf8").trim()
+				: null,
+			previousRestored: previous ? existsSync(join(dir, previous)) : false,
+			backup: existsSync(join(dir, "epicnext-cms-rollback")),
+		};
+	} finally {
+		rmSync(dir, { recursive: true, force: true });
+	}
+}
+
+describe("deployment transaction", () => {
+	it("publishes the commit image only after migrations, health and smoke tests", () => {
+		const result = simulate("success");
+		expect(result.status, result.output).toBe(0);
+		expect(result.app).toBe("new");
+		expect(result.backup).toBe(false);
+		expect(result.calls.indexOf("pnpm db:migrate")).toBeLessThan(
+			result.calls.indexOf("docker stop"),
+		);
+		expect(result.calls.indexOf("pnpm test:e2e")).toBeLessThan(
+			result.calls.indexOf(
+				`docker tag epicnext-cms:${sha} epicnext-cms:latest`,
+			),
+		);
+		expect(result.calls).toContain(
+			"docker tag sha256:old epicnext-cms:previous",
+		);
+	});
+	it.each(["run-failure", "health-failure", "smoke-failure"])(
+		"restores the exact previous container after %s",
+		(scenario) => {
+			const result = simulate(scenario);
+			expect(result.status).not.toBe(0);
+			expect(result.app).toBe("old");
+			expect(result.output).toContain("Rollback verified: sha256:old");
+			expect(result.calls).not.toContain(
+				`docker tag epicnext-cms:${sha} epicnext-cms:latest`,
+			);
+		},
+	);
+	it("restores the legacy compose container on failure", () => {
+		const result = simulate("smoke-failure", "epicnext-cms");
+		expect(result.status).not.toBe(0);
+		expect(result.app).toBeNull();
+		expect(result.previousRestored).toBe(true);
+		expect(result.calls).toContain("docker start epicnext-cms");
+	});
+	it.each([
+		"lock-failure",
+		"remote-failure",
+		"build-failure",
+		"migration-failure",
+	])("leaves the active container untouched after %s", (scenario) => {
+		const result = simulate(scenario);
+		expect(result.status).not.toBe(0);
+		expect(result.app).toBe("old");
+		expect(result.calls).not.toContain("docker stop");
+	});
+	it.each(["stale", "superseded"])(
+		"skips a %s commit without touching production",
+		(scenario) => {
+			const result = simulate(scenario);
+			expect(result.status, result.output).toBe(0);
+			expect(result.app).toBe("old");
+			expect(result.calls).not.toContain("pnpm db:migrate");
+			expect(result.calls).not.toContain("docker stop");
+		},
+	);
+	it("reports a failed first deployment without inventing a rollback", () => {
+		const result = simulate("smoke-failure", "");
+		expect(result.status).not.toBe(0);
+		expect(result.output).toContain("No previous container exists");
+		expect(result.app).toBeNull();
+	});
+});