fix(ops): stop a compose replica from blocking the blue/green release
Gitea Actions Runner Test / test-job (push) Successful in 1s
CI / check (push) Successful in 30s
CI / tests-integration (push) Successful in 1m57s
CI / tests-unit (push) Successful in 2m3s
CI / tests-ui (push) Successful in 2m49s
CI / preflight (push) Skipped
CI / deploy (push) Successful in 2m48s

The deploy failed after the build, the migrations and the browser gate:
"Port 3002 is already in use". The holder was `epicnext-cms`, a compose
replica of release 6bffc537 that the daily scripts/docker-update.sh cron
had recreated at 03:30 with restart=unless-stopped. nginx serves the green
slot on 3003, so that replica was squatting the blue slot the next
candidate needed, and live traffic never noticed.

It got there because the updater's CI-ownership guard only tested
epicnext-cms-app. After a cutover to the green slot that container is
stopped, renamed and deleted, so the guard stopped firing while the host
stayed CI-managed.

- scripts/docker-update.sh: refuse a compose deployment on a CI host by
  checking both slot containers and the nginx upstream, which is the only
  thing that still marks the host as blue/green while a slot is idle.
- scripts/ci-deploy.sh: retire a compose replica of this checkout from
  the candidate port before starting the candidate, so a stray replica
  can never block a release again. Never a slot container, never the port
  nginx serves; anything else still fails loudly in assert_port_free.
- Tests cover both directions: a squatting replica is removed and the
  release lands, a replica on the live port is left alone.
This commit is contained in:
openhands committed 2026-10-05 21:13:49 +02:00
1 parent 11ad6d4376
commit 5b2eb91c5c
7 files changed
+186 -9

No files matched your search

+32
View File
@@ -218,6 +218,11 @@ function simulateBlueGreen(scenario: string, livePort = 3002) {
writeFileSync(join(dir, "nginx-site.conf"), "upstream cms_app { }\n");
// Het live container-bestand, zodat `docker inspect` hem als draaiend ziet.
writeFileSync(join(dir, "epicnext-cms-app"), "old\n");
// Een compose-replica uit deze checkout (scenario 'port-taken-compose' of
// 'compose-live'); retire_compose_replicas() in het script zoekt hem op bij
// zijn container-id en beslist aan zijn PORT of hij live draait.
if (scenario === "port-taken-compose" || scenario === "compose-live")
writeFileSync(join(dir, "epicnext-cms"), "compose\n");
const result = spawnSync(bash, [resolve(root, "scripts/ci-deploy.sh")], {
cwd: dir,
encoding: "utf8",
@@ -316,6 +321,33 @@ describe("blue/green cutover", () => {
expect(r.upstream).not.toContain("127.0.0.1:3003");
});
it("removes a compose replica that squats the candidate port and then deploys", () => {
// Live op 3003, dus de kandidaat moet op 3002. Een compose-replica uit
// deze checkout zit daar al: precies de situatie die de release tegenhield
// totdat iemand de container met de hand verwijderde.
const r = simulateBlueGreen("port-taken-compose", 3003);
expect(r.status, r.output).toBe(0);
expect(r.output).toContain(
"Removing compose replica epicnext-cms on port 3002",
);
// Vóór het starten van de kandidaat, anders blijft 3002 bezet.
expect(r.calls.indexOf("docker rm -f epicnext-cms\n")).toBeGreaterThan(-1);
expect(r.calls.indexOf("docker rm -f epicnext-cms\n")).toBeLessThan(
r.calls.indexOf("docker run -d --name epicnext-cms-app"),
);
expect(r.upstream).toContain("127.0.0.1:3002");
});
it("leaves a compose replica alone when it is the live release", () => {
// De replica draait hier op 3003, de poort waar nginx naar wijst. Hoe
// onheilijk een compose-container ook is, hij serveert het verkeer en
// wordt dus niet weggenomen.
const r = simulateBlueGreen("compose-live", 3003);
expect(r.status, r.output).toBe(0);
expect(r.calls).not.toContain("docker rm -f epicnext-cms\n");
expect(r.upstream).toContain("127.0.0.1:3002");
});
it.each([
"run-failure",
"health-failure",
+38
View File
@@ -56,6 +56,22 @@ function simulate(scenario: string, args: string[] = []) {
join(dir, "scripts/docker-prune.sh"),
);
writeFileSync(join(dir, ".env"), "HOTEL_NAME=Test\n");
// De CI-eigendomscontrole leest het nginx-upstream-bestand, net als
// scripts/ci-deploy.sh. Tests draaien op de productiehost, dus het
// scenario 'ci-upstream' levert zelf een bestand met een slot erin en alle
// andere scenario's een leeg bestand — anders zou elke test hier op een
// echte blue/green-host onterecht stoppen.
writeFileSync(
join(
dir,
scenario === "ci-upstream"
? "cms_upstream_servers.conf"
: "no-such-upstream.conf",
),
scenario === "ci-upstream"
? "server 127.0.0.1:3003 max_fails=2 fail_timeout=10s;\n"
: "",
);
if (scenario.startsWith("saved-"))
writeFileSync(
join(dir, ".docker-install"),
@@ -82,6 +98,12 @@ function simulate(scenario: string, args: string[] = []) {
: scenario.startsWith("registry")
? "registry.test/team/cms"
: "",
CMS_UPSTREAM_FILE: join(
dir,
scenario === "ci-upstream"
? "cms_upstream_servers.conf"
: "no-such-upstream.conf",
).replaceAll("\\", "/"),
},
},
);
@@ -169,6 +191,8 @@ describe("Docker clone updates", () => {
it.each([
"dirty",
"ci-active",
"ci-green",
"ci-upstream",
"pull-failure",
"build-failure",
"migration-failure",
@@ -177,6 +201,20 @@ describe("Docker clone updates", () => {
expect(r.status, r.output).not.toBe(0);
expect(r.calls).not.toContain("compose up");
});
// Regressie: na een cutover naar het groene slot bestaat epicnext-cms-app
// niet meer en zag deze controle alleen een vrijgekomen blauwe container. De
// dagelijkse `docker-update.sh` startte toen een compose-replica op poort
// 3002 en blokkeerde elke volgende release. De blauwe container alleen
// controleren is dus geen bewijs dat de host niet door CI beheerd wordt.
it.each(["ci-green", "ci-upstream"])(
"refuses a compose deployment while CI owns the host (%s)",
(scenario) => {
const r = simulate(scenario);
expect(r.status, r.output).not.toBe(0);
expect(r.output).toContain("This host is managed by CI");
expect(r.calls).not.toContain("compose up");
},
);
it.each(["wrong-image", "wrong-release", "wrong-public", "recreate-failure"])(
"never reports success for %s",
(scenario) => {