fix(deploy): replace both legacy and CI containers during cutover
This commit is contained in:
1 parent
6cbcb50191
commit
fe1ee8a6fe
2 files changed
+51
-5
No files matched your search
+18
-4
@@ -13,6 +13,9 @@ sha="$(git rev-parse HEAD)"
|
|||||||
image="epicnext-cms:$sha"
|
image="epicnext-cms:$sha"
|
||||||
previous_name=""
|
previous_name=""
|
||||||
previous_image=""
|
previous_image=""
|
||||||
|
secondary_name=""
|
||||||
|
secondary_backup="epicnext-cms-rollback-secondary"
|
||||||
|
secondary_backup_created=0
|
||||||
backup_name="epicnext-cms-rollback"
|
backup_name="epicnext-cms-rollback"
|
||||||
cutover_started=0
|
cutover_started=0
|
||||||
candidate_attempted=0
|
candidate_attempted=0
|
||||||
@@ -52,6 +55,9 @@ finish() {
|
|||||||
if command -v ss >/dev/null 2>&1; then ss -ltnp 'sport = :3002' >&2 || true; fi
|
if command -v ss >/dev/null 2>&1; then ss -ltnp 'sport = :3002' >&2 || true; fi
|
||||||
if [ "$candidate_attempted" -eq 1 ]; then docker rm -f epicnext-cms-app || true; fi
|
if [ "$candidate_attempted" -eq 1 ]; then docker rm -f epicnext-cms-app || true; fi
|
||||||
if [ "$backup_created" -eq 1 ]; then docker rename "$backup_name" "$previous_name" || true; fi
|
if [ "$backup_created" -eq 1 ]; then docker rename "$backup_name" "$previous_name" || true; fi
|
||||||
|
if [ "$secondary_backup_created" -eq 1 ]; then
|
||||||
|
docker rename "$secondary_backup" "$secondary_name" || true
|
||||||
|
fi
|
||||||
if [ -n "$previous_name" ]; then
|
if [ -n "$previous_name" ]; then
|
||||||
if docker start "$previous_name" && healthy; then
|
if docker start "$previous_name" && healthy; then
|
||||||
echo "Rollback verified: $previous_image"
|
echo "Rollback verified: $previous_image"
|
||||||
@@ -61,6 +67,7 @@ finish() {
|
|||||||
else
|
else
|
||||||
echo "No previous container exists; rollback is unavailable" >&2
|
echo "No previous container exists; rollback is unavailable" >&2
|
||||||
fi
|
fi
|
||||||
|
if [ "$secondary_backup_created" -eq 1 ]; then docker start "$secondary_name" || true; fi
|
||||||
fi
|
fi
|
||||||
exit "$status"
|
exit "$status"
|
||||||
}
|
}
|
||||||
@@ -97,10 +104,9 @@ pnpm db:migrate
|
|||||||
check_current
|
check_current
|
||||||
|
|
||||||
# Prefer the active CI container, or the active legacy compose container.
|
# Prefer the active CI container, or the active legacy compose container.
|
||||||
for name in epicnext-cms-app epicnext-cms; do
|
for name in epicnext-cms epicnext-cms-app; do
|
||||||
if [ "$(docker inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)" = true ]; then
|
if [ "$(docker inspect --format '{{.State.Running}}' "$name" 2>/dev/null || true)" = true ]; then
|
||||||
previous_name="$name"
|
if [ -z "$previous_name" ]; then previous_name="$name"; else secondary_name="$name"; fi
|
||||||
break
|
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
if [ -z "$previous_name" ]; then
|
if [ -z "$previous_name" ]; then
|
||||||
@@ -117,11 +123,18 @@ if [ -n "$previous_name" ]; then
|
|||||||
docker tag "$previous_image" epicnext-cms:previous
|
docker tag "$previous_image" epicnext-cms:previous
|
||||||
fi
|
fi
|
||||||
# Remove a stopped leftover CI container when the compose container is active.
|
# Remove a stopped leftover CI container when the compose container is active.
|
||||||
if [ "$previous_name" != epicnext-cms-app ] && docker inspect epicnext-cms-app >/dev/null 2>&1; then
|
if [ "$previous_name" != epicnext-cms-app ] && [ "$secondary_name" != epicnext-cms-app ] && docker inspect epicnext-cms-app >/dev/null 2>&1; then
|
||||||
docker rm epicnext-cms-app
|
docker rm epicnext-cms-app
|
||||||
fi
|
fi
|
||||||
|
|
||||||
cutover_started=1
|
cutover_started=1
|
||||||
|
# Both legacy Compose and CI containers can exist after earlier failed updates.
|
||||||
|
# Preserve each before releasing the shared host port; never kill an arbitrary PID.
|
||||||
|
if [ -n "$secondary_name" ]; then
|
||||||
|
docker stop "$secondary_name"
|
||||||
|
docker rename "$secondary_name" "$secondary_backup"
|
||||||
|
secondary_backup_created=1
|
||||||
|
fi
|
||||||
if [ -n "$previous_name" ]; then
|
if [ -n "$previous_name" ]; then
|
||||||
docker stop "$previous_name"
|
docker stop "$previous_name"
|
||||||
docker rename "$previous_name" "$backup_name"
|
docker rename "$previous_name" "$backup_name"
|
||||||
@@ -153,6 +166,7 @@ PLAYWRIGHT_BASE_URL=http://127.0.0.1:3002 pnpm test:e2e
|
|||||||
docker tag "$image" epicnext-cms:latest
|
docker tag "$image" epicnext-cms:latest
|
||||||
cutover_started=0
|
cutover_started=0
|
||||||
if [ "$backup_created" -eq 1 ]; then docker rm "$backup_name" || true; fi
|
if [ "$backup_created" -eq 1 ]; then docker rm "$backup_name" || true; fi
|
||||||
|
if [ "$secondary_backup_created" -eq 1 ]; then docker rm "$secondary_backup" || true; fi
|
||||||
|
|
||||||
echo "Deployment verified: $sha"
|
echo "Deployment verified: $sha"
|
||||||
# Retain the current and previous releases; do not remove arbitrary named tags.
|
# Retain the current and previous releases; do not remove arbitrary named tags.
|
||||||
|
|||||||
@@ -24,12 +24,17 @@ const bash =
|
|||||||
.find((path) => existsSync(path)) ?? "bash")
|
.find((path) => existsSync(path)) ?? "bash")
|
||||||
: "bash";
|
: "bash";
|
||||||
const sha = "a".repeat(40);
|
const sha = "a".repeat(40);
|
||||||
function simulate(scenario: string, previous = "epicnext-cms-app") {
|
function simulate(
|
||||||
|
scenario: string,
|
||||||
|
previous = "epicnext-cms-app",
|
||||||
|
both = false,
|
||||||
|
) {
|
||||||
const dir = mkdtempSync(join(tmpdir(), "cms-deploy-test-"));
|
const dir = mkdtempSync(join(tmpdir(), "cms-deploy-test-"));
|
||||||
try {
|
try {
|
||||||
mkdirSync(join(dir, "production"));
|
mkdirSync(join(dir, "production"));
|
||||||
writeFileSync(join(dir, "production", ".env"), 'HOTEL_NAME="Test Hotel"\n');
|
writeFileSync(join(dir, "production", ".env"), 'HOTEL_NAME="Test Hotel"\n');
|
||||||
if (previous) writeFileSync(join(dir, previous), "old\n");
|
if (previous) writeFileSync(join(dir, previous), "old\n");
|
||||||
|
if (both) writeFileSync(join(dir, "epicnext-cms-app"), "old\n");
|
||||||
const result = spawnSync(bash, [resolve(root, "scripts/ci-deploy.sh")], {
|
const result = spawnSync(bash, [resolve(root, "scripts/ci-deploy.sh")], {
|
||||||
cwd: dir,
|
cwd: dir,
|
||||||
encoding: "utf8",
|
encoding: "utf8",
|
||||||
@@ -56,6 +61,7 @@ function simulate(scenario: string, previous = "epicnext-cms-app") {
|
|||||||
: null,
|
: null,
|
||||||
previousRestored: previous ? existsSync(join(dir, previous)) : false,
|
previousRestored: previous ? existsSync(join(dir, previous)) : false,
|
||||||
backup: existsSync(join(dir, "epicnext-cms-rollback")),
|
backup: existsSync(join(dir, "epicnext-cms-rollback")),
|
||||||
|
secondaryBackup: existsSync(join(dir, "epicnext-cms-rollback-secondary")),
|
||||||
};
|
};
|
||||||
} finally {
|
} finally {
|
||||||
rmSync(dir, { recursive: true, force: true });
|
rmSync(dir, { recursive: true, force: true });
|
||||||
@@ -133,3 +139,29 @@ describe("deployment transaction", () => {
|
|||||||
expect(result.app).toBeNull();
|
expect(result.app).toBeNull();
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
|
describe("legacy and CI container coexistence", () => {
|
||||||
|
it("stops and preserves both before starting the candidate, then removes backups only after verification", () => {
|
||||||
|
const r = simulate("success", "epicnext-cms", true);
|
||||||
|
expect(r.status, r.output).toBe(0);
|
||||||
|
expect(r.calls).toContain("docker stop epicnext-cms-app");
|
||||||
|
expect(r.calls).toContain("docker stop epicnext-cms\n");
|
||||||
|
expect(r.calls.indexOf("docker stop epicnext-cms\n")).toBeLessThan(
|
||||||
|
r.calls.indexOf("docker run"),
|
||||||
|
);
|
||||||
|
expect(r.calls.indexOf("docker stop epicnext-cms-app")).toBeLessThan(
|
||||||
|
r.calls.indexOf("docker run"),
|
||||||
|
);
|
||||||
|
expect(r.backup).toBe(false);
|
||||||
|
expect(r.secondaryBackup).toBe(false);
|
||||||
|
expect(r.app).toBe("new");
|
||||||
|
});
|
||||||
|
it("restores both original containers when candidate release validation fails", () => {
|
||||||
|
const r = simulate("release-failure", "epicnext-cms", true);
|
||||||
|
expect(r.status, r.output).not.toBe(0);
|
||||||
|
expect(r.previousRestored).toBe(true);
|
||||||
|
expect(r.app).toBe("old");
|
||||||
|
expect(r.secondaryBackup).toBe(false);
|
||||||
|
expect(r.calls).toContain("docker start epicnext-cms");
|
||||||
|
});
|
||||||
|
});
|
||||||
Reference in new issue
Block a user