ci: serialize deployments and restore previous release on smoke failure
This commit is contained in:
Simo committed
1 parent
85a6df162b
commit
7c1f109a9d
7 files changed
+369
-234
No files matched your search
@@ -0,0 +1,129 @@
|
||||
import { spawnSync } from "node:child_process";
|
||||
import {
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { delimiter, dirname, join, resolve } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
const root = process.cwd();
|
||||
const bash =
|
||||
process.platform === "win32"
|
||||
? ((process.env.PATH ?? "")
|
||||
.split(delimiter)
|
||||
.flatMap((dir) => [
|
||||
join(dir, "bash.exe"),
|
||||
join(dirname(dir), "bin", "bash.exe"),
|
||||
join(dirname(dirname(dir)), "bin", "bash.exe"),
|
||||
])
|
||||
.find((path) => existsSync(path)) ?? "bash")
|
||||
: "bash";
|
||||
const sha = "a".repeat(40);
|
||||
function simulate(scenario: string, previous = "epicnext-cms-app") {
|
||||
const dir = mkdtempSync(join(tmpdir(), "cms-deploy-test-"));
|
||||
try {
|
||||
mkdirSync(join(dir, "production"));
|
||||
writeFileSync(join(dir, "production", ".env"), 'HOTEL_NAME="Test Hotel"\n');
|
||||
if (previous) writeFileSync(join(dir, previous), "old\n");
|
||||
const result = spawnSync(bash, [resolve(root, "scripts/ci-deploy.sh")], {
|
||||
cwd: dir,
|
||||
encoding: "utf8",
|
||||
timeout: 15000,
|
||||
env: {
|
||||
...process.env,
|
||||
BASH_ENV: resolve(root, "src/test/ci-deploy-harness.sh"),
|
||||
TEST_DIR: dir.replaceAll("\\", "/"),
|
||||
CMS_DEPLOY_DIR: join(dir, "production").replaceAll("\\", "/"),
|
||||
TEST_SHA: sha,
|
||||
SCENARIO: scenario,
|
||||
DEPLOY_BRANCH: "main",
|
||||
},
|
||||
});
|
||||
if (result.error) throw result.error;
|
||||
return {
|
||||
status: result.status,
|
||||
output: result.stdout + result.stderr,
|
||||
calls: existsSync(join(dir, "calls"))
|
||||
? readFileSync(join(dir, "calls"), "utf8")
|
||||
: "",
|
||||
app: existsSync(join(dir, "epicnext-cms-app"))
|
||||
? readFileSync(join(dir, "epicnext-cms-app"), "utf8").trim()
|
||||
: null,
|
||||
previousRestored: previous ? existsSync(join(dir, previous)) : false,
|
||||
backup: existsSync(join(dir, "epicnext-cms-rollback")),
|
||||
};
|
||||
} finally {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
describe("deployment transaction", () => {
|
||||
it("publishes the commit image only after migrations, health and smoke tests", () => {
|
||||
const result = simulate("success");
|
||||
expect(result.status, result.output).toBe(0);
|
||||
expect(result.app).toBe("new");
|
||||
expect(result.backup).toBe(false);
|
||||
expect(result.calls.indexOf("pnpm db:migrate")).toBeLessThan(
|
||||
result.calls.indexOf("docker stop"),
|
||||
);
|
||||
expect(result.calls.indexOf("pnpm test:e2e")).toBeLessThan(
|
||||
result.calls.indexOf(
|
||||
`docker tag epicnext-cms:${sha} epicnext-cms:latest`,
|
||||
),
|
||||
);
|
||||
expect(result.calls).toContain(
|
||||
"docker tag sha256:old epicnext-cms:previous",
|
||||
);
|
||||
});
|
||||
it.each(["run-failure", "health-failure", "smoke-failure"])(
|
||||
"restores the exact previous container after %s",
|
||||
(scenario) => {
|
||||
const result = simulate(scenario);
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.app).toBe("old");
|
||||
expect(result.output).toContain("Rollback verified: sha256:old");
|
||||
expect(result.calls).not.toContain(
|
||||
`docker tag epicnext-cms:${sha} epicnext-cms:latest`,
|
||||
);
|
||||
},
|
||||
);
|
||||
it("restores the legacy compose container on failure", () => {
|
||||
const result = simulate("smoke-failure", "epicnext-cms");
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.app).toBeNull();
|
||||
expect(result.previousRestored).toBe(true);
|
||||
expect(result.calls).toContain("docker start epicnext-cms");
|
||||
});
|
||||
it.each([
|
||||
"lock-failure",
|
||||
"remote-failure",
|
||||
"build-failure",
|
||||
"migration-failure",
|
||||
])("leaves the active container untouched after %s", (scenario) => {
|
||||
const result = simulate(scenario);
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.app).toBe("old");
|
||||
expect(result.calls).not.toContain("docker stop");
|
||||
});
|
||||
it.each(["stale", "superseded"])(
|
||||
"skips a %s commit without touching production",
|
||||
(scenario) => {
|
||||
const result = simulate(scenario);
|
||||
expect(result.status, result.output).toBe(0);
|
||||
expect(result.app).toBe("old");
|
||||
expect(result.calls).not.toContain("pnpm db:migrate");
|
||||
expect(result.calls).not.toContain("docker stop");
|
||||
},
|
||||
);
|
||||
it("reports a failed first deployment without inventing a rollback", () => {
|
||||
const result = simulate("smoke-failure", "");
|
||||
expect(result.status).not.toBe(0);
|
||||
expect(result.output).toContain("No previous container exists");
|
||||
expect(result.app).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -58,10 +58,10 @@ describe("CI workflow", () => {
|
||||
expect(workflow).toContain('tags: ["v*"]');
|
||||
});
|
||||
|
||||
it("has e2e job that smoke-tests the deployed app", () => {
|
||||
expect(workflow).toContain("e2e:");
|
||||
expect(workflow).toContain("needs: deploy");
|
||||
expect(workflow).toContain("pnpm test:e2e");
|
||||
expect(workflow).toContain("PLAYWRIGHT_BASE_URL");
|
||||
it("smoke-tests the deployed app within the deployment transaction", () => {
|
||||
expect(workflow).toContain("bash scripts/ci-deploy.sh");
|
||||
const deploy = readFileSync("scripts/ci-deploy.sh", "utf8");
|
||||
expect(deploy).toContain("pnpm test:e2e");
|
||||
expect(deploy).toContain("PLAYWRIGHT_BASE_URL");
|
||||
});
|
||||
});
|
||||
@@ -1,104 +1,33 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { expect, it } from "vitest";
|
||||
|
||||
const workflow = readFileSync(
|
||||
resolve(process.cwd(), ".gitea/workflows/ci.yaml"),
|
||||
"utf8",
|
||||
);
|
||||
const deployStart = workflow.indexOf("\n deploy:");
|
||||
const e2eStart = workflow.indexOf("\n e2e:");
|
||||
const deployJob =
|
||||
deployStart > -1
|
||||
? workflow.slice(deployStart, e2eStart > deployStart ? e2eStart : undefined)
|
||||
: "";
|
||||
|
||||
describe("deploy job", () => {
|
||||
it("runs on self-hosted for Docker access", () => {
|
||||
expect(deployJob).toContain("runs-on: self-hosted");
|
||||
});
|
||||
|
||||
it("has checkout step", () => {
|
||||
expect(deployJob).toContain("actions/checkout@v4");
|
||||
});
|
||||
|
||||
it("builds Docker image with BuildKit", () => {
|
||||
expect(deployJob).toContain("DOCKER_BUILDKIT=1");
|
||||
expect(deployJob).toContain("docker build");
|
||||
expect(deployJob).toContain("-t epicnext-cms:latest");
|
||||
});
|
||||
|
||||
it("builds on host network for registry access", () => {
|
||||
expect(deployJob).toContain("--network=host");
|
||||
});
|
||||
|
||||
it("stops old container before starting new one", () => {
|
||||
expect(deployJob).toContain("docker stop epicnext-cms-app");
|
||||
expect(deployJob).toContain("docker rm epicnext-cms-app");
|
||||
});
|
||||
|
||||
it("starts container with correct settings", () => {
|
||||
expect(deployJob).toContain("--restart always");
|
||||
expect(deployJob).toContain("--net=host");
|
||||
expect(deployJob).toContain("--name epicnext-cms-app");
|
||||
});
|
||||
|
||||
it("provides production .env to the build", () => {
|
||||
expect(deployJob).toContain("cp /var/www/atom-nexst/.env .env");
|
||||
});
|
||||
|
||||
it("runs container with production env and volumes", () => {
|
||||
expect(deployJob).toContain(". /var/www/atom-nexst/.env");
|
||||
// biome-ignore lint/suspicious/noTemplateCurlyInString: intentional literal shell snippet
|
||||
expect(deployJob).toContain('"${ENV_ARGS[@]}"');
|
||||
expect(deployJob).toContain("/var/www/Gamedata:/var/www/Gamedata");
|
||||
expect(deployJob).toContain("/app/storage");
|
||||
});
|
||||
|
||||
it("does not use --env-file (it keeps literal quotes)", () => {
|
||||
expect(deployJob).not.toContain("--env-file");
|
||||
});
|
||||
|
||||
it("frees port 3002 by stopping the compose container", () => {
|
||||
expect(deployJob).toContain("docker stop epicnext-cms 2>/dev/null || true");
|
||||
});
|
||||
|
||||
it("rolls back to compose on health check failure", () => {
|
||||
expect(deployJob).toContain("docker compose");
|
||||
});
|
||||
|
||||
it("runs database migrations before replacing the container", () => {
|
||||
expect(deployJob).toContain("pnpm db:migrate");
|
||||
expect(deployJob.indexOf("pnpm db:migrate")).toBeLessThan(
|
||||
deployJob.indexOf("docker stop epicnext-cms-app"),
|
||||
);
|
||||
});
|
||||
|
||||
it("runs health check", () => {
|
||||
expect(deployJob).toContain("/api/health");
|
||||
expect(deployJob).toContain('"database":true');
|
||||
});
|
||||
|
||||
it("preserves recent build cache and avoids pruning application volumes", () => {
|
||||
expect(deployJob).toContain(
|
||||
'docker builder prune -af --filter "until=72h" --keep-storage=2g',
|
||||
);
|
||||
expect(deployJob).toContain('docker image prune -f --filter "until=168h"');
|
||||
expect(deployJob).not.toContain("docker volume prune");
|
||||
expect(deployJob).not.toContain("docker network prune");
|
||||
expect(deployJob).not.toContain("docker container prune");
|
||||
expect(deployJob).not.toContain("docker image prune -af");
|
||||
});
|
||||
|
||||
it("does not run pnpm test in deploy", () => {
|
||||
expect(deployJob).not.toContain("pnpm test");
|
||||
});
|
||||
|
||||
it("shows docker logs on failure", () => {
|
||||
expect(deployJob).toContain("docker logs epicnext-cms-app");
|
||||
});
|
||||
|
||||
it("exits with error on health check failure", () => {
|
||||
expect(deployJob).toContain("exit 1");
|
||||
});
|
||||
const workflow = readFileSync(".gitea/workflows/ci.yaml", "utf8");
|
||||
const deploy = readFileSync("scripts/ci-deploy.sh", "utf8");
|
||||
it("uses two test workers and runs smoke checks in the deployment transaction", () => {
|
||||
expect(workflow).toContain("pnpm test --maxWorkers=2");
|
||||
expect(workflow).not.toContain("\n e2e:");
|
||||
expect(workflow).toContain("bash scripts/ci-deploy.sh");
|
||||
expect(deploy).toContain(
|
||||
"PLAYWRIGHT_BASE_URL=http://127.0.0.1:3002 pnpm test:e2e",
|
||||
);
|
||||
});
|
||||
it("locks CI and scheduled deployments using the same production lock", () => {
|
||||
expect(deploy).toContain('exec 9>"$deploy_dir/.deploy.lock"');
|
||||
expect(deploy).toContain("flock -w 1800 9");
|
||||
const scheduled = readFileSync("scripts/docker-update.sh", "utf8");
|
||||
expect(scheduled).toContain('exec 9>"$DIR/.deploy.lock"');
|
||||
expect(scheduled.indexOf("flock -w 1800 9")).toBeLessThan(
|
||||
scheduled.indexOf("git pull --ff-only"),
|
||||
);
|
||||
});
|
||||
it("preserves production runtime configuration and recent cache", () => {
|
||||
expect(deploy).toContain("--net=host");
|
||||
expect(deploy).toContain("--restart always");
|
||||
expect(deploy).toContain("/var/www/Gamedata:/var/www/Gamedata");
|
||||
expect(deploy).toContain("/app/storage");
|
||||
expect(deploy).not.toContain("--env-file");
|
||||
expect(deploy).toContain(
|
||||
'docker builder prune -af --filter "until=72h" --keep-storage=2g',
|
||||
);
|
||||
expect(deploy).not.toContain("docker volume prune");
|
||||
});
|
||||
@@ -0,0 +1,44 @@
|
||||
# Sourced only by the deployment simulation tests; no external services are used.
|
||||
git() {
|
||||
if [ "$1" = rev-parse ]; then printf '%s\n' "$TEST_SHA"; return; fi
|
||||
local n=0
|
||||
if [ -f "$TEST_DIR/remote-count" ]; then read -r n < "$TEST_DIR/remote-count"; fi
|
||||
n=$((n+1)); printf '%s\n' "$n" > "$TEST_DIR/remote-count"
|
||||
if [ "$SCENARIO" = remote-failure ]; then return 1; fi
|
||||
if [ "$SCENARIO" = stale ] || { [ "$SCENARIO" = superseded ] && [ "$n" -gt 1 ]; }; then
|
||||
printf '%s\trefs/heads/main\n' bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb
|
||||
else printf '%s\trefs/heads/main\n' "$TEST_SHA"; fi
|
||||
}
|
||||
flock() { echo "lock" >> "$TEST_DIR/calls"; [ "$SCENARIO" != lock-failure ]; }
|
||||
pnpm() {
|
||||
echo "pnpm $*" >> "$TEST_DIR/calls"
|
||||
if [ "$1" = db:migrate ] && [ "$SCENARIO" = migration-failure ]; then return 1; fi
|
||||
if [ "$1" = test:e2e ] && [ "$SCENARIO" = smoke-failure ]; then return 1; fi
|
||||
return 0
|
||||
}
|
||||
curl() {
|
||||
if [ "$SCENARIO" = health-failure ] && [ "$(cat "$TEST_DIR/epicnext-cms-app" 2>/dev/null)" = new ]; then return 1; fi
|
||||
echo '{"database":true}'
|
||||
}
|
||||
sleep() { :; }
|
||||
docker() {
|
||||
echo "docker $*" >> "$TEST_DIR/calls"
|
||||
local name="${@: -1}"
|
||||
case "$1" in
|
||||
inspect)
|
||||
[ -f "$TEST_DIR/$name" ] || return 1
|
||||
if [ "${3:-}" = '{{.State.Running}}' ]; then echo true
|
||||
elif [ "${3:-}" = '{{.Image}}' ]; then echo sha256:old
|
||||
fi ;;
|
||||
build) [ "$SCENARIO" != build-failure ] ;;
|
||||
stop) return 0 ;;
|
||||
rename) mv "$TEST_DIR/$2" "$TEST_DIR/$3" ;;
|
||||
run)
|
||||
echo new > "$TEST_DIR/epicnext-cms-app"
|
||||
[ "$SCENARIO" != run-failure ] ;;
|
||||
start) [ -f "$TEST_DIR/$2" ] ;;
|
||||
rm) rm -f "$TEST_DIR/$name" ;;
|
||||
*) return 0 ;;
|
||||
esac
|
||||
}
|
||||
export -f git flock pnpm curl sleep docker
|
||||
Reference in new issue
Block a user