diff --git a/.gitea/workflows/ci.yaml b/.gitea/workflows/ci.yaml index 99e36b70..ce52ad5e 100644 --- a/.gitea/workflows/ci.yaml +++ b/.gitea/workflows/ci.yaml @@ -70,16 +70,19 @@ jobs: error_handler() { cd /var/www/atom-nexst 2>/dev/null || cd / || true echo "!!! DEPLOYMENT FAILED on line $1 !!!" >&2 - # Roll back the build artifact if cutover already moved .next into place. - if [ "${CUTOVER_STARTED}" = "1" ] && [ -d "${LIVE}/.next.prev" ]; then - echo "Rolling back .next to previous artifact..." >&2 - rm -rf "${LIVE}/.next" || true - mv "${LIVE}/.next.prev" "${LIVE}/.next" || true + # Leave the healthy current process untouched when staging fails. + # Restart only when cutover has already stopped or replaced it. + if [ "${CUTOVER_STARTED}" = "1" ]; then + if [ -d "${LIVE}/.next.prev" ]; then + echo "Rolling back .next to previous artifact..." >&2 + rm -rf "${LIVE}/.next" || true + mv "${LIVE}/.next.prev" "${LIVE}/.next" || true + fi + pm2 restart next --update-env 2>/dev/null || pm2 start pnpm --name "next" -- start 2>/dev/null || true fi if [ -n "${STAGE}" ] && [ -d "${STAGE}" ]; then git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true fi - pm2 restart next --update-env 2>/dev/null || pm2 start pnpm --name "next" -- start 2>/dev/null || true exit 1 } trap 'error_handler $LINENO' ERR @@ -150,14 +153,7 @@ jobs: exit 1 fi - echo "Cutover: stop service (free DB connections), migrate, swap .next..." - CUTOVER_STARTED=1 - pm2 stop next --kill-timeout 10000 || true - # Wait for PM2 to fully exit and MariaDB to reclaim connections. - sleep 10 - - # Migrate only after live is stopped — avoids ER_CON_COUNT_ERROR while - # the old process still holds DATABASE_POOL_SIZE connections. + echo "Migrating staged release while the current app stays online..." cd "${STAGE}" MIGRATE_OK=0 for i in $(seq 1 10); do @@ -173,6 +169,10 @@ jobs: exit 1 fi + echo "Cutover: stop service and swap .next..." + CUTOVER_STARTED=1 + pm2 stop next --kill-timeout 10000 || true + cd "${LIVE}" echo "Hard reset live tree to origin/main (no nuclear src wipe)..." git reset --hard origin/main diff --git a/src/lib/deploy-workflow-contract.test.ts b/src/lib/deploy-workflow-contract.test.ts index 83bbef57..aeea2b39 100644 --- a/src/lib/deploy-workflow-contract.test.ts +++ b/src/lib/deploy-workflow-contract.test.ts @@ -6,6 +6,7 @@ const liveVar = "${" + "LIVE}"; const stageVar = "${" + "STAGE}"; const userVar = "${" + "DEPLOY_USER}"; const groupVar = "${" + "DEPLOY_GROUP}"; +const cutoverStartedVar = "${" + "CUTOVER_STARTED}"; function toContainLiteral(workflow: string, literal: string) { return workflow.indexOf(literal) >= 0; @@ -66,7 +67,7 @@ describe("production deploy workflow", () => { expect(deployJob).toContain("pnpm typecheck"); }); - it("swaps a built .next artifact during a short service cutover", () => { + it("migrates while the current app is online, then swaps the built artifact", () => { expect(deployJob).toContain("mv .next .next.prev"); expect(toContainLiteral(deployJob, `mv "${stageVar}/.next" .next`)).toBe( true, @@ -84,12 +85,31 @@ describe("production deploy workflow", () => { startLabelAt, ); expect(buildAt).toBeGreaterThan(-1); - expect(stopAt).toBeGreaterThan(buildAt); - expect(migrateAt).toBeGreaterThan(stopAt); - expect(startLabelAt).toBeGreaterThan(migrateAt); + expect(migrateAt).toBeGreaterThan(buildAt); + expect(stopAt).toBeGreaterThan(migrateAt); + expect(startLabelAt).toBeGreaterThan(stopAt); expect(startAt).toBeGreaterThan(startLabelAt); }); + it("does not restart the healthy app when deployment fails before cutover", () => { + const handlerStart = deployJob.indexOf("error_handler() {"); + const handlerEnd = deployJob.indexOf("trap 'error_handler", handlerStart); + const handler = deployJob.slice(handlerStart, handlerEnd); + const cutoverGuardAt = handler.indexOf( + `if [ "${cutoverStartedVar}" = "1" ]; then`, + ); + const restartAt = handler.indexOf("pm2 restart next", cutoverGuardAt); + const stageCleanupAt = handler.indexOf(`if [ -n "${stageVar}"`, restartAt); + + expect(handlerStart).toBeGreaterThan(-1); + expect(cutoverGuardAt).toBeGreaterThan(-1); + expect(restartAt).toBeGreaterThan(cutoverGuardAt); + expect(stageCleanupAt).toBeGreaterThan(restartAt); + expect(handler.slice(restartAt, stageCleanupAt)).toContain( + "\n fi", + ); + }); + it("does not override onlyBuiltDependencies (uses pnpm-workspace.yaml)", () => { expect(workflow).not.toContain("PNPM_CONFIG_ONLY_BUILT_DEPENDENCIES"); });