fix: keep app online during deploy preparation

.gitea/workflows/ci.yaml

@@ -70,16 +70,19 @@ jobs:
           error_handler() {
             cd /var/www/atom-nexst 2>/dev/null || cd / || true
             echo "!!! DEPLOYMENT FAILED on line $1 !!!" >&2
-            # Roll back the build artifact if cutover already moved .next into place.
-            if [ "${CUTOVER_STARTED}" = "1" ] && [ -d "${LIVE}/.next.prev" ]; then
-              echo "Rolling back .next to previous artifact..." >&2
-              rm -rf "${LIVE}/.next" || true
-              mv "${LIVE}/.next.prev" "${LIVE}/.next" || true
+            # Leave the healthy current process untouched when staging fails.
+            # Restart only when cutover has already stopped or replaced it.
+            if [ "${CUTOVER_STARTED}" = "1" ]; then
+              if [ -d "${LIVE}/.next.prev" ]; then
+                echo "Rolling back .next to previous artifact..." >&2
+                rm -rf "${LIVE}/.next" || true
+                mv "${LIVE}/.next.prev" "${LIVE}/.next" || true
+              fi
+              pm2 restart next --update-env 2>/dev/null || pm2 start pnpm --name "next" -- start 2>/dev/null || true
             fi
             if [ -n "${STAGE}" ] && [ -d "${STAGE}" ]; then
               git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true
             fi
-            pm2 restart next --update-env 2>/dev/null || pm2 start pnpm --name "next" -- start 2>/dev/null || true
             exit 1
           }
           trap 'error_handler $LINENO' ERR
@@ -150,14 +153,7 @@ jobs:
             exit 1
           fi
 
-          echo "Cutover: stop service (free DB connections), migrate, swap .next..."
-          CUTOVER_STARTED=1
-          pm2 stop next --kill-timeout 10000 || true
-          # Wait for PM2 to fully exit and MariaDB to reclaim connections.
-          sleep 10
-
-          # Migrate only after live is stopped — avoids ER_CON_COUNT_ERROR while
-          # the old process still holds DATABASE_POOL_SIZE connections.
+          echo "Migrating staged release while the current app stays online..."
           cd "${STAGE}"
           MIGRATE_OK=0
           for i in $(seq 1 10); do
@@ -173,6 +169,10 @@ jobs:
             exit 1
           fi
 
+          echo "Cutover: stop service and swap .next..."
+          CUTOVER_STARTED=1
+          pm2 stop next --kill-timeout 10000 || true
+
           cd "${LIVE}"
           echo "Hard reset live tree to origin/main (no nuclear src wipe)..."
           git reset --hard origin/main

src/lib/deploy-workflow-contract.test.ts

@@ -6,6 +6,7 @@ const liveVar = "${" + "LIVE}";
 const stageVar = "${" + "STAGE}";
 const userVar = "${" + "DEPLOY_USER}";
 const groupVar = "${" + "DEPLOY_GROUP}";
+const cutoverStartedVar = "${" + "CUTOVER_STARTED}";
 
 function toContainLiteral(workflow: string, literal: string) {
 	return workflow.indexOf(literal) >= 0;
@@ -66,7 +67,7 @@ describe("production deploy workflow", () => {
 		expect(deployJob).toContain("pnpm typecheck");
 	});
 
-	it("swaps a built .next artifact during a short service cutover", () => {
+	it("migrates while the current app is online, then swaps the built artifact", () => {
 		expect(deployJob).toContain("mv .next .next.prev");
 		expect(toContainLiteral(deployJob, `mv "${stageVar}/.next" .next`)).toBe(
 			true,
@@ -84,12 +85,31 @@ describe("production deploy workflow", () => {
 			startLabelAt,
 		);
 		expect(buildAt).toBeGreaterThan(-1);
-		expect(stopAt).toBeGreaterThan(buildAt);
-		expect(migrateAt).toBeGreaterThan(stopAt);
-		expect(startLabelAt).toBeGreaterThan(migrateAt);
+		expect(migrateAt).toBeGreaterThan(buildAt);
+		expect(stopAt).toBeGreaterThan(migrateAt);
+		expect(startLabelAt).toBeGreaterThan(stopAt);
 		expect(startAt).toBeGreaterThan(startLabelAt);
 	});
 
+	it("does not restart the healthy app when deployment fails before cutover", () => {
+		const handlerStart = deployJob.indexOf("error_handler() {");
+		const handlerEnd = deployJob.indexOf("trap 'error_handler", handlerStart);
+		const handler = deployJob.slice(handlerStart, handlerEnd);
+		const cutoverGuardAt = handler.indexOf(
+			`if [ "${cutoverStartedVar}" = "1" ]; then`,
+		);
+		const restartAt = handler.indexOf("pm2 restart next", cutoverGuardAt);
+		const stageCleanupAt = handler.indexOf(`if [ -n "${stageVar}"`, restartAt);
+
+		expect(handlerStart).toBeGreaterThan(-1);
+		expect(cutoverGuardAt).toBeGreaterThan(-1);
+		expect(restartAt).toBeGreaterThan(cutoverGuardAt);
+		expect(stageCleanupAt).toBeGreaterThan(restartAt);
+		expect(handler.slice(restartAt, stageCleanupAt)).toContain(
+			"\n            fi",
+		);
+	});
+
 	it("does not override onlyBuiltDependencies (uses pnpm-workspace.yaml)", () => {
 		expect(workflow).not.toContain("PNPM_CONFIG_ONLY_BUILT_DEPENDENCIES");
 	});