fix: minimize deploy cutover downtime
This commit is contained in:
1 parent
828fda63e7
commit
bfc951cfd9
2 files changed
+34
-25
No files matched your search
+25
-24
@@ -78,6 +78,11 @@ jobs:
|
||||
rm -rf "${LIVE}/.next" || true
|
||||
mv "${LIVE}/.next.prev" "${LIVE}/.next" || true
|
||||
fi
|
||||
if [ -d "${LIVE}/node_modules.prev" ]; then
|
||||
echo "Rolling back node_modules to previous artifact..." >&2
|
||||
rm -rf "${LIVE}/node_modules" || true
|
||||
mv "${LIVE}/node_modules.prev" "${LIVE}/node_modules" || true
|
||||
fi
|
||||
pm2 restart next --update-env 2>/dev/null || pm2 start pnpm --name "next" -- start 2>/dev/null || true
|
||||
fi
|
||||
if [ -n "${STAGE}" ] && [ -d "${STAGE}" ]; then
|
||||
@@ -169,17 +174,14 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Cutover: stop service and swap .next..."
|
||||
CUTOVER_STARTED=1
|
||||
pm2 stop next --kill-timeout 10000 || true
|
||||
|
||||
cd "${LIVE}"
|
||||
echo "Hard reset live tree to origin/main (no nuclear src wipe)..."
|
||||
git reset --hard origin/main
|
||||
# Keep env, uploads, and deps we are about to replace from stage.
|
||||
git clean -fd \
|
||||
-e .env -e .env.local -e .env.production -e .env*.local \
|
||||
-e storage -e public/cache -e node_modules -e .next -e .next.prev
|
||||
-e storage -e public/cache \
|
||||
-e node_modules -e node_modules.prev -e .next -e .next.prev
|
||||
|
||||
if ! git diff --exit-code HEAD -- src >/dev/null; then
|
||||
echo "ERROR: live src/ still differs from HEAD after reset:" >&2
|
||||
@@ -188,18 +190,6 @@ jobs:
|
||||
fi
|
||||
echo "Verified live src/ matches HEAD"
|
||||
|
||||
# Save current .next as backup before swapping (kept until health check passes).
|
||||
if [ -d .next ]; then
|
||||
mv .next .next.prev
|
||||
fi
|
||||
mv "${STAGE}/.next" .next
|
||||
|
||||
# Use the exact node_modules the stage build resolved against.
|
||||
rm -rf node_modules
|
||||
mv "${STAGE}/node_modules" node_modules
|
||||
|
||||
sudo chown -R "${DEPLOY_USER}:${DEPLOY_GROUP}" "${LIVE}" 2>/dev/null || true
|
||||
|
||||
# Next.js prefers an already-set process PORT over .env. PM2 may still
|
||||
# have PORT=3000 from an older start, while .env (and nginx) expect 3002.
|
||||
# Export PORT before start so the process matches health checks.
|
||||
@@ -220,14 +210,25 @@ jobs:
|
||||
fi
|
||||
}
|
||||
|
||||
echo "Starting PM2 with a clean PORT=${PORT} listener..."
|
||||
echo "Cutover: atomically swap artifacts and restart on PORT=${PORT}..."
|
||||
CUTOVER_STARTED=1
|
||||
pm2 stop next --kill-timeout 10000 || true
|
||||
|
||||
cd "${LIVE}"
|
||||
# Cutover already stopped the app, but failed deploys can leave orphans
|
||||
# on 3002 (or an old PM2 env still bound to 3000).
|
||||
pm2 stop next --kill-timeout 10000 2>/dev/null || true
|
||||
|
||||
# Rename both current artifacts so cutover and rollback stay fast.
|
||||
if [ -d .next ]; then
|
||||
mv .next .next.prev
|
||||
fi
|
||||
mv "${STAGE}/.next" .next
|
||||
if [ -d node_modules ]; then
|
||||
mv node_modules node_modules.prev
|
||||
fi
|
||||
mv "${STAGE}/node_modules" node_modules
|
||||
|
||||
free_tcp_port "${PORT}"
|
||||
free_tcp_port 3000
|
||||
sleep 1
|
||||
echo "Starting PM2 with a clean PORT=${PORT} listener..."
|
||||
pm2 delete next 2>/dev/null || true
|
||||
PORT="${PORT}" pm2 start pnpm --name next -- start
|
||||
pm2 save 2>/dev/null || true
|
||||
@@ -262,8 +263,8 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Cleaning stage worktree and previous .next backup..."
|
||||
rm -rf "${LIVE}/.next.prev"
|
||||
echo "Cleaning stage worktree and previous artifact backups..."
|
||||
rm -rf "${LIVE}/.next.prev" "${LIVE}/node_modules.prev"
|
||||
git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true
|
||||
STAGE=""
|
||||
|
||||
|
||||
@@ -75,10 +75,15 @@ describe("production deploy workflow", () => {
|
||||
expect(
|
||||
toContainLiteral(deployJob, `mv "${stageVar}/node_modules" node_modules`),
|
||||
).toBe(true);
|
||||
expect(deployJob).toContain("mv node_modules node_modules.prev");
|
||||
expect(deployJob).toContain("Rolling back .next to previous artifact");
|
||||
expect(deployJob).toContain(
|
||||
"Rolling back node_modules to previous artifact",
|
||||
);
|
||||
const buildAt = deployJob.indexOf("pnpm build");
|
||||
const stopAt = deployJob.indexOf("pm2 stop next");
|
||||
const migrateAt = deployJob.indexOf("pnpm db:migrate");
|
||||
const resetAt = deployJob.indexOf("git reset --hard origin/main");
|
||||
const startLabelAt = deployJob.indexOf("Starting PM2");
|
||||
const startAt = deployJob.indexOf(
|
||||
"pm2 start pnpm --name next -- start",
|
||||
@@ -86,9 +91,12 @@ describe("production deploy workflow", () => {
|
||||
);
|
||||
expect(buildAt).toBeGreaterThan(-1);
|
||||
expect(migrateAt).toBeGreaterThan(buildAt);
|
||||
expect(stopAt).toBeGreaterThan(migrateAt);
|
||||
expect(resetAt).toBeGreaterThan(migrateAt);
|
||||
expect(stopAt).toBeGreaterThan(resetAt);
|
||||
expect(startLabelAt).toBeGreaterThan(stopAt);
|
||||
expect(startAt).toBeGreaterThan(startLabelAt);
|
||||
expect(deployJob.match(/pm2 stop next/g)).toHaveLength(1);
|
||||
expect(deployJob.slice(stopAt, startAt)).not.toContain("sleep ");
|
||||
});
|
||||
|
||||
it("does not restart the healthy app when deployment fails before cutover", () => {
|
||||
|
||||
Reference in new issue
Block a user