fix(deploy): stage worktree build and short .next cutover
Build install/test/migrate in a detached worktree while the live site keeps serving, then swap .next and node_modules during a brief stop. Drops nuclear rm -rf src and rolls back .next.prev on cutover failure. Co-authored-by: Cursor <[email protected]>
This commit is contained in:
SimoandCursor committed
1 parent
77931db775
commit
687e1f9fb0
2 files changed
+124
-58
No files matched your search
@@ -236,58 +236,65 @@ jobs:
|
||||
|
||||
echo "--- Deploying ---"
|
||||
|
||||
LIVE="/var/www/atom-nexst"
|
||||
STAGE=""
|
||||
CUTOVER_STARTED=0
|
||||
|
||||
error_handler() {
|
||||
echo "!!! DEPLOYMENT FAILED on line $1 !!!" >&2
|
||||
# Roll back the build artifact if cutover already moved .next into place.
|
||||
if [ "${CUTOVER_STARTED}" = "1" ] && [ -d "${LIVE}/.next.prev" ]; then
|
||||
echo "Rolling back .next to previous artifact..." >&2
|
||||
rm -rf "${LIVE}/.next" || true
|
||||
mv "${LIVE}/.next.prev" "${LIVE}/.next" || true
|
||||
fi
|
||||
if [ -n "${STAGE}" ] && [ -d "${STAGE}" ]; then
|
||||
git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true
|
||||
fi
|
||||
sudo systemctl start atom-nexst.service || true
|
||||
exit 1
|
||||
}
|
||||
trap 'error_handler $LINENO' ERR
|
||||
|
||||
docker image prune -f
|
||||
cd /var/www/atom-nexst/
|
||||
|
||||
DEPLOY_USER="$(id -un)"
|
||||
DEPLOY_GROUP="$(id -gn)"
|
||||
sudo chown -R "${DEPLOY_USER}:${DEPLOY_GROUP}" /var/www/atom-nexst/
|
||||
git config --global --add safe.directory /var/www/atom-nexst
|
||||
git remote set-url origin /docker/gitea/gitea/git/repositories/remco/epicnext-cms.git/
|
||||
sudo chown -R "${DEPLOY_USER}:${DEPLOY_GROUP}" "${LIVE}"
|
||||
git config --global --add safe.directory "${LIVE}"
|
||||
git -C "${LIVE}" remote set-url origin /docker/gitea/gitea/git/repositories/remco/epicnext-cms.git/
|
||||
|
||||
echo "Fetching origin/main..."
|
||||
git fetch origin --prune
|
||||
git -C "${LIVE}" fetch origin --prune
|
||||
|
||||
echo "Clearing sticky git index bits (if any)..."
|
||||
STICKY_LIST="$(git ls-files -v | awk '/^[a-zS]/ {print substr($0,3)}' || true)"
|
||||
STICKY_LIST="$(git -C "${LIVE}" ls-files -v | awk '/^[a-zS]/ {print substr($0,3)}' || true)"
|
||||
if [ -n "${STICKY_LIST}" ]; then
|
||||
echo "${STICKY_LIST}" | while IFS= read -r f; do
|
||||
[ -n "$f" ] || continue
|
||||
git update-index --no-skip-worktree --no-assume-unchanged -- "$f" 2>/dev/null || true
|
||||
git -C "${LIVE}" update-index --no-skip-worktree --no-assume-unchanged -- "$f" 2>/dev/null || true
|
||||
done
|
||||
fi
|
||||
|
||||
echo "Hard reset to origin/main..."
|
||||
git reset --hard origin/main
|
||||
|
||||
echo "Nuclear-replacing src/ from HEAD..."
|
||||
rm -rf src
|
||||
git checkout -f HEAD -- src
|
||||
git clean -fd -e .env -e .env.local -e .env.production -e .env*.local
|
||||
|
||||
if ! git diff --exit-code HEAD -- src >/dev/null; then
|
||||
echo "ERROR: src/ still differs from HEAD after nuclear checkout:" >&2
|
||||
git diff --stat HEAD -- src >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Verified src/ matches HEAD"
|
||||
|
||||
rm -f tsconfig.tsbuildinfo .tsbuildinfo
|
||||
find . -maxdepth 3 -name '*.tsbuildinfo' -delete 2>/dev/null || true
|
||||
rm -rf .output dist .next/types .next/dev
|
||||
|
||||
export APP_VERSION="$(git rev-parse --short HEAD)"
|
||||
export APP_VERSION="$(git -C "${LIVE}" rev-parse --short origin/main)"
|
||||
export NEXT_PUBLIC_APP_VERSION="${APP_VERSION}"
|
||||
echo "APP_VERSION=${APP_VERSION}"
|
||||
|
||||
STAGE="/var/tmp/atom-nexst-stage-${APP_VERSION}"
|
||||
echo "Preparing stage worktree at ${STAGE} (live site stays up)..."
|
||||
git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true
|
||||
git -C "${LIVE}" worktree add --detach "${STAGE}" origin/main
|
||||
|
||||
# Production env stays on the live tree; stage only needs a symlink for build/migrate.
|
||||
ln -sfn "${LIVE}/.env" "${STAGE}/.env"
|
||||
|
||||
cd "${STAGE}"
|
||||
rm -f tsconfig.tsbuildinfo .tsbuildinfo
|
||||
find . -maxdepth 3 -name '*.tsbuildinfo' -delete 2>/dev/null || true
|
||||
rm -rf .output dist .next .next/types .next/dev
|
||||
|
||||
pnpm install --frozen-lockfile
|
||||
# Additive migrations while the old build still serves traffic.
|
||||
pnpm db:migrate
|
||||
pnpm prisma:generate
|
||||
pnpm typecheck
|
||||
@@ -296,11 +303,48 @@ jobs:
|
||||
# Do not set SKIP_ENV_VALIDATION here — that flag is for tests/tooling only.
|
||||
pnpm build
|
||||
|
||||
sudo chown -R www-data:www-data /var/www/atom-nexst/
|
||||
if [ ! -d "${STAGE}/.next" ]; then
|
||||
echo "ERROR: stage build produced no .next/" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Hard resetting systemd service..."
|
||||
echo "Cutover: stop service, sync code, swap .next artifact..."
|
||||
CUTOVER_STARTED=1
|
||||
sudo systemctl stop atom-nexst.service || true
|
||||
pkill -f 'next-server' || true
|
||||
|
||||
cd "${LIVE}"
|
||||
echo "Hard reset live tree to origin/main (no nuclear src wipe)..."
|
||||
git reset --hard origin/main
|
||||
# Keep env, uploads, and deps we are about to replace from stage.
|
||||
git clean -fd \
|
||||
-e .env -e .env.local -e .env.production -e .env*.local \
|
||||
-e storage -e public/cache -e node_modules -e .next -e .next.prev
|
||||
|
||||
if ! git diff --exit-code HEAD -- src >/dev/null; then
|
||||
echo "ERROR: live src/ still differs from HEAD after reset:" >&2
|
||||
git diff --stat HEAD -- src >&2 || true
|
||||
exit 1
|
||||
fi
|
||||
echo "Verified live src/ matches HEAD"
|
||||
|
||||
# Atomic-ish artifact swap: keep previous .next until the new one is in place.
|
||||
rm -rf .next.prev
|
||||
if [ -d .next ]; then
|
||||
mv .next .next.prev
|
||||
fi
|
||||
mv "${STAGE}/.next" .next
|
||||
|
||||
# Use the exact node_modules the stage build resolved against.
|
||||
rm -rf node_modules
|
||||
mv "${STAGE}/node_modules" node_modules
|
||||
|
||||
# Prisma client is gitignored — regenerate into live src/generated.
|
||||
pnpm prisma:generate
|
||||
|
||||
sudo chown -R www-data:www-data "${LIVE}"
|
||||
|
||||
echo "Starting systemd service..."
|
||||
sudo systemctl start atom-nexst.service
|
||||
|
||||
sleep 2
|
||||
@@ -329,4 +373,9 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Cleaning stage worktree and previous .next..."
|
||||
rm -rf "${LIVE}/.next.prev"
|
||||
git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true
|
||||
STAGE=""
|
||||
|
||||
echo "--- Deployed successfully ---"
|
||||
@@ -7,47 +7,63 @@ describe("production deploy workflow", () => {
|
||||
resolve(process.cwd(), ".gitea/workflows/deploy.yaml"),
|
||||
"utf8",
|
||||
);
|
||||
const deployJob = workflow.slice(workflow.indexOf("\n deploy:"));
|
||||
|
||||
it("preserves the Next.js incremental build cache", () => {
|
||||
// May clear .next/types or .next/dev, but must not wipe the whole .next tree.
|
||||
expect(workflow).not.toMatch(/rm\s+-rf\s+\.next(?:\s|$)/);
|
||||
expect(workflow).toContain("rm -rf .output dist .next/types .next/dev");
|
||||
expect(workflow).toContain("pnpm install --frozen-lockfile");
|
||||
// Production builds must validate env (AUTH_SECRET, DATABASE_URL, …).
|
||||
expect(workflow).not.toContain("SKIP_ENV_VALIDATION=1");
|
||||
it("builds in a stage worktree while preserving live .env and storage", () => {
|
||||
expect(deployJob).toContain("worktree add --detach");
|
||||
expect(deployJob).toContain("/var/tmp/atom-nexst-stage-");
|
||||
expect(deployJob).toContain('ln -sfn "${LIVE}/.env"');
|
||||
expect(deployJob).toContain("-e storage");
|
||||
expect(deployJob).not.toContain("SKIP_ENV_VALIDATION=1");
|
||||
expect(deployJob).toContain("pnpm install --frozen-lockfile");
|
||||
});
|
||||
|
||||
it("reclaims ownership before git reset so www-data files can be overwritten", () => {
|
||||
expect(workflow).toContain('sudo chown -R "$' + "{DEPLOY_USER}:" + '$' + '{DEPLOY_GROUP}"');
|
||||
const reclaimAt = workflow.indexOf(
|
||||
it("reclaims ownership before git operations so www-data files can be overwritten", () => {
|
||||
expect(workflow).toContain(
|
||||
'sudo chown -R "$' + "{DEPLOY_USER}:" + '$' + '{DEPLOY_GROUP}"',
|
||||
);
|
||||
const resetAt = workflow.indexOf("git reset --hard origin/main");
|
||||
const reclaimAt = deployJob.indexOf(
|
||||
'sudo chown -R "$' + "{DEPLOY_USER}:" + '$' + '{DEPLOY_GROUP}"',
|
||||
);
|
||||
const fetchAt = deployJob.indexOf("git -C \"${LIVE}\" fetch origin --prune");
|
||||
expect(reclaimAt).toBeGreaterThan(-1);
|
||||
expect(resetAt).toBeGreaterThan(reclaimAt);
|
||||
expect(fetchAt).toBeGreaterThan(reclaimAt);
|
||||
});
|
||||
|
||||
it("nuclear-replaces src/ and clears sticky bits without scanning every path", () => {
|
||||
expect(workflow).toContain("rm -rf src");
|
||||
expect(workflow).toContain("git checkout -f HEAD -- src");
|
||||
expect(workflow).toContain("no-skip-worktree");
|
||||
expect(workflow).toContain("no-assume-unchanged");
|
||||
expect(workflow).toContain("Verified src/ matches HEAD");
|
||||
expect(workflow).toContain("git ls-files -v");
|
||||
expect(workflow).not.toContain("git ls-files -z");
|
||||
expect(workflow).toContain("pnpm typecheck");
|
||||
it("avoids nuclear src wipe and verifies live src after cutover reset", () => {
|
||||
expect(deployJob).not.toContain("rm -rf src");
|
||||
expect(deployJob).not.toContain("Nuclear-replacing src/");
|
||||
expect(deployJob).toContain("no-skip-worktree");
|
||||
expect(deployJob).toContain("no-assume-unchanged");
|
||||
expect(deployJob).toContain("Verified live src/ matches HEAD");
|
||||
expect(deployJob).toContain("ls-files -v");
|
||||
expect(deployJob).not.toContain("git ls-files -z");
|
||||
expect(deployJob).toContain("pnpm typecheck");
|
||||
});
|
||||
|
||||
it("swaps a built .next artifact during a short service cutover", () => {
|
||||
expect(deployJob).toContain("mv .next .next.prev");
|
||||
expect(deployJob).toContain('mv "${STAGE}/.next" .next');
|
||||
expect(deployJob).toContain('mv "${STAGE}/node_modules" node_modules');
|
||||
expect(deployJob).toContain("Rolling back .next to previous artifact");
|
||||
const buildAt = deployJob.indexOf("pnpm build");
|
||||
const stopAt = deployJob.indexOf("sudo systemctl stop atom-nexst.service");
|
||||
const startLabelAt = deployJob.indexOf("Starting systemd service...");
|
||||
const startAt = deployJob.indexOf(
|
||||
"sudo systemctl start atom-nexst.service",
|
||||
startLabelAt,
|
||||
);
|
||||
expect(buildAt).toBeGreaterThan(-1);
|
||||
expect(stopAt).toBeGreaterThan(buildAt);
|
||||
expect(startLabelAt).toBeGreaterThan(stopAt);
|
||||
expect(startAt).toBeGreaterThan(startLabelAt);
|
||||
});
|
||||
|
||||
it("does not override onlyBuiltDependencies (uses pnpm-workspace.yaml)", () => {
|
||||
expect(workflow).not.toContain("PNPM_CONFIG_ONLY_BUILT_DEPENDENCIES");
|
||||
});
|
||||
|
||||
it("runs typecheck and tests before build", () => {
|
||||
expect(workflow).toContain("pnpm typecheck");
|
||||
expect(workflow).toContain("pnpm test");
|
||||
// Scope to the deploy job: the release job's documentation body also
|
||||
// mentions these commands, which must not affect this contract.
|
||||
const deployJob = workflow.slice(workflow.indexOf("\n deploy:"));
|
||||
it("runs typecheck and tests before build in the stage", () => {
|
||||
const typecheckAt = deployJob.indexOf("pnpm typecheck");
|
||||
const testAt = deployJob.indexOf("pnpm test");
|
||||
const buildAt = deployJob.indexOf("pnpm build");
|
||||
@@ -57,7 +73,9 @@ describe("production deploy workflow", () => {
|
||||
});
|
||||
|
||||
it("exports APP_VERSION from git for Sentry releases", () => {
|
||||
expect(workflow).toContain('export APP_VERSION="$(git rev-parse --short HEAD)"');
|
||||
expect(workflow).toContain(
|
||||
'export APP_VERSION="$(git -C "${LIVE}" rev-parse --short origin/main)"',
|
||||
);
|
||||
expect(workflow).toContain(
|
||||
'export NEXT_PUBLIC_APP_VERSION="$' + "{APP_VERSION}\"",
|
||||
);
|
||||
@@ -66,7 +84,6 @@ describe("production deploy workflow", () => {
|
||||
it("runs an HTTP health check before declaring deploy success", () => {
|
||||
expect(workflow).toContain("/api/health");
|
||||
expect(workflow).toContain('"database":true');
|
||||
const deployJob = workflow.slice(workflow.indexOf("\n deploy:"));
|
||||
const startAt = deployJob.indexOf("systemctl start atom-nexst.service");
|
||||
const healthAt = deployJob.indexOf("/api/health");
|
||||
const successAt = deployJob.indexOf("--- Deployed successfully ---");
|
||||
|
||||
Reference in new issue
Block a user