diff --git a/.gitea/workflows/deploy.yaml b/.gitea/workflows/deploy.yaml index 5fc9390e15..e29191e12d 100644 --- a/.gitea/workflows/deploy.yaml +++ b/.gitea/workflows/deploy.yaml @@ -236,58 +236,65 @@ jobs: echo "--- Deploying ---" + LIVE="/var/www/atom-nexst" + STAGE="" + CUTOVER_STARTED=0 + error_handler() { echo "!!! DEPLOYMENT FAILED on line $1 !!!" >&2 + # Roll back the build artifact if cutover already moved .next into place. + if [ "${CUTOVER_STARTED}" = "1" ] && [ -d "${LIVE}/.next.prev" ]; then + echo "Rolling back .next to previous artifact..." >&2 + rm -rf "${LIVE}/.next" || true + mv "${LIVE}/.next.prev" "${LIVE}/.next" || true + fi + if [ -n "${STAGE}" ] && [ -d "${STAGE}" ]; then + git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true + fi sudo systemctl start atom-nexst.service || true exit 1 } trap 'error_handler $LINENO' ERR docker image prune -f - cd /var/www/atom-nexst/ DEPLOY_USER="$(id -un)" DEPLOY_GROUP="$(id -gn)" - sudo chown -R "${DEPLOY_USER}:${DEPLOY_GROUP}" /var/www/atom-nexst/ - git config --global --add safe.directory /var/www/atom-nexst - git remote set-url origin /docker/gitea/gitea/git/repositories/remco/epicnext-cms.git/ + sudo chown -R "${DEPLOY_USER}:${DEPLOY_GROUP}" "${LIVE}" + git config --global --add safe.directory "${LIVE}" + git -C "${LIVE}" remote set-url origin /docker/gitea/gitea/git/repositories/remco/epicnext-cms.git/ echo "Fetching origin/main..." - git fetch origin --prune + git -C "${LIVE}" fetch origin --prune echo "Clearing sticky git index bits (if any)..." - STICKY_LIST="$(git ls-files -v | awk '/^[a-zS]/ {print substr($0,3)}' || true)" + STICKY_LIST="$(git -C "${LIVE}" ls-files -v | awk '/^[a-zS]/ {print substr($0,3)}' || true)" if [ -n "${STICKY_LIST}" ]; then echo "${STICKY_LIST}" | while IFS= read -r f; do [ -n "$f" ] || continue - git update-index --no-skip-worktree --no-assume-unchanged -- "$f" 2>/dev/null || true + git -C "${LIVE}" update-index --no-skip-worktree --no-assume-unchanged -- "$f" 2>/dev/null || true done fi - echo "Hard reset to origin/main..." - git reset --hard origin/main - - echo "Nuclear-replacing src/ from HEAD..." - rm -rf src - git checkout -f HEAD -- src - git clean -fd -e .env -e .env.local -e .env.production -e .env*.local - - if ! git diff --exit-code HEAD -- src >/dev/null; then - echo "ERROR: src/ still differs from HEAD after nuclear checkout:" >&2 - git diff --stat HEAD -- src >&2 || true - exit 1 - fi - echo "Verified src/ matches HEAD" - - rm -f tsconfig.tsbuildinfo .tsbuildinfo - find . -maxdepth 3 -name '*.tsbuildinfo' -delete 2>/dev/null || true - rm -rf .output dist .next/types .next/dev - - export APP_VERSION="$(git rev-parse --short HEAD)" + export APP_VERSION="$(git -C "${LIVE}" rev-parse --short origin/main)" export NEXT_PUBLIC_APP_VERSION="${APP_VERSION}" echo "APP_VERSION=${APP_VERSION}" + STAGE="/var/tmp/atom-nexst-stage-${APP_VERSION}" + echo "Preparing stage worktree at ${STAGE} (live site stays up)..." + git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true + git -C "${LIVE}" worktree add --detach "${STAGE}" origin/main + + # Production env stays on the live tree; stage only needs a symlink for build/migrate. + ln -sfn "${LIVE}/.env" "${STAGE}/.env" + + cd "${STAGE}" + rm -f tsconfig.tsbuildinfo .tsbuildinfo + find . -maxdepth 3 -name '*.tsbuildinfo' -delete 2>/dev/null || true + rm -rf .output dist .next .next/types .next/dev + pnpm install --frozen-lockfile + # Additive migrations while the old build still serves traffic. pnpm db:migrate pnpm prisma:generate pnpm typecheck @@ -296,11 +303,48 @@ jobs: # Do not set SKIP_ENV_VALIDATION here — that flag is for tests/tooling only. pnpm build - sudo chown -R www-data:www-data /var/www/atom-nexst/ + if [ ! -d "${STAGE}/.next" ]; then + echo "ERROR: stage build produced no .next/" >&2 + exit 1 + fi - echo "Hard resetting systemd service..." + echo "Cutover: stop service, sync code, swap .next artifact..." + CUTOVER_STARTED=1 sudo systemctl stop atom-nexst.service || true pkill -f 'next-server' || true + + cd "${LIVE}" + echo "Hard reset live tree to origin/main (no nuclear src wipe)..." + git reset --hard origin/main + # Keep env, uploads, and deps we are about to replace from stage. + git clean -fd \ + -e .env -e .env.local -e .env.production -e .env*.local \ + -e storage -e public/cache -e node_modules -e .next -e .next.prev + + if ! git diff --exit-code HEAD -- src >/dev/null; then + echo "ERROR: live src/ still differs from HEAD after reset:" >&2 + git diff --stat HEAD -- src >&2 || true + exit 1 + fi + echo "Verified live src/ matches HEAD" + + # Atomic-ish artifact swap: keep previous .next until the new one is in place. + rm -rf .next.prev + if [ -d .next ]; then + mv .next .next.prev + fi + mv "${STAGE}/.next" .next + + # Use the exact node_modules the stage build resolved against. + rm -rf node_modules + mv "${STAGE}/node_modules" node_modules + + # Prisma client is gitignored — regenerate into live src/generated. + pnpm prisma:generate + + sudo chown -R www-data:www-data "${LIVE}" + + echo "Starting systemd service..." sudo systemctl start atom-nexst.service sleep 2 @@ -329,4 +373,9 @@ jobs: exit 1 fi + echo "Cleaning stage worktree and previous .next..." + rm -rf "${LIVE}/.next.prev" + git -C "${LIVE}" worktree remove --force "${STAGE}" 2>/dev/null || rm -rf "${STAGE}" || true + STAGE="" + echo "--- Deployed successfully ---" diff --git a/src/lib/deploy-workflow-contract.test.ts b/src/lib/deploy-workflow-contract.test.ts index 423bb8c589..299ca03276 100644 --- a/src/lib/deploy-workflow-contract.test.ts +++ b/src/lib/deploy-workflow-contract.test.ts @@ -7,47 +7,63 @@ describe("production deploy workflow", () => { resolve(process.cwd(), ".gitea/workflows/deploy.yaml"), "utf8", ); + const deployJob = workflow.slice(workflow.indexOf("\n deploy:")); - it("preserves the Next.js incremental build cache", () => { - // May clear .next/types or .next/dev, but must not wipe the whole .next tree. - expect(workflow).not.toMatch(/rm\s+-rf\s+\.next(?:\s|$)/); - expect(workflow).toContain("rm -rf .output dist .next/types .next/dev"); - expect(workflow).toContain("pnpm install --frozen-lockfile"); - // Production builds must validate env (AUTH_SECRET, DATABASE_URL, …). - expect(workflow).not.toContain("SKIP_ENV_VALIDATION=1"); + it("builds in a stage worktree while preserving live .env and storage", () => { + expect(deployJob).toContain("worktree add --detach"); + expect(deployJob).toContain("/var/tmp/atom-nexst-stage-"); + expect(deployJob).toContain('ln -sfn "${LIVE}/.env"'); + expect(deployJob).toContain("-e storage"); + expect(deployJob).not.toContain("SKIP_ENV_VALIDATION=1"); + expect(deployJob).toContain("pnpm install --frozen-lockfile"); }); - it("reclaims ownership before git reset so www-data files can be overwritten", () => { - expect(workflow).toContain('sudo chown -R "$' + "{DEPLOY_USER}:" + '$' + '{DEPLOY_GROUP}"'); - const reclaimAt = workflow.indexOf( + it("reclaims ownership before git operations so www-data files can be overwritten", () => { + expect(workflow).toContain( 'sudo chown -R "$' + "{DEPLOY_USER}:" + '$' + '{DEPLOY_GROUP}"', ); - const resetAt = workflow.indexOf("git reset --hard origin/main"); + const reclaimAt = deployJob.indexOf( + 'sudo chown -R "$' + "{DEPLOY_USER}:" + '$' + '{DEPLOY_GROUP}"', + ); + const fetchAt = deployJob.indexOf("git -C \"${LIVE}\" fetch origin --prune"); expect(reclaimAt).toBeGreaterThan(-1); - expect(resetAt).toBeGreaterThan(reclaimAt); + expect(fetchAt).toBeGreaterThan(reclaimAt); }); - it("nuclear-replaces src/ and clears sticky bits without scanning every path", () => { - expect(workflow).toContain("rm -rf src"); - expect(workflow).toContain("git checkout -f HEAD -- src"); - expect(workflow).toContain("no-skip-worktree"); - expect(workflow).toContain("no-assume-unchanged"); - expect(workflow).toContain("Verified src/ matches HEAD"); - expect(workflow).toContain("git ls-files -v"); - expect(workflow).not.toContain("git ls-files -z"); - expect(workflow).toContain("pnpm typecheck"); + it("avoids nuclear src wipe and verifies live src after cutover reset", () => { + expect(deployJob).not.toContain("rm -rf src"); + expect(deployJob).not.toContain("Nuclear-replacing src/"); + expect(deployJob).toContain("no-skip-worktree"); + expect(deployJob).toContain("no-assume-unchanged"); + expect(deployJob).toContain("Verified live src/ matches HEAD"); + expect(deployJob).toContain("ls-files -v"); + expect(deployJob).not.toContain("git ls-files -z"); + expect(deployJob).toContain("pnpm typecheck"); + }); + + it("swaps a built .next artifact during a short service cutover", () => { + expect(deployJob).toContain("mv .next .next.prev"); + expect(deployJob).toContain('mv "${STAGE}/.next" .next'); + expect(deployJob).toContain('mv "${STAGE}/node_modules" node_modules'); + expect(deployJob).toContain("Rolling back .next to previous artifact"); + const buildAt = deployJob.indexOf("pnpm build"); + const stopAt = deployJob.indexOf("sudo systemctl stop atom-nexst.service"); + const startLabelAt = deployJob.indexOf("Starting systemd service..."); + const startAt = deployJob.indexOf( + "sudo systemctl start atom-nexst.service", + startLabelAt, + ); + expect(buildAt).toBeGreaterThan(-1); + expect(stopAt).toBeGreaterThan(buildAt); + expect(startLabelAt).toBeGreaterThan(stopAt); + expect(startAt).toBeGreaterThan(startLabelAt); }); it("does not override onlyBuiltDependencies (uses pnpm-workspace.yaml)", () => { expect(workflow).not.toContain("PNPM_CONFIG_ONLY_BUILT_DEPENDENCIES"); }); - it("runs typecheck and tests before build", () => { - expect(workflow).toContain("pnpm typecheck"); - expect(workflow).toContain("pnpm test"); - // Scope to the deploy job: the release job's documentation body also - // mentions these commands, which must not affect this contract. - const deployJob = workflow.slice(workflow.indexOf("\n deploy:")); + it("runs typecheck and tests before build in the stage", () => { const typecheckAt = deployJob.indexOf("pnpm typecheck"); const testAt = deployJob.indexOf("pnpm test"); const buildAt = deployJob.indexOf("pnpm build"); @@ -57,7 +73,9 @@ describe("production deploy workflow", () => { }); it("exports APP_VERSION from git for Sentry releases", () => { - expect(workflow).toContain('export APP_VERSION="$(git rev-parse --short HEAD)"'); + expect(workflow).toContain( + 'export APP_VERSION="$(git -C "${LIVE}" rev-parse --short origin/main)"', + ); expect(workflow).toContain( 'export NEXT_PUBLIC_APP_VERSION="$' + "{APP_VERSION}\"", ); @@ -66,7 +84,6 @@ describe("production deploy workflow", () => { it("runs an HTTP health check before declaring deploy success", () => { expect(workflow).toContain("/api/health"); expect(workflow).toContain('"database":true'); - const deployJob = workflow.slice(workflow.indexOf("\n deploy:")); const startAt = deployJob.indexOf("systemctl start atom-nexst.service"); const healthAt = deployJob.indexOf("/api/health"); const successAt = deployJob.indexOf("--- Deployed successfully ---");