feat: export Catalog Studio assets and SQL to catalog repository

2026-09-05 11:49:00 +02:00
1 parent bfbf244141
commit 9ec9ab31ad
@@ -76,3 +76,8 @@ LOG_LEVEL=error

 # --- FLARESOLVERR (Cloudflare bypass for clone sources) ---
 FLARESOLVERR_URL=http://localhost:8191
+
+# Catalog Studio export: dedicated clean clone on Beta-3 with Git push credentials.
+CATALOG_GIT_CHECKOUT=
+# Persistent directory shared by CMS and worker, outside the catalog clone.
+CATALOG_GIT_STATE_DIR=
@@ -0,0 +1,73 @@
+# Catalog Studio repository export
+
+Studio mutations automatically queue an export to
+`https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git`, branch
+`Beta-3`. The jobs worker processes pending exports every minute. Streaming
+imports stay active until their stream completes. Server actions for catalog
+pages, offers, deletion and maintenance are covered as well.
+
+## Server setup
+
+1. Create a **dedicated clean clone** of the repository on `Beta-3`, outside the
+   CMS directory. Configure non-interactive Git push authentication for the
+   worker OS account using its credential helper. Do not put tokens in URLs.
+2. Set `CATALOG_GIT_CHECKOUT` to that absolute clone directory in the CMS and
+   worker environments. Set `CATALOG_GIT_STATE_DIR` to a persistent, writable
+   directory outside the clone, shared by both processes on the same host.
+3. Run `pnpm jobs:worker` alongside the CMS under your process supervisor.
+   Both processes must have access to the configured asset directories and DB.
+   The worker command enables the React server condition for server-only modules.
+4. Restart the CMS after setting the environment. In **Studio → Sync**, use
+   **Export now / retry** for the initial export. Subsequent mutations queue
+   automatically. Status shows pending/active operations and the last commit.
+
+Leaving `CATALOG_GIT_CHECKOUT` empty disables export. No credentials are shipped.
+This source change alone does not configure or deploy the production service.
+
+## Exported content
+
+| Source | Repository destination |
+| --- | --- |
+| Configured Nitro bundles, including furniture, figures, effects and pets | `Gamedata/bundled` |
+| Configured furni icons | `Gamedata/icons` |
+| Existing badge/catalog images | `Gamedata/c_images` |
+| FurnitureData and supported public game-data JSON files | `Gamedata/config` |
+| Existing localized FurnitureData files | `catalogue version 2 ( Final (Dev)/langs furnidata` |
+| `items_base`, `catalog_pages`, `catalog_items` | `catalogue version 2 ( Final (Dev)/sqls` |
+| `catalog_pages_bc`, `catalog_items_bc`, when present | Same SQL directory |
+
+SQL is read in one consistent, read-only InnoDB transaction. Dumps contain table
+definitions and deterministic upserts with hexadecimal UTF-8 string literals.
+Import `items_base.sql`, then `catalog_pages.sql`, then `catalog_items.sql`.
+Existing schemas are not migrated by these dumps. Rows absent from the source
+are omitted; importing an upsert dump into another existing database does not
+delete that database's extra rows. No user, session or credential tables are exported.
+
+Only existing local assets are exported. Translation generation follows Studio's
+existing setting; export does not generate missing languages or download assets.
+Public JSON is explicitly allowlisted so translation caches and private runtime
+files cannot enter the repository. Invalid JSON or concurrent Studio changes
+prevent publication of that snapshot.
+
+## Failure and concurrency behavior
+
+- Pending events survive process restarts; events added during publication remain
+  pending for the next run. Partial imports are exported as their settled local
+  state, including successful items from a batch containing failures.
+- A single filesystem lock serializes the worker. Dead local process markers are
+  recovered on the next run. For an unreadable marker or a marker from another
+  host, stop the CMS and worker before repairing the queue directory.
+- A failed push retains the local commit and pending events for retry. Concurrent
+  upstream commits are rebased; a conflict aborts the rebase and leaves the event
+  pending. Resolve conflicts in the dedicated clone, then retry.
+- The checkout must be clean before each run. Unrelated files are preserved and
+  no force push is used. Missing local files do not cause remote deletions.
+- Status errors omit raw Git output to avoid exposing authentication material.
+
+## Verification
+
+Run the `catalog-git-*` service tests and `src/lib/catalog-export-api.test.ts` with
+Vitest. The integration test creates a temporary bare remote and verifies push,
+failed-push recovery, no-op export and preservation of unrelated files. SQL
+snapshot tests mock the database; production DB import and production push need
+verification in the deployed environment.
@@ -16,7 +16,7 @@
 		"format": "biome format --write .",
 		"knip": "knip --include files,dependencies,devDependencies,unlisted,binaries",
 		"diag:permissions": "tsx scripts/diagnose-permission-page.ts",
-		"jobs:worker": "tsx scripts/jobs-worker.ts",
+		"jobs:worker": "node --conditions=react-server --import tsx scripts/jobs-worker.ts",
 		"test": "vitest run",
 		"test:e2e": "playwright test",
 		"typecheck": "tsc --noEmit",
@@ -11,6 +11,7 @@ import {
 import { logger } from "../src/lib/logger";
 import { redis } from "../src/lib/redis";
 import { emulatorOffline, healthDegraded } from "../src/lib/services/alert";
+import { runCatalogExport } from "../src/lib/services/catalog-git-export";
 import { rcon } from "../src/lib/services/rcon";

 function captureWorkerError(err: unknown, context: string): void {
@@ -261,6 +262,11 @@ async function publishScheduledArticles(): Promise<void> {
 }

 async function main() {
+	new Cron("* * * * *", () => {
+		runCatalogExport().catch((e) =>
+			captureWorkerError(e, "Catalog export failed"),
+		);
+	});
 	logger.info("Worker started", { module: "jobs" });

 	if (env.EMULATOR_JAR_PATH && env.EMULATOR_BACKUP_DIR) {
@@ -7,6 +7,7 @@ import {
 	actionOk,
 	handleActionError,
 } from "@/lib/safe-action-shared";
+import { withCatalogExport } from "@/lib/services/catalog-git-queue";
 import type {
 	AlignResult,
 	DedupResult,
@@ -38,7 +39,9 @@ export async function fixSpriteIdsAction(): Promise<
 > {
 	try {
 		await guard();
-		return actionOk(await maintenance.fixSpriteIds());
+		return await withCatalogExport(async () => {
+			return actionOk(await maintenance.fixSpriteIds());
+		});
 	} catch (e) {
 		return handleActionError(e);
 	}
@@ -49,7 +52,9 @@ export async function fixCatalogOffersAction(): Promise<
 > {
 	try {
 		await guard();
-		return actionOk(await maintenance.fixCatalogOffers());
+		return await withCatalogExport(async () => {
+			return actionOk(await maintenance.fixCatalogOffers());
+		});
 	} catch (e) {
 		return handleActionError(e);
 	}
@@ -60,7 +65,9 @@ export async function reconcileIdsAction(): Promise<
 > {
 	try {
 		await guard();
-		return actionOk(await maintenance.reconcileIds());
+		return await withCatalogExport(async () => {
+			return actionOk(await maintenance.reconcileIds());
+		});
 	} catch (e) {
 		return handleActionError(e);
 	}
@@ -71,7 +78,9 @@ export async function removeDuplicateItemsBaseAction(): Promise<
 > {
 	try {
 		await guard();
-		return actionOk(await maintenance.removeDuplicates());
+		return await withCatalogExport(async () => {
+			return actionOk(await maintenance.removeDuplicates());
+		});
 	} catch (e) {
 		return handleActionError(e);
 	}
@@ -93,7 +102,9 @@ export async function applyAlignIdsAction(): Promise<
 > {
 	try {
 		await guard();
-		return actionOk(await maintenance.forceItemsBaseIdsToFurnidata(true));
+		return await withCatalogExport(async () => {
+			return actionOk(await maintenance.forceItemsBaseIdsToFurnidata(true));
+		});
 	} catch (e) {
 		return handleActionError(e);
 	}
@@ -104,7 +115,9 @@ export async function fixEverythingAction(input?: {
 }): Promise<ActionResult<maintenance.FixAllResult>> {
 	try {
 		await guard();
-		return actionOk(await maintenance.fixEverything(input ?? {}));
+		return await withCatalogExport(async () => {
+			return actionOk(await maintenance.fixEverything(input ?? {}));
+		});
 	} catch (e) {
 		return handleActionError(e);
 	}
@@ -5,6 +5,7 @@ import { db, WebsiteSetting } from "@/lib/db";
 import { PERMS } from "@/lib/permissions";
 import { adminAction } from "@/lib/safe-action";
 import { actionOk } from "@/lib/safe-action-shared";
+import { withCatalogExport } from "@/lib/services/catalog-git-queue";
 import { deleteImportedItem } from "@/lib/services/furni-import";
 import { siteSettings } from "@/lib/services/site-settings";

@@ -14,10 +15,9 @@ export const deleteImportedFurni = adminAction(
 	{ permission: PERMS.ASSETS_IMPORT, schema: deleteSchema },
 	async (ctx) =>
 		actionOk(
-			(await deleteImportedItem(ctx.data.classname)) as unknown as Record<
-				string,
-				unknown
-			>,
+			(await withCatalogExport(() =>
+				deleteImportedItem(ctx.data.classname),
+			)) as unknown as Record<string, unknown>,
 		),
 );

@@ -3,6 +3,8 @@ import { OfficialSyncClient } from "@/app/admin/import/official/official-sync-cl
 import { SyncAllClient } from "@/app/admin/import/sync/sync-all-client";
 import { canAccess, getAdminContext, PERMS } from "@/lib/permissions";

+import { CatalogExportClient } from "./catalog-export-client";
+
 export default async function StudioSyncPage(_props: {
 	params: Promise<{ locale: string }>;
 }) {
@@ -12,6 +14,7 @@ export default async function StudioSyncPage(_props: {
 	}
 	return (
 		<div className="space-y-8">
+			<CatalogExportClient />
 			<h1 className="text-2xl font-bold tracking-tight">Sync All Sources</h1>
 			<p className="text-muted-foreground mt-2">
 				Scan configured sources and automatically import any furniture that is
@@ -0,0 +1,18 @@
+import { withAdmin } from "@/lib/api-handler";
+import { apiError, apiOk } from "@/lib/api-response";
+import { PERMS } from "@/lib/permissions";
+import {
+	catalogExportEnabled,
+	catalogExportQueue,
+	catalogExportStatus,
+} from "@/lib/services/catalog-git-queue";
+
+export const GET = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () =>
+	apiOk(await catalogExportStatus()),
+);
+export const POST = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => {
+	if (!catalogExportEnabled())
+		return apiError("Catalog repository export is not configured", 400);
+	await catalogExportQueue().request();
+	return apiOk(await catalogExportStatus());
+});
@@ -1,9 +1,15 @@
-import { type NextRequest, NextResponse } from "next/server";
+import { after, type NextRequest, NextResponse } from "next/server";
 import { logAuthorizationEvent } from "@/lib/admin/authorization-events";
 import { validateCsrfToken } from "@/lib/foundation/security";
 import { canAccess, getApiAdminContext } from "@/lib/permissions";
 import { logServerError } from "@/lib/server-log";

+import {
+	catalogExportEnabled,
+	catalogExportQueue,
+	isCatalogMutation,
+} from "@/lib/services/catalog-git-queue";
+
 const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);
 const MAX_BODY_BYTES = 10 * 1024 * 1024; // 10 MB

@@ -80,9 +86,42 @@ export function withAdmin(
 				{ status: 403 },
 			);
 		}
+		let finishExport: (() => Promise<void>) | undefined;
 		try {
-			return await handler(request, context, routeContext);
+			if (
+				catalogExportEnabled() &&
+				isCatalogMutation(request.method, request.nextUrl.pathname)
+			) {
+				finishExport = await catalogExportQueue().begin();
+			}
+			const response = await handler(request, context, routeContext);
+			if (
+				finishExport &&
+				response.body &&
+				response.headers.get("content-type")?.includes("text/event-stream")
+			) {
+				const [client, completion] = response.body.tee();
+				const finish = finishExport;
+				after(async () => {
+					const reader = completion.getReader();
+					try {
+						while (!(await reader.read()).done) {
+							/* Wait for all import results. */
+						}
+					} finally {
+						reader.releaseLock();
+						await finish();
+					}
+				});
+				return new Response(client, {
+					status: response.status,
+					headers: response.headers,
+				});
+			}
+			await finishExport?.();
+			return response;
 		} catch (error) {
+			await finishExport?.().catch(() => undefined);
 			logServerError("admin.api_failed", error, {
 				path: request.nextUrl.pathname,
 				userId: context.session.user.id,
@@ -0,0 +1,87 @@
+import { mkdtemp } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { NextRequest } from "next/server";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+	after: vi.fn(),
+	auth: vi.fn(),
+	csrf: vi.fn(),
+}));
+vi.mock("next/server", async (original) => ({
+	...(await original<typeof import("next/server")>()),
+	after: mocks.after,
+}));
+vi.mock("@/lib/admin/authorization-events", () => ({
+	logAuthorizationEvent: vi.fn(),
+}));
+vi.mock("@/lib/foundation/security", () => ({ validateCsrfToken: mocks.csrf }));
+vi.mock("@/lib/permissions", () => ({
+	getApiAdminContext: mocks.auth,
+	canAccess: () => true,
+}));
+vi.mock("@/lib/server-log", () => ({ logServerError: vi.fn() }));
+
+import { withAdmin } from "./api-handler";
+import { catalogExportQueue } from "./services/catalog-git-queue";
+
+describe("catalog API completion tracking", () => {
+	beforeEach(async () => {
+		vi.stubEnv("CATALOG_GIT_CHECKOUT", "/catalog");
+		vi.stubEnv(
+			"CATALOG_GIT_STATE_DIR",
+			await mkdtemp(path.join(os.tmpdir(), "catalog-api-")),
+		);
+		mocks.auth.mockResolvedValue({
+			session: { user: { id: 1, rank: 7 } },
+			permissions: [],
+		});
+		mocks.csrf.mockResolvedValue(true);
+		mocks.after.mockReset();
+	});
+	afterEach(() => vi.unstubAllEnvs());
+	it("waits for the last SSE event before making changes publishable", async () => {
+		let end: (() => void) | undefined;
+		const response = new Response(
+			new ReadableStream({
+				start(controller) {
+					controller.enqueue(
+						new TextEncoder().encode('data: {"type":"progress"}\n\n'),
+					);
+					end = () => controller.close();
+				},
+			}),
+			{ headers: { "content-type": "text/event-stream" } },
+		);
+		const route = withAdmin({}, async () => response);
+		const returned = await route(
+			new NextRequest("http://localhost/api/admin/import/furni/batch", {
+				method: "POST",
+			}),
+		);
+		expect(await catalogExportQueue().batch()).toBeNull();
+		const complete = mocks.after.mock.calls[0][0]();
+		end?.();
+		await returned.text();
+		await complete;
+		expect(await catalogExportQueue().batch()).toHaveLength(1);
+	});
+	it("does not queue unauthenticated requests or reads", async () => {
+		const handler = vi.fn(async () => new Response("{}"));
+		const route = withAdmin({}, handler);
+		mocks.auth.mockResolvedValueOnce(null);
+		expect(
+			(
+				await route(
+					new NextRequest("http://localhost/api/admin/import/furni", {
+						method: "POST",
+					}),
+				)
+			).status,
+		).toBe(401);
+		expect(handler).not.toHaveBeenCalled();
+		await route(new NextRequest("http://localhost/api/admin/import/furni"));
+		expect(await catalogExportQueue().batch()).toHaveLength(0);
+	});
+});
@@ -0,0 +1,57 @@
+import { mkdtemp, readFile } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({ snapshot: vi.fn(), publish: vi.fn() }));
+vi.mock("./catalog-git-snapshot", () => ({
+	createCatalogSnapshot: mocks.snapshot,
+}));
+vi.mock("./catalog-git-core", async (original) => ({
+	...(await original<typeof import("./catalog-git-core")>()),
+	publishCatalogFiles: mocks.publish,
+}));
+
+import { runCatalogExport } from "./catalog-git-export";
+import { catalogExportQueue, withCatalogExport } from "./catalog-git-queue";
+
+describe("export worker", () => {
+	beforeEach(async () => {
+		vi.stubEnv("CATALOG_GIT_CHECKOUT", "/configured/checkout");
+		vi.stubEnv(
+			"CATALOG_GIT_STATE_DIR",
+			await mkdtemp(path.join(os.tmpdir(), "catalog-worker-")),
+		);
+		mocks.snapshot.mockReset().mockResolvedValue([]);
+		mocks.publish.mockReset().mockResolvedValue("abc123");
+	});
+	afterEach(() => vi.unstubAllEnvs());
+	it("retains failed work, retries it and clears only completed events", async () => {
+		const q = catalogExportQueue();
+		await q.request();
+		mocks.publish.mockRejectedValueOnce(new Error("credential secret"));
+		await runCatalogExport();
+		expect(await q.batch()).toHaveLength(1);
+		expect(
+			await readFile(path.join(q.root, "status.json"), "utf8"),
+		).not.toContain("credential secret");
+		await runCatalogExport();
+		expect(await q.batch()).toHaveLength(0);
+		expect(mocks.publish).toHaveBeenCalledTimes(2);
+	});
+	it("does not publish during an operation or if one starts while capturing", async () => {
+		const q = catalogExportQueue();
+		await q.request();
+		await withCatalogExport(async () => {
+			await runCatalogExport();
+		});
+		expect(mocks.snapshot).not.toHaveBeenCalled();
+		mocks.snapshot.mockImplementationOnce(async () => {
+			await q.request();
+			return [];
+		});
+		await runCatalogExport();
+		expect(mocks.publish).not.toHaveBeenCalled();
+		expect(await q.batch()).toHaveLength(3);
+	});
+});
@@ -0,0 +1,82 @@
+import { promises as fs } from "node:fs";
+import { hostname } from "node:os";
+import path from "node:path";
+import {
+	CATALOG_BRANCH,
+	CATALOG_REMOTE,
+	publishCatalogFiles,
+	recoverCatalogQueue,
+} from "./catalog-git-core";
+import { catalogExportEnabled, catalogExportQueue } from "./catalog-git-queue";
+import { createCatalogSnapshot } from "./catalog-git-snapshot";
+
+export async function runCatalogExport() {
+	const checkout = process.env.CATALOG_GIT_CHECKOUT?.trim();
+	if (!catalogExportEnabled() || !checkout) return;
+	const queue = catalogExportQueue();
+	await queue.entries();
+	await recoverCatalogQueue(queue.root);
+	const lock = await fs
+		.open(path.join(queue.root, "worker.lock"), "wx")
+		.catch((error: NodeJS.ErrnoException) => {
+			if (error.code === "EEXIST") return null;
+			throw error;
+		});
+	if (!lock) return;
+	let snapshot: string | undefined;
+	const status = async (value: Record<string, unknown>) => {
+		const temp = path.join(queue.root, "status.tmp");
+		let previous: Record<string, unknown> = {};
+		try {
+			previous = JSON.parse(
+				await fs.readFile(path.join(queue.root, "status.json"), "utf8"),
+			);
+		} catch {
+			/* First attempt. */
+		}
+		await fs.writeFile(
+			temp,
+			JSON.stringify({ ...previous, error: undefined, ...value }),
+		);
+		await fs.rename(temp, path.join(queue.root, "status.json"));
+	};
+	try {
+		await lock.writeFile(
+			JSON.stringify({ pid: process.pid, host: hostname() }),
+		);
+		const batch = await queue.batch();
+		if (!batch?.length) return;
+		snapshot = await fs.mkdtemp(path.join(queue.root, "snapshot-"));
+		const files = await createCatalogSnapshot(snapshot);
+		// Any operation started/completed during capture invalidates the snapshot.
+		const current = await queue.batch();
+		if (!current || JSON.stringify(current) !== JSON.stringify(batch)) return;
+		const commit = await publishCatalogFiles({
+			checkout,
+			remote: CATALOG_REMOTE,
+			branch: CATALOG_BRANCH,
+			files,
+		});
+		await queue.complete(batch);
+		await status({ commit, finishedAt: new Date().toISOString() });
+	} catch {
+		// Do not expose credentials or Git stderr through the admin API.
+		await status({
+			error:
+				"Export failed. Check repository access, branch, clean checkout and asset/database availability. Pending changes will be retried.",
+			finishedAt: new Date().toISOString(),
+		});
+	} finally {
+		try {
+			if (
+				snapshot &&
+				path.resolve(snapshot).startsWith(path.resolve(queue.root) + path.sep)
+			) {
+				await fs.rm(snapshot, { recursive: true, force: true });
+			}
+		} finally {
+			await lock.close();
+			await fs.rm(path.join(queue.root, "worker.lock"), { force: true });
+		}
+	}
+}
@@ -0,0 +1,54 @@
+import { promises as fs } from "node:fs";
+import path from "node:path";
+import { CatalogExportQueue } from "./catalog-git-core";
+
+export function catalogExportEnabled() {
+	return Boolean(process.env.CATALOG_GIT_CHECKOUT?.trim());
+}
+export function catalogExportQueue() {
+	return new CatalogExportQueue(
+		process.env.CATALOG_GIT_STATE_DIR ||
+			path.join(process.cwd(), "storage", "catalog-git"),
+	);
+}
+export function isCatalogMutation(method: string, pathname: string) {
+	return (
+		["POST", "PUT", "PATCH", "DELETE"].includes(method) &&
+		(pathname.startsWith("/api/admin/import/") ||
+			pathname.startsWith("/api/admin/catalog/") ||
+			pathname.startsWith("/api/admin/furni/") ||
+			pathname.startsWith("/api/admin/furniture/"))
+	);
+}
+export async function catalogExportStatus() {
+	const q = catalogExportQueue();
+	if (!catalogExportEnabled()) return { enabled: false, pending: 0, active: 0 };
+	const entries = await q.entries();
+	let last: { commit?: string; finishedAt?: string; error?: string } = {};
+	try {
+		last = JSON.parse(
+			await fs.readFile(path.join(q.root, "status.json"), "utf8"),
+		);
+	} catch (error) {
+		if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
+	}
+	return {
+		enabled: true,
+		pending: entries.filter((f) => f.endsWith(".pending")).length,
+		active: entries.filter((f) => f.endsWith(".active")).length,
+		running: entries.includes("worker.lock"),
+		...last,
+	};
+}
+
+export async function withCatalogExport<T>(
+	operation: () => Promise<T>,
+): Promise<T> {
+	if (!catalogExportEnabled()) return operation();
+	const finish = await catalogExportQueue().begin();
+	try {
+		return await operation();
+	} finally {
+		await finish();
+	}
+}