diff --git a/.env.example b/.env.example index 285ab245b4..14d9448f36 100644 --- a/.env.example +++ b/.env.example @@ -76,3 +76,8 @@ LOG_LEVEL=error # --- FLARESOLVERR (Cloudflare bypass for clone sources) --- FLARESOLVERR_URL=http://localhost:8191 + +# Catalog Studio export: dedicated clean clone on Beta-3 with Git push credentials. +CATALOG_GIT_CHECKOUT= +# Persistent directory shared by CMS and worker, outside the catalog clone. +CATALOG_GIT_STATE_DIR= diff --git a/CATALOG_EXPORT.md b/CATALOG_EXPORT.md new file mode 100644 index 0000000000..d69639f556 --- /dev/null +++ b/CATALOG_EXPORT.md @@ -0,0 +1,73 @@ +# Catalog Studio repository export + +Studio mutations automatically queue an export to +`https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git`, branch +`Beta-3`. The jobs worker processes pending exports every minute. Streaming +imports stay active until their stream completes. Server actions for catalog +pages, offers, deletion and maintenance are covered as well. + +## Server setup + +1. Create a **dedicated clean clone** of the repository on `Beta-3`, outside the + CMS directory. Configure non-interactive Git push authentication for the + worker OS account using its credential helper. Do not put tokens in URLs. +2. Set `CATALOG_GIT_CHECKOUT` to that absolute clone directory in the CMS and + worker environments. Set `CATALOG_GIT_STATE_DIR` to a persistent, writable + directory outside the clone, shared by both processes on the same host. +3. Run `pnpm jobs:worker` alongside the CMS under your process supervisor. + Both processes must have access to the configured asset directories and DB. + The worker command enables the React server condition for server-only modules. +4. Restart the CMS after setting the environment. In **Studio → Sync**, use + **Export now / retry** for the initial export. Subsequent mutations queue + automatically. Status shows pending/active operations and the last commit. + +Leaving `CATALOG_GIT_CHECKOUT` empty disables export. No credentials are shipped. +This source change alone does not configure or deploy the production service. + +## Exported content + +| Source | Repository destination | +| --- | --- | +| Configured Nitro bundles, including furniture, figures, effects and pets | `Gamedata/bundled` | +| Configured furni icons | `Gamedata/icons` | +| Existing badge/catalog images | `Gamedata/c_images` | +| FurnitureData and supported public game-data JSON files | `Gamedata/config` | +| Existing localized FurnitureData files | `catalogue version 2 ( Final (Dev)/langs furnidata` | +| `items_base`, `catalog_pages`, `catalog_items` | `catalogue version 2 ( Final (Dev)/sqls` | +| `catalog_pages_bc`, `catalog_items_bc`, when present | Same SQL directory | + +SQL is read in one consistent, read-only InnoDB transaction. Dumps contain table +definitions and deterministic upserts with hexadecimal UTF-8 string literals. +Import `items_base.sql`, then `catalog_pages.sql`, then `catalog_items.sql`. +Existing schemas are not migrated by these dumps. Rows absent from the source +are omitted; importing an upsert dump into another existing database does not +delete that database's extra rows. No user, session or credential tables are exported. + +Only existing local assets are exported. Translation generation follows Studio's +existing setting; export does not generate missing languages or download assets. +Public JSON is explicitly allowlisted so translation caches and private runtime +files cannot enter the repository. Invalid JSON or concurrent Studio changes +prevent publication of that snapshot. + +## Failure and concurrency behavior + +- Pending events survive process restarts; events added during publication remain + pending for the next run. Partial imports are exported as their settled local + state, including successful items from a batch containing failures. +- A single filesystem lock serializes the worker. Dead local process markers are + recovered on the next run. For an unreadable marker or a marker from another + host, stop the CMS and worker before repairing the queue directory. +- A failed push retains the local commit and pending events for retry. Concurrent + upstream commits are rebased; a conflict aborts the rebase and leaves the event + pending. Resolve conflicts in the dedicated clone, then retry. +- The checkout must be clean before each run. Unrelated files are preserved and + no force push is used. Missing local files do not cause remote deletions. +- Status errors omit raw Git output to avoid exposing authentication material. + +## Verification + +Run the `catalog-git-*` service tests and `src/lib/catalog-export-api.test.ts` with +Vitest. The integration test creates a temporary bare remote and verifies push, +failed-push recovery, no-op export and preservation of unrelated files. SQL +snapshot tests mock the database; production DB import and production push need +verification in the deployed environment. diff --git a/package.json b/package.json index 99a34088ae..6468d2bb4f 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "format": "biome format --write .", "knip": "knip --include files,dependencies,devDependencies,unlisted,binaries", "diag:permissions": "tsx scripts/diagnose-permission-page.ts", - "jobs:worker": "tsx scripts/jobs-worker.ts", + "jobs:worker": "node --conditions=react-server --import tsx scripts/jobs-worker.ts", "test": "vitest run", "test:e2e": "playwright test", "typecheck": "tsc --noEmit", diff --git a/scripts/jobs-worker.ts b/scripts/jobs-worker.ts index 97b8f377df..7f3b50b272 100644 --- a/scripts/jobs-worker.ts +++ b/scripts/jobs-worker.ts @@ -11,6 +11,7 @@ import { import { logger } from "../src/lib/logger"; import { redis } from "../src/lib/redis"; import { emulatorOffline, healthDegraded } from "../src/lib/services/alert"; +import { runCatalogExport } from "../src/lib/services/catalog-git-export"; import { rcon } from "../src/lib/services/rcon"; function captureWorkerError(err: unknown, context: string): void { @@ -261,6 +262,11 @@ async function publishScheduledArticles(): Promise { } async function main() { + new Cron("* * * * *", () => { + runCatalogExport().catch((e) => + captureWorkerError(e, "Catalog export failed"), + ); + }); logger.info("Worker started", { module: "jobs" }); if (env.EMULATOR_JAR_PATH && env.EMULATOR_BACKUP_DIR) { diff --git a/src/actions/catalog-bc.ts b/src/actions/catalog-bc.ts index de924b9f1a..c6a862c52b 100644 --- a/src/actions/catalog-bc.ts +++ b/src/actions/catalog-bc.ts @@ -5,6 +5,7 @@ import { revalidatePath } from "next/cache"; import { requirePermission } from "@/lib/admin/guard"; import { CatalogItemsBc, CatalogPagesBc, db } from "@/lib/db"; import { PERMS } from "@/lib/permissions"; +import { withCatalogExport } from "@/lib/services/catalog-git-queue"; import { rcon } from "@/lib/services/rcon"; import { logStaffActivity } from "@/lib/services/staff-activity"; @@ -52,39 +53,43 @@ export async function updateBcPage({ ...fields }: { id: number } & Record) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - const data = pickAllowed(fields, BC_PAGE_FIELDS); - if (Object.keys(data).length === 0) { - return { ok: false as const, error: "No valid fields to update" }; - } - await db - .update(CatalogPagesBc) - .set(data as Partial) - .where(eq(CatalogPagesBc.id, id)); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "bc_page_update", - description: `Updated BC catalog page #${id}`, - targetType: "catalog_page_bc", - targetId: id, + return await withCatalogExport(async () => { + const data = pickAllowed(fields, BC_PAGE_FIELDS); + if (Object.keys(data).length === 0) { + return { ok: false as const, error: "No valid fields to update" }; + } + await db + .update(CatalogPagesBc) + .set(data as Partial) + .where(eq(CatalogPagesBc.id, id)); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "bc_page_update", + description: `Updated BC catalog page #${id}`, + targetType: "catalog_page_bc", + targetId: id, + }); + revalidatePath("/admin/catalog/builder-club"); + return { ok: true as const }; }); - revalidatePath("/admin/catalog/builder-club"); - return { ok: true as const }; } export async function deleteBcItem({ id }: { id: number }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - await db.delete(CatalogItemsBc).where(eq(CatalogItemsBc.id, id)); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "bc_item_delete", - description: `Deleted BC catalog item #${id}`, - targetType: "catalog_item_bc", - targetId: id, + return await withCatalogExport(async () => { + await db.delete(CatalogItemsBc).where(eq(CatalogItemsBc.id, id)); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "bc_item_delete", + description: `Deleted BC catalog item #${id}`, + targetType: "catalog_item_bc", + targetId: id, + }); + revalidatePath("/admin/catalog/builder-club"); + return { ok: true as const }; }); - revalidatePath("/admin/catalog/builder-club"); - return { ok: true as const }; } export async function updateBcItem({ @@ -98,24 +103,26 @@ export async function updateBcItem({ extradata?: string; }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - const safe = pickAllowed(data as Record, BC_ITEM_FIELDS); - if (Object.keys(safe).length === 0) { - return { ok: false as const, error: "No valid fields to update" }; - } - await db - .update(CatalogItemsBc) - .set(safe as Partial) - .where(eq(CatalogItemsBc.id, id)); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "bc_item_update", - description: `Updated BC catalog item #${id}`, - targetType: "catalog_item_bc", - targetId: id, + return await withCatalogExport(async () => { + const safe = pickAllowed(data as Record, BC_ITEM_FIELDS); + if (Object.keys(safe).length === 0) { + return { ok: false as const, error: "No valid fields to update" }; + } + await db + .update(CatalogItemsBc) + .set(safe as Partial) + .where(eq(CatalogItemsBc.id, id)); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "bc_item_update", + description: `Updated BC catalog item #${id}`, + targetType: "catalog_item_bc", + targetId: id, + }); + revalidatePath("/admin/catalog/builder-club"); + return { ok: true as const }; }); - revalidatePath("/admin/catalog/builder-club"); - return { ok: true as const }; } export async function createBcItem({ @@ -129,18 +136,22 @@ export async function createBcItem({ extradata: string; }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - const [result] = await db.insert(CatalogItemsBc).values({ pageId, ...data }); - const createdId = Number(result.insertId); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "bc_item_create", - description: `Created BC catalog item #${createdId}`, - targetType: "catalog_item_bc", - targetId: createdId, + return await withCatalogExport(async () => { + const [result] = await db + .insert(CatalogItemsBc) + .values({ pageId, ...data }); + const createdId = Number(result.insertId); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "bc_item_create", + description: `Created BC catalog item #${createdId}`, + targetType: "catalog_item_bc", + targetId: createdId, + }); + revalidatePath("/admin/catalog/builder-club"); + return { ok: true as const, data: { id: createdId } }; }); - revalidatePath("/admin/catalog/builder-club"); - return { ok: true as const, data: { id: createdId } }; } export async function toggleBcPage({ @@ -151,22 +162,24 @@ export async function toggleBcPage({ field: "enabled" | "visible"; }) { await requirePermission(PERMS.CATALOG_EDIT); - const [page] = await db - .select({ - enabled: CatalogPagesBc.enabled, - visible: CatalogPagesBc.visible, - }) - .from(CatalogPagesBc) - .where(eq(CatalogPagesBc.id, id)) - .limit(1); - if (!page) return { ok: false as const, error: "Page not found" }; - await db - .update(CatalogPagesBc) - .set({ [field]: page[field] === "1" ? "0" : "1" }) - .where(eq(CatalogPagesBc.id, id)); - await rcon.updateCatalog(); - revalidatePath("/admin/catalog/builder-club"); - return { ok: true as const }; + return await withCatalogExport(async () => { + const [page] = await db + .select({ + enabled: CatalogPagesBc.enabled, + visible: CatalogPagesBc.visible, + }) + .from(CatalogPagesBc) + .where(eq(CatalogPagesBc.id, id)) + .limit(1); + if (!page) return { ok: false as const, error: "Page not found" }; + await db + .update(CatalogPagesBc) + .set({ [field]: page[field] === "1" ? "0" : "1" }) + .where(eq(CatalogPagesBc.id, id)); + await rcon.updateCatalog(); + revalidatePath("/admin/catalog/builder-club"); + return { ok: true as const }; + }); } export async function createBcPage(input: { @@ -180,30 +193,32 @@ export async function createBcPage(input: { orderNum?: number; }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - const [result] = await db.insert(CatalogPagesBc).values({ - caption: input.caption, - parentId: input.parentId, - pageLayout: input.pageLayout ?? "default_3x3", - iconColor: input.iconColor ?? 0, - iconImage: input.iconImage ?? 0, - orderNum: input.orderNum ?? 0, - visible: input.visible ?? "1", - enabled: input.enabled ?? "1", - pageHeadline: "", - pageTeaser: "", + return await withCatalogExport(async () => { + const [result] = await db.insert(CatalogPagesBc).values({ + caption: input.caption, + parentId: input.parentId, + pageLayout: input.pageLayout ?? "default_3x3", + iconColor: input.iconColor ?? 0, + iconImage: input.iconImage ?? 0, + orderNum: input.orderNum ?? 0, + visible: input.visible ?? "1", + enabled: input.enabled ?? "1", + pageHeadline: "", + pageTeaser: "", + }); + const createdId = Number(result.insertId); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "bc_page_create", + description: `Created BC catalog page "${input.caption}"`, + targetType: "catalog_page_bc", + targetId: createdId, + }); + revalidatePath("/admin/catalog"); + revalidatePath("/admin/catalog/builder-club"); + return { ok: true as const, data: { id: createdId } }; }); - const createdId = Number(result.insertId); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "bc_page_create", - description: `Created BC catalog page "${input.caption}"`, - targetType: "catalog_page_bc", - targetId: createdId, - }); - revalidatePath("/admin/catalog"); - revalidatePath("/admin/catalog/builder-club"); - return { ok: true as const, data: { id: createdId } }; } async function moveBcPage(pageId: number, newParentId: number): Promise { @@ -234,24 +249,26 @@ export async function reorderBcTreePage(input: { newOrderNum: number; }) { await requirePermission(PERMS.CATALOG_EDIT); - if (input.newParentId !== undefined) { - try { - await moveBcPage(input.pageId, input.newParentId); - } catch (err) { - return { - ok: false as const, - error: err instanceof Error ? err.message : "Invalid move", - }; + return await withCatalogExport(async () => { + if (input.newParentId !== undefined) { + try { + await moveBcPage(input.pageId, input.newParentId); + } catch (err) { + return { + ok: false as const, + error: err instanceof Error ? err.message : "Invalid move", + }; + } } - } - await db - .update(CatalogPagesBc) - .set({ orderNum: input.newOrderNum }) - .where(eq(CatalogPagesBc.id, input.pageId)); - await rcon.updateCatalog(); - revalidatePath("/admin/catalog"); - revalidatePath("/admin/catalog/builder-club"); - return { ok: true as const, data: {} }; + await db + .update(CatalogPagesBc) + .set({ orderNum: input.newOrderNum }) + .where(eq(CatalogPagesBc.id, input.pageId)); + await rcon.updateCatalog(); + revalidatePath("/admin/catalog"); + revalidatePath("/admin/catalog/builder-club"); + return { ok: true as const, data: {} }; + }); } export async function deleteBcTreePage(input: { @@ -259,52 +276,54 @@ export async function deleteBcTreePage(input: { mode: "reparent" | "cascade"; }) { await requirePermission(PERMS.CATALOG_EDIT); - const [page] = await db - .select({ parentId: CatalogPagesBc.parentId }) - .from(CatalogPagesBc) - .where(eq(CatalogPagesBc.id, input.pageId)) - .limit(1); - if (!page) return { ok: false as const, error: "Page not found" }; + return await withCatalogExport(async () => { + const [page] = await db + .select({ parentId: CatalogPagesBc.parentId }) + .from(CatalogPagesBc) + .where(eq(CatalogPagesBc.id, input.pageId)) + .limit(1); + if (!page) return { ok: false as const, error: "Page not found" }; - if (input.mode === "reparent") { - await db.transaction(async (tx) => { - await tx - .update(CatalogPagesBc) - .set({ parentId: page.parentId }) - .where(eq(CatalogPagesBc.parentId, input.pageId)); - await tx - .delete(CatalogItemsBc) - .where(eq(CatalogItemsBc.pageId, input.pageId)); - await tx - .delete(CatalogPagesBc) - .where(eq(CatalogPagesBc.id, input.pageId)); - }); - } else { - const toDelete: number[] = [input.pageId]; - const queue: number[] = [input.pageId]; - while (queue.length > 0) { - const children = await db - .select({ id: CatalogPagesBc.id }) - .from(CatalogPagesBc) - .where(inArray(CatalogPagesBc.parentId, queue)); - queue.length = 0; - for (const child of children) { - toDelete.push(child.id); - queue.push(child.id); + if (input.mode === "reparent") { + await db.transaction(async (tx) => { + await tx + .update(CatalogPagesBc) + .set({ parentId: page.parentId }) + .where(eq(CatalogPagesBc.parentId, input.pageId)); + await tx + .delete(CatalogItemsBc) + .where(eq(CatalogItemsBc.pageId, input.pageId)); + await tx + .delete(CatalogPagesBc) + .where(eq(CatalogPagesBc.id, input.pageId)); + }); + } else { + const toDelete: number[] = [input.pageId]; + const queue: number[] = [input.pageId]; + while (queue.length > 0) { + const children = await db + .select({ id: CatalogPagesBc.id }) + .from(CatalogPagesBc) + .where(inArray(CatalogPagesBc.parentId, queue)); + queue.length = 0; + for (const child of children) { + toDelete.push(child.id); + queue.push(child.id); + } } + await db.transaction(async (tx) => { + await tx + .delete(CatalogItemsBc) + .where(inArray(CatalogItemsBc.pageId, toDelete)); + await tx + .delete(CatalogPagesBc) + .where(inArray(CatalogPagesBc.id, toDelete)); + }); } - await db.transaction(async (tx) => { - await tx - .delete(CatalogItemsBc) - .where(inArray(CatalogItemsBc.pageId, toDelete)); - await tx - .delete(CatalogPagesBc) - .where(inArray(CatalogPagesBc.id, toDelete)); - }); - } - await rcon.updateCatalog(); - revalidatePath("/admin/catalog"); - revalidatePath("/admin/catalog/builder-club"); - return { ok: true as const, data: {} }; + await rcon.updateCatalog(); + revalidatePath("/admin/catalog"); + revalidatePath("/admin/catalog/builder-club"); + return { ok: true as const, data: {} }; + }); } diff --git a/src/actions/catalog-items.ts b/src/actions/catalog-items.ts index b4b2c069ef..1340de3c5f 100644 --- a/src/actions/catalog-items.ts +++ b/src/actions/catalog-items.ts @@ -6,6 +6,7 @@ import { requirePermission } from "@/lib/admin/guard"; import { CatalogItems, db, ItemsBase } from "@/lib/db"; import { PERMS } from "@/lib/permissions"; import { logAudit } from "@/lib/services/audit"; +import { withCatalogExport } from "@/lib/services/catalog-git-queue"; import { allocateCatalogItemId } from "@/lib/services/furni-import"; import { rcon } from "@/lib/services/rcon"; import { logStaffActivity } from "@/lib/services/staff-activity"; @@ -123,32 +124,34 @@ export async function createCatalogItem(data: { clubOnly: "0" | "1"; }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - let catalogName = data.catalogName.trim(); - if (!catalogName) { - const firstId = Number.parseInt(data.itemIds.split(";")[0] || "", 10); - if (firstId > 0) { - const [base] = await db - .select({ - publicName: ItemsBase.publicName, - itemName: ItemsBase.itemName, - }) - .from(ItemsBase) - .where(eq(ItemsBase.id, firstId)) - .limit(1); - catalogName = base?.publicName || base?.itemName || String(firstId); + return await withCatalogExport(async () => { + let catalogName = data.catalogName.trim(); + if (!catalogName) { + const firstId = Number.parseInt(data.itemIds.split(";")[0] || "", 10); + if (firstId > 0) { + const [base] = await db + .select({ + publicName: ItemsBase.publicName, + itemName: ItemsBase.itemName, + }) + .from(ItemsBase) + .where(eq(ItemsBase.id, firstId)) + .limit(1); + catalogName = base?.publicName || base?.itemName || String(firstId); + } } - } - const id = await insertCatalogItemRow({ ...data, catalogName }); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "catalog_item_create", - description: `Created catalog item #${id}`, - targetType: "catalog_item", - targetId: id, + const id = await insertCatalogItemRow({ ...data, catalogName }); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "catalog_item_create", + description: `Created catalog item #${id}`, + targetType: "catalog_item", + targetId: id, + }); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: { id } }; }); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: { id } }; } /** Bulk create with one RCON refresh at the end. */ @@ -165,84 +168,88 @@ export async function bulkCreateCatalogItems({ }>; }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - if (rows.length === 0) { - return { ok: true as const, data: { created: 0, failed: 0 } }; - } - if (rows.length > 500) { - return { ok: false as const, error: "Max 500 items per bulk import" }; - } - - const baseIds = [...new Set(rows.map((r) => r.baseId))]; - const bases = await db - .select({ - id: ItemsBase.id, - publicName: ItemsBase.publicName, - itemName: ItemsBase.itemName, - }) - .from(ItemsBase) - .where(inArray(ItemsBase.id, baseIds)); - const baseMap = new Map(bases.map((b) => [b.id, b])); - - let created = 0; - let failed = 0; - - for (const row of rows) { - const base = baseMap.get(row.baseId); - if (!base) { - failed++; - continue; + return await withCatalogExport(async () => { + if (rows.length === 0) { + return { ok: true as const, data: { created: 0, failed: 0 } }; } - try { - await insertCatalogItemRow({ - pageId, - itemIds: String(row.baseId), - catalogName: base.publicName || base.itemName || String(row.baseId), - costCredits: row.credits ?? 0, - costPoints: row.points ?? 0, - pointsType: row.pointsType ?? 0, - amount: 1, - limitedSells: 0, - limitedStack: 0, - orderNumber: 1, - offerId: -1, - songId: 0, - haveOffer: "1", - clubOnly: "0", - extradata: "", + if (rows.length > 500) { + return { ok: false as const, error: "Max 500 items per bulk import" }; + } + + const baseIds = [...new Set(rows.map((r) => r.baseId))]; + const bases = await db + .select({ + id: ItemsBase.id, + publicName: ItemsBase.publicName, + itemName: ItemsBase.itemName, + }) + .from(ItemsBase) + .where(inArray(ItemsBase.id, baseIds)); + const baseMap = new Map(bases.map((b) => [b.id, b])); + + let created = 0; + let failed = 0; + + for (const row of rows) { + const base = baseMap.get(row.baseId); + if (!base) { + failed++; + continue; + } + try { + await insertCatalogItemRow({ + pageId, + itemIds: String(row.baseId), + catalogName: base.publicName || base.itemName || String(row.baseId), + costCredits: row.credits ?? 0, + costPoints: row.points ?? 0, + pointsType: row.pointsType ?? 0, + amount: 1, + limitedSells: 0, + limitedStack: 0, + orderNumber: 1, + offerId: -1, + songId: 0, + haveOffer: "1", + clubOnly: "0", + extradata: "", + }); + created++; + } catch { + failed++; + } + } + + if (created > 0) { + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "catalog_items_bulk_create", + description: `Bulk imported ${created} catalog item(s) on page #${pageId}`, + targetType: "catalog_page", + targetId: pageId, }); - created++; - } catch { - failed++; + revalidatePath("/admin/catalog"); } - } - if (created > 0) { - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "catalog_items_bulk_create", - description: `Bulk imported ${created} catalog item(s) on page #${pageId}`, - targetType: "catalog_page", - targetId: pageId, - }); - revalidatePath("/admin/catalog"); - } - - return { ok: true as const, data: { created, failed } }; + return { ok: true as const, data: { created, failed } }; + }); } export async function deleteCatalogItems({ ids }: { ids: number[] }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - await db.delete(CatalogItems).where(inArray(CatalogItems.id, ids)); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "catalog_items_delete", - description: `Deleted catalog items: ${ids.join(", ")}`, - targetType: "catalog_item", + return await withCatalogExport(async () => { + await db.delete(CatalogItems).where(inArray(CatalogItems.id, ids)); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "catalog_items_delete", + description: `Deleted catalog items: ${ids.join(", ")}`, + targetType: "catalog_item", + }); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; }); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; } export async function moveCatalogItems({ @@ -253,11 +260,12 @@ export async function moveCatalogItems({ targetPageId: number; }) { await requirePermission(PERMS.CATALOG_EDIT); - if (ids.length === 0) { - return { ok: true as const, data: {} }; - } - const pageIdStr = String(targetPageId); - await db.execute(sql` + return await withCatalogExport(async () => { + if (ids.length === 0) { + return { ok: true as const, data: {} }; + } + const pageIdStr = String(targetPageId); + await db.execute(sql` UPDATE catalog_items SET page_id = ${pageIdStr} WHERE id IN (${sql.join( @@ -265,9 +273,10 @@ export async function moveCatalogItems({ sql`, `, )}) `); - await rcon.updateCatalog(); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; + await rcon.updateCatalog(); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; + }); } export async function reorderCatalogItems({ @@ -276,15 +285,17 @@ export async function reorderCatalogItems({ orders: Array<{ id: number; orderNumber: number }>; }) { await requirePermission(PERMS.CATALOG_EDIT); - for (const { id, orderNumber } of orders) { - await db - .update(CatalogItems) - .set({ orderNumber }) - .where(eq(CatalogItems.id, id)); - } - await rcon.updateCatalog(); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; + return await withCatalogExport(async () => { + for (const { id, orderNumber } of orders) { + await db + .update(CatalogItems) + .set({ orderNumber }) + .where(eq(CatalogItems.id, id)); + } + await rcon.updateCatalog(); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; + }); } export async function updateCatalogItem({ @@ -297,46 +308,48 @@ export async function updateCatalogItem({ baseItem?: { id: number; fields: Record }; }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - const safeCatalog = pickAllowed(catalogFields, CATALOG_ITEM_FIELDS); - if (Object.keys(safeCatalog).length === 0 && !baseItem) { - return { ok: false as const, error: "No valid fields to update" }; - } + return await withCatalogExport(async () => { + const safeCatalog = pickAllowed(catalogFields, CATALOG_ITEM_FIELDS); + if (Object.keys(safeCatalog).length === 0 && !baseItem) { + return { ok: false as const, error: "No valid fields to update" }; + } - // page_id is often VARCHAR — update it via raw SQL when present. - const pageIdRaw = safeCatalog.pageId; - if (pageIdRaw !== undefined) { - const pageIdStr = String(pageIdRaw); - await db.execute(sql` + // page_id is often VARCHAR — update it via raw SQL when present. + const pageIdRaw = safeCatalog.pageId; + if (pageIdRaw !== undefined) { + const pageIdStr = String(pageIdRaw); + await db.execute(sql` UPDATE catalog_items SET page_id = ${pageIdStr} WHERE id = ${id} `); - delete safeCatalog.pageId; - } - - if (Object.keys(safeCatalog).length > 0) { - await db - .update(CatalogItems) - .set(safeCatalog as Partial) - .where(eq(CatalogItems.id, id)); - } - if (baseItem) { - const safeBase = pickAllowed(baseItem.fields, ITEMS_BASE_FIELDS); - if (Object.keys(safeBase).length > 0) { - await db - .update(ItemsBase) - .set(safeBase as Partial) - .where(eq(ItemsBase.id, baseItem.id)); + delete safeCatalog.pageId; } - } - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "catalog_item_update", - description: `Updated catalog item #${id}`, - targetType: "catalog_item", - targetId: id, + + if (Object.keys(safeCatalog).length > 0) { + await db + .update(CatalogItems) + .set(safeCatalog as Partial) + .where(eq(CatalogItems.id, id)); + } + if (baseItem) { + const safeBase = pickAllowed(baseItem.fields, ITEMS_BASE_FIELDS); + if (Object.keys(safeBase).length > 0) { + await db + .update(ItemsBase) + .set(safeBase as Partial) + .where(eq(ItemsBase.id, baseItem.id)); + } + } + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "catalog_item_update", + description: `Updated catalog item #${id}`, + targetType: "catalog_item", + targetId: id, + }); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; }); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; } export async function translateCatalogItems(input: { @@ -344,121 +357,124 @@ export async function translateCatalogItems(input: { items: Array<{ id: number; publicName: string; description?: string }>; }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - const parsed = translateItemsSchema.safeParse(input); - if (!parsed.success) { - return { - ok: false as const, - error: parsed.error.issues[0]?.message ?? "Invalid translate payload", - }; - } - const { items } = parsed.data; - const { invalidateFurniDataCache } = await import( - "@/lib/services/catalog-items-loader" - ); - const { patchFurniEntryNames } = await import("@/lib/services/furni-data"); + return await withCatalogExport(async () => { + const parsed = translateItemsSchema.safeParse(input); + if (!parsed.success) { + return { + ok: false as const, + error: parsed.error.issues[0]?.message ?? "Invalid translate payload", + }; + } + const { items } = parsed.data; + const { invalidateFurniDataCache } = await import( + "@/lib/services/catalog-items-loader" + ); + const { patchFurniEntryNames } = await import("@/lib/services/furni-data"); - let namesUpdated = 0; - let descriptionsUpdated = 0; - const furniPatches: Array<{ - classname: string; - itemType: string; - name?: string; - description?: string; - spriteId?: number; - createIfMissing?: boolean; - }> = []; + let namesUpdated = 0; + let descriptionsUpdated = 0; + const furniPatches: Array<{ + classname: string; + itemType: string; + name?: string; + description?: string; + spriteId?: number; + createIfMissing?: boolean; + }> = []; - for (const item of items) { - const [base] = await db - .select({ - id: ItemsBase.id, - publicName: ItemsBase.publicName, - itemName: ItemsBase.itemName, - type: ItemsBase.type, - spriteId: ItemsBase.spriteId, - }) - .from(ItemsBase) - .where(eq(ItemsBase.id, item.id)) - .limit(1); - if (!base) continue; - - const nextName = item.publicName?.trim() ?? ""; - const nextDesc = item.description ?? ""; - const nameChanged = nextName !== "" && nextName !== (base.publicName ?? ""); - - if (nameChanged) { - await db - .update(ItemsBase) - .set({ publicName: nextName }) - .where(eq(ItemsBase.id, base.id)); - const idStr = String(base.id); - const related = await db + for (const item of items) { + const [base] = await db .select({ - id: CatalogItems.id, - catalogName: CatalogItems.catalogName, + id: ItemsBase.id, + publicName: ItemsBase.publicName, + itemName: ItemsBase.itemName, + type: ItemsBase.type, + spriteId: ItemsBase.spriteId, }) - .from(CatalogItems) - .where( - or( - eq(CatalogItems.itemIds, idStr), - like(CatalogItems.itemIds, `${idStr};%`), - like(CatalogItems.itemIds, `%;${idStr};%`), - like(CatalogItems.itemIds, `%;${idStr}`), - ), - ); - for (const row of related) { - if (row.catalogName !== nextName) { - await db - .update(CatalogItems) - .set({ catalogName: nextName }) - .where(eq(CatalogItems.id, row.id)); + .from(ItemsBase) + .where(eq(ItemsBase.id, item.id)) + .limit(1); + if (!base) continue; + + const nextName = item.publicName?.trim() ?? ""; + const nextDesc = item.description ?? ""; + const nameChanged = + nextName !== "" && nextName !== (base.publicName ?? ""); + + if (nameChanged) { + await db + .update(ItemsBase) + .set({ publicName: nextName }) + .where(eq(ItemsBase.id, base.id)); + const idStr = String(base.id); + const related = await db + .select({ + id: CatalogItems.id, + catalogName: CatalogItems.catalogName, + }) + .from(CatalogItems) + .where( + or( + eq(CatalogItems.itemIds, idStr), + like(CatalogItems.itemIds, `${idStr};%`), + like(CatalogItems.itemIds, `%;${idStr};%`), + like(CatalogItems.itemIds, `%;${idStr}`), + ), + ); + for (const row of related) { + if (row.catalogName !== nextName) { + await db + .update(CatalogItems) + .set({ catalogName: nextName }) + .where(eq(CatalogItems.id, row.id)); + } } + namesUpdated++; + } + + if (nextDesc !== "" || nameChanged) { + descriptionsUpdated += nextDesc !== "" ? 1 : 0; + furniPatches.push({ + classname: base.itemName, + itemType: base.type || "s", + name: nextName || base.publicName || base.itemName, + description: nextDesc, + spriteId: base.spriteId, + createIfMissing: true, + }); } - namesUpdated++; } - if (nextDesc !== "" || nameChanged) { - descriptionsUpdated += nextDesc !== "" ? 1 : 0; - furniPatches.push({ - classname: base.itemName, - itemType: base.type || "s", - name: nextName || base.publicName || base.itemName, - description: nextDesc, - spriteId: base.spriteId, - createIfMissing: true, - }); + const furniResult = + furniPatches.length > 0 + ? await patchFurniEntryNames(furniPatches) + : { updated: 0, inserted: 0 }; + if (furniResult.updated > 0 || furniResult.inserted > 0) { + invalidateFurniDataCache(); } - } - const furniResult = - furniPatches.length > 0 - ? await patchFurniEntryNames(furniPatches) - : { updated: 0, inserted: 0 }; - if (furniResult.updated > 0 || furniResult.inserted > 0) { - invalidateFurniDataCache(); - } - - await rcon.updateCatalog(); - await logAudit({ - userId: staff.id, - action: "items_base_translate", - target: "ItemsBase", - after: { - namesUpdated, - descriptionsUpdated, - furniDataUpdated: furniResult.updated > 0, - furniDataInserted: furniResult.inserted, - }, + await rcon.updateCatalog(); + await logAudit({ + userId: staff.id, + action: "items_base_translate", + target: "ItemsBase", + after: { + namesUpdated, + descriptionsUpdated, + furniDataUpdated: furniResult.updated > 0, + furniDataInserted: furniResult.inserted, + }, + }); + revalidatePath("/admin/catalog"); + return { + ok: true as const, + data: { + namesUpdated, + descriptionsUpdated, + furniDataUpdated: furniResult.updated, + furniDataInserted: furniResult.inserted, + updated: items.length, + }, + }; }); - revalidatePath("/admin/catalog"); - return { - ok: true as const, - data: { - namesUpdated, - descriptionsUpdated, - furniDataUpdated: furniResult.updated, - furniDataInserted: furniResult.inserted, - updated: items.length, - }, - }; } diff --git a/src/actions/catalog.ts b/src/actions/catalog.ts index 8a4b14bd6a..c4511877f5 100644 --- a/src/actions/catalog.ts +++ b/src/actions/catalog.ts @@ -6,6 +6,7 @@ import { requirePermission } from "@/lib/admin/guard"; import { CatalogPages, db } from "@/lib/db"; import { PERMS } from "@/lib/permissions"; import type { ActionResult } from "@/lib/safe-action-shared"; +import { withCatalogExport } from "@/lib/services/catalog-git-queue"; import { deletePage, movePage } from "@/lib/services/catalog-tree"; import { rcon } from "@/lib/services/rcon"; import { logStaffActivity } from "@/lib/services/staff-activity"; @@ -48,42 +49,46 @@ export async function updateCatalogPage({ ...fields }: { id: number } & Record): Promise { const staff = await requirePermission(PERMS.CATALOG_EDIT); - const data = pickPageFields(fields); - if (Object.keys(data).length === 0) { - return { ok: false as const, error: "No valid fields to update" }; - } - if (typeof data.caption === "string" && !data.captionSave) { - data.captionSave = data.caption.slice(0, 25); - } - await db - .update(CatalogPages) - .set(data as Partial) - .where(eq(CatalogPages.id, id)); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "catalog_page_update", - description: `Updated catalog page #${id}`, - targetType: "catalog_page", - targetId: id, + return await withCatalogExport(async () => { + const data = pickPageFields(fields); + if (Object.keys(data).length === 0) { + return { ok: false as const, error: "No valid fields to update" }; + } + if (typeof data.caption === "string" && !data.captionSave) { + data.captionSave = data.caption.slice(0, 25); + } + await db + .update(CatalogPages) + .set(data as Partial) + .where(eq(CatalogPages.id, id)); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "catalog_page_update", + description: `Updated catalog page #${id}`, + targetType: "catalog_page", + targetId: id, + }); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; }); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; } export async function deleteCatalogPage({ id }: { id: number }) { const staff = await requirePermission(PERMS.CATALOG_EDIT); - await deletePage(id, "reparent"); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "catalog_page_delete", - description: `Deleted catalog page #${id}`, - targetType: "catalog_page", - targetId: id, + return await withCatalogExport(async () => { + await deletePage(id, "reparent"); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "catalog_page_delete", + description: `Deleted catalog page #${id}`, + targetType: "catalog_page", + targetId: id, + }); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; }); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; } export async function toggleCatalogPage({ @@ -94,24 +99,26 @@ export async function toggleCatalogPage({ action: "toggleEnabled" | "toggleVisible"; }) { await requirePermission(PERMS.CATALOG_EDIT); - const [page] = await db - .select({ - enabled: CatalogPages.enabled, - visible: CatalogPages.visible, - }) - .from(CatalogPages) - .where(eq(CatalogPages.id, id)) - .limit(1); - if (!page) return { ok: false as const, error: "Catalog page not found" }; - const field = action === "toggleEnabled" ? "enabled" : "visible"; - const current = action === "toggleEnabled" ? page.enabled : page.visible; - await db - .update(CatalogPages) - .set({ [field]: current === "1" ? "0" : "1" }) - .where(eq(CatalogPages.id, id)); - await rcon.updateCatalog(); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; + return await withCatalogExport(async () => { + const [page] = await db + .select({ + enabled: CatalogPages.enabled, + visible: CatalogPages.visible, + }) + .from(CatalogPages) + .where(eq(CatalogPages.id, id)) + .limit(1); + if (!page) return { ok: false as const, error: "Catalog page not found" }; + const field = action === "toggleEnabled" ? "enabled" : "visible"; + const current = action === "toggleEnabled" ? page.enabled : page.visible; + await db + .update(CatalogPages) + .set({ [field]: current === "1" ? "0" : "1" }) + .where(eq(CatalogPages.id, id)); + await rcon.updateCatalog(); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; + }); } export async function createCatalogPage(input: { @@ -126,34 +133,36 @@ export async function createCatalogPage(input: { orderNum?: number; }): Promise> { const staff = await requirePermission(PERMS.CATALOG_EDIT); - const [result] = await db.insert(CatalogPages).values({ - caption: input.caption, - parentId: input.parentId, - pageLayout: input.pageLayout ?? "default_3x3", - captionSave: input.caption.slice(0, 25), - iconColor: input.iconColor ?? 0, - iconImage: input.iconImage ?? 0, - minRank: input.minRank ?? 1, - orderNum: input.orderNum ?? 0, - visible: input.visible ?? "1", - enabled: input.enabled ?? "1", - clubOnly: "0", - vipOnly: "0", - pageHeadline: "", - pageTeaser: "", - includes: "", + return await withCatalogExport(async () => { + const [result] = await db.insert(CatalogPages).values({ + caption: input.caption, + parentId: input.parentId, + pageLayout: input.pageLayout ?? "default_3x3", + captionSave: input.caption.slice(0, 25), + iconColor: input.iconColor ?? 0, + iconImage: input.iconImage ?? 0, + minRank: input.minRank ?? 1, + orderNum: input.orderNum ?? 0, + visible: input.visible ?? "1", + enabled: input.enabled ?? "1", + clubOnly: "0", + vipOnly: "0", + pageHeadline: "", + pageTeaser: "", + includes: "", + }); + const createdId = Number(result.insertId); + await rcon.updateCatalog(); + await logStaffActivity({ + staffId: staff.id, + action: "catalog_page_create", + description: `Created catalog page "${input.caption}"`, + targetType: "catalog_page", + targetId: createdId, + }); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: { id: createdId } }; }); - const createdId = Number(result.insertId); - await rcon.updateCatalog(); - await logStaffActivity({ - staffId: staff.id, - action: "catalog_page_create", - description: `Created catalog page "${input.caption}"`, - targetType: "catalog_page", - targetId: createdId, - }); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: { id: createdId } }; } export async function reorderTreePage(input: { @@ -162,23 +171,25 @@ export async function reorderTreePage(input: { newOrderNum: number; }) { await requirePermission(PERMS.CATALOG_EDIT); - if (input.newParentId !== undefined) { - try { - await movePage(input.pageId, input.newParentId); - } catch (err) { - return { - ok: false as const, - error: err instanceof Error ? err.message : "Invalid move", - }; + return await withCatalogExport(async () => { + if (input.newParentId !== undefined) { + try { + await movePage(input.pageId, input.newParentId); + } catch (err) { + return { + ok: false as const, + error: err instanceof Error ? err.message : "Invalid move", + }; + } } - } - await db - .update(CatalogPages) - .set({ orderNum: input.newOrderNum }) - .where(eq(CatalogPages.id, input.pageId)); - await rcon.updateCatalog(); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; + await db + .update(CatalogPages) + .set({ orderNum: input.newOrderNum }) + .where(eq(CatalogPages.id, input.pageId)); + await rcon.updateCatalog(); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; + }); } export async function deleteTreePage(input: { @@ -186,8 +197,10 @@ export async function deleteTreePage(input: { mode: "reparent" | "cascade"; }) { await requirePermission(PERMS.CATALOG_EDIT); - await deletePage(input.pageId, input.mode); - await rcon.updateCatalog(); - revalidatePath("/admin/catalog"); - return { ok: true as const, data: {} }; + return await withCatalogExport(async () => { + await deletePage(input.pageId, input.mode); + await rcon.updateCatalog(); + revalidatePath("/admin/catalog"); + return { ok: true as const, data: {} }; + }); } diff --git a/src/actions/furni-maintenance.ts b/src/actions/furni-maintenance.ts index 2cbea7ac3b..382b3d1f99 100644 --- a/src/actions/furni-maintenance.ts +++ b/src/actions/furni-maintenance.ts @@ -7,6 +7,7 @@ import { actionOk, handleActionError, } from "@/lib/safe-action-shared"; +import { withCatalogExport } from "@/lib/services/catalog-git-queue"; import type { AlignResult, DedupResult, @@ -38,7 +39,9 @@ export async function fixSpriteIdsAction(): Promise< > { try { await guard(); - return actionOk(await maintenance.fixSpriteIds()); + return await withCatalogExport(async () => { + return actionOk(await maintenance.fixSpriteIds()); + }); } catch (e) { return handleActionError(e); } @@ -49,7 +52,9 @@ export async function fixCatalogOffersAction(): Promise< > { try { await guard(); - return actionOk(await maintenance.fixCatalogOffers()); + return await withCatalogExport(async () => { + return actionOk(await maintenance.fixCatalogOffers()); + }); } catch (e) { return handleActionError(e); } @@ -60,7 +65,9 @@ export async function reconcileIdsAction(): Promise< > { try { await guard(); - return actionOk(await maintenance.reconcileIds()); + return await withCatalogExport(async () => { + return actionOk(await maintenance.reconcileIds()); + }); } catch (e) { return handleActionError(e); } @@ -71,7 +78,9 @@ export async function removeDuplicateItemsBaseAction(): Promise< > { try { await guard(); - return actionOk(await maintenance.removeDuplicates()); + return await withCatalogExport(async () => { + return actionOk(await maintenance.removeDuplicates()); + }); } catch (e) { return handleActionError(e); } @@ -93,7 +102,9 @@ export async function applyAlignIdsAction(): Promise< > { try { await guard(); - return actionOk(await maintenance.forceItemsBaseIdsToFurnidata(true)); + return await withCatalogExport(async () => { + return actionOk(await maintenance.forceItemsBaseIdsToFurnidata(true)); + }); } catch (e) { return handleActionError(e); } @@ -104,7 +115,9 @@ export async function fixEverythingAction(input?: { }): Promise> { try { await guard(); - return actionOk(await maintenance.fixEverything(input ?? {})); + return await withCatalogExport(async () => { + return actionOk(await maintenance.fixEverything(input ?? {})); + }); } catch (e) { return handleActionError(e); } diff --git a/src/actions/import-furni.ts b/src/actions/import-furni.ts index f41b1a289a..cf84896e6b 100644 --- a/src/actions/import-furni.ts +++ b/src/actions/import-furni.ts @@ -5,6 +5,7 @@ import { db, WebsiteSetting } from "@/lib/db"; import { PERMS } from "@/lib/permissions"; import { adminAction } from "@/lib/safe-action"; import { actionOk } from "@/lib/safe-action-shared"; +import { withCatalogExport } from "@/lib/services/catalog-git-queue"; import { deleteImportedItem } from "@/lib/services/furni-import"; import { siteSettings } from "@/lib/services/site-settings"; @@ -14,10 +15,9 @@ export const deleteImportedFurni = adminAction( { permission: PERMS.ASSETS_IMPORT, schema: deleteSchema }, async (ctx) => actionOk( - (await deleteImportedItem(ctx.data.classname)) as unknown as Record< - string, - unknown - >, + (await withCatalogExport(() => + deleteImportedItem(ctx.data.classname), + )) as unknown as Record, ), ); diff --git a/src/app/admin/studio/sync/catalog-export-client.tsx b/src/app/admin/studio/sync/catalog-export-client.tsx new file mode 100644 index 0000000000..9de95d237c --- /dev/null +++ b/src/app/admin/studio/sync/catalog-export-client.tsx @@ -0,0 +1,102 @@ +"use client"; + +import { useCallback, useEffect, useState } from "react"; +import { adminFetch } from "@/lib/admin-fetch"; + +type ExportStatus = { + enabled: boolean; + pending: number; + active: number; + running?: boolean; + commit?: string; + finishedAt?: string; + error?: string; +}; + +export function CatalogExportClient() { + const [status, setStatus] = useState(null); + const [error, setError] = useState(""); + const [busy, setBusy] = useState(false); + const refresh = useCallback(async () => { + try { + const response = await adminFetch("/api/admin/catalog-export", { + cache: "no-store", + }); + if (!response.ok) throw new Error("Unable to load export status"); + setStatus(await response.json()); + setError(""); + } catch { + setError("Unable to load export status"); + } + }, []); + useEffect(() => { + void refresh(); + const timer = setInterval(() => { + void refresh(); + }, 15_000); + return () => clearInterval(timer); + }, [refresh]); + async function retry() { + setBusy(true); + try { + const response = await adminFetch("/api/admin/catalog-export", { + method: "POST", + }); + if (!response.ok) throw new Error("Unable to queue export"); + await refresh(); + } catch { + setError("Unable to queue export"); + } finally { + setBusy(false); + } + } + return ( +
+

Catalog repository export

+

+ Automatically publishes bundles, icons, gamedata, translations and + catalog SQL to Epicnabbo-Catalogus-Updated-Daily (Beta-3). +

+
+ {!status + ? "Loading…" + : !status.enabled + ? "Not configured on this server." + : status.active + ? `Waiting for ${status.active} operation(s) to finish.` + : status.running + ? "Export in progress…" + : status.pending + ? `${status.pending} update(s) queued for the worker.` + : "No pending updates."} + {status?.commit && ( +

+ Last published commit:{" "} + + {status.commit.slice(0, 12)} + +

+ )} + {status?.finishedAt &&

Last attempt: {status.finishedAt}

} +
+ {(error || status?.error) && ( +

+ {error || status?.error} +

+ )} + +
+ ); +} diff --git a/src/app/admin/studio/sync/page.tsx b/src/app/admin/studio/sync/page.tsx index 8abdf58d5c..d796d9495f 100644 --- a/src/app/admin/studio/sync/page.tsx +++ b/src/app/admin/studio/sync/page.tsx @@ -3,6 +3,8 @@ import { OfficialSyncClient } from "@/app/admin/import/official/official-sync-cl import { SyncAllClient } from "@/app/admin/import/sync/sync-all-client"; import { canAccess, getAdminContext, PERMS } from "@/lib/permissions"; +import { CatalogExportClient } from "./catalog-export-client"; + export default async function StudioSyncPage(_props: { params: Promise<{ locale: string }>; }) { @@ -12,6 +14,7 @@ export default async function StudioSyncPage(_props: { } return (
+

Sync All Sources

Scan configured sources and automatically import any furniture that is diff --git a/src/app/api/admin/catalog-export/route.ts b/src/app/api/admin/catalog-export/route.ts new file mode 100644 index 0000000000..b72afd28cd --- /dev/null +++ b/src/app/api/admin/catalog-export/route.ts @@ -0,0 +1,18 @@ +import { withAdmin } from "@/lib/api-handler"; +import { apiError, apiOk } from "@/lib/api-response"; +import { PERMS } from "@/lib/permissions"; +import { + catalogExportEnabled, + catalogExportQueue, + catalogExportStatus, +} from "@/lib/services/catalog-git-queue"; + +export const GET = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => + apiOk(await catalogExportStatus()), +); +export const POST = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => { + if (!catalogExportEnabled()) + return apiError("Catalog repository export is not configured", 400); + await catalogExportQueue().request(); + return apiOk(await catalogExportStatus()); +}); diff --git a/src/lib/api-handler.ts b/src/lib/api-handler.ts index 560b9b0a24..14398642ab 100644 --- a/src/lib/api-handler.ts +++ b/src/lib/api-handler.ts @@ -1,9 +1,15 @@ -import { type NextRequest, NextResponse } from "next/server"; +import { after, type NextRequest, NextResponse } from "next/server"; import { logAuthorizationEvent } from "@/lib/admin/authorization-events"; import { validateCsrfToken } from "@/lib/foundation/security"; import { canAccess, getApiAdminContext } from "@/lib/permissions"; import { logServerError } from "@/lib/server-log"; +import { + catalogExportEnabled, + catalogExportQueue, + isCatalogMutation, +} from "@/lib/services/catalog-git-queue"; + const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]); const MAX_BODY_BYTES = 10 * 1024 * 1024; // 10 MB @@ -80,9 +86,42 @@ export function withAdmin( { status: 403 }, ); } + let finishExport: (() => Promise) | undefined; try { - return await handler(request, context, routeContext); + if ( + catalogExportEnabled() && + isCatalogMutation(request.method, request.nextUrl.pathname) + ) { + finishExport = await catalogExportQueue().begin(); + } + const response = await handler(request, context, routeContext); + if ( + finishExport && + response.body && + response.headers.get("content-type")?.includes("text/event-stream") + ) { + const [client, completion] = response.body.tee(); + const finish = finishExport; + after(async () => { + const reader = completion.getReader(); + try { + while (!(await reader.read()).done) { + /* Wait for all import results. */ + } + } finally { + reader.releaseLock(); + await finish(); + } + }); + return new Response(client, { + status: response.status, + headers: response.headers, + }); + } + await finishExport?.(); + return response; } catch (error) { + await finishExport?.().catch(() => undefined); logServerError("admin.api_failed", error, { path: request.nextUrl.pathname, userId: context.session.user.id, diff --git a/src/lib/catalog-export-api.test.ts b/src/lib/catalog-export-api.test.ts new file mode 100644 index 0000000000..c93c0b9f2e --- /dev/null +++ b/src/lib/catalog-export-api.test.ts @@ -0,0 +1,87 @@ +import { mkdtemp } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { NextRequest } from "next/server"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + after: vi.fn(), + auth: vi.fn(), + csrf: vi.fn(), +})); +vi.mock("next/server", async (original) => ({ + ...(await original()), + after: mocks.after, +})); +vi.mock("@/lib/admin/authorization-events", () => ({ + logAuthorizationEvent: vi.fn(), +})); +vi.mock("@/lib/foundation/security", () => ({ validateCsrfToken: mocks.csrf })); +vi.mock("@/lib/permissions", () => ({ + getApiAdminContext: mocks.auth, + canAccess: () => true, +})); +vi.mock("@/lib/server-log", () => ({ logServerError: vi.fn() })); + +import { withAdmin } from "./api-handler"; +import { catalogExportQueue } from "./services/catalog-git-queue"; + +describe("catalog API completion tracking", () => { + beforeEach(async () => { + vi.stubEnv("CATALOG_GIT_CHECKOUT", "/catalog"); + vi.stubEnv( + "CATALOG_GIT_STATE_DIR", + await mkdtemp(path.join(os.tmpdir(), "catalog-api-")), + ); + mocks.auth.mockResolvedValue({ + session: { user: { id: 1, rank: 7 } }, + permissions: [], + }); + mocks.csrf.mockResolvedValue(true); + mocks.after.mockReset(); + }); + afterEach(() => vi.unstubAllEnvs()); + it("waits for the last SSE event before making changes publishable", async () => { + let end: (() => void) | undefined; + const response = new Response( + new ReadableStream({ + start(controller) { + controller.enqueue( + new TextEncoder().encode('data: {"type":"progress"}\n\n'), + ); + end = () => controller.close(); + }, + }), + { headers: { "content-type": "text/event-stream" } }, + ); + const route = withAdmin({}, async () => response); + const returned = await route( + new NextRequest("http://localhost/api/admin/import/furni/batch", { + method: "POST", + }), + ); + expect(await catalogExportQueue().batch()).toBeNull(); + const complete = mocks.after.mock.calls[0][0](); + end?.(); + await returned.text(); + await complete; + expect(await catalogExportQueue().batch()).toHaveLength(1); + }); + it("does not queue unauthenticated requests or reads", async () => { + const handler = vi.fn(async () => new Response("{}")); + const route = withAdmin({}, handler); + mocks.auth.mockResolvedValueOnce(null); + expect( + ( + await route( + new NextRequest("http://localhost/api/admin/import/furni", { + method: "POST", + }), + ) + ).status, + ).toBe(401); + expect(handler).not.toHaveBeenCalled(); + await route(new NextRequest("http://localhost/api/admin/import/furni")); + expect(await catalogExportQueue().batch()).toHaveLength(0); + }); +}); diff --git a/src/lib/services/catalog-git-core.test.ts b/src/lib/services/catalog-git-core.test.ts new file mode 100644 index 0000000000..244f65e73a --- /dev/null +++ b/src/lib/services/catalog-git-core.test.ts @@ -0,0 +1,103 @@ +import { execFileSync } from "node:child_process"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { describe, expect, it } from "vitest"; +import { + CatalogExportQueue, + publishCatalogFiles, + recoverCatalogQueue, + sqlValue, +} from "./catalog-git-core"; + +describe("catalog export", () => { + it("recovers queue entries owned by a terminated local process", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "catalog-recovery-")); + const deadPid = Number( + execFileSync( + process.execPath, + ["-e", "process.stdout.write(String(process.pid))"], + { encoding: "utf8" }, + ), + ); + const owner = JSON.stringify({ pid: deadPid, host: os.hostname() }); + await writeFile(path.join(root, "12345678.active"), owner); + await writeFile(path.join(root, "worker.lock"), owner); + await recoverCatalogQueue(root); + const q = new CatalogExportQueue(root); + expect(await q.batch()).toEqual(["12345678.pending"]); + expect(await q.entries()).not.toContain("worker.lock"); + }); + + it("keeps new requests pending when an older snapshot completes", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "catalog-queue-")); + const q = new CatalogExportQueue(root); + const finish = await q.begin(); + expect(await q.batch()).toBeNull(); + await finish(); + const batch = await q.batch(); + expect(batch).toHaveLength(1); + await q.request(); + if (!batch) throw new Error("Expected pending batch"); + await q.complete(batch); + expect(await q.batch()).toHaveLength(1); + }); + it("encodes SQL strings without quote or backslash ambiguity", () => { + expect(sqlValue("Valentine's \\ Day\n")).toBe( + "CONVERT(X'56616c656e74696e652773205c204461790a' USING utf8mb4)", + ); + expect(sqlValue(null)).toBe("NULL"); + expect(sqlValue(42)).toBe("42"); + expect(() => sqlValue(Number.NaN)).toThrow(); + }); + it("publishes to a bare remote, retries a failed push and avoids empty commits", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "catalog-git-")); + const remote = path.join(root, "remote.git"); + const checkout = path.join(root, "checkout"); + const git = (cwd: string, ...args: string[]) => + execFileSync("git", args, { cwd, encoding: "utf8" }).trim(); + git(root, "init", "--bare", remote); + git(root, "clone", remote, checkout); + git(checkout, "checkout", "-b", "Beta-3"); + git(checkout, "config", "user.name", "Catalog test"); + git(checkout, "config", "user.email", "catalog@example.test"); + await writeFile(path.join(checkout, "readme.md"), "preserved"); + git(checkout, "add", "readme.md"); + git(checkout, "commit", "-m", "initial"); + git(checkout, "push", "-u", "origin", "Beta-3"); + const source = path.join(root, "chair.nitro"); + await writeFile(source, "bundle"); + const files = [ + { source, target: "Gamedata/bundled/furniture/chair.nitro" }, + ]; + const options = { checkout, remote, branch: "Beta-3", files }; + const first = await publishCatalogFiles(options); + expect(await publishCatalogFiles(options)).toBe(first); + await writeFile(source, "updated"); + git( + checkout, + "remote", + "set-url", + "--push", + "origin", + path.join(root, "missing.git"), + ); + await expect(publishCatalogFiles(options)).rejects.toThrow(); + git(checkout, "remote", "set-url", "--push", "origin", remote); + const retried = await publishCatalogFiles(options); + expect(retried).not.toBe(first); + expect(git(remote, "rev-parse", "Beta-3")).toBe(retried); + expect(await readFile(path.join(checkout, "readme.md"), "utf8")).toBe( + "preserved", + ); + await expect( + publishCatalogFiles({ + ...options, + files: [{ source, target: "../escape" }], + }), + ).rejects.toThrow(); + await mkdir(path.join(checkout, "unrelated")); + await writeFile(path.join(checkout, "unrelated", "local.txt"), "local"); + await expect(publishCatalogFiles(options)).rejects.toThrow(/clean/); + }); +}); diff --git a/src/lib/services/catalog-git-core.ts b/src/lib/services/catalog-git-core.ts new file mode 100644 index 0000000000..95c6def23c --- /dev/null +++ b/src/lib/services/catalog-git-core.ts @@ -0,0 +1,230 @@ +import { execFile } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { promises as fs } from "node:fs"; +import { hostname } from "node:os"; +import path from "node:path"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); +export const CATALOG_REMOTE = + "https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git"; +export const CATALOG_BRANCH = "Beta-3"; +export const CATALOG_SQL_ROOT = "catalogue version 2 ( Final (Dev)/sqls"; +export const CATALOG_LANGUAGE_ROOT = + "catalogue version 2 ( Final (Dev)/langs furnidata"; + +export class CatalogExportQueue { + constructor(readonly root: string) {} + async entries() { + await fs.mkdir(this.root, { recursive: true }); + return (await fs.readdir(this.root)).sort(); + } + async begin() { + await this.entries(); + const id = randomUUID(); + await fs.writeFile( + path.join(this.root, `${id}.active`), + JSON.stringify({ + pid: process.pid, + host: hostname(), + startedAt: new Date().toISOString(), + }), + { flag: "wx" }, + ); + let finished = false; + return async () => { + if (finished) return; + await fs.rename( + path.join(this.root, `${id}.active`), + path.join(this.root, `${id}.pending`), + ); + finished = true; + }; + } + async request() { + await (await this.begin())(); + } + async batch(): Promise { + const entries = await this.entries(); + if (entries.some((f) => f.endsWith(".active"))) return null; + return entries.filter((f) => f.endsWith(".pending")); + } + async complete(batch: string[]) { + for (const file of batch) { + if (!/^[a-f0-9-]+\.pending$/.test(file)) + throw new Error("Invalid queue entry"); + await fs.rm(path.join(this.root, file), { force: true }); + } + } +} + +export function sqlValue(value: unknown): string { + if (value === null || value === undefined) return "NULL"; + if (typeof value === "number") { + if (!Number.isFinite(value)) throw new Error("Invalid SQL number"); + return String(value); + } + if (typeof value === "bigint") return String(value); + if (typeof value === "boolean") return value ? "1" : "0"; + if (Buffer.isBuffer(value)) return `X'${value.toString("hex")}'`; + const text = + value instanceof Date + ? value.toISOString().slice(0, 23).replace("T", " ") + : String(value); + return text.length + ? `CONVERT(X'${Buffer.from(text, "utf8").toString("hex")}' USING utf8mb4)` + : "''"; +} + +export interface CatalogFile { + source: string; + target: string; +} + +export function catalogTarget(root: string, target: string) { + if ( + target.includes("\\") || + target.split("/").some((s) => s === ".." || s === ".git") || + path.isAbsolute(target) + ) { + throw new Error("Invalid catalog target"); + } + const result = path.resolve(root, target); + if (!result.startsWith(`${path.resolve(root)}${path.sep}`)) + throw new Error("Invalid catalog target"); + if ( + !target.startsWith("Gamedata/") && + !target.startsWith(`${CATALOG_SQL_ROOT}/`) && + !target.startsWith(`${CATALOG_LANGUAGE_ROOT}/`) + ) { + throw new Error("Unmanaged catalog target"); + } + return result; +} + +export async function publishCatalogFiles(options: { + checkout: string; + remote: string; + branch: string; + files: CatalogFile[]; +}) { + const { checkout, remote, branch, files } = options; + const git = async (...args: string[]) => { + const { stdout } = await exec("git", args, { + cwd: checkout, + timeout: 120_000, + maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, GIT_TERMINAL_PROMPT: "0" }, + }); + return stdout.trim(); + }; + for (const file of files) catalogTarget(checkout, file.target); + if ((await git("remote", "get-url", "origin")) !== remote) + throw new Error("Unexpected catalog origin"); + if ((await git("branch", "--show-current")) !== branch) + throw new Error("Unexpected catalog branch"); + if (await git("status", "--porcelain")) + throw new Error("Catalog checkout must be clean"); + try { + await git("pull", "--rebase", "origin", branch); + } catch (error) { + await git("rebase", "--abort").catch(() => undefined); + throw error; + } + const created: string[] = []; + let committed = false; + try { + for (const file of files) { + const target = catalogTarget(checkout, file.target); + // Reject symlink ancestors, including the target itself. + let ancestor = target; + while (ancestor !== path.resolve(checkout)) { + const stat = await fs + .lstat(ancestor) + .catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return null; + throw error; + }); + if (stat?.isSymbolicLink()) + throw new Error("Symlink in catalog target"); + ancestor = path.dirname(ancestor); + } + const exists = await fs.stat(target).then( + () => true, + () => false, + ); + if (!exists) created.push(target); + await fs.mkdir(path.dirname(target), { recursive: true }); + await fs.copyFile(file.source, target); + } + // Only explicitly exported paths are eligible for staging. + for (let i = 0; i < files.length; i += 50) { + await git( + "--literal-pathspecs", + "add", + "--", + ...files.slice(i, i + 50).map((f) => f.target), + ); + } + if (await git("diff", "--cached", "--name-only")) { + await git( + "-c", + "user.name=Catalog Studio", + "-c", + "user.email=catalog-studio@epicnabbo.nl", + "commit", + "-m", + "Update catalog assets and SQL from Catalog Studio", + ); + } + committed = true; + // Also retries commits retained after a previous failed push. + await git("push", "origin", `HEAD:refs/heads/${branch}`); + return await git("rev-parse", "HEAD"); + } catch (error) { + if (!committed) { + // The checkout was clean at entry; restore only this attempt's changes. + await git("restore", "--staged", "--worktree", "."); + for (const file of created) await fs.rm(file, { force: true }); + } + throw error; + } +} + +export async function recoverCatalogQueue(root: string) { + for (const name of await fs.readdir(root)) { + if (!name.endsWith(".active") && name !== "worker.lock") continue; + const file = path.join(root, name); + let owner: { pid: number; host: string }; + try { + owner = JSON.parse(await fs.readFile(file, "utf8")); + } catch { + continue; + } + if ( + owner.host !== hostname() || + !Number.isInteger(owner.pid) || + owner.pid <= 0 + ) + continue; + try { + process.kill(owner.pid, 0); + continue; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") continue; + } + // Recheck ownership before recovering a dead process's entry. + if ( + (await fs.readFile(file, "utf8").catch(() => "")) !== + JSON.stringify(owner) + ) + continue; + if (name === "worker.lock") await fs.rm(file, { force: true }); + else + await fs + .rename(file, path.join(root, name.replace(/\.active$/, ".pending"))) + .catch((error: NodeJS.ErrnoException) => { + if (error.code !== "ENOENT") throw error; + }); + } +} diff --git a/src/lib/services/catalog-git-export.test.ts b/src/lib/services/catalog-git-export.test.ts new file mode 100644 index 0000000000..7ea3c6fa79 --- /dev/null +++ b/src/lib/services/catalog-git-export.test.ts @@ -0,0 +1,57 @@ +import { mkdtemp, readFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ snapshot: vi.fn(), publish: vi.fn() })); +vi.mock("./catalog-git-snapshot", () => ({ + createCatalogSnapshot: mocks.snapshot, +})); +vi.mock("./catalog-git-core", async (original) => ({ + ...(await original()), + publishCatalogFiles: mocks.publish, +})); + +import { runCatalogExport } from "./catalog-git-export"; +import { catalogExportQueue, withCatalogExport } from "./catalog-git-queue"; + +describe("export worker", () => { + beforeEach(async () => { + vi.stubEnv("CATALOG_GIT_CHECKOUT", "/configured/checkout"); + vi.stubEnv( + "CATALOG_GIT_STATE_DIR", + await mkdtemp(path.join(os.tmpdir(), "catalog-worker-")), + ); + mocks.snapshot.mockReset().mockResolvedValue([]); + mocks.publish.mockReset().mockResolvedValue("abc123"); + }); + afterEach(() => vi.unstubAllEnvs()); + it("retains failed work, retries it and clears only completed events", async () => { + const q = catalogExportQueue(); + await q.request(); + mocks.publish.mockRejectedValueOnce(new Error("credential secret")); + await runCatalogExport(); + expect(await q.batch()).toHaveLength(1); + expect( + await readFile(path.join(q.root, "status.json"), "utf8"), + ).not.toContain("credential secret"); + await runCatalogExport(); + expect(await q.batch()).toHaveLength(0); + expect(mocks.publish).toHaveBeenCalledTimes(2); + }); + it("does not publish during an operation or if one starts while capturing", async () => { + const q = catalogExportQueue(); + await q.request(); + await withCatalogExport(async () => { + await runCatalogExport(); + }); + expect(mocks.snapshot).not.toHaveBeenCalled(); + mocks.snapshot.mockImplementationOnce(async () => { + await q.request(); + return []; + }); + await runCatalogExport(); + expect(mocks.publish).not.toHaveBeenCalled(); + expect(await q.batch()).toHaveLength(3); + }); +}); diff --git a/src/lib/services/catalog-git-export.ts b/src/lib/services/catalog-git-export.ts new file mode 100644 index 0000000000..b7b097dcc7 --- /dev/null +++ b/src/lib/services/catalog-git-export.ts @@ -0,0 +1,82 @@ +import { promises as fs } from "node:fs"; +import { hostname } from "node:os"; +import path from "node:path"; +import { + CATALOG_BRANCH, + CATALOG_REMOTE, + publishCatalogFiles, + recoverCatalogQueue, +} from "./catalog-git-core"; +import { catalogExportEnabled, catalogExportQueue } from "./catalog-git-queue"; +import { createCatalogSnapshot } from "./catalog-git-snapshot"; + +export async function runCatalogExport() { + const checkout = process.env.CATALOG_GIT_CHECKOUT?.trim(); + if (!catalogExportEnabled() || !checkout) return; + const queue = catalogExportQueue(); + await queue.entries(); + await recoverCatalogQueue(queue.root); + const lock = await fs + .open(path.join(queue.root, "worker.lock"), "wx") + .catch((error: NodeJS.ErrnoException) => { + if (error.code === "EEXIST") return null; + throw error; + }); + if (!lock) return; + let snapshot: string | undefined; + const status = async (value: Record) => { + const temp = path.join(queue.root, "status.tmp"); + let previous: Record = {}; + try { + previous = JSON.parse( + await fs.readFile(path.join(queue.root, "status.json"), "utf8"), + ); + } catch { + /* First attempt. */ + } + await fs.writeFile( + temp, + JSON.stringify({ ...previous, error: undefined, ...value }), + ); + await fs.rename(temp, path.join(queue.root, "status.json")); + }; + try { + await lock.writeFile( + JSON.stringify({ pid: process.pid, host: hostname() }), + ); + const batch = await queue.batch(); + if (!batch?.length) return; + snapshot = await fs.mkdtemp(path.join(queue.root, "snapshot-")); + const files = await createCatalogSnapshot(snapshot); + // Any operation started/completed during capture invalidates the snapshot. + const current = await queue.batch(); + if (!current || JSON.stringify(current) !== JSON.stringify(batch)) return; + const commit = await publishCatalogFiles({ + checkout, + remote: CATALOG_REMOTE, + branch: CATALOG_BRANCH, + files, + }); + await queue.complete(batch); + await status({ commit, finishedAt: new Date().toISOString() }); + } catch { + // Do not expose credentials or Git stderr through the admin API. + await status({ + error: + "Export failed. Check repository access, branch, clean checkout and asset/database availability. Pending changes will be retried.", + finishedAt: new Date().toISOString(), + }); + } finally { + try { + if ( + snapshot && + path.resolve(snapshot).startsWith(path.resolve(queue.root) + path.sep) + ) { + await fs.rm(snapshot, { recursive: true, force: true }); + } + } finally { + await lock.close(); + await fs.rm(path.join(queue.root, "worker.lock"), { force: true }); + } + } +} diff --git a/src/lib/services/catalog-git-queue.ts b/src/lib/services/catalog-git-queue.ts new file mode 100644 index 0000000000..1038436b3d --- /dev/null +++ b/src/lib/services/catalog-git-queue.ts @@ -0,0 +1,54 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { CatalogExportQueue } from "./catalog-git-core"; + +export function catalogExportEnabled() { + return Boolean(process.env.CATALOG_GIT_CHECKOUT?.trim()); +} +export function catalogExportQueue() { + return new CatalogExportQueue( + process.env.CATALOG_GIT_STATE_DIR || + path.join(process.cwd(), "storage", "catalog-git"), + ); +} +export function isCatalogMutation(method: string, pathname: string) { + return ( + ["POST", "PUT", "PATCH", "DELETE"].includes(method) && + (pathname.startsWith("/api/admin/import/") || + pathname.startsWith("/api/admin/catalog/") || + pathname.startsWith("/api/admin/furni/") || + pathname.startsWith("/api/admin/furniture/")) + ); +} +export async function catalogExportStatus() { + const q = catalogExportQueue(); + if (!catalogExportEnabled()) return { enabled: false, pending: 0, active: 0 }; + const entries = await q.entries(); + let last: { commit?: string; finishedAt?: string; error?: string } = {}; + try { + last = JSON.parse( + await fs.readFile(path.join(q.root, "status.json"), "utf8"), + ); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error; + } + return { + enabled: true, + pending: entries.filter((f) => f.endsWith(".pending")).length, + active: entries.filter((f) => f.endsWith(".active")).length, + running: entries.includes("worker.lock"), + ...last, + }; +} + +export async function withCatalogExport( + operation: () => Promise, +): Promise { + if (!catalogExportEnabled()) return operation(); + const finish = await catalogExportQueue().begin(); + try { + return await operation(); + } finally { + await finish(); + } +} diff --git a/src/lib/services/catalog-git-snapshot.test.ts b/src/lib/services/catalog-git-snapshot.test.ts new file mode 100644 index 0000000000..81498ae33a --- /dev/null +++ b/src/lib/services/catalog-git-snapshot.test.ts @@ -0,0 +1,161 @@ +vi.mock("@/lib/services/figuredata", () => ({ + getFigureDataPath: async () => + path.join(await mocks.gamedata(), "custom/FigureData.json"), +})); +vi.mock("@/lib/services/figuremap", () => ({ + getFigureMapPath: async () => + path.join(await mocks.gamedata(), "custom/FigureMap.json"), +})); +vi.mock("@/lib/services/effectmap", () => ({ + getEffectMapPath: async () => + path.join(await mocks.gamedata(), "custom/EffectMap.json"), +})); + +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + query: vi.fn(), + commit: vi.fn(), + rollback: vi.fn(), + release: vi.fn(), + gamedata: vi.fn(), + dirs: vi.fn(), + data: vi.fn(), + settings: vi.fn(), +})); +vi.mock("@/lib/db", () => ({ + db: { $client: { getConnection: async () => mocks } }, +})); +vi.mock("@/lib/services/furni-asset-dirs", () => ({ + getGamedataRoot: mocks.gamedata, + getFurniAssetDirs: mocks.dirs, +})); +vi.mock("@/lib/services/furni-data", () => ({ + getFurnitureDataPath: mocks.data, +})); +vi.mock("@/lib/services/site-settings", () => ({ + siteSettings: { get: mocks.settings }, +})); + +import { createCatalogSnapshot } from "./catalog-git-snapshot"; + +describe("catalog snapshot", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.settings.mockResolvedValue(""); + mocks.query.mockImplementation(async (sql: string) => { + if (sql === "SHOW TABLES") + return [ + [ + { name: "items_base" }, + { name: "catalog_pages" }, + { name: "catalog_items" }, + ], + ]; + if (sql.startsWith("SHOW CREATE")) + return [ + [ + { + "Create Table": + "CREATE TABLE `test` (`id` int PRIMARY KEY, `name` text) ENGINE=InnoDB", + }, + ], + ]; + if (sql.startsWith("SHOW COLUMNS")) + return [ + [ + { Field: "id", Extra: "" }, + { Field: "name", Extra: "" }, + ], + ]; + if (sql.startsWith("SELECT")) + return [[{ id: 1, name: "Valentine's chair" }]]; + return [[]]; + }); + }); + async function fixture() { + const root = await mkdtemp(path.join(os.tmpdir(), "catalog-snapshot-")); + for (const folder of [ + "config", + "custom", + "icons", + "bundled/furniture", + "c_images/catalogue", + "output", + ]) + await mkdir(path.join(root, folder), { recursive: true }); + await writeFile( + path.join(root, "config/FurnitureData.json"), + '{"roomitemtypes":{"furnitype":[]}}', + ); + await writeFile(path.join(root, "config/FurnitureData_it.json"), "{}"); + await writeFile( + path.join(root, "config/private-cache.json"), + '{"secret":"not exported"}', + ); + await writeFile(path.join(root, "icons/chair_icon.png"), "image"); + await writeFile(path.join(root, "bundled/furniture/chair.nitro"), "bundle"); + await writeFile( + path.join(root, "c_images/catalogue/icon_1.png"), + "catalog icon", + ); + await writeFile( + path.join(root, "custom/FigureData.json"), + '{"configured":true}', + ); + await writeFile(path.join(root, "custom/FigureMap.json"), "{}"); + await writeFile(path.join(root, "custom/EffectMap.json"), "{}"); + mocks.gamedata.mockResolvedValue(root); + mocks.dirs.mockResolvedValue({ + nitroDir: path.join(root, "bundled/furniture"), + iconDir: path.join(root, "icons"), + }); + mocks.data.mockResolvedValue(path.join(root, "config/FurnitureData.json")); + return root; + } + it("includes bundles, both icon types, translations and SQL but excludes private caches", async () => { + const root = await fixture(); + const files = await createCatalogSnapshot(path.join(root, "output")); + const targets = files.map((f) => f.target); + expect(targets).toContain("Gamedata/bundled/furniture/chair.nitro"); + expect(targets).toContain("Gamedata/icons/chair_icon.png"); + expect(targets).toContain("Gamedata/c_images/catalogue/icon_1.png"); + expect(targets).toContain( + "catalogue version 2 ( Final (Dev)/langs furnidata/FurnitureData_it.json", + ); + expect(targets.some((t) => t.includes("private-cache"))).toBe(false); + expect(targets.filter((t) => t.endsWith(".sql"))).toHaveLength(3); + const sqlFile = files.find((f) => f.target.endsWith("/items_base.sql")); + expect(sqlFile).toBeDefined(); + expect(await readFile(sqlFile?.source ?? "", "utf8")).toContain( + "ON DUPLICATE KEY UPDATE", + ); + const figure = files.find( + (f) => f.target === "Gamedata/config/FigureData.json", + ); + expect(figure).toBeDefined(); + expect(await readFile(figure?.source ?? "", "utf8")).toContain( + '"configured":true', + ); + expect(targets).toContain("Gamedata/config/FigureMap.json"); + expect(targets).toContain("Gamedata/config/EffectMap.json"); + expect(mocks.commit).toHaveBeenCalledOnce(); + expect(mocks.release).toHaveBeenCalledOnce(); + }); + it("rejects invalid JSON before publishing and releases failed SQL snapshots", async () => { + const root = await fixture(); + mocks.query.mockRejectedValueOnce(new Error("DB unavailable")); + await expect( + createCatalogSnapshot(path.join(root, "output")), + ).rejects.toThrow("DB unavailable"); + expect(mocks.rollback).toHaveBeenCalledOnce(); + expect(mocks.release).toHaveBeenCalledOnce(); + await writeFile(path.join(root, "config/FurnitureData.json"), "{"); + await expect( + createCatalogSnapshot(path.join(root, "output")), + ).rejects.toThrow(); + }); +}); diff --git a/src/lib/services/catalog-git-snapshot.ts b/src/lib/services/catalog-git-snapshot.ts new file mode 100644 index 0000000000..51a7eded05 --- /dev/null +++ b/src/lib/services/catalog-git-snapshot.ts @@ -0,0 +1,216 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import type { RowDataPacket } from "mysql2"; +import { db } from "@/lib/db"; +import { getEffectMapPath } from "@/lib/services/effectmap"; +import { getFigureDataPath } from "@/lib/services/figuredata"; +import { getFigureMapPath } from "@/lib/services/figuremap"; +import { + getFurniAssetDirs, + getGamedataRoot, +} from "@/lib/services/furni-asset-dirs"; +import { getFurnitureDataPath } from "@/lib/services/furni-data"; +import { siteSettings } from "@/lib/services/site-settings"; +import { + CATALOG_LANGUAGE_ROOT, + CATALOG_SQL_ROOT, + type CatalogFile, + sqlValue, +} from "./catalog-git-core"; + +const CONFIG_FILES = [ + "FurnitureData.json", + "EffectMap.json", + "ExternalTexts.json", + "FigureData.json", + "FigureMap.json", + "HabboAvatarActions.json", + "ProductData.json", +]; +const TABLES = ["items_base", "catalog_pages", "catalog_items"] as const; +const identifier = (name: string) => `\`${name.replaceAll("`", "``")}\``; + +export async function createCatalogSnapshot( + directory: string, +): Promise { + const sources = new Map(); + const addTree = async (root: string, target: string, extensions: RegExp) => { + async function walk(source: string, relative = "") { + const entries = await fs + .readdir(source, { withFileTypes: true }) + .catch((error: NodeJS.ErrnoException) => { + if (error.code === "ENOENT") return []; + throw error; + }); + for (const entry of entries) { + if (entry.isSymbolicLink()) + throw new Error("Symlink in catalog source"); + const local = path.join(source, entry.name); + const dest = relative ? `${relative}/${entry.name}` : entry.name; + if (entry.isDirectory()) await walk(local, dest); + else if (entry.isFile() && extensions.test(entry.name)) + sources.set(`${target}/${dest}`, local); + } + } + await walk(root); + }; + const gamedata = await getGamedataRoot(); + const furni = await getFurniAssetDirs(); + const furnitureData = await getFurnitureDataPath(); + // Default files first; configured live locations take precedence. + await addTree( + path.join(process.cwd(), "public/swf/c_images"), + "Gamedata/c_images", + /\.(png|gif|webp|jpe?g)$/i, + ); + await addTree( + path.join(process.cwd(), "public/nitro-assets/bundled"), + "Gamedata/bundled", + /\.nitro$/i, + ); + if (gamedata) { + await addTree( + path.join(gamedata, "bundled"), + "Gamedata/bundled", + /\.nitro$/i, + ); + await addTree( + path.join(gamedata, "icons"), + "Gamedata/icons", + /\.(png|gif|webp|jpe?g)$/i, + ); + await addTree( + path.join(gamedata, "c_images"), + "Gamedata/c_images", + /\.(png|gif|webp|jpe?g)$/i, + ); + } + await addTree(furni.nitroDir, "Gamedata/bundled/furniture", /\.nitro$/i); + await addTree(furni.iconDir, "Gamedata/icons", /\.(png|gif|webp|jpe?g)$/i); + for (const type of ["figure", "effect"]) { + const configured = await siteSettings.get(`${type}_nitro_dir`, ""); + if (configured?.trim()) + await addTree(configured, `Gamedata/bundled/${type}`, /\.nitro$/i); + } + // Pets currently write directly to the CMS directory. + await addTree( + path.join(process.cwd(), "public/nitro-assets/bundled/pet"), + "Gamedata/bundled/pet", + /\.nitro$/i, + ); + const configRoot = gamedata + ? path.join(gamedata, "config") + : path.dirname(furnitureData); + const configuredJson: Record = { + FurnitureData: furnitureData, + FigureData: await getFigureDataPath(), + FigureMap: await getFigureMapPath(), + EffectMap: await getEffectMapPath(), + }; + for (const name of CONFIG_FILES) { + const source = + configuredJson[name.replace(/\.json$/, "")] ?? + path.join(configRoot, name); + try { + await fs.access(source); + sources.set(`Gamedata/config/${name}`, source); + } catch (error) { + if ( + name === "FurnitureData.json" || + (error as NodeJS.ErrnoException).code !== "ENOENT" + ) + throw error; + } + } + for (const name of await fs.readdir(path.dirname(furnitureData))) { + if (/^FurnitureData_[a-z]{2}\.json$/.test(name)) { + sources.set( + `${CATALOG_LANGUAGE_ROOT}/${name}`, + path.join(path.dirname(furnitureData), name), + ); + } + } + const files: CatalogFile[] = []; + const fingerprints = new Map(); + const fingerprint = async (source: string) => { + const stat = await fs.lstat(source); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error("Invalid catalog source"); + return `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`; + }; + for (const [target, source] of sources) { + fingerprints.set(source, await fingerprint(source)); + const copy = path.join(directory, String(files.length)); + await fs.copyFile(source, copy); + if (target.endsWith(".json")) JSON.parse(await fs.readFile(copy, "utf8")); + files.push({ source: copy, target }); + } + const connection = await db.$client.getConnection(); + try { + await connection.query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ"); + await connection.query( + "START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY", + ); + const [available] = await connection.query("SHOW TABLES"); + const tableNames = new Set( + available.map((row) => String(Object.values(row)[0])), + ); + const tables = [ + ...TABLES, + ...["catalog_pages_bc", "catalog_items_bc"].filter((name) => + tableNames.has(name), + ), + ]; + for (const table of tables) { + const [schemaRows] = await connection.query( + `SHOW CREATE TABLE ${identifier(table)}`, + ); + const definition = String(schemaRows[0]["Create Table"]); + if (!/ENGINE=InnoDB\b/i.test(definition)) + throw new Error("Catalog SQL snapshots require InnoDB tables"); + const [columns] = await connection.query( + `SHOW COLUMNS FROM ${identifier(table)}`, + ); + const names = columns + .filter( + (c) => !/\b(?:VIRTUAL|STORED) GENERATED\b/.test(String(c.Extra)), + ) + .map((c) => String(c.Field)); + const quoted = names.map(identifier).join(", "); + const [rows] = await connection.query( + `SELECT ${quoted} FROM ${identifier(table)} ORDER BY id`, + ); + const source = path.join(directory, `${table}.sql`); + const output = await fs.open(source, "wx"); + try { + await output.writeFile( + `-- Catalog Studio snapshot. Existing rows are updated; absent rows are not deleted.\nSET NAMES utf8mb4;\n${definition.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS")};\nSTART TRANSACTION;\n`, + ); + const update = names + .filter((n) => n !== "id") + .map((n) => `${identifier(n)}=VALUES(${identifier(n)})`) + .join(", "); + for (const row of rows) { + await output.writeFile( + `INSERT INTO ${identifier(table)} (${quoted}) VALUES (${names.map((n) => sqlValue(row[n])).join(", ")}) ON DUPLICATE KEY UPDATE ${update};\n`, + ); + } + await output.writeFile("COMMIT;\n"); + } finally { + await output.close(); + } + files.push({ source, target: `${CATALOG_SQL_ROOT}/${table}.sql` }); + } + await connection.commit(); + } catch (error) { + await connection.rollback(); + throw error; + } finally { + connection.release(); + } + for (const [source, stamp] of fingerprints) { + if ((await fingerprint(source)) !== stamp) + throw new Error("Assets changed during export; retry required"); + } + return files; +}