feat: export Catalog Studio assets and SQL to catalog repository
CI / check (pull_request) Failing after 1m21s
CI / deploy (pull_request) Skipped
CI / e2e (pull_request) Skipped

This commit is contained in:
Simo committed 2026-09-05 11:49:00 +02:00
1 parent bfbf244141
commit 9ec9ab31ad
21 files changed
+1811 -514

No files matched your search

+5
View File
@@ -76,3 +76,8 @@ LOG_LEVEL=error
# --- FLARESOLVERR (Cloudflare bypass for clone sources) ---
FLARESOLVERR_URL=http://localhost:8191
# Catalog Studio export: dedicated clean clone on Beta-3 with Git push credentials.
CATALOG_GIT_CHECKOUT=
# Persistent directory shared by CMS and worker, outside the catalog clone.
CATALOG_GIT_STATE_DIR=
+73
View File
@@ -0,0 +1,73 @@
# Catalog Studio repository export
Studio mutations automatically queue an export to
`https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git`, branch
`Beta-3`. The jobs worker processes pending exports every minute. Streaming
imports stay active until their stream completes. Server actions for catalog
pages, offers, deletion and maintenance are covered as well.
## Server setup
1. Create a **dedicated clean clone** of the repository on `Beta-3`, outside the
CMS directory. Configure non-interactive Git push authentication for the
worker OS account using its credential helper. Do not put tokens in URLs.
2. Set `CATALOG_GIT_CHECKOUT` to that absolute clone directory in the CMS and
worker environments. Set `CATALOG_GIT_STATE_DIR` to a persistent, writable
directory outside the clone, shared by both processes on the same host.
3. Run `pnpm jobs:worker` alongside the CMS under your process supervisor.
Both processes must have access to the configured asset directories and DB.
The worker command enables the React server condition for server-only modules.
4. Restart the CMS after setting the environment. In **Studio → Sync**, use
**Export now / retry** for the initial export. Subsequent mutations queue
automatically. Status shows pending/active operations and the last commit.
Leaving `CATALOG_GIT_CHECKOUT` empty disables export. No credentials are shipped.
This source change alone does not configure or deploy the production service.
## Exported content
| Source | Repository destination |
| --- | --- |
| Configured Nitro bundles, including furniture, figures, effects and pets | `Gamedata/bundled` |
| Configured furni icons | `Gamedata/icons` |
| Existing badge/catalog images | `Gamedata/c_images` |
| FurnitureData and supported public game-data JSON files | `Gamedata/config` |
| Existing localized FurnitureData files | `catalogue version 2 ( Final (Dev)/langs furnidata` |
| `items_base`, `catalog_pages`, `catalog_items` | `catalogue version 2 ( Final (Dev)/sqls` |
| `catalog_pages_bc`, `catalog_items_bc`, when present | Same SQL directory |
SQL is read in one consistent, read-only InnoDB transaction. Dumps contain table
definitions and deterministic upserts with hexadecimal UTF-8 string literals.
Import `items_base.sql`, then `catalog_pages.sql`, then `catalog_items.sql`.
Existing schemas are not migrated by these dumps. Rows absent from the source
are omitted; importing an upsert dump into another existing database does not
delete that database's extra rows. No user, session or credential tables are exported.
Only existing local assets are exported. Translation generation follows Studio's
existing setting; export does not generate missing languages or download assets.
Public JSON is explicitly allowlisted so translation caches and private runtime
files cannot enter the repository. Invalid JSON or concurrent Studio changes
prevent publication of that snapshot.
## Failure and concurrency behavior
- Pending events survive process restarts; events added during publication remain
pending for the next run. Partial imports are exported as their settled local
state, including successful items from a batch containing failures.
- A single filesystem lock serializes the worker. Dead local process markers are
recovered on the next run. For an unreadable marker or a marker from another
host, stop the CMS and worker before repairing the queue directory.
- A failed push retains the local commit and pending events for retry. Concurrent
upstream commits are rebased; a conflict aborts the rebase and leaves the event
pending. Resolve conflicts in the dedicated clone, then retry.
- The checkout must be clean before each run. Unrelated files are preserved and
no force push is used. Missing local files do not cause remote deletions.
- Status errors omit raw Git output to avoid exposing authentication material.
## Verification
Run the `catalog-git-*` service tests and `src/lib/catalog-export-api.test.ts` with
Vitest. The integration test creates a temporary bare remote and verifies push,
failed-push recovery, no-op export and preservation of unrelated files. SQL
snapshot tests mock the database; production DB import and production push need
verification in the deployed environment.
+1 -1
View File
@@ -16,7 +16,7 @@
"format": "biome format --write .",
"knip": "knip --include files,dependencies,devDependencies,unlisted,binaries",
"diag:permissions": "tsx scripts/diagnose-permission-page.ts",
"jobs:worker": "tsx scripts/jobs-worker.ts",
"jobs:worker": "node --conditions=react-server --import tsx scripts/jobs-worker.ts",
"test": "vitest run",
"test:e2e": "playwright test",
"typecheck": "tsc --noEmit",
+6
View File
@@ -11,6 +11,7 @@ import {
import { logger } from "../src/lib/logger";
import { redis } from "../src/lib/redis";
import { emulatorOffline, healthDegraded } from "../src/lib/services/alert";
import { runCatalogExport } from "../src/lib/services/catalog-git-export";
import { rcon } from "../src/lib/services/rcon";
function captureWorkerError(err: unknown, context: string): void {
@@ -261,6 +262,11 @@ async function publishScheduledArticles(): Promise<void> {
}
async function main() {
new Cron("* * * * *", () => {
runCatalogExport().catch((e) =>
captureWorkerError(e, "Catalog export failed"),
);
});
logger.info("Worker started", { module: "jobs" });
if (env.EMULATOR_JAR_PATH && env.EMULATOR_BACKUP_DIR) {
+174 -155
View File
@@ -5,6 +5,7 @@ import { revalidatePath } from "next/cache";
import { requirePermission } from "@/lib/admin/guard";
import { CatalogItemsBc, CatalogPagesBc, db } from "@/lib/db";
import { PERMS } from "@/lib/permissions";
import { withCatalogExport } from "@/lib/services/catalog-git-queue";
import { rcon } from "@/lib/services/rcon";
import { logStaffActivity } from "@/lib/services/staff-activity";
@@ -52,39 +53,43 @@ export async function updateBcPage({
...fields
}: { id: number } & Record<string, unknown>) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
const data = pickAllowed(fields, BC_PAGE_FIELDS);
if (Object.keys(data).length === 0) {
return { ok: false as const, error: "No valid fields to update" };
}
await db
.update(CatalogPagesBc)
.set(data as Partial<typeof CatalogPagesBc.$inferInsert>)
.where(eq(CatalogPagesBc.id, id));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_page_update",
description: `Updated BC catalog page #${id}`,
targetType: "catalog_page_bc",
targetId: id,
return await withCatalogExport(async () => {
const data = pickAllowed(fields, BC_PAGE_FIELDS);
if (Object.keys(data).length === 0) {
return { ok: false as const, error: "No valid fields to update" };
}
await db
.update(CatalogPagesBc)
.set(data as Partial<typeof CatalogPagesBc.$inferInsert>)
.where(eq(CatalogPagesBc.id, id));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_page_update",
description: `Updated BC catalog page #${id}`,
targetType: "catalog_page_bc",
targetId: id,
});
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const };
});
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const };
}
export async function deleteBcItem({ id }: { id: number }) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
await db.delete(CatalogItemsBc).where(eq(CatalogItemsBc.id, id));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_item_delete",
description: `Deleted BC catalog item #${id}`,
targetType: "catalog_item_bc",
targetId: id,
return await withCatalogExport(async () => {
await db.delete(CatalogItemsBc).where(eq(CatalogItemsBc.id, id));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_item_delete",
description: `Deleted BC catalog item #${id}`,
targetType: "catalog_item_bc",
targetId: id,
});
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const };
});
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const };
}
export async function updateBcItem({
@@ -98,24 +103,26 @@ export async function updateBcItem({
extradata?: string;
}) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
const safe = pickAllowed(data as Record<string, unknown>, BC_ITEM_FIELDS);
if (Object.keys(safe).length === 0) {
return { ok: false as const, error: "No valid fields to update" };
}
await db
.update(CatalogItemsBc)
.set(safe as Partial<typeof CatalogItemsBc.$inferInsert>)
.where(eq(CatalogItemsBc.id, id));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_item_update",
description: `Updated BC catalog item #${id}`,
targetType: "catalog_item_bc",
targetId: id,
return await withCatalogExport(async () => {
const safe = pickAllowed(data as Record<string, unknown>, BC_ITEM_FIELDS);
if (Object.keys(safe).length === 0) {
return { ok: false as const, error: "No valid fields to update" };
}
await db
.update(CatalogItemsBc)
.set(safe as Partial<typeof CatalogItemsBc.$inferInsert>)
.where(eq(CatalogItemsBc.id, id));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_item_update",
description: `Updated BC catalog item #${id}`,
targetType: "catalog_item_bc",
targetId: id,
});
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const };
});
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const };
}
export async function createBcItem({
@@ -129,18 +136,22 @@ export async function createBcItem({
extradata: string;
}) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
const [result] = await db.insert(CatalogItemsBc).values({ pageId, ...data });
const createdId = Number(result.insertId);
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_item_create",
description: `Created BC catalog item #${createdId}`,
targetType: "catalog_item_bc",
targetId: createdId,
return await withCatalogExport(async () => {
const [result] = await db
.insert(CatalogItemsBc)
.values({ pageId, ...data });
const createdId = Number(result.insertId);
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_item_create",
description: `Created BC catalog item #${createdId}`,
targetType: "catalog_item_bc",
targetId: createdId,
});
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const, data: { id: createdId } };
});
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const, data: { id: createdId } };
}
export async function toggleBcPage({
@@ -151,22 +162,24 @@ export async function toggleBcPage({
field: "enabled" | "visible";
}) {
await requirePermission(PERMS.CATALOG_EDIT);
const [page] = await db
.select({
enabled: CatalogPagesBc.enabled,
visible: CatalogPagesBc.visible,
})
.from(CatalogPagesBc)
.where(eq(CatalogPagesBc.id, id))
.limit(1);
if (!page) return { ok: false as const, error: "Page not found" };
await db
.update(CatalogPagesBc)
.set({ [field]: page[field] === "1" ? "0" : "1" })
.where(eq(CatalogPagesBc.id, id));
await rcon.updateCatalog();
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const };
return await withCatalogExport(async () => {
const [page] = await db
.select({
enabled: CatalogPagesBc.enabled,
visible: CatalogPagesBc.visible,
})
.from(CatalogPagesBc)
.where(eq(CatalogPagesBc.id, id))
.limit(1);
if (!page) return { ok: false as const, error: "Page not found" };
await db
.update(CatalogPagesBc)
.set({ [field]: page[field] === "1" ? "0" : "1" })
.where(eq(CatalogPagesBc.id, id));
await rcon.updateCatalog();
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const };
});
}
export async function createBcPage(input: {
@@ -180,30 +193,32 @@ export async function createBcPage(input: {
orderNum?: number;
}) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
const [result] = await db.insert(CatalogPagesBc).values({
caption: input.caption,
parentId: input.parentId,
pageLayout: input.pageLayout ?? "default_3x3",
iconColor: input.iconColor ?? 0,
iconImage: input.iconImage ?? 0,
orderNum: input.orderNum ?? 0,
visible: input.visible ?? "1",
enabled: input.enabled ?? "1",
pageHeadline: "",
pageTeaser: "",
return await withCatalogExport(async () => {
const [result] = await db.insert(CatalogPagesBc).values({
caption: input.caption,
parentId: input.parentId,
pageLayout: input.pageLayout ?? "default_3x3",
iconColor: input.iconColor ?? 0,
iconImage: input.iconImage ?? 0,
orderNum: input.orderNum ?? 0,
visible: input.visible ?? "1",
enabled: input.enabled ?? "1",
pageHeadline: "",
pageTeaser: "",
});
const createdId = Number(result.insertId);
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_page_create",
description: `Created BC catalog page "${input.caption}"`,
targetType: "catalog_page_bc",
targetId: createdId,
});
revalidatePath("/admin/catalog");
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const, data: { id: createdId } };
});
const createdId = Number(result.insertId);
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "bc_page_create",
description: `Created BC catalog page "${input.caption}"`,
targetType: "catalog_page_bc",
targetId: createdId,
});
revalidatePath("/admin/catalog");
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const, data: { id: createdId } };
}
async function moveBcPage(pageId: number, newParentId: number): Promise<void> {
@@ -234,24 +249,26 @@ export async function reorderBcTreePage(input: {
newOrderNum: number;
}) {
await requirePermission(PERMS.CATALOG_EDIT);
if (input.newParentId !== undefined) {
try {
await moveBcPage(input.pageId, input.newParentId);
} catch (err) {
return {
ok: false as const,
error: err instanceof Error ? err.message : "Invalid move",
};
return await withCatalogExport(async () => {
if (input.newParentId !== undefined) {
try {
await moveBcPage(input.pageId, input.newParentId);
} catch (err) {
return {
ok: false as const,
error: err instanceof Error ? err.message : "Invalid move",
};
}
}
}
await db
.update(CatalogPagesBc)
.set({ orderNum: input.newOrderNum })
.where(eq(CatalogPagesBc.id, input.pageId));
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const, data: {} };
await db
.update(CatalogPagesBc)
.set({ orderNum: input.newOrderNum })
.where(eq(CatalogPagesBc.id, input.pageId));
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const, data: {} };
});
}
export async function deleteBcTreePage(input: {
@@ -259,52 +276,54 @@ export async function deleteBcTreePage(input: {
mode: "reparent" | "cascade";
}) {
await requirePermission(PERMS.CATALOG_EDIT);
const [page] = await db
.select({ parentId: CatalogPagesBc.parentId })
.from(CatalogPagesBc)
.where(eq(CatalogPagesBc.id, input.pageId))
.limit(1);
if (!page) return { ok: false as const, error: "Page not found" };
return await withCatalogExport(async () => {
const [page] = await db
.select({ parentId: CatalogPagesBc.parentId })
.from(CatalogPagesBc)
.where(eq(CatalogPagesBc.id, input.pageId))
.limit(1);
if (!page) return { ok: false as const, error: "Page not found" };
if (input.mode === "reparent") {
await db.transaction(async (tx) => {
await tx
.update(CatalogPagesBc)
.set({ parentId: page.parentId })
.where(eq(CatalogPagesBc.parentId, input.pageId));
await tx
.delete(CatalogItemsBc)
.where(eq(CatalogItemsBc.pageId, input.pageId));
await tx
.delete(CatalogPagesBc)
.where(eq(CatalogPagesBc.id, input.pageId));
});
} else {
const toDelete: number[] = [input.pageId];
const queue: number[] = [input.pageId];
while (queue.length > 0) {
const children = await db
.select({ id: CatalogPagesBc.id })
.from(CatalogPagesBc)
.where(inArray(CatalogPagesBc.parentId, queue));
queue.length = 0;
for (const child of children) {
toDelete.push(child.id);
queue.push(child.id);
if (input.mode === "reparent") {
await db.transaction(async (tx) => {
await tx
.update(CatalogPagesBc)
.set({ parentId: page.parentId })
.where(eq(CatalogPagesBc.parentId, input.pageId));
await tx
.delete(CatalogItemsBc)
.where(eq(CatalogItemsBc.pageId, input.pageId));
await tx
.delete(CatalogPagesBc)
.where(eq(CatalogPagesBc.id, input.pageId));
});
} else {
const toDelete: number[] = [input.pageId];
const queue: number[] = [input.pageId];
while (queue.length > 0) {
const children = await db
.select({ id: CatalogPagesBc.id })
.from(CatalogPagesBc)
.where(inArray(CatalogPagesBc.parentId, queue));
queue.length = 0;
for (const child of children) {
toDelete.push(child.id);
queue.push(child.id);
}
}
await db.transaction(async (tx) => {
await tx
.delete(CatalogItemsBc)
.where(inArray(CatalogItemsBc.pageId, toDelete));
await tx
.delete(CatalogPagesBc)
.where(inArray(CatalogPagesBc.id, toDelete));
});
}
await db.transaction(async (tx) => {
await tx
.delete(CatalogItemsBc)
.where(inArray(CatalogItemsBc.pageId, toDelete));
await tx
.delete(CatalogPagesBc)
.where(inArray(CatalogPagesBc.id, toDelete));
});
}
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const, data: {} };
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
revalidatePath("/admin/catalog/builder-club");
return { ok: true as const, data: {} };
});
}
+267 -251
View File
@@ -6,6 +6,7 @@ import { requirePermission } from "@/lib/admin/guard";
import { CatalogItems, db, ItemsBase } from "@/lib/db";
import { PERMS } from "@/lib/permissions";
import { logAudit } from "@/lib/services/audit";
import { withCatalogExport } from "@/lib/services/catalog-git-queue";
import { allocateCatalogItemId } from "@/lib/services/furni-import";
import { rcon } from "@/lib/services/rcon";
import { logStaffActivity } from "@/lib/services/staff-activity";
@@ -123,32 +124,34 @@ export async function createCatalogItem(data: {
clubOnly: "0" | "1";
}) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
let catalogName = data.catalogName.trim();
if (!catalogName) {
const firstId = Number.parseInt(data.itemIds.split(";")[0] || "", 10);
if (firstId > 0) {
const [base] = await db
.select({
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
})
.from(ItemsBase)
.where(eq(ItemsBase.id, firstId))
.limit(1);
catalogName = base?.publicName || base?.itemName || String(firstId);
return await withCatalogExport(async () => {
let catalogName = data.catalogName.trim();
if (!catalogName) {
const firstId = Number.parseInt(data.itemIds.split(";")[0] || "", 10);
if (firstId > 0) {
const [base] = await db
.select({
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
})
.from(ItemsBase)
.where(eq(ItemsBase.id, firstId))
.limit(1);
catalogName = base?.publicName || base?.itemName || String(firstId);
}
}
}
const id = await insertCatalogItemRow({ ...data, catalogName });
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_item_create",
description: `Created catalog item #${id}`,
targetType: "catalog_item",
targetId: id,
const id = await insertCatalogItemRow({ ...data, catalogName });
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_item_create",
description: `Created catalog item #${id}`,
targetType: "catalog_item",
targetId: id,
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: { id } };
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: { id } };
}
/** Bulk create with one RCON refresh at the end. */
@@ -165,84 +168,88 @@ export async function bulkCreateCatalogItems({
}>;
}) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
if (rows.length === 0) {
return { ok: true as const, data: { created: 0, failed: 0 } };
}
if (rows.length > 500) {
return { ok: false as const, error: "Max 500 items per bulk import" };
}
const baseIds = [...new Set(rows.map((r) => r.baseId))];
const bases = await db
.select({
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
})
.from(ItemsBase)
.where(inArray(ItemsBase.id, baseIds));
const baseMap = new Map(bases.map((b) => [b.id, b]));
let created = 0;
let failed = 0;
for (const row of rows) {
const base = baseMap.get(row.baseId);
if (!base) {
failed++;
continue;
return await withCatalogExport(async () => {
if (rows.length === 0) {
return { ok: true as const, data: { created: 0, failed: 0 } };
}
try {
await insertCatalogItemRow({
pageId,
itemIds: String(row.baseId),
catalogName: base.publicName || base.itemName || String(row.baseId),
costCredits: row.credits ?? 0,
costPoints: row.points ?? 0,
pointsType: row.pointsType ?? 0,
amount: 1,
limitedSells: 0,
limitedStack: 0,
orderNumber: 1,
offerId: -1,
songId: 0,
haveOffer: "1",
clubOnly: "0",
extradata: "",
if (rows.length > 500) {
return { ok: false as const, error: "Max 500 items per bulk import" };
}
const baseIds = [...new Set(rows.map((r) => r.baseId))];
const bases = await db
.select({
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
})
.from(ItemsBase)
.where(inArray(ItemsBase.id, baseIds));
const baseMap = new Map(bases.map((b) => [b.id, b]));
let created = 0;
let failed = 0;
for (const row of rows) {
const base = baseMap.get(row.baseId);
if (!base) {
failed++;
continue;
}
try {
await insertCatalogItemRow({
pageId,
itemIds: String(row.baseId),
catalogName: base.publicName || base.itemName || String(row.baseId),
costCredits: row.credits ?? 0,
costPoints: row.points ?? 0,
pointsType: row.pointsType ?? 0,
amount: 1,
limitedSells: 0,
limitedStack: 0,
orderNumber: 1,
offerId: -1,
songId: 0,
haveOffer: "1",
clubOnly: "0",
extradata: "",
});
created++;
} catch {
failed++;
}
}
if (created > 0) {
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_items_bulk_create",
description: `Bulk imported ${created} catalog item(s) on page #${pageId}`,
targetType: "catalog_page",
targetId: pageId,
});
created++;
} catch {
failed++;
revalidatePath("/admin/catalog");
}
}
if (created > 0) {
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_items_bulk_create",
description: `Bulk imported ${created} catalog item(s) on page #${pageId}`,
targetType: "catalog_page",
targetId: pageId,
});
revalidatePath("/admin/catalog");
}
return { ok: true as const, data: { created, failed } };
return { ok: true as const, data: { created, failed } };
});
}
export async function deleteCatalogItems({ ids }: { ids: number[] }) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
await db.delete(CatalogItems).where(inArray(CatalogItems.id, ids));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_items_delete",
description: `Deleted catalog items: ${ids.join(", ")}`,
targetType: "catalog_item",
return await withCatalogExport(async () => {
await db.delete(CatalogItems).where(inArray(CatalogItems.id, ids));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_items_delete",
description: `Deleted catalog items: ${ids.join(", ")}`,
targetType: "catalog_item",
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
}
export async function moveCatalogItems({
@@ -253,11 +260,12 @@ export async function moveCatalogItems({
targetPageId: number;
}) {
await requirePermission(PERMS.CATALOG_EDIT);
if (ids.length === 0) {
return { ok: true as const, data: {} };
}
const pageIdStr = String(targetPageId);
await db.execute(sql`
return await withCatalogExport(async () => {
if (ids.length === 0) {
return { ok: true as const, data: {} };
}
const pageIdStr = String(targetPageId);
await db.execute(sql`
UPDATE catalog_items
SET page_id = ${pageIdStr}
WHERE id IN (${sql.join(
@@ -265,9 +273,10 @@ export async function moveCatalogItems({
sql`, `,
)})
`);
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
}
export async function reorderCatalogItems({
@@ -276,15 +285,17 @@ export async function reorderCatalogItems({
orders: Array<{ id: number; orderNumber: number }>;
}) {
await requirePermission(PERMS.CATALOG_EDIT);
for (const { id, orderNumber } of orders) {
await db
.update(CatalogItems)
.set({ orderNumber })
.where(eq(CatalogItems.id, id));
}
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
return await withCatalogExport(async () => {
for (const { id, orderNumber } of orders) {
await db
.update(CatalogItems)
.set({ orderNumber })
.where(eq(CatalogItems.id, id));
}
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
}
export async function updateCatalogItem({
@@ -297,46 +308,48 @@ export async function updateCatalogItem({
baseItem?: { id: number; fields: Record<string, unknown> };
}) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
const safeCatalog = pickAllowed(catalogFields, CATALOG_ITEM_FIELDS);
if (Object.keys(safeCatalog).length === 0 && !baseItem) {
return { ok: false as const, error: "No valid fields to update" };
}
return await withCatalogExport(async () => {
const safeCatalog = pickAllowed(catalogFields, CATALOG_ITEM_FIELDS);
if (Object.keys(safeCatalog).length === 0 && !baseItem) {
return { ok: false as const, error: "No valid fields to update" };
}
// page_id is often VARCHAR — update it via raw SQL when present.
const pageIdRaw = safeCatalog.pageId;
if (pageIdRaw !== undefined) {
const pageIdStr = String(pageIdRaw);
await db.execute(sql`
// page_id is often VARCHAR — update it via raw SQL when present.
const pageIdRaw = safeCatalog.pageId;
if (pageIdRaw !== undefined) {
const pageIdStr = String(pageIdRaw);
await db.execute(sql`
UPDATE catalog_items SET page_id = ${pageIdStr} WHERE id = ${id}
`);
delete safeCatalog.pageId;
}
if (Object.keys(safeCatalog).length > 0) {
await db
.update(CatalogItems)
.set(safeCatalog as Partial<typeof CatalogItems.$inferInsert>)
.where(eq(CatalogItems.id, id));
}
if (baseItem) {
const safeBase = pickAllowed(baseItem.fields, ITEMS_BASE_FIELDS);
if (Object.keys(safeBase).length > 0) {
await db
.update(ItemsBase)
.set(safeBase as Partial<typeof ItemsBase.$inferInsert>)
.where(eq(ItemsBase.id, baseItem.id));
delete safeCatalog.pageId;
}
}
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_item_update",
description: `Updated catalog item #${id}`,
targetType: "catalog_item",
targetId: id,
if (Object.keys(safeCatalog).length > 0) {
await db
.update(CatalogItems)
.set(safeCatalog as Partial<typeof CatalogItems.$inferInsert>)
.where(eq(CatalogItems.id, id));
}
if (baseItem) {
const safeBase = pickAllowed(baseItem.fields, ITEMS_BASE_FIELDS);
if (Object.keys(safeBase).length > 0) {
await db
.update(ItemsBase)
.set(safeBase as Partial<typeof ItemsBase.$inferInsert>)
.where(eq(ItemsBase.id, baseItem.id));
}
}
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_item_update",
description: `Updated catalog item #${id}`,
targetType: "catalog_item",
targetId: id,
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
}
export async function translateCatalogItems(input: {
@@ -344,121 +357,124 @@ export async function translateCatalogItems(input: {
items: Array<{ id: number; publicName: string; description?: string }>;
}) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
const parsed = translateItemsSchema.safeParse(input);
if (!parsed.success) {
return {
ok: false as const,
error: parsed.error.issues[0]?.message ?? "Invalid translate payload",
};
}
const { items } = parsed.data;
const { invalidateFurniDataCache } = await import(
"@/lib/services/catalog-items-loader"
);
const { patchFurniEntryNames } = await import("@/lib/services/furni-data");
return await withCatalogExport(async () => {
const parsed = translateItemsSchema.safeParse(input);
if (!parsed.success) {
return {
ok: false as const,
error: parsed.error.issues[0]?.message ?? "Invalid translate payload",
};
}
const { items } = parsed.data;
const { invalidateFurniDataCache } = await import(
"@/lib/services/catalog-items-loader"
);
const { patchFurniEntryNames } = await import("@/lib/services/furni-data");
let namesUpdated = 0;
let descriptionsUpdated = 0;
const furniPatches: Array<{
classname: string;
itemType: string;
name?: string;
description?: string;
spriteId?: number;
createIfMissing?: boolean;
}> = [];
let namesUpdated = 0;
let descriptionsUpdated = 0;
const furniPatches: Array<{
classname: string;
itemType: string;
name?: string;
description?: string;
spriteId?: number;
createIfMissing?: boolean;
}> = [];
for (const item of items) {
const [base] = await db
.select({
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
type: ItemsBase.type,
spriteId: ItemsBase.spriteId,
})
.from(ItemsBase)
.where(eq(ItemsBase.id, item.id))
.limit(1);
if (!base) continue;
const nextName = item.publicName?.trim() ?? "";
const nextDesc = item.description ?? "";
const nameChanged = nextName !== "" && nextName !== (base.publicName ?? "");
if (nameChanged) {
await db
.update(ItemsBase)
.set({ publicName: nextName })
.where(eq(ItemsBase.id, base.id));
const idStr = String(base.id);
const related = await db
for (const item of items) {
const [base] = await db
.select({
id: CatalogItems.id,
catalogName: CatalogItems.catalogName,
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
type: ItemsBase.type,
spriteId: ItemsBase.spriteId,
})
.from(CatalogItems)
.where(
or(
eq(CatalogItems.itemIds, idStr),
like(CatalogItems.itemIds, `${idStr};%`),
like(CatalogItems.itemIds, `%;${idStr};%`),
like(CatalogItems.itemIds, `%;${idStr}`),
),
);
for (const row of related) {
if (row.catalogName !== nextName) {
await db
.update(CatalogItems)
.set({ catalogName: nextName })
.where(eq(CatalogItems.id, row.id));
.from(ItemsBase)
.where(eq(ItemsBase.id, item.id))
.limit(1);
if (!base) continue;
const nextName = item.publicName?.trim() ?? "";
const nextDesc = item.description ?? "";
const nameChanged =
nextName !== "" && nextName !== (base.publicName ?? "");
if (nameChanged) {
await db
.update(ItemsBase)
.set({ publicName: nextName })
.where(eq(ItemsBase.id, base.id));
const idStr = String(base.id);
const related = await db
.select({
id: CatalogItems.id,
catalogName: CatalogItems.catalogName,
})
.from(CatalogItems)
.where(
or(
eq(CatalogItems.itemIds, idStr),
like(CatalogItems.itemIds, `${idStr};%`),
like(CatalogItems.itemIds, `%;${idStr};%`),
like(CatalogItems.itemIds, `%;${idStr}`),
),
);
for (const row of related) {
if (row.catalogName !== nextName) {
await db
.update(CatalogItems)
.set({ catalogName: nextName })
.where(eq(CatalogItems.id, row.id));
}
}
namesUpdated++;
}
if (nextDesc !== "" || nameChanged) {
descriptionsUpdated += nextDesc !== "" ? 1 : 0;
furniPatches.push({
classname: base.itemName,
itemType: base.type || "s",
name: nextName || base.publicName || base.itemName,
description: nextDesc,
spriteId: base.spriteId,
createIfMissing: true,
});
}
namesUpdated++;
}
if (nextDesc !== "" || nameChanged) {
descriptionsUpdated += nextDesc !== "" ? 1 : 0;
furniPatches.push({
classname: base.itemName,
itemType: base.type || "s",
name: nextName || base.publicName || base.itemName,
description: nextDesc,
spriteId: base.spriteId,
createIfMissing: true,
});
const furniResult =
furniPatches.length > 0
? await patchFurniEntryNames(furniPatches)
: { updated: 0, inserted: 0 };
if (furniResult.updated > 0 || furniResult.inserted > 0) {
invalidateFurniDataCache();
}
}
const furniResult =
furniPatches.length > 0
? await patchFurniEntryNames(furniPatches)
: { updated: 0, inserted: 0 };
if (furniResult.updated > 0 || furniResult.inserted > 0) {
invalidateFurniDataCache();
}
await rcon.updateCatalog();
await logAudit({
userId: staff.id,
action: "items_base_translate",
target: "ItemsBase",
after: {
namesUpdated,
descriptionsUpdated,
furniDataUpdated: furniResult.updated > 0,
furniDataInserted: furniResult.inserted,
},
await rcon.updateCatalog();
await logAudit({
userId: staff.id,
action: "items_base_translate",
target: "ItemsBase",
after: {
namesUpdated,
descriptionsUpdated,
furniDataUpdated: furniResult.updated > 0,
furniDataInserted: furniResult.inserted,
},
});
revalidatePath("/admin/catalog");
return {
ok: true as const,
data: {
namesUpdated,
descriptionsUpdated,
furniDataUpdated: furniResult.updated,
furniDataInserted: furniResult.inserted,
updated: items.length,
},
};
});
revalidatePath("/admin/catalog");
return {
ok: true as const,
data: {
namesUpdated,
descriptionsUpdated,
furniDataUpdated: furniResult.updated,
furniDataInserted: furniResult.inserted,
updated: items.length,
},
};
}
+108 -95
View File
@@ -6,6 +6,7 @@ import { requirePermission } from "@/lib/admin/guard";
import { CatalogPages, db } from "@/lib/db";
import { PERMS } from "@/lib/permissions";
import type { ActionResult } from "@/lib/safe-action-shared";
import { withCatalogExport } from "@/lib/services/catalog-git-queue";
import { deletePage, movePage } from "@/lib/services/catalog-tree";
import { rcon } from "@/lib/services/rcon";
import { logStaffActivity } from "@/lib/services/staff-activity";
@@ -48,42 +49,46 @@ export async function updateCatalogPage({
...fields
}: { id: number } & Record<string, unknown>): Promise<ActionResult> {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
const data = pickPageFields(fields);
if (Object.keys(data).length === 0) {
return { ok: false as const, error: "No valid fields to update" };
}
if (typeof data.caption === "string" && !data.captionSave) {
data.captionSave = data.caption.slice(0, 25);
}
await db
.update(CatalogPages)
.set(data as Partial<typeof CatalogPages.$inferInsert>)
.where(eq(CatalogPages.id, id));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_page_update",
description: `Updated catalog page #${id}`,
targetType: "catalog_page",
targetId: id,
return await withCatalogExport(async () => {
const data = pickPageFields(fields);
if (Object.keys(data).length === 0) {
return { ok: false as const, error: "No valid fields to update" };
}
if (typeof data.caption === "string" && !data.captionSave) {
data.captionSave = data.caption.slice(0, 25);
}
await db
.update(CatalogPages)
.set(data as Partial<typeof CatalogPages.$inferInsert>)
.where(eq(CatalogPages.id, id));
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_page_update",
description: `Updated catalog page #${id}`,
targetType: "catalog_page",
targetId: id,
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
}
export async function deleteCatalogPage({ id }: { id: number }) {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
await deletePage(id, "reparent");
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_page_delete",
description: `Deleted catalog page #${id}`,
targetType: "catalog_page",
targetId: id,
return await withCatalogExport(async () => {
await deletePage(id, "reparent");
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_page_delete",
description: `Deleted catalog page #${id}`,
targetType: "catalog_page",
targetId: id,
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
}
export async function toggleCatalogPage({
@@ -94,24 +99,26 @@ export async function toggleCatalogPage({
action: "toggleEnabled" | "toggleVisible";
}) {
await requirePermission(PERMS.CATALOG_EDIT);
const [page] = await db
.select({
enabled: CatalogPages.enabled,
visible: CatalogPages.visible,
})
.from(CatalogPages)
.where(eq(CatalogPages.id, id))
.limit(1);
if (!page) return { ok: false as const, error: "Catalog page not found" };
const field = action === "toggleEnabled" ? "enabled" : "visible";
const current = action === "toggleEnabled" ? page.enabled : page.visible;
await db
.update(CatalogPages)
.set({ [field]: current === "1" ? "0" : "1" })
.where(eq(CatalogPages.id, id));
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
return await withCatalogExport(async () => {
const [page] = await db
.select({
enabled: CatalogPages.enabled,
visible: CatalogPages.visible,
})
.from(CatalogPages)
.where(eq(CatalogPages.id, id))
.limit(1);
if (!page) return { ok: false as const, error: "Catalog page not found" };
const field = action === "toggleEnabled" ? "enabled" : "visible";
const current = action === "toggleEnabled" ? page.enabled : page.visible;
await db
.update(CatalogPages)
.set({ [field]: current === "1" ? "0" : "1" })
.where(eq(CatalogPages.id, id));
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
}
export async function createCatalogPage(input: {
@@ -126,34 +133,36 @@ export async function createCatalogPage(input: {
orderNum?: number;
}): Promise<ActionResult<{ id: number }>> {
const staff = await requirePermission(PERMS.CATALOG_EDIT);
const [result] = await db.insert(CatalogPages).values({
caption: input.caption,
parentId: input.parentId,
pageLayout: input.pageLayout ?? "default_3x3",
captionSave: input.caption.slice(0, 25),
iconColor: input.iconColor ?? 0,
iconImage: input.iconImage ?? 0,
minRank: input.minRank ?? 1,
orderNum: input.orderNum ?? 0,
visible: input.visible ?? "1",
enabled: input.enabled ?? "1",
clubOnly: "0",
vipOnly: "0",
pageHeadline: "",
pageTeaser: "",
includes: "",
return await withCatalogExport(async () => {
const [result] = await db.insert(CatalogPages).values({
caption: input.caption,
parentId: input.parentId,
pageLayout: input.pageLayout ?? "default_3x3",
captionSave: input.caption.slice(0, 25),
iconColor: input.iconColor ?? 0,
iconImage: input.iconImage ?? 0,
minRank: input.minRank ?? 1,
orderNum: input.orderNum ?? 0,
visible: input.visible ?? "1",
enabled: input.enabled ?? "1",
clubOnly: "0",
vipOnly: "0",
pageHeadline: "",
pageTeaser: "",
includes: "",
});
const createdId = Number(result.insertId);
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_page_create",
description: `Created catalog page "${input.caption}"`,
targetType: "catalog_page",
targetId: createdId,
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: { id: createdId } };
});
const createdId = Number(result.insertId);
await rcon.updateCatalog();
await logStaffActivity({
staffId: staff.id,
action: "catalog_page_create",
description: `Created catalog page "${input.caption}"`,
targetType: "catalog_page",
targetId: createdId,
});
revalidatePath("/admin/catalog");
return { ok: true as const, data: { id: createdId } };
}
export async function reorderTreePage(input: {
@@ -162,23 +171,25 @@ export async function reorderTreePage(input: {
newOrderNum: number;
}) {
await requirePermission(PERMS.CATALOG_EDIT);
if (input.newParentId !== undefined) {
try {
await movePage(input.pageId, input.newParentId);
} catch (err) {
return {
ok: false as const,
error: err instanceof Error ? err.message : "Invalid move",
};
return await withCatalogExport(async () => {
if (input.newParentId !== undefined) {
try {
await movePage(input.pageId, input.newParentId);
} catch (err) {
return {
ok: false as const,
error: err instanceof Error ? err.message : "Invalid move",
};
}
}
}
await db
.update(CatalogPages)
.set({ orderNum: input.newOrderNum })
.where(eq(CatalogPages.id, input.pageId));
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
await db
.update(CatalogPages)
.set({ orderNum: input.newOrderNum })
.where(eq(CatalogPages.id, input.pageId));
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
}
export async function deleteTreePage(input: {
@@ -186,8 +197,10 @@ export async function deleteTreePage(input: {
mode: "reparent" | "cascade";
}) {
await requirePermission(PERMS.CATALOG_EDIT);
await deletePage(input.pageId, input.mode);
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
return await withCatalogExport(async () => {
await deletePage(input.pageId, input.mode);
await rcon.updateCatalog();
revalidatePath("/admin/catalog");
return { ok: true as const, data: {} };
});
}
+19 -6
View File
@@ -7,6 +7,7 @@ import {
actionOk,
handleActionError,
} from "@/lib/safe-action-shared";
import { withCatalogExport } from "@/lib/services/catalog-git-queue";
import type {
AlignResult,
DedupResult,
@@ -38,7 +39,9 @@ export async function fixSpriteIdsAction(): Promise<
> {
try {
await guard();
return actionOk(await maintenance.fixSpriteIds());
return await withCatalogExport(async () => {
return actionOk(await maintenance.fixSpriteIds());
});
} catch (e) {
return handleActionError(e);
}
@@ -49,7 +52,9 @@ export async function fixCatalogOffersAction(): Promise<
> {
try {
await guard();
return actionOk(await maintenance.fixCatalogOffers());
return await withCatalogExport(async () => {
return actionOk(await maintenance.fixCatalogOffers());
});
} catch (e) {
return handleActionError(e);
}
@@ -60,7 +65,9 @@ export async function reconcileIdsAction(): Promise<
> {
try {
await guard();
return actionOk(await maintenance.reconcileIds());
return await withCatalogExport(async () => {
return actionOk(await maintenance.reconcileIds());
});
} catch (e) {
return handleActionError(e);
}
@@ -71,7 +78,9 @@ export async function removeDuplicateItemsBaseAction(): Promise<
> {
try {
await guard();
return actionOk(await maintenance.removeDuplicates());
return await withCatalogExport(async () => {
return actionOk(await maintenance.removeDuplicates());
});
} catch (e) {
return handleActionError(e);
}
@@ -93,7 +102,9 @@ export async function applyAlignIdsAction(): Promise<
> {
try {
await guard();
return actionOk(await maintenance.forceItemsBaseIdsToFurnidata(true));
return await withCatalogExport(async () => {
return actionOk(await maintenance.forceItemsBaseIdsToFurnidata(true));
});
} catch (e) {
return handleActionError(e);
}
@@ -104,7 +115,9 @@ export async function fixEverythingAction(input?: {
}): Promise<ActionResult<maintenance.FixAllResult>> {
try {
await guard();
return actionOk(await maintenance.fixEverything(input ?? {}));
return await withCatalogExport(async () => {
return actionOk(await maintenance.fixEverything(input ?? {}));
});
} catch (e) {
return handleActionError(e);
}
+4 -4
View File
@@ -5,6 +5,7 @@ import { db, WebsiteSetting } from "@/lib/db";
import { PERMS } from "@/lib/permissions";
import { adminAction } from "@/lib/safe-action";
import { actionOk } from "@/lib/safe-action-shared";
import { withCatalogExport } from "@/lib/services/catalog-git-queue";
import { deleteImportedItem } from "@/lib/services/furni-import";
import { siteSettings } from "@/lib/services/site-settings";
@@ -14,10 +15,9 @@ export const deleteImportedFurni = adminAction(
{ permission: PERMS.ASSETS_IMPORT, schema: deleteSchema },
async (ctx) =>
actionOk(
(await deleteImportedItem(ctx.data.classname)) as unknown as Record<
string,
unknown
>,
(await withCatalogExport(() =>
deleteImportedItem(ctx.data.classname),
)) as unknown as Record<string, unknown>,
),
);
@@ -0,0 +1,102 @@
"use client";
import { useCallback, useEffect, useState } from "react";
import { adminFetch } from "@/lib/admin-fetch";
type ExportStatus = {
enabled: boolean;
pending: number;
active: number;
running?: boolean;
commit?: string;
finishedAt?: string;
error?: string;
};
export function CatalogExportClient() {
const [status, setStatus] = useState<ExportStatus | null>(null);
const [error, setError] = useState("");
const [busy, setBusy] = useState(false);
const refresh = useCallback(async () => {
try {
const response = await adminFetch("/api/admin/catalog-export", {
cache: "no-store",
});
if (!response.ok) throw new Error("Unable to load export status");
setStatus(await response.json());
setError("");
} catch {
setError("Unable to load export status");
}
}, []);
useEffect(() => {
void refresh();
const timer = setInterval(() => {
void refresh();
}, 15_000);
return () => clearInterval(timer);
}, [refresh]);
async function retry() {
setBusy(true);
try {
const response = await adminFetch("/api/admin/catalog-export", {
method: "POST",
});
if (!response.ok) throw new Error("Unable to queue export");
await refresh();
} catch {
setError("Unable to queue export");
} finally {
setBusy(false);
}
}
return (
<section className="space-y-3 rounded-lg border border-[var(--admin-border)] p-4">
<h2 className="text-lg font-semibold">Catalog repository export</h2>
<p className="text-sm text-muted-foreground">
Automatically publishes bundles, icons, gamedata, translations and
catalog SQL to Epicnabbo-Catalogus-Updated-Daily (Beta-3).
</p>
<div role="status" className="text-sm" aria-live="polite">
{!status
? "Loading…"
: !status.enabled
? "Not configured on this server."
: status.active
? `Waiting for ${status.active} operation(s) to finish.`
: status.running
? "Export in progress…"
: status.pending
? `${status.pending} update(s) queued for the worker.`
: "No pending updates."}
{status?.commit && (
<p>
Last published commit:{" "}
<a
className="underline"
href={`https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily/commit/${status.commit}`}
target="_blank"
rel="noreferrer"
>
{status.commit.slice(0, 12)}
</a>
</p>
)}
{status?.finishedAt && <p>Last attempt: {status.finishedAt}</p>}
</div>
{(error || status?.error) && (
<p role="alert" className="text-sm">
{error || status?.error}
</p>
)}
<button
type="button"
disabled={!status?.enabled || busy}
onClick={retry}
className="rounded-md border border-[var(--admin-border)] px-3 py-2 text-sm disabled:opacity-50"
>
{busy ? "Queuing…" : "Export now / retry"}
</button>
</section>
);
}
+3
View File
@@ -3,6 +3,8 @@ import { OfficialSyncClient } from "@/app/admin/import/official/official-sync-cl
import { SyncAllClient } from "@/app/admin/import/sync/sync-all-client";
import { canAccess, getAdminContext, PERMS } from "@/lib/permissions";
import { CatalogExportClient } from "./catalog-export-client";
export default async function StudioSyncPage(_props: {
params: Promise<{ locale: string }>;
}) {
@@ -12,6 +14,7 @@ export default async function StudioSyncPage(_props: {
}
return (
<div className="space-y-8">
<CatalogExportClient />
<h1 className="text-2xl font-bold tracking-tight">Sync All Sources</h1>
<p className="text-muted-foreground mt-2">
Scan configured sources and automatically import any furniture that is
+18
View File
@@ -0,0 +1,18 @@
import { withAdmin } from "@/lib/api-handler";
import { apiError, apiOk } from "@/lib/api-response";
import { PERMS } from "@/lib/permissions";
import {
catalogExportEnabled,
catalogExportQueue,
catalogExportStatus,
} from "@/lib/services/catalog-git-queue";
export const GET = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () =>
apiOk(await catalogExportStatus()),
);
export const POST = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => {
if (!catalogExportEnabled())
return apiError("Catalog repository export is not configured", 400);
await catalogExportQueue().request();
return apiOk(await catalogExportStatus());
});
+41 -2
View File
@@ -1,9 +1,15 @@
import { type NextRequest, NextResponse } from "next/server";
import { after, type NextRequest, NextResponse } from "next/server";
import { logAuthorizationEvent } from "@/lib/admin/authorization-events";
import { validateCsrfToken } from "@/lib/foundation/security";
import { canAccess, getApiAdminContext } from "@/lib/permissions";
import { logServerError } from "@/lib/server-log";
import {
catalogExportEnabled,
catalogExportQueue,
isCatalogMutation,
} from "@/lib/services/catalog-git-queue";
const MUTATING_METHODS = new Set(["POST", "PUT", "PATCH", "DELETE"]);
const MAX_BODY_BYTES = 10 * 1024 * 1024; // 10 MB
@@ -80,9 +86,42 @@ export function withAdmin(
{ status: 403 },
);
}
let finishExport: (() => Promise<void>) | undefined;
try {
return await handler(request, context, routeContext);
if (
catalogExportEnabled() &&
isCatalogMutation(request.method, request.nextUrl.pathname)
) {
finishExport = await catalogExportQueue().begin();
}
const response = await handler(request, context, routeContext);
if (
finishExport &&
response.body &&
response.headers.get("content-type")?.includes("text/event-stream")
) {
const [client, completion] = response.body.tee();
const finish = finishExport;
after(async () => {
const reader = completion.getReader();
try {
while (!(await reader.read()).done) {
/* Wait for all import results. */
}
} finally {
reader.releaseLock();
await finish();
}
});
return new Response(client, {
status: response.status,
headers: response.headers,
});
}
await finishExport?.();
return response;
} catch (error) {
await finishExport?.().catch(() => undefined);
logServerError("admin.api_failed", error, {
path: request.nextUrl.pathname,
userId: context.session.user.id,
+87
View File
@@ -0,0 +1,87 @@
import { mkdtemp } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { NextRequest } from "next/server";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
after: vi.fn(),
auth: vi.fn(),
csrf: vi.fn(),
}));
vi.mock("next/server", async (original) => ({
...(await original<typeof import("next/server")>()),
after: mocks.after,
}));
vi.mock("@/lib/admin/authorization-events", () => ({
logAuthorizationEvent: vi.fn(),
}));
vi.mock("@/lib/foundation/security", () => ({ validateCsrfToken: mocks.csrf }));
vi.mock("@/lib/permissions", () => ({
getApiAdminContext: mocks.auth,
canAccess: () => true,
}));
vi.mock("@/lib/server-log", () => ({ logServerError: vi.fn() }));
import { withAdmin } from "./api-handler";
import { catalogExportQueue } from "./services/catalog-git-queue";
describe("catalog API completion tracking", () => {
beforeEach(async () => {
vi.stubEnv("CATALOG_GIT_CHECKOUT", "/catalog");
vi.stubEnv(
"CATALOG_GIT_STATE_DIR",
await mkdtemp(path.join(os.tmpdir(), "catalog-api-")),
);
mocks.auth.mockResolvedValue({
session: { user: { id: 1, rank: 7 } },
permissions: [],
});
mocks.csrf.mockResolvedValue(true);
mocks.after.mockReset();
});
afterEach(() => vi.unstubAllEnvs());
it("waits for the last SSE event before making changes publishable", async () => {
let end: (() => void) | undefined;
const response = new Response(
new ReadableStream({
start(controller) {
controller.enqueue(
new TextEncoder().encode('data: {"type":"progress"}\n\n'),
);
end = () => controller.close();
},
}),
{ headers: { "content-type": "text/event-stream" } },
);
const route = withAdmin({}, async () => response);
const returned = await route(
new NextRequest("http://localhost/api/admin/import/furni/batch", {
method: "POST",
}),
);
expect(await catalogExportQueue().batch()).toBeNull();
const complete = mocks.after.mock.calls[0][0]();
end?.();
await returned.text();
await complete;
expect(await catalogExportQueue().batch()).toHaveLength(1);
});
it("does not queue unauthenticated requests or reads", async () => {
const handler = vi.fn(async () => new Response("{}"));
const route = withAdmin({}, handler);
mocks.auth.mockResolvedValueOnce(null);
expect(
(
await route(
new NextRequest("http://localhost/api/admin/import/furni", {
method: "POST",
}),
)
).status,
).toBe(401);
expect(handler).not.toHaveBeenCalled();
await route(new NextRequest("http://localhost/api/admin/import/furni"));
expect(await catalogExportQueue().batch()).toHaveLength(0);
});
});
+103
View File
@@ -0,0 +1,103 @@
import { execFileSync } from "node:child_process";
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";
import {
CatalogExportQueue,
publishCatalogFiles,
recoverCatalogQueue,
sqlValue,
} from "./catalog-git-core";
describe("catalog export", () => {
it("recovers queue entries owned by a terminated local process", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "catalog-recovery-"));
const deadPid = Number(
execFileSync(
process.execPath,
["-e", "process.stdout.write(String(process.pid))"],
{ encoding: "utf8" },
),
);
const owner = JSON.stringify({ pid: deadPid, host: os.hostname() });
await writeFile(path.join(root, "12345678.active"), owner);
await writeFile(path.join(root, "worker.lock"), owner);
await recoverCatalogQueue(root);
const q = new CatalogExportQueue(root);
expect(await q.batch()).toEqual(["12345678.pending"]);
expect(await q.entries()).not.toContain("worker.lock");
});
it("keeps new requests pending when an older snapshot completes", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "catalog-queue-"));
const q = new CatalogExportQueue(root);
const finish = await q.begin();
expect(await q.batch()).toBeNull();
await finish();
const batch = await q.batch();
expect(batch).toHaveLength(1);
await q.request();
if (!batch) throw new Error("Expected pending batch");
await q.complete(batch);
expect(await q.batch()).toHaveLength(1);
});
it("encodes SQL strings without quote or backslash ambiguity", () => {
expect(sqlValue("Valentine's \\ Day\n")).toBe(
"CONVERT(X'56616c656e74696e652773205c204461790a' USING utf8mb4)",
);
expect(sqlValue(null)).toBe("NULL");
expect(sqlValue(42)).toBe("42");
expect(() => sqlValue(Number.NaN)).toThrow();
});
it("publishes to a bare remote, retries a failed push and avoids empty commits", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "catalog-git-"));
const remote = path.join(root, "remote.git");
const checkout = path.join(root, "checkout");
const git = (cwd: string, ...args: string[]) =>
execFileSync("git", args, { cwd, encoding: "utf8" }).trim();
git(root, "init", "--bare", remote);
git(root, "clone", remote, checkout);
git(checkout, "checkout", "-b", "Beta-3");
git(checkout, "config", "user.name", "Catalog test");
git(checkout, "config", "user.email", "[email protected]");
await writeFile(path.join(checkout, "readme.md"), "preserved");
git(checkout, "add", "readme.md");
git(checkout, "commit", "-m", "initial");
git(checkout, "push", "-u", "origin", "Beta-3");
const source = path.join(root, "chair.nitro");
await writeFile(source, "bundle");
const files = [
{ source, target: "Gamedata/bundled/furniture/chair.nitro" },
];
const options = { checkout, remote, branch: "Beta-3", files };
const first = await publishCatalogFiles(options);
expect(await publishCatalogFiles(options)).toBe(first);
await writeFile(source, "updated");
git(
checkout,
"remote",
"set-url",
"--push",
"origin",
path.join(root, "missing.git"),
);
await expect(publishCatalogFiles(options)).rejects.toThrow();
git(checkout, "remote", "set-url", "--push", "origin", remote);
const retried = await publishCatalogFiles(options);
expect(retried).not.toBe(first);
expect(git(remote, "rev-parse", "Beta-3")).toBe(retried);
expect(await readFile(path.join(checkout, "readme.md"), "utf8")).toBe(
"preserved",
);
await expect(
publishCatalogFiles({
...options,
files: [{ source, target: "../escape" }],
}),
).rejects.toThrow();
await mkdir(path.join(checkout, "unrelated"));
await writeFile(path.join(checkout, "unrelated", "local.txt"), "local");
await expect(publishCatalogFiles(options)).rejects.toThrow(/clean/);
});
});
+230
View File
@@ -0,0 +1,230 @@
import { execFile } from "node:child_process";
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import { hostname } from "node:os";
import path from "node:path";
import { promisify } from "node:util";
const exec = promisify(execFile);
export const CATALOG_REMOTE =
"https://gitlab.epicnabbo.nl/remco/Epicnabbo-Catalogus-Updated-Daily.git";
export const CATALOG_BRANCH = "Beta-3";
export const CATALOG_SQL_ROOT = "catalogue version 2 ( Final (Dev)/sqls";
export const CATALOG_LANGUAGE_ROOT =
"catalogue version 2 ( Final (Dev)/langs furnidata";
export class CatalogExportQueue {
constructor(readonly root: string) {}
async entries() {
await fs.mkdir(this.root, { recursive: true });
return (await fs.readdir(this.root)).sort();
}
async begin() {
await this.entries();
const id = randomUUID();
await fs.writeFile(
path.join(this.root, `${id}.active`),
JSON.stringify({
pid: process.pid,
host: hostname(),
startedAt: new Date().toISOString(),
}),
{ flag: "wx" },
);
let finished = false;
return async () => {
if (finished) return;
await fs.rename(
path.join(this.root, `${id}.active`),
path.join(this.root, `${id}.pending`),
);
finished = true;
};
}
async request() {
await (await this.begin())();
}
async batch(): Promise<string[] | null> {
const entries = await this.entries();
if (entries.some((f) => f.endsWith(".active"))) return null;
return entries.filter((f) => f.endsWith(".pending"));
}
async complete(batch: string[]) {
for (const file of batch) {
if (!/^[a-f0-9-]+\.pending$/.test(file))
throw new Error("Invalid queue entry");
await fs.rm(path.join(this.root, file), { force: true });
}
}
}
export function sqlValue(value: unknown): string {
if (value === null || value === undefined) return "NULL";
if (typeof value === "number") {
if (!Number.isFinite(value)) throw new Error("Invalid SQL number");
return String(value);
}
if (typeof value === "bigint") return String(value);
if (typeof value === "boolean") return value ? "1" : "0";
if (Buffer.isBuffer(value)) return `X'${value.toString("hex")}'`;
const text =
value instanceof Date
? value.toISOString().slice(0, 23).replace("T", " ")
: String(value);
return text.length
? `CONVERT(X'${Buffer.from(text, "utf8").toString("hex")}' USING utf8mb4)`
: "''";
}
export interface CatalogFile {
source: string;
target: string;
}
export function catalogTarget(root: string, target: string) {
if (
target.includes("\\") ||
target.split("/").some((s) => s === ".." || s === ".git") ||
path.isAbsolute(target)
) {
throw new Error("Invalid catalog target");
}
const result = path.resolve(root, target);
if (!result.startsWith(`${path.resolve(root)}${path.sep}`))
throw new Error("Invalid catalog target");
if (
!target.startsWith("Gamedata/") &&
!target.startsWith(`${CATALOG_SQL_ROOT}/`) &&
!target.startsWith(`${CATALOG_LANGUAGE_ROOT}/`)
) {
throw new Error("Unmanaged catalog target");
}
return result;
}
export async function publishCatalogFiles(options: {
checkout: string;
remote: string;
branch: string;
files: CatalogFile[];
}) {
const { checkout, remote, branch, files } = options;
const git = async (...args: string[]) => {
const { stdout } = await exec("git", args, {
cwd: checkout,
timeout: 120_000,
maxBuffer: 16 * 1024 * 1024,
env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
});
return stdout.trim();
};
for (const file of files) catalogTarget(checkout, file.target);
if ((await git("remote", "get-url", "origin")) !== remote)
throw new Error("Unexpected catalog origin");
if ((await git("branch", "--show-current")) !== branch)
throw new Error("Unexpected catalog branch");
if (await git("status", "--porcelain"))
throw new Error("Catalog checkout must be clean");
try {
await git("pull", "--rebase", "origin", branch);
} catch (error) {
await git("rebase", "--abort").catch(() => undefined);
throw error;
}
const created: string[] = [];
let committed = false;
try {
for (const file of files) {
const target = catalogTarget(checkout, file.target);
// Reject symlink ancestors, including the target itself.
let ancestor = target;
while (ancestor !== path.resolve(checkout)) {
const stat = await fs
.lstat(ancestor)
.catch((error: NodeJS.ErrnoException) => {
if (error.code === "ENOENT") return null;
throw error;
});
if (stat?.isSymbolicLink())
throw new Error("Symlink in catalog target");
ancestor = path.dirname(ancestor);
}
const exists = await fs.stat(target).then(
() => true,
() => false,
);
if (!exists) created.push(target);
await fs.mkdir(path.dirname(target), { recursive: true });
await fs.copyFile(file.source, target);
}
// Only explicitly exported paths are eligible for staging.
for (let i = 0; i < files.length; i += 50) {
await git(
"--literal-pathspecs",
"add",
"--",
...files.slice(i, i + 50).map((f) => f.target),
);
}
if (await git("diff", "--cached", "--name-only")) {
await git(
"-c",
"user.name=Catalog Studio",
"-c",
"[email protected]",
"commit",
"-m",
"Update catalog assets and SQL from Catalog Studio",
);
}
committed = true;
// Also retries commits retained after a previous failed push.
await git("push", "origin", `HEAD:refs/heads/${branch}`);
return await git("rev-parse", "HEAD");
} catch (error) {
if (!committed) {
// The checkout was clean at entry; restore only this attempt's changes.
await git("restore", "--staged", "--worktree", ".");
for (const file of created) await fs.rm(file, { force: true });
}
throw error;
}
}
export async function recoverCatalogQueue(root: string) {
for (const name of await fs.readdir(root)) {
if (!name.endsWith(".active") && name !== "worker.lock") continue;
const file = path.join(root, name);
let owner: { pid: number; host: string };
try {
owner = JSON.parse(await fs.readFile(file, "utf8"));
} catch {
continue;
}
if (
owner.host !== hostname() ||
!Number.isInteger(owner.pid) ||
owner.pid <= 0
)
continue;
try {
process.kill(owner.pid, 0);
continue;
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ESRCH") continue;
}
// Recheck ownership before recovering a dead process's entry.
if (
(await fs.readFile(file, "utf8").catch(() => "")) !==
JSON.stringify(owner)
)
continue;
if (name === "worker.lock") await fs.rm(file, { force: true });
else
await fs
.rename(file, path.join(root, name.replace(/\.active$/, ".pending")))
.catch((error: NodeJS.ErrnoException) => {
if (error.code !== "ENOENT") throw error;
});
}
}
@@ -0,0 +1,57 @@
import { mkdtemp, readFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({ snapshot: vi.fn(), publish: vi.fn() }));
vi.mock("./catalog-git-snapshot", () => ({
createCatalogSnapshot: mocks.snapshot,
}));
vi.mock("./catalog-git-core", async (original) => ({
...(await original<typeof import("./catalog-git-core")>()),
publishCatalogFiles: mocks.publish,
}));
import { runCatalogExport } from "./catalog-git-export";
import { catalogExportQueue, withCatalogExport } from "./catalog-git-queue";
describe("export worker", () => {
beforeEach(async () => {
vi.stubEnv("CATALOG_GIT_CHECKOUT", "/configured/checkout");
vi.stubEnv(
"CATALOG_GIT_STATE_DIR",
await mkdtemp(path.join(os.tmpdir(), "catalog-worker-")),
);
mocks.snapshot.mockReset().mockResolvedValue([]);
mocks.publish.mockReset().mockResolvedValue("abc123");
});
afterEach(() => vi.unstubAllEnvs());
it("retains failed work, retries it and clears only completed events", async () => {
const q = catalogExportQueue();
await q.request();
mocks.publish.mockRejectedValueOnce(new Error("credential secret"));
await runCatalogExport();
expect(await q.batch()).toHaveLength(1);
expect(
await readFile(path.join(q.root, "status.json"), "utf8"),
).not.toContain("credential secret");
await runCatalogExport();
expect(await q.batch()).toHaveLength(0);
expect(mocks.publish).toHaveBeenCalledTimes(2);
});
it("does not publish during an operation or if one starts while capturing", async () => {
const q = catalogExportQueue();
await q.request();
await withCatalogExport(async () => {
await runCatalogExport();
});
expect(mocks.snapshot).not.toHaveBeenCalled();
mocks.snapshot.mockImplementationOnce(async () => {
await q.request();
return [];
});
await runCatalogExport();
expect(mocks.publish).not.toHaveBeenCalled();
expect(await q.batch()).toHaveLength(3);
});
});
+82
View File
@@ -0,0 +1,82 @@
import { promises as fs } from "node:fs";
import { hostname } from "node:os";
import path from "node:path";
import {
CATALOG_BRANCH,
CATALOG_REMOTE,
publishCatalogFiles,
recoverCatalogQueue,
} from "./catalog-git-core";
import { catalogExportEnabled, catalogExportQueue } from "./catalog-git-queue";
import { createCatalogSnapshot } from "./catalog-git-snapshot";
export async function runCatalogExport() {
const checkout = process.env.CATALOG_GIT_CHECKOUT?.trim();
if (!catalogExportEnabled() || !checkout) return;
const queue = catalogExportQueue();
await queue.entries();
await recoverCatalogQueue(queue.root);
const lock = await fs
.open(path.join(queue.root, "worker.lock"), "wx")
.catch((error: NodeJS.ErrnoException) => {
if (error.code === "EEXIST") return null;
throw error;
});
if (!lock) return;
let snapshot: string | undefined;
const status = async (value: Record<string, unknown>) => {
const temp = path.join(queue.root, "status.tmp");
let previous: Record<string, unknown> = {};
try {
previous = JSON.parse(
await fs.readFile(path.join(queue.root, "status.json"), "utf8"),
);
} catch {
/* First attempt. */
}
await fs.writeFile(
temp,
JSON.stringify({ ...previous, error: undefined, ...value }),
);
await fs.rename(temp, path.join(queue.root, "status.json"));
};
try {
await lock.writeFile(
JSON.stringify({ pid: process.pid, host: hostname() }),
);
const batch = await queue.batch();
if (!batch?.length) return;
snapshot = await fs.mkdtemp(path.join(queue.root, "snapshot-"));
const files = await createCatalogSnapshot(snapshot);
// Any operation started/completed during capture invalidates the snapshot.
const current = await queue.batch();
if (!current || JSON.stringify(current) !== JSON.stringify(batch)) return;
const commit = await publishCatalogFiles({
checkout,
remote: CATALOG_REMOTE,
branch: CATALOG_BRANCH,
files,
});
await queue.complete(batch);
await status({ commit, finishedAt: new Date().toISOString() });
} catch {
// Do not expose credentials or Git stderr through the admin API.
await status({
error:
"Export failed. Check repository access, branch, clean checkout and asset/database availability. Pending changes will be retried.",
finishedAt: new Date().toISOString(),
});
} finally {
try {
if (
snapshot &&
path.resolve(snapshot).startsWith(path.resolve(queue.root) + path.sep)
) {
await fs.rm(snapshot, { recursive: true, force: true });
}
} finally {
await lock.close();
await fs.rm(path.join(queue.root, "worker.lock"), { force: true });
}
}
}
+54
View File
@@ -0,0 +1,54 @@
import { promises as fs } from "node:fs";
import path from "node:path";
import { CatalogExportQueue } from "./catalog-git-core";
export function catalogExportEnabled() {
return Boolean(process.env.CATALOG_GIT_CHECKOUT?.trim());
}
export function catalogExportQueue() {
return new CatalogExportQueue(
process.env.CATALOG_GIT_STATE_DIR ||
path.join(process.cwd(), "storage", "catalog-git"),
);
}
export function isCatalogMutation(method: string, pathname: string) {
return (
["POST", "PUT", "PATCH", "DELETE"].includes(method) &&
(pathname.startsWith("/api/admin/import/") ||
pathname.startsWith("/api/admin/catalog/") ||
pathname.startsWith("/api/admin/furni/") ||
pathname.startsWith("/api/admin/furniture/"))
);
}
export async function catalogExportStatus() {
const q = catalogExportQueue();
if (!catalogExportEnabled()) return { enabled: false, pending: 0, active: 0 };
const entries = await q.entries();
let last: { commit?: string; finishedAt?: string; error?: string } = {};
try {
last = JSON.parse(
await fs.readFile(path.join(q.root, "status.json"), "utf8"),
);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== "ENOENT") throw error;
}
return {
enabled: true,
pending: entries.filter((f) => f.endsWith(".pending")).length,
active: entries.filter((f) => f.endsWith(".active")).length,
running: entries.includes("worker.lock"),
...last,
};
}
export async function withCatalogExport<T>(
operation: () => Promise<T>,
): Promise<T> {
if (!catalogExportEnabled()) return operation();
const finish = await catalogExportQueue().begin();
try {
return await operation();
} finally {
await finish();
}
}
@@ -0,0 +1,161 @@
vi.mock("@/lib/services/figuredata", () => ({
getFigureDataPath: async () =>
path.join(await mocks.gamedata(), "custom/FigureData.json"),
}));
vi.mock("@/lib/services/figuremap", () => ({
getFigureMapPath: async () =>
path.join(await mocks.gamedata(), "custom/FigureMap.json"),
}));
vi.mock("@/lib/services/effectmap", () => ({
getEffectMapPath: async () =>
path.join(await mocks.gamedata(), "custom/EffectMap.json"),
}));
import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { beforeEach, describe, expect, it, vi } from "vitest";
const mocks = vi.hoisted(() => ({
query: vi.fn(),
commit: vi.fn(),
rollback: vi.fn(),
release: vi.fn(),
gamedata: vi.fn(),
dirs: vi.fn(),
data: vi.fn(),
settings: vi.fn(),
}));
vi.mock("@/lib/db", () => ({
db: { $client: { getConnection: async () => mocks } },
}));
vi.mock("@/lib/services/furni-asset-dirs", () => ({
getGamedataRoot: mocks.gamedata,
getFurniAssetDirs: mocks.dirs,
}));
vi.mock("@/lib/services/furni-data", () => ({
getFurnitureDataPath: mocks.data,
}));
vi.mock("@/lib/services/site-settings", () => ({
siteSettings: { get: mocks.settings },
}));
import { createCatalogSnapshot } from "./catalog-git-snapshot";
describe("catalog snapshot", () => {
beforeEach(() => {
vi.clearAllMocks();
mocks.settings.mockResolvedValue("");
mocks.query.mockImplementation(async (sql: string) => {
if (sql === "SHOW TABLES")
return [
[
{ name: "items_base" },
{ name: "catalog_pages" },
{ name: "catalog_items" },
],
];
if (sql.startsWith("SHOW CREATE"))
return [
[
{
"Create Table":
"CREATE TABLE `test` (`id` int PRIMARY KEY, `name` text) ENGINE=InnoDB",
},
],
];
if (sql.startsWith("SHOW COLUMNS"))
return [
[
{ Field: "id", Extra: "" },
{ Field: "name", Extra: "" },
],
];
if (sql.startsWith("SELECT"))
return [[{ id: 1, name: "Valentine's chair" }]];
return [[]];
});
});
async function fixture() {
const root = await mkdtemp(path.join(os.tmpdir(), "catalog-snapshot-"));
for (const folder of [
"config",
"custom",
"icons",
"bundled/furniture",
"c_images/catalogue",
"output",
])
await mkdir(path.join(root, folder), { recursive: true });
await writeFile(
path.join(root, "config/FurnitureData.json"),
'{"roomitemtypes":{"furnitype":[]}}',
);
await writeFile(path.join(root, "config/FurnitureData_it.json"), "{}");
await writeFile(
path.join(root, "config/private-cache.json"),
'{"secret":"not exported"}',
);
await writeFile(path.join(root, "icons/chair_icon.png"), "image");
await writeFile(path.join(root, "bundled/furniture/chair.nitro"), "bundle");
await writeFile(
path.join(root, "c_images/catalogue/icon_1.png"),
"catalog icon",
);
await writeFile(
path.join(root, "custom/FigureData.json"),
'{"configured":true}',
);
await writeFile(path.join(root, "custom/FigureMap.json"), "{}");
await writeFile(path.join(root, "custom/EffectMap.json"), "{}");
mocks.gamedata.mockResolvedValue(root);
mocks.dirs.mockResolvedValue({
nitroDir: path.join(root, "bundled/furniture"),
iconDir: path.join(root, "icons"),
});
mocks.data.mockResolvedValue(path.join(root, "config/FurnitureData.json"));
return root;
}
it("includes bundles, both icon types, translations and SQL but excludes private caches", async () => {
const root = await fixture();
const files = await createCatalogSnapshot(path.join(root, "output"));
const targets = files.map((f) => f.target);
expect(targets).toContain("Gamedata/bundled/furniture/chair.nitro");
expect(targets).toContain("Gamedata/icons/chair_icon.png");
expect(targets).toContain("Gamedata/c_images/catalogue/icon_1.png");
expect(targets).toContain(
"catalogue version 2 ( Final (Dev)/langs furnidata/FurnitureData_it.json",
);
expect(targets.some((t) => t.includes("private-cache"))).toBe(false);
expect(targets.filter((t) => t.endsWith(".sql"))).toHaveLength(3);
const sqlFile = files.find((f) => f.target.endsWith("/items_base.sql"));
expect(sqlFile).toBeDefined();
expect(await readFile(sqlFile?.source ?? "", "utf8")).toContain(
"ON DUPLICATE KEY UPDATE",
);
const figure = files.find(
(f) => f.target === "Gamedata/config/FigureData.json",
);
expect(figure).toBeDefined();
expect(await readFile(figure?.source ?? "", "utf8")).toContain(
'"configured":true',
);
expect(targets).toContain("Gamedata/config/FigureMap.json");
expect(targets).toContain("Gamedata/config/EffectMap.json");
expect(mocks.commit).toHaveBeenCalledOnce();
expect(mocks.release).toHaveBeenCalledOnce();
});
it("rejects invalid JSON before publishing and releases failed SQL snapshots", async () => {
const root = await fixture();
mocks.query.mockRejectedValueOnce(new Error("DB unavailable"));
await expect(
createCatalogSnapshot(path.join(root, "output")),
).rejects.toThrow("DB unavailable");
expect(mocks.rollback).toHaveBeenCalledOnce();
expect(mocks.release).toHaveBeenCalledOnce();
await writeFile(path.join(root, "config/FurnitureData.json"), "{");
await expect(
createCatalogSnapshot(path.join(root, "output")),
).rejects.toThrow();
});
});
+216
View File
@@ -0,0 +1,216 @@
import { promises as fs } from "node:fs";
import path from "node:path";
import type { RowDataPacket } from "mysql2";
import { db } from "@/lib/db";
import { getEffectMapPath } from "@/lib/services/effectmap";
import { getFigureDataPath } from "@/lib/services/figuredata";
import { getFigureMapPath } from "@/lib/services/figuremap";
import {
getFurniAssetDirs,
getGamedataRoot,
} from "@/lib/services/furni-asset-dirs";
import { getFurnitureDataPath } from "@/lib/services/furni-data";
import { siteSettings } from "@/lib/services/site-settings";
import {
CATALOG_LANGUAGE_ROOT,
CATALOG_SQL_ROOT,
type CatalogFile,
sqlValue,
} from "./catalog-git-core";
const CONFIG_FILES = [
"FurnitureData.json",
"EffectMap.json",
"ExternalTexts.json",
"FigureData.json",
"FigureMap.json",
"HabboAvatarActions.json",
"ProductData.json",
];
const TABLES = ["items_base", "catalog_pages", "catalog_items"] as const;
const identifier = (name: string) => `\`${name.replaceAll("`", "``")}\``;
export async function createCatalogSnapshot(
directory: string,
): Promise<CatalogFile[]> {
const sources = new Map<string, string>();
const addTree = async (root: string, target: string, extensions: RegExp) => {
async function walk(source: string, relative = "") {
const entries = await fs
.readdir(source, { withFileTypes: true })
.catch((error: NodeJS.ErrnoException) => {
if (error.code === "ENOENT") return [];
throw error;
});
for (const entry of entries) {
if (entry.isSymbolicLink())
throw new Error("Symlink in catalog source");
const local = path.join(source, entry.name);
const dest = relative ? `${relative}/${entry.name}` : entry.name;
if (entry.isDirectory()) await walk(local, dest);
else if (entry.isFile() && extensions.test(entry.name))
sources.set(`${target}/${dest}`, local);
}
}
await walk(root);
};
const gamedata = await getGamedataRoot();
const furni = await getFurniAssetDirs();
const furnitureData = await getFurnitureDataPath();
// Default files first; configured live locations take precedence.
await addTree(
path.join(process.cwd(), "public/swf/c_images"),
"Gamedata/c_images",
/\.(png|gif|webp|jpe?g)$/i,
);
await addTree(
path.join(process.cwd(), "public/nitro-assets/bundled"),
"Gamedata/bundled",
/\.nitro$/i,
);
if (gamedata) {
await addTree(
path.join(gamedata, "bundled"),
"Gamedata/bundled",
/\.nitro$/i,
);
await addTree(
path.join(gamedata, "icons"),
"Gamedata/icons",
/\.(png|gif|webp|jpe?g)$/i,
);
await addTree(
path.join(gamedata, "c_images"),
"Gamedata/c_images",
/\.(png|gif|webp|jpe?g)$/i,
);
}
await addTree(furni.nitroDir, "Gamedata/bundled/furniture", /\.nitro$/i);
await addTree(furni.iconDir, "Gamedata/icons", /\.(png|gif|webp|jpe?g)$/i);
for (const type of ["figure", "effect"]) {
const configured = await siteSettings.get(`${type}_nitro_dir`, "");
if (configured?.trim())
await addTree(configured, `Gamedata/bundled/${type}`, /\.nitro$/i);
}
// Pets currently write directly to the CMS directory.
await addTree(
path.join(process.cwd(), "public/nitro-assets/bundled/pet"),
"Gamedata/bundled/pet",
/\.nitro$/i,
);
const configRoot = gamedata
? path.join(gamedata, "config")
: path.dirname(furnitureData);
const configuredJson: Record<string, string> = {
FurnitureData: furnitureData,
FigureData: await getFigureDataPath(),
FigureMap: await getFigureMapPath(),
EffectMap: await getEffectMapPath(),
};
for (const name of CONFIG_FILES) {
const source =
configuredJson[name.replace(/\.json$/, "")] ??
path.join(configRoot, name);
try {
await fs.access(source);
sources.set(`Gamedata/config/${name}`, source);
} catch (error) {
if (
name === "FurnitureData.json" ||
(error as NodeJS.ErrnoException).code !== "ENOENT"
)
throw error;
}
}
for (const name of await fs.readdir(path.dirname(furnitureData))) {
if (/^FurnitureData_[a-z]{2}\.json$/.test(name)) {
sources.set(
`${CATALOG_LANGUAGE_ROOT}/${name}`,
path.join(path.dirname(furnitureData), name),
);
}
}
const files: CatalogFile[] = [];
const fingerprints = new Map<string, string>();
const fingerprint = async (source: string) => {
const stat = await fs.lstat(source);
if (!stat.isFile() || stat.isSymbolicLink())
throw new Error("Invalid catalog source");
return `${stat.size}:${stat.mtimeMs}:${stat.ctimeMs}`;
};
for (const [target, source] of sources) {
fingerprints.set(source, await fingerprint(source));
const copy = path.join(directory, String(files.length));
await fs.copyFile(source, copy);
if (target.endsWith(".json")) JSON.parse(await fs.readFile(copy, "utf8"));
files.push({ source: copy, target });
}
const connection = await db.$client.getConnection();
try {
await connection.query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ");
await connection.query(
"START TRANSACTION WITH CONSISTENT SNAPSHOT, READ ONLY",
);
const [available] = await connection.query<RowDataPacket[]>("SHOW TABLES");
const tableNames = new Set(
available.map((row) => String(Object.values(row)[0])),
);
const tables = [
...TABLES,
...["catalog_pages_bc", "catalog_items_bc"].filter((name) =>
tableNames.has(name),
),
];
for (const table of tables) {
const [schemaRows] = await connection.query<RowDataPacket[]>(
`SHOW CREATE TABLE ${identifier(table)}`,
);
const definition = String(schemaRows[0]["Create Table"]);
if (!/ENGINE=InnoDB\b/i.test(definition))
throw new Error("Catalog SQL snapshots require InnoDB tables");
const [columns] = await connection.query<RowDataPacket[]>(
`SHOW COLUMNS FROM ${identifier(table)}`,
);
const names = columns
.filter(
(c) => !/\b(?:VIRTUAL|STORED) GENERATED\b/.test(String(c.Extra)),
)
.map((c) => String(c.Field));
const quoted = names.map(identifier).join(", ");
const [rows] = await connection.query<RowDataPacket[]>(
`SELECT ${quoted} FROM ${identifier(table)} ORDER BY id`,
);
const source = path.join(directory, `${table}.sql`);
const output = await fs.open(source, "wx");
try {
await output.writeFile(
`-- Catalog Studio snapshot. Existing rows are updated; absent rows are not deleted.\nSET NAMES utf8mb4;\n${definition.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS")};\nSTART TRANSACTION;\n`,
);
const update = names
.filter((n) => n !== "id")
.map((n) => `${identifier(n)}=VALUES(${identifier(n)})`)
.join(", ");
for (const row of rows) {
await output.writeFile(
`INSERT INTO ${identifier(table)} (${quoted}) VALUES (${names.map((n) => sqlValue(row[n])).join(", ")}) ON DUPLICATE KEY UPDATE ${update};\n`,
);
}
await output.writeFile("COMMIT;\n");
} finally {
await output.close();
}
files.push({ source, target: `${CATALOG_SQL_ROOT}/${table}.sql` });
}
await connection.commit();
} catch (error) {
await connection.rollback();
throw error;
} finally {
connection.release();
}
for (const [source, stamp] of fingerprints) {
if ((await fingerprint(source)) !== stamp)
throw new Error("Assets changed during export; retry required");
}
return files;
}