diff --git a/src/lib/services/catalog-audit.ts b/src/lib/services/catalog-audit.ts index 18b3ed13..039de870 100644 --- a/src/lib/services/catalog-audit.ts +++ b/src/lib/services/catalog-audit.ts @@ -662,6 +662,65 @@ export async function runCatalogAudit( remappedReferences, }, }); + + // Structural repair can DELETE items_base rows (duplicate classname + // merge) and rewrite catalog_items.item_ids, so the missing-catalog + // list computed from the pre-repair snapshot is stale: entries may + // reference ids that no longer exist. Re-derive it from the repaired + // state, otherwise the generated catalog SQL would insert orphaned + // catalog_items rows (item_ids pointing at deleted sprite ids). + if (duplicateRowsRemoved > 0 || orphanedRemoved > 0) { + const [postItems] = (await db.execute(sql` + SELECT id, item_name, public_name, type, interaction_type + FROM items_base ORDER BY id + `)) as unknown as [ + Array<{ + id: number; + item_name: string; + public_name: string; + type: string; + interaction_type: string; + }>, + unknown, + ]; + items = postItems; + + const [postCatalog] = (await db.execute(sql` + SELECT id, item_ids, catalog_name FROM catalog_items ORDER BY id + `)) as unknown as [ + Array<{ id: number; item_ids: string; catalog_name: string }>, + unknown, + ]; + const postCatalogIds = new Set(); + for (const ci of postCatalog) { + const ids = ci.item_ids.split(";").map(Number).filter(Boolean); + for (const id of ids) postCatalogIds.add(id); + } + + missingCatalogEntries.length = 0; + for (const item of items) { + let kind = classifyCatalogItem(item); + if ( + kind === "furni" && + (isBadgeType(item.type) || badgeGifSet.has(item.item_name)) + ) { + kind = "badge"; + } + if (NO_CATALOG_KINDS.has(kind)) continue; + if (postCatalogIds.has(item.id)) continue; + const hasNitro = assetNameCandidates(item.item_name).nitro.some((n) => + nitroSet.has(n), + ); + if (kind === "pet" || hasNitro) { + missingCatalogEntries.push({ + classname: item.item_name, + itemId: item.id, + itemType: item.type, + publicName: item.public_name, + }); + } + } + } } catch (err) { onEvent?.({ type: "error", diff --git a/src/lib/services/catalog-repair-live.test.ts b/src/lib/services/catalog-repair-live.test.ts new file mode 100644 index 00000000..defa4949 --- /dev/null +++ b/src/lib/services/catalog-repair-live.test.ts @@ -0,0 +1,243 @@ +// @vitest-environment node +// Live repair rehearsal for the catalog studio: runs the full repair pipeline +// (orphaned refs, duplicate classnames, missing catalog SQL, furnidata repair + +// reconcile) against a sandbox copy of the production DB and verifies the end +// state is consistent. Writes go to a temp gamedata dir; the production +// FurnitureData.json is only ever read (a copy is made first). +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { sql } from "drizzle-orm"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; +import { db } from "@/lib/db"; +import { + generateCatalogSql, + repairDuplicateClassnames, + repairFurniData, + repairOrphanedCatalog, +} from "@/lib/services/catalog-repair"; +import { reconcileFurniDataWithItemsBase } from "@/lib/services/furni-data"; +import { reconcileImportedOfferIds } from "@/lib/services/furni-import"; +import { + assetNameCandidates, + classifyCatalogItem, + NO_CATALOG_KINDS, +} from "@/lib/services/item-kind"; + +const { getGamedataRootMock, getNitroFilesRootMock, settingGetMock } = + vi.hoisted(() => ({ + getGamedataRootMock: vi.fn(), + getNitroFilesRootMock: vi.fn(), + settingGetMock: vi.fn(), + })); + +vi.mock("@/lib/services/furni-asset-dirs", async (importOriginal) => { + const actual = + await importOriginal(); + return { + ...actual, + getGamedataRoot: getGamedataRootMock, + getNitroFilesRoot: getNitroFilesRootMock, + }; +}); + +vi.mock("@/lib/services/site-settings", () => ({ + siteSettings: { get: settingGetMock }, +})); + +const isSandbox = + process.env.DATABASE_URL?.includes("@127.0.0.1:3307") || + process.env.DATABASE_URL?.includes(":@127.0.0.1:3307/") || + process.env.DATABASE_URL?.includes("localhost:3307"); + +const describeGuard = isSandbox ? describe : describe.skip; + +const REAL_NITRO_DIR = "/var/www/Gamedata/bundled/furniture"; +const REAL_FURNIDATA = "/var/www/Gamedata/config/FurnitureData.json"; + +let tmpRoot: string; +let gamedataDir: string; + +beforeAll(async () => { + tmpRoot = await fs.mkdtemp(path.join(os.tmpdir(), "furni-repair-rehearsal-")); + gamedataDir = path.join(tmpRoot, "Gamedata"); + await fs.mkdir(path.join(gamedataDir, "config"), { recursive: true }); + getGamedataRootMock.mockResolvedValue(gamedataDir); + getNitroFilesRootMock.mockResolvedValue(""); + settingGetMock.mockImplementation( + async (key: string, fallback: string | null = null) => { + if (key === "furni_data_url") + return "/gamedata/config/FurnitureData.json"; + if (key === "furni_data_mirror_path") return ""; + return fallback; + }, + ); + try { + await fs.copyFile( + REAL_FURNIDATA, + path.join(gamedataDir, "config", "FurnitureData.json"), + ); + } catch { + // missing copy — repair still runs against an empty file + } +}); + +afterAll(async () => { + await fs.rm(tmpRoot, { recursive: true, force: true }); + vi.clearAllMocks(); +}); + +describeGuard("catalog studio live repair rehearsal (sandbox)", () => { + it("runs the full repair pipeline and converges to a consistent state", async () => { + const itemTypeMap = (t: string) => (t === "i" || t === "s" ? "i" : "s"); + + const [itemRows] = (await db.execute(sql` + SELECT id, item_name, public_name, type FROM items_base + `)) as unknown as [ + Array<{ + id: number; + item_name: string; + public_name: string | null; + type: string; + }>, + unknown, + ]; + const [catalogRows] = (await db.execute(sql` + SELECT item_ids FROM catalog_items + `)) as unknown as [Array<{ item_ids: string }>, unknown]; + + const nitroFiles = new Set( + (await fs.readdir(REAL_NITRO_DIR).catch(() => [] as string[])) + .filter((f) => f.endsWith(".nitro")) + .map((f) => f.slice(0, -6)), + ); + + const audit = async () => { + const [itemRowsNow] = (await db.execute(sql` + SELECT id, item_name, type FROM items_base + `)) as unknown as [ + Array<{ id: number; item_name: string; type: string }>, + unknown, + ]; + const [catalogRowsNow] = (await db.execute(sql` + SELECT item_ids FROM catalog_items + `)) as unknown as [Array<{ item_ids: string }>, unknown]; + const withCatalog = new Set(); + for (const r of catalogRowsNow) + for (const p of String(r.item_ids ?? "").split(";")) { + const n = Number(p.trim()); + if (n) withCatalog.add(n); + } + const byName = new Map(); + for (const r of itemRowsNow) + byName.set(r.item_name, (byName.get(r.item_name) ?? 0) + 1); + const dupGroups = [...byName].filter(([, c]) => c > 1).length; + const itemIds = new Set(itemRowsNow.map((r) => Number(r.id))); + let orphaned = 0; + for (const r of catalogRowsNow) { + const ids = String(r.item_ids ?? "") + .split(";") + .map(Number) + .filter(Boolean); + if (ids.some((i) => !itemIds.has(i))) orphaned++; + } + let missingNonSystem = 0; + for (const r of itemRowsNow) { + if (withCatalog.has(Number(r.id))) continue; + const kind = classifyCatalogItem(r as never); + if (NO_CATALOG_KINDS.has(kind)) continue; + missingNonSystem++; + } + return { dupGroups, orphaned, missingNonSystem }; + }; + + const beforeState = await audit(); + console.log("REHEARSAL before:", JSON.stringify(beforeState)); + + const orphan = await repairOrphanedCatalog(); + console.log("REHEARSAL orphaned:", JSON.stringify(orphan)); + + const dup = await repairDuplicateClassnames(); + console.log("REHEARSAL duplicates:", JSON.stringify(dup)); + + // Missing catalog entries (audit filter: non-system kind AND (pet OR has nitro)). + const entries = []; + const withCatalog = new Set(); + for (const r of catalogRows) + for (const p of String(r.item_ids ?? "").split(";")) { + const n = Number(p.trim()); + if (n) withCatalog.add(n); + } + for (const r of itemRows) { + if (withCatalog.has(Number(r.id))) continue; + const kind = classifyCatalogItem(r as never); + if (NO_CATALOG_KINDS.has(kind)) continue; + const hasNitro = assetNameCandidates(r.item_name).nitro.some((n) => + nitroFiles.has(n), + ); + if (kind === "pet" || hasNitro) { + entries.push({ + classname: r.item_name, + itemId: Number(r.id), + itemType: itemTypeMap(r.type), + publicName: r.public_name ?? r.item_name, + }); + } + } + console.log("REHEARSAL missing catalog entries:", entries.length); + const catalogSql = await generateCatalogSql(entries, { + organizeSql: true, + }); + let applied = 0; + let failed = 0; + const failures: string[] = []; + for (const stmt of catalogSql.statements) { + const trimmed = stmt.trim(); + if ( + !trimmed || + trimmed.startsWith("--") || + trimmed === "START TRANSACTION;" || + trimmed === "COMMIT;" + ) + continue; + try { + await db.execute(sql.raw(trimmed)); + applied++; + } catch (err) { + failed++; + const e = err as { cause?: unknown; message?: string }; + const cause = + typeof e.cause === "object" && e.cause !== null + ? JSON.stringify(e.cause, Object.getOwnPropertyNames(e.cause)) + : String(e.cause ?? ""); + failures.push( + `${trimmed.slice(0, 60)} → CAUSE: ${cause.slice(0, 500)}`, + ); + } + } + console.log( + "REHEARSAL catalog sql:", + JSON.stringify({ count: catalogSql.count, applied, failed }), + ); + for (const f of failures) console.log("REHEARSAL sql FAILURE:", f); + + const furniDataRepair = await repairFurniData(itemRows as never); + console.log("REHEARSAL furnidata repair:", JSON.stringify(furniDataRepair)); + + const reconcile = await reconcileFurniDataWithItemsBase(); + console.log("REHEARSAL furnidata reconcile:", JSON.stringify(reconcile)); + + const offer = await reconcileImportedOfferIds(); + console.log("REHEARSAL offer reconcile:", JSON.stringify(offer)); + + const afterState = await audit(); + console.log("REHEARSAL after:", JSON.stringify(afterState)); + + expect(orphan.removed).toBeGreaterThanOrEqual(0); + expect(orphan.cleaned).toBeGreaterThanOrEqual(0); + expect(dup.merged).toBeGreaterThan(0); + expect(dup.rowsRemoved).toBeGreaterThan(0); + expect(afterState.dupGroups).toBe(0); + expect(afterState.orphaned).toBe(0); + }, 600000); +}); diff --git a/src/lib/services/catalog-repair.ts b/src/lib/services/catalog-repair.ts index 36934530..701e925d 100644 --- a/src/lib/services/catalog-repair.ts +++ b/src/lib/services/catalog-repair.ts @@ -24,6 +24,7 @@ export interface CatalogSqlResult { statements: string[]; sql: string; count: number; + skipped: number; } function escSqlLiteral(value: string): string { @@ -341,12 +342,25 @@ export async function generateCatalogSql( `)) as unknown as [Array<{ maxId: number | bigint | null }>, unknown]; let nextId = Number(maxIdResult[0]?.maxId ?? 0) + 1; + // Skip entries whose sprite id no longer exists in items_base (e.g. because + // a duplicate-classname merge ran between the audit snapshot and this + // generation). Inserting them would create orphaned catalog_items rows. + const [validIds] = (await db.execute(sql` + SELECT id FROM items_base + `)) as unknown as [Array<{ id: number }>, unknown]; + const validItemIds = new Set(validIds.map((r) => Number(r.id))); + let skipped = 0; + // Build a map of caption_save → pageId so multiple items in the same // category share one page (and we emit the page INSERT only once). const pageMap = new Map(); let generated = 0; for (const entry of entries) { + if (!validItemIds.has(entry.itemId)) { + skipped++; + continue; + } let pageId: number; if (organizeSql && parentPageId) { const catKey = classifyFurni(entry.classname, entry.itemType); @@ -367,7 +381,7 @@ export async function generateCatalogSql( const newId = nextPageId++; pageMap.set(captionSave, newId); statements.push( - `INSERT INTO catalog_pages (id, caption_save, caption, page_layout, parent_id, min_rank, order_num, icon_image, enabled, visible, includes, page_headline, page_teaser, page_special, page_text1, page_text2, page_text_details, page_text_teaser) VALUES (${newId}, '${escSqlLiteral(captionSave)}', '${escSqlLiteral(caption)}', 'default_3x3', ${parentPageId}, '1', ${catInfo.order}, ${catInfo.icon}, '1', '1', '', '', '', '', '', '', '');`, + `INSERT INTO catalog_pages (id, caption_save, caption, page_layout, parent_id, min_rank, order_num, icon_image, enabled, visible, includes, page_headline, page_teaser, page_special, page_text1, page_text2, page_text_details, page_text_teaser) VALUES (${newId}, '${escSqlLiteral(captionSave)}', '${escSqlLiteral(caption)}', 'default_3x3', ${parentPageId}, '1', ${catInfo.order}, ${catInfo.icon}, '1', '1', '', '', '', '', '', '', '', '');`, ); } existing = pageMap.get(captionSave); @@ -394,7 +408,7 @@ export async function generateCatalogSql( } statements.push("COMMIT;"); - return { statements, sql: statements.join("\n"), count: generated }; + return { statements, sql: statements.join("\n"), count: generated, skipped }; } /** Execute generated INSERT statements directly against the database. */