diff --git a/src/app/api/admin/studio/nitro-cleanup/gamedata/route.ts b/src/app/api/admin/studio/nitro-cleanup/gamedata/route.ts new file mode 100644 index 00000000..440fa099 --- /dev/null +++ b/src/app/api/admin/studio/nitro-cleanup/gamedata/route.ts @@ -0,0 +1,24 @@ +import { apiError, apiOk } from "@/lib/api"; +import { withAdmin } from "@/lib/api-handler"; +import { planGamedataCleanup } from "@/lib/furni/gamedata-dedupe"; +import { PERMS } from "@/lib/permissions"; + +/** + * Read-only preview of the gamedata cleanup: duplicate FurnitureData rows, + * FurnitureData rows whose `.nitro` is gone, and the dangling/duplicated + * FigureMap parts plus the FigureData entries nothing points at any more. + * + * Nothing here writes. The apply endpoint re-derives every decision under its + * own lock, so this response is a report for the operator rather than a plan + * that is replayed later. + */ +export const GET = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => { + try { + return apiOk({ plan: await planGamedataCleanup() }); + } catch (err) { + return apiError( + `Gamedata cleanup preview failed: ${(err as Error).message}`, + 500, + ); + } +}); diff --git a/src/app/api/admin/studio/nitro-cleanup/gamedata/run/route.ts b/src/app/api/admin/studio/nitro-cleanup/gamedata/run/route.ts new file mode 100644 index 00000000..318a07a8 --- /dev/null +++ b/src/app/api/admin/studio/nitro-cleanup/gamedata/run/route.ts @@ -0,0 +1,140 @@ +import { apiError, apiJson, apiOk } from "@/lib/api"; +import { withAdmin } from "@/lib/api-handler"; +import { + cleanFigureData, + cleanFigureMap, + cleanFurnitureData, + dedupeFurnitureData, + GamedataCleanupLimitError, + planGamedataCleanup, +} from "@/lib/furni/gamedata-dedupe"; +import { PERMS } from "@/lib/permissions"; +import { logAudit } from "@/lib/services/audit"; + +/** + * The passes, in the only order that is correct. + * + * `cleanFigureMap` has to run before `cleanFigureData`: dropping the part that + * points at a set is exactly what makes that set unreferenced, so the other way + * round deletes a set the map still names. Dedupe runs first so the orphan pass + * judges a single row per classname. + */ +const PASSES = [ + "dedupeFurnitureData", + "cleanFurnitureData", + "cleanFigureMap", + "cleanFigureData", +] as const; +type Pass = (typeof PASSES)[number]; + +function isPass(value: unknown): value is Pass { + return PASSES.includes(value as Pass); +} + +/** + * Apply the selected gamedata cleanup passes. Destructive — the caller must + * send `confirm: true` after showing the operator the preview from the sibling + * GET route. `passes` defaults to all of them. + * + * Each pass caps how many rows it will delete (`maxRemovals`, default 500) and + * writes a timestamped backup of every file it touches. A pass that would + * exceed the cap throws before writing and the route answers 409 with what it + * found, so raising the cap is always a separate, deliberate second call. + */ +export const POST = withAdmin( + { permission: PERMS.ASSETS_IMPORT }, + async (request, ctx) => { + let body: { + confirm?: unknown; + passes?: unknown; + maxRemovals?: unknown; + }; + try { + body = await request.json(); + } catch { + return apiError("Invalid JSON body", 400); + } + + if (body.confirm !== true) + return apiError( + "This rewrites gamedata JSON files; send confirm: true after reviewing the preview.", + 400, + ); + + let maxRemovals: number | undefined; + if (body.maxRemovals !== undefined) { + if ( + typeof body.maxRemovals !== "number" || + !Number.isFinite(body.maxRemovals) || + body.maxRemovals <= 0 + ) + return apiError("`maxRemovals` must be a positive number", 400); + maxRemovals = Math.floor(body.maxRemovals); + } + + let requested: Pass[] = [...PASSES]; + if (body.passes !== undefined) { + if (!Array.isArray(body.passes)) + return apiError("`passes` must be an array", 400); + const unknown = body.passes.filter((p) => !isPass(p)); + if (unknown.length > 0) + return apiError(`Unknown pass: ${String(unknown[0])}`, 400); + requested = PASSES.filter((pass) => + (body.passes as unknown[]).includes(pass), + ); + } + if (requested.length === 0) + return apiError("No cleanup passes selected", 400); + + const options = maxRemovals === undefined ? undefined : { maxRemovals }; + const result: Record = {}; + try { + // Run in the fixed order above, not in the order the client listed + // them, so a client cannot break the dependency by reordering. + if (requested.includes("dedupeFurnitureData")) + result.dedupeFurnitureData = await dedupeFurnitureData(options); + if (requested.includes("cleanFurnitureData")) + result.cleanFurnitureData = await cleanFurnitureData(options); + if (requested.includes("cleanFigureMap")) + result.cleanFigureMap = await cleanFigureMap(options); + if (requested.includes("cleanFigureData")) + result.cleanFigureData = await cleanFigureData(options); + } catch (err) { + if (err instanceof GamedataCleanupLimitError) { + // Nothing was written by the pass that refused, so the files are + // as they were. A preview tells the operator what to expect. + const plan = await planGamedataCleanup(); + logAudit({ + userId: ctx.session.user.id, + action: "nitro_cleanup_gamedata", + target: "FurnitureData/FigureData/FigureMap", + after: { passes: requested, refused: err.message, result }, + }); + return apiJson( + { + ok: false, + error: err.message, + pass: err.pass, + wouldRemove: err.wouldRemove, + maxRemovals: err.maxRemovals, + partial: result, + plan, + }, + { status: 409 }, + ); + } + return apiError( + `Gamedata cleanup failed: ${(err as Error).message}`, + 500, + ); + } + + logAudit({ + userId: ctx.session.user.id, + action: "nitro_cleanup_gamedata", + target: "FurnitureData/FigureData/FigureMap", + after: { passes: requested, result }, + }); + return apiOk({ result }); + }, +); diff --git a/src/components/admin/studio/nitro-cleanup-panel.tsx b/src/components/admin/studio/nitro-cleanup-panel.tsx index 49c5c66f..43309fa6 100644 --- a/src/components/admin/studio/nitro-cleanup-panel.tsx +++ b/src/components/admin/studio/nitro-cleanup-panel.tsx @@ -58,6 +58,50 @@ interface ScanSession { error?: string; } +/** + * The gamedata cleanup reports its own shape instead of importing the scan + * module, which pulls in `node:fs` and cannot be bundled for the browser. + */ +interface GamedataPlan { + duplicates: Array<{ + classname: string; + section: string; + removeIndexes: number[]; + keepIndex: number; + hasNitro: boolean; + hasIcon: boolean; + differsFromSurvivor: boolean; + }>; + crossSection: Array<{ classname: string; sections: string[] }>; + orphans: Array<{ classname: string; section: string }>; + figureMapIssues: Array<{ + libraryId: string; + reason: string; + partId: number | null; + partType: string; + }>; + figureDataIssues: Array<{ + reason: string; + id: number; + setType?: string; + }>; + warnings: string[]; +} + +type GamedataPass = + | "dedupeFurnitureData" + | "cleanFurnitureData" + | "cleanFigureMap" + | "cleanFigureData"; + +/** Display order also documents that the passes are not interchangeable. */ +const GAMEDATA_PASSES: GamedataPass[] = [ + "dedupeFurnitureData", + "cleanFurnitureData", + "cleanFigureMap", + "cleanFigureData", +]; + interface DeleteResponse { deleted: number; copiesRemoved: number; @@ -318,6 +362,19 @@ export function NitroCleanupPanel() { } | null>(null); const abortRef = useRef(null); const loadAbortRef = useRef(null); + const [gamedataPlan, setGamedataPlan] = useState(null); + const [gamedataLoading, setGamedataLoading] = useState(false); + const [gamedataRunning, setGamedataRunning] = useState(false); + const [gamedataPasses, setGamedataPasses] = useState>( + new Set(GAMEDATA_PASSES), + ); + const [gamedataMaxRemovals, setGamedataMaxRemovals] = useState(0); + const [gamedataLimitError, setGamedataLimitError] = useState<{ + pass: string; + wouldRemove: number; + maxRemovals: number; + partial: Record>; + } | null>(null); const { confirm, dialog } = useConfirmDialog(); // Latest filter values, read from effects without putting them in deps. const filterRef = useRef({ onlyOld, minAgeDays }); @@ -325,7 +382,13 @@ export function NitroCleanupPanel() { filterRef.current = { onlyOld, minAgeDays }; }); - const busy = scanning || deleting || repairing || autoCleaning || exporting; + const busy = + scanning || + deleting || + repairing || + autoCleaning || + exporting || + gamedataRunning; const running = session?.state === "running" || (scanning && !session); const fetchSession = useCallback(async (): Promise => { @@ -801,6 +864,116 @@ export function NitroCleanupPanel() { } }; + const loadGamedataPlan = + useCallback(async (): Promise => { + setGamedataLoading(true); + try { + const res = await adminFetch( + "/api/admin/studio/nitro-cleanup/gamedata", + { headers: { Accept: "application/json" } }, + ); + const data = (await res.json()) as { + plan?: GamedataPlan; + error?: string; + }; + if (!res.ok) throw new Error(data.error || "Preview failed"); + setGamedataPlan(data.plan ?? null); + return data.plan ?? null; + } catch (err) { + toast.error( + err instanceof Error ? err.message : "Gamedata preview failed", + ); + return null; + } finally { + setGamedataLoading(false); + } + }, []); + + const toggleGamedataPass = (pass: GamedataPass) => { + setGamedataPasses((prev) => { + const next = new Set(prev); + if (next.has(pass)) next.delete(pass); + else next.add(pass); + return next; + }); + }; + + const runGamedataCleanup = async (maxRemovals?: number) => { + const plan = gamedataPlan ?? (await loadGamedataPlan()); + if (!plan) return; + if (gamedataPasses.size === 0) { + toast.error(t("gamedataNothingSelected")); + return; + } + const confirmed = await confirm({ + title: t("gamedataConfirmTitle"), + description: t("gamedataConfirmDescription", { + duplicates: plan.duplicates.length, + orphans: plan.orphans.length, + figureMap: plan.figureMapIssues.length, + figureData: plan.figureDataIssues.length, + passes: gamedataPasses.size, + }), + confirmLabel: t("gamedataConfirmLabel"), + variant: "danger", + }); + if (!confirmed) return; + setGamedataRunning(true); + setGamedataLimitError(null); + try { + const res = await adminFetch( + "/api/admin/studio/nitro-cleanup/gamedata/run", + { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + confirm: true, + passes: [...gamedataPasses], + ...(maxRemovals === undefined ? {} : { maxRemovals }), + }), + }, + ); + const data = (await res.json()) as { + result?: Record>; + error?: string; + pass?: string; + wouldRemove?: number; + maxRemovals?: number; + partial?: Record>; + }; + if (res.status === 409) { + // A pass refused to write. Nothing was written by that pass, so + // the files are as they were; the operator decides whether the + // limit was wrong or the paths are. + setGamedataLimitError({ + pass: data.pass ?? "", + wouldRemove: data.wouldRemove ?? 0, + maxRemovals: data.maxRemovals ?? 0, + partial: data.partial ?? {}, + }); + toast.error(data.error || "Gamedata cleanup refused"); + await loadGamedataPlan(); + return; + } + if (!res.ok) throw new Error(data.error || "Gamedata cleanup failed"); + const removed = Object.values(data.result ?? {}).reduce( + (total, counts) => + total + Object.values(counts).reduce((sum, n) => sum + n, 0), + 0, + ); + if (removed > 0) toast.success(t("gamedataToastDone", { removed })); + else toast.info(t("gamedataToastNothing")); + // Re-read: the plan is derived from files this just rewrote. + await loadGamedataPlan(); + } catch (err) { + toast.error( + err instanceof Error ? err.message : "Gamedata cleanup failed", + ); + } finally { + setGamedataRunning(false); + } + }; + const exportManifest = async () => { if (scan === null) return; setExporting(true); @@ -1001,6 +1174,176 @@ export function NitroCleanupPanel() {

)} +
+
+
+ + {t("gamedataTitle")} +
+
+ + +
+
+ +

+ {t("gamedataHint")} +

+ + {gamedataPlan === null ? ( +

+ {t("gamedataNoResult")} +

+ ) : ( + <> +
+ {( + [ + ["duplicates", gamedataPlan.duplicates.length], + ["orphans", gamedataPlan.orphans.length], + ["figureMap", gamedataPlan.figureMapIssues.length], + ["figureData", gamedataPlan.figureDataIssues.length], + ] as const + ).map(([key, count]) => ( +
+
+ {t(`gamedataCount_${key}`)} +
+
{count}
+
+ ))} +
+ +
+ {GAMEDATA_PASSES.map((pass) => ( +
+ toggleGamedataPass(pass)} + className="mt-0.5" + /> + +
+ ))} +
+ + {gamedataPlan.duplicates.some((d) => d.differsFromSurvivor) && ( +

+ + {t("gamedataDifferingWarning")} +

+ )} + {gamedataPlan.crossSection.length > 0 && ( +

+ {t("gamedataCrossSectionNote", { + count: gamedataPlan.crossSection.length, + })} +

+ )} + {gamedataPlan.warnings.map((warning) => ( +

+ + {warning} +

+ ))} + +
+ + setGamedataMaxRemovals(Number(e.target.value))} + aria-label={t("gamedataMaxRemovalsLabel")} + /> + + {t("gamedataMaxRemovalsHint")} + +
+ + {gamedataLimitError && ( +
+

+ {t("gamedataLimitTitle", { + pass: gamedataLimitError.pass, + wouldRemove: gamedataLimitError.wouldRemove, + maxRemovals: gamedataLimitError.maxRemovals, + })} +

+

+ {t("gamedataLimitDescription")} +

+ +
+ )} + + )} +
+ {scan && (
diff --git a/src/lib/furni/gamedata-dedupe.test.ts b/src/lib/furni/gamedata-dedupe.test.ts new file mode 100644 index 00000000..bc96e0e6 --- /dev/null +++ b/src/lib/furni/gamedata-dedupe.test.ts @@ -0,0 +1,712 @@ +// @vitest-environment node +import { promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { getRuntimePath } from "@/lib/utils/runtime-path"; + +const dirs = vi.hoisted(() => ({ + gamedataRoot: "", + furnitureData: "", + figureData: "", + figureMap: "", + nitroDir: "", + iconDir: "", +})); + +vi.mock("@/lib/services/site-settings", () => ({ + siteSettings: { + get: vi.fn(async (key: string, fallback: string) => { + if (key === "gamedata_root") return dirs.gamedataRoot; + if (key === "furni_data_url") return "/gamedata/FurnitureData.json"; + if (key === "figuredata_url") return "/gamedata/FigureData.json"; + if (key === "figuremap_url") return "/gamedata/FigureMap.json"; + if (key === "furni_nitro_dir") return dirs.nitroDir; + if (key === "furni_icon_dir") return dirs.iconDir; + return fallback; + }), + }, +})); + +import { + __resetGamedataCleanupCache, + cleanFigureData, + cleanFigureMap, + cleanFurnitureData, + dedupeFurnitureData, + GamedataCleanupLimitError, + planGamedataCleanup, + scanFurnitureDataDuplicates, + scanFurnitureDataOrphans, +} from "./gamedata-dedupe"; + +const readJson = async (file: string) => + JSON.parse(await fs.readFile(file, "utf-8")) as Record; +const writeJson = async (file: string, data: unknown) => + fs.writeFile(file, JSON.stringify(data, null, 2), "utf-8"); + +const room = (entries: Array>) => ({ + roomitemtypes: { furnitype: entries }, +}); +const figureData = (setTypes: unknown[], palettes: unknown[] = []) => ({ + palettes, + setTypes, +}); + +async function addNitro(...classnames: string[]) { + for (const classname of classnames) + await fs.writeFile(path.join(dirs.nitroDir, `${classname}.nitro`), "x"); +} +async function addIcon(...classnames: string[]) { + for (const classname of classnames) + await fs.writeFile(path.join(dirs.iconDir, `${classname}_icon.png`), "x"); +} + +beforeEach(async () => { + // `gamedata_root` drives the real path resolution, so the fixtures sit + // where the production resolvers put them: FurnitureData at the root, + // FigureData/FigureMap in `config/`. + const root = getRuntimePath(os.tmpdir(), "gamedata-dedupe-"); + await fs.rm(root, { recursive: true, force: true }); + dirs.gamedataRoot = root; + dirs.furnitureData = path.join(root, "FurnitureData.json"); + dirs.figureData = path.join(root, "config", "FigureData.json"); + dirs.figureMap = path.join(root, "config", "FigureMap.json"); + dirs.nitroDir = path.join(root, "nitro"); + dirs.iconDir = path.join(root, "icons"); + await fs.mkdir(dirs.nitroDir, { recursive: true }); + await fs.mkdir(dirs.iconDir, { recursive: true }); + await fs.mkdir(path.dirname(dirs.figureData), { recursive: true }); + + await writeJson(dirs.furnitureData, { roomitemtypes: { furnitype: [] } }); + await writeJson(dirs.figureData, { palettes: [], setTypes: [] }); + await writeJson(dirs.figureMap, { libraries: [] }); + __resetGamedataCleanupCache(); +}); + +afterEach(async () => { + await fs.rm(dirs.gamedataRoot, { recursive: true, force: true }); + __resetGamedataCleanupCache(); +}); + +describe("scanFurnitureDataDuplicates", () => { + it("finds nothing in a clean file", async () => { + await writeJson(dirs.furnitureData, room([{ classname: "chair" }])); + const { duplicates, crossSection } = await scanFurnitureDataDuplicates(); + expect(duplicates).toEqual([]); + expect(crossSection).toEqual([]); + }); + + it("groups a repeated classname and keeps the newest import", async () => { + // Two rows, one classname, so both name the same `.nitro` and the file + // check cannot separate them: the higher `revision` decides. + await writeJson( + dirs.furnitureData, + room([ + { classname: "chair", revision: 1 }, + { classname: "sofa", revision: 1 }, + { classname: "chair", revision: 3 }, + ]), + ); + await addNitro("sofa", "chair"); + const { duplicates } = await scanFurnitureDataDuplicates(); + expect(duplicates).toHaveLength(1); + expect(duplicates[0].classname).toBe("chair"); + expect(duplicates[0].keepIndex).toBe(2); + expect(duplicates[0].removeIndexes).toEqual([0]); + expect(duplicates[0].hasNitro).toBe(true); + expect(duplicates[0].hasIcon).toBe(false); + expect(duplicates[0].differsFromSurvivor).toBe(true); + }); + + it("keeps the first of two rows with the same revision", async () => { + await writeJson( + dirs.furnitureData, + room([ + { classname: "chair", revision: 2 }, + { classname: "chair", revision: 2 }, + ]), + ); + const { duplicates } = await scanFurnitureDataDuplicates(); + expect(duplicates[0].keepIndex).toBe(0); + // Byte-identical rows, so dropping one loses nothing. + expect(duplicates[0].differsFromSurvivor).toBe(false); + }); + + it("matches classnames case-insensitively", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "chair" }, { classname: "CHAIR" }]), + ); + const { duplicates } = await scanFurnitureDataDuplicates(); + expect(duplicates).toHaveLength(1); + }); + + it("reports, but does not group, a classname in both sections", async () => { + await writeJson(dirs.furnitureData, { + roomitemtypes: { furnitype: [{ classname: "chair" }] }, + wallitemtypes: { furnitype: [{ classname: "chair" }] }, + }); + const { duplicates, crossSection } = await scanFurnitureDataDuplicates(); + // Removing one would silently change the item's type. + expect(duplicates).toEqual([]); + expect(crossSection).toEqual([ + { classname: "chair", sections: ["roomitemtypes", "wallitemtypes"] }, + ]); + }); + + it("flags a removed duplicate that is not an exact copy", async () => { + await writeJson( + dirs.furnitureData, + room([ + { classname: "chair", revision: 1, name: "old" }, + { classname: "chair", revision: 2, name: "new" }, + ]), + ); + await addNitro("chair"); + const { duplicates } = await scanFurnitureDataDuplicates(); + expect(duplicates[0].differsFromSurvivor).toBe(true); + }); +}); + +describe("dedupeFurnitureData", () => { + it("removes the duplicate and keeps the freshest entry", async () => { + await writeJson( + dirs.furnitureData, + room([ + { classname: "chair", name: "old", revision: 1 }, + { classname: "sofa" }, + { classname: "chair", name: "new", revision: 4 }, + ]), + ); + await addNitro("chair", "sofa"); + const result = await dedupeFurnitureData(); + expect(result.removedDuplicates).toBe(1); + expect(result.differingGroups).toBe(1); + + const after = (await readJson(dirs.furnitureData)) as { + roomitemtypes: { furnitype: Array<{ classname: string; name: string }> }; + }; + expect(after.roomitemtypes.furnitype).toEqual([ + { classname: "sofa" }, + { classname: "chair", name: "new", revision: 4 }, + ]); + }); + + it("keeps the first when neither duplicate has a bundle", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "chair", name: "first" }, { classname: "chair" }]), + ); + await dedupeFurnitureData(); + const after = (await readJson(dirs.furnitureData)) as { + roomitemtypes: { furnitype: Array<{ name: string }> }; + }; + expect(after.roomitemtypes.furnitype).toEqual([ + { classname: "chair", name: "first" }, + ]); + }); + + it("preserves the other section and other top-level keys", async () => { + await writeJson(dirs.furnitureData, { + roomitemtypes: { + furnitype: [{ classname: "chair" }, { classname: "chair" }], + }, + wallitemtypes: { furnitype: [{ classname: "poster" }] }, + palette: { custom: true }, + }); + await dedupeFurnitureData(); + const after = (await readJson(dirs.furnitureData)) as Record; + expect(after.wallitemtypes).toEqual({ + furnitype: [{ classname: "poster" }], + }); + expect(after.palette).toEqual({ custom: true }); + }); + + it("reports zero differing groups when nothing conflicts", async () => { + await writeJson(dirs.furnitureData, room([{ classname: "chair" }])); + await addNitro("chair"); + const result = await dedupeFurnitureData(); + expect(result).toMatchObject({ removedDuplicates: 0, differingGroups: 0 }); + }); + + it("leaves entries without a classname alone", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "chair" }, { id: 9 }, { classname: "chair" }]), + ); + await addNitro("chair"); + await dedupeFurnitureData(); + const after = (await readJson(dirs.furnitureData)) as { + roomitemtypes: { furnitype: Array<{ id: number }> }; + }; + expect(after.roomitemtypes.furnitype).toEqual([ + { classname: "chair" }, + { id: 9 }, + ]); + }); +}); + +describe("cleanFurnitureData", () => { + it("removes entries whose bundle is missing", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "chair" }, { classname: "ghost" }]), + ); + await addNitro("chair"); + expect(await cleanFurnitureData()).toMatchObject({ removedOrphans: 1 }); + const after = (await readJson(dirs.furnitureData)) as { + roomitemtypes: { furnitype: Array<{ classname: string }> }; + }; + expect(after.roomitemtypes.furnitype).toEqual([{ classname: "chair" }]); + }); + + it("treats an icon without a bundle as an orphan", async () => { + // The icon is tracked, but only the `.nitro` makes an item render. + await writeJson(dirs.furnitureData, room([{ classname: "chair" }])); + await addIcon("chair"); + expect(await cleanFurnitureData()).toMatchObject({ removedOrphans: 1 }); + const after = (await readJson(dirs.furnitureData)) as { + roomitemtypes: { furnitype: unknown[] }; + }; + expect(after.roomitemtypes.furnitype).toEqual([]); + }); + + it("cleans both sections", async () => { + await writeJson(dirs.furnitureData, { + roomitemtypes: { furnitype: [{ classname: "chair" }] }, + wallitemtypes: { furnitype: [{ classname: "poster" }] }, + }); + await cleanFurnitureData(); + const after = (await readJson(dirs.furnitureData)) as Record; + expect(after.roomitemtypes).toEqual({ furnitype: [] }); + expect(after.wallitemtypes).toEqual({ furnitype: [] }); + }); +}); + +describe("figure map and figure data", () => { + it("removes a part whose set is missing from FigureData", async () => { + await writeJson( + dirs.figureData, + figureData([{ type: "set", paletteId: 1, sets: [{ id: 10 }] }]), + ); + await writeJson(dirs.figureMap, { + libraries: [ + { + id: "lib1", + revision: 1, + parts: [ + { id: 10, type: "set" }, + { id: 99, type: "set" }, + ], + }, + ], + }); + expect(await cleanFigureMap()).toMatchObject({ + removedParts: 1, + removedLibraries: 0, + }); + const after = (await readJson(dirs.figureMap)) as { + libraries: Array<{ parts: unknown[] }>; + }; + expect(after.libraries[0].parts).toEqual([{ id: 10, type: "set" }]); + }); + + it("removes an exactly repeated part but keeps the first", async () => { + // Both `set:10` and `other:10` are valid, so the only fault is the + // repeated `set:10`. + await writeJson( + dirs.figureData, + figureData([ + { type: "set", paletteId: 1, sets: [{ id: 10 }] }, + { type: "other", paletteId: 1, sets: [{ id: 10 }] }, + ]), + ); + await writeJson(dirs.figureMap, { + libraries: [ + { + id: "lib1", + revision: 1, + parts: [ + { id: 10, type: "set" }, + { id: 10, type: "set" }, + { id: 10, type: "other" }, + ], + }, + ], + }); + expect(await cleanFigureMap()).toMatchObject({ + removedParts: 1, + removedLibraries: 0, + }); + const after = (await readJson(dirs.figureMap)) as { + libraries: Array<{ parts: Array<{ id: number; type: string }> }>; + }; + expect(after.libraries[0].parts).toEqual([ + { id: 10, type: "set" }, + { id: 10, type: "other" }, + ]); + }); + + it("drops a library left with no parts at all", async () => { + // FigureData must hold a set, otherwise the "no information" guard + // refuses to touch the map at all. + await writeJson( + dirs.figureData, + figureData([{ type: "set", paletteId: 1, sets: [{ id: 10 }] }]), + ); + await writeJson(dirs.figureMap, { + libraries: [{ id: "lib1", revision: 1, parts: [{ id: 1, type: "set" }] }], + }); + expect(await cleanFigureMap()).toMatchObject({ + removedParts: 1, + removedLibraries: 1, + }); + const after = (await readJson(dirs.figureMap)) as { libraries: unknown[] }; + expect(after.libraries).toEqual([]); + }); + + it("leaves a library that never had parts alone", async () => { + await writeJson(dirs.figureData, figureData([])); + await writeJson(dirs.figureMap, { + libraries: [{ id: "nude", revision: 1 }], + }); + expect(await cleanFigureMap()).toMatchObject({ + removedParts: 0, + removedLibraries: 0, + }); + const after = (await readJson(dirs.figureMap)) as { + libraries: Array<{ id: string }>; + }; + expect(after.libraries).toEqual([{ id: "nude", revision: 1 }]); + }); + + it("removes unreferenced sets and then unreferenced palettes", async () => { + await writeJson(dirs.figureMap, { + libraries: [ + { id: "lib1", revision: 1, parts: [{ id: 10, type: "set" }] }, + ], + }); + await writeJson( + dirs.figureData, + figureData( + [ + { type: "set", paletteId: 1, sets: [{ id: 10 }, { id: 11 }] }, + { type: "other", paletteId: 2, sets: [] }, + ], + [ + { id: 1, colors: [] }, + { id: 2, colors: [] }, + { id: 3, colors: [] }, + ], + ), + ); + // Set 11 is unreferenced. Palette 2 is still used by the `other` set + // type, so it stays even though that type has no sets. Palette 3 is + // used by nobody. + expect(await cleanFigureData()).toMatchObject({ + removedSets: 1, + removedPalettes: 1, + }); + const after = (await readJson(dirs.figureData)) as { + palettes: Array<{ id: number }>; + setTypes: Array<{ sets: Array<{ id: number }> }>; + }; + expect(after.setTypes[0].sets).toEqual([{ id: 10 }]); + expect(after.palettes.map((p) => p.id)).toEqual([1, 2]); + }); + + it("is a no-op when everything is referenced", async () => { + await writeJson(dirs.figureMap, { + libraries: [ + { id: "lib1", revision: 1, parts: [{ id: 10, type: "set" }] }, + ], + }); + await writeJson( + dirs.figureData, + figureData( + [{ type: "set", paletteId: 1, sets: [{ id: 10 }] }], + [{ id: 1, colors: [] }], + ), + ); + expect(await cleanFigureData()).toMatchObject({ + removedSets: 0, + removedPalettes: 0, + }); + }); +}); + +describe("planGamedataCleanup", () => { + it("gives the same answer from a preloaded copy of the file", async () => { + // A 59 MB file parsed twice in one pass is the cost this avoids, so + // both entry points have to accept the caller's copy. + await writeJson( + dirs.furnitureData, + room([ + { classname: "chair" }, + { classname: "chair" }, + { classname: "ghost" }, + ]), + ); + await addNitro("chair"); + const parsed = (await readJson(dirs.furnitureData)) as Record< + string, + never + >; + + const { duplicates } = await scanFurnitureDataDuplicates(parsed); + expect(duplicates).toHaveLength(1); + const orphans = await scanFurnitureDataOrphans(parsed); + expect(orphans.map((o) => o.classname)).toEqual(["ghost"]); + // The loaded copy is only read from, never rewritten. + expect(await fs.readFile(dirs.furnitureData, "utf-8")).toBe( + JSON.stringify(parsed, null, 2), + ); + }); + + it("summarises every pass without writing anything", async () => { + await writeJson( + dirs.furnitureData, + room([ + { classname: "chair" }, + { classname: "chair" }, + { classname: "ghost" }, + ]), + ); + await addNitro("chair"); + await writeJson( + dirs.figureData, + figureData([{ type: "set", paletteId: 1, sets: [{ id: 10 }] }]), + ); + await writeJson(dirs.figureMap, { + libraries: [ + { id: "lib1", revision: 1, parts: [{ id: 99, type: "set" }] }, + ], + }); + + const before = await fs.readFile(dirs.furnitureData, "utf-8"); + const plan = await planGamedataCleanup(); + expect(plan.duplicates).toHaveLength(1); + expect(plan.orphans.map((o) => o.classname)).toEqual(["ghost"]); + expect(plan.figureMapIssues).toEqual([ + { libraryId: "lib1", reason: "missing-set", partId: 99, partType: "set" }, + ]); + expect(plan.figureDataIssues).toContainEqual({ + reason: "unreferenced-set", + id: 10, + setType: "set", + }); + // A plan must not touch the file. + expect(await fs.readFile(dirs.furnitureData, "utf-8")).toBe(before); + }); +}); + +describe("safety guards", () => { + /** Write a file so a test can prove it came back byte-for-byte. */ + const snapshot = async (file: string) => await fs.readFile(file, "utf-8"); + + describe("removal limit", () => { + it("refuses cleanFurnitureData above the limit and writes nothing", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "a" }, { classname: "b" }, { classname: "c" }]), + ); + // No .nitro anywhere: all three look like orphans. + const before = await snapshot(dirs.furnitureData); + + await expect(cleanFurnitureData({ maxRemovals: 2 })).rejects.toThrow( + GamedataCleanupLimitError, + ); + expect(await snapshot(dirs.furnitureData)).toBe(before); + }); + + it("names the pass, the count and the limit in the error", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "a" }, { classname: "b" }]), + ); + const err = (await cleanFurnitureData({ maxRemovals: 1 }).catch( + (e) => e, + )) as GamedataCleanupLimitError; + expect(err).toBeInstanceOf(GamedataCleanupLimitError); + expect(err.pass).toBe("cleanFurnitureData"); + expect(err.wouldRemove).toBe(2); + expect(err.maxRemovals).toBe(1); + }); + + it("proceeds once the limit is raised on purpose", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "a" }, { classname: "b" }]), + ); + const result = await cleanFurnitureData({ maxRemovals: 2 }); + expect(result.removedOrphans).toBe(2); + }); + + it("refuses dedupe above the limit too", async () => { + await writeJson( + dirs.furnitureData, + room([ + { classname: "chair", revision: 1 }, + { classname: "chair", revision: 2 }, + { classname: "chair", revision: 3 }, + ]), + ); + const before = await snapshot(dirs.furnitureData); + await expect(dedupeFurnitureData({ maxRemovals: 1 })).rejects.toThrow( + GamedataCleanupLimitError, + ); + expect(await snapshot(dirs.furnitureData)).toBe(before); + }); + + it("leaves a normal-sized cleanup alone", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "chair" }, { classname: "ghost" }]), + ); + await addNitro("chair"); + const result = await cleanFurnitureData({ maxRemovals: 1 }); + expect(result.removedOrphans).toBe(1); + }); + }); + + describe("empty input", () => { + it("does not strip FigureMap when FigureData holds no sets", async () => { + await writeJson(dirs.figureData, figureData([])); + await writeJson(dirs.figureMap, { + libraries: [ + { id: "lib1", revision: 1, parts: [{ id: 1, type: "set" }] }, + ], + }); + const before = await snapshot(dirs.figureMap); + + const result = await cleanFigureMap(); + expect(result.removedParts).toBe(0); + // An empty FigureData is a wrong path or a half-finished import, not + // proof that every part is broken. + expect(result.skipped).toBeTruthy(); + expect(await snapshot(dirs.figureMap)).toBe(before); + }); + + it("does not delete every set when FigureMap references nothing", async () => { + await writeJson( + dirs.figureData, + figureData([{ type: "set", paletteId: 1, sets: [{ id: 10 }] }]), + ); + await writeJson(dirs.figureMap, { libraries: [] }); + const before = await snapshot(dirs.figureData); + + const result = await cleanFigureData(); + expect(result.removedSets).toBe(0); + expect(result.skipped).toBeTruthy(); + expect(await snapshot(dirs.figureData)).toBe(before); + }); + + it("warns in the plan when a wrong asset path would orphan the catalog", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "a" }, { classname: "b" }, { classname: "c" }]), + ); + const plan = await planGamedataCleanup(); + expect(plan.orphans).toHaveLength(3); + expect(plan.warnings.join(" ")).toMatch(/furni_nitro_dir/); + }); + + it("warns in the plan when FigureMap is empty but FigureData has sets", async () => { + await writeJson( + dirs.figureData, + figureData([{ type: "set", paletteId: 1, sets: [{ id: 10 }] }]), + ); + await writeJson(dirs.figureMap, { libraries: [] }); + const plan = await planGamedataCleanup(); + expect(plan.warnings.join(" ")).toMatch(/figuremap_url/); + }); + + it("reports no warning on a healthy file", async () => { + await writeJson(dirs.furnitureData, room([{ classname: "chair" }])); + await addNitro("chair"); + await writeJson( + dirs.figureData, + figureData([{ type: "set", paletteId: 1, sets: [{ id: 10 }] }]), + ); + await writeJson(dirs.figureMap, { + libraries: [ + { id: "lib1", revision: 1, parts: [{ id: 10, type: "set" }] }, + ], + }); + const plan = await planGamedataCleanup(); + expect(plan.warnings).toEqual([]); + }); + }); + + describe("backups", () => { + it("copies FurnitureData aside before rewriting it", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "chair" }, { classname: "ghost" }]), + ); + await addNitro("chair"); + const before = await snapshot(dirs.furnitureData); + + const result = await cleanFurnitureData(); + expect(result.backups).toHaveLength(1); + const backup = result.backups[0]; + expect(backup).toMatch(/FurnitureData\.json\.bak-/); + // The backup holds the file as it was before the pass ran. + expect(await fs.readFile(backup, "utf-8")).toBe(before); + expect(await snapshot(dirs.furnitureData)).not.toBe(before); + }); + + it("copies FigureData and FigureMap aside too", async () => { + // The map keeps one valid part so the FigureData pass still has a + // reference to work with, and drops one invalid part so it writes. + await writeJson(dirs.figureMap, { + libraries: [ + { + id: "lib1", + revision: 1, + parts: [ + { id: 99, type: "set" }, + { id: 10, type: "set" }, + ], + }, + ], + }); + await writeJson( + dirs.figureData, + figureData([ + { type: "set", paletteId: 1, sets: [{ id: 10 }, { id: 11 }] }, + ]), + ); + const mapBefore = await snapshot(dirs.figureMap); + const dataBefore = await snapshot(dirs.figureData); + + const map = await cleanFigureMap(); + const data = await cleanFigureData(); + expect(map.backups).toHaveLength(1); + expect(await fs.readFile(map.backups[0], "utf-8")).toBe(mapBefore); + expect(await fs.readFile(data.backups[0], "utf-8")).toBe(dataBefore); + }); + + it("creates no backup when a pass has nothing to do", async () => { + await writeJson(dirs.furnitureData, room([{ classname: "chair" }])); + await addNitro("chair"); + const result = await cleanFurnitureData(); + expect(result.removedOrphans).toBe(0); + expect(result.backups).toEqual([]); + }); + + it("restoring the backup brings the removed rows back", async () => { + await writeJson( + dirs.furnitureData, + room([{ classname: "chair" }, { classname: "ghost" }]), + ); + await addNitro("chair"); + const before = await snapshot(dirs.furnitureData); + + const result = await cleanFurnitureData(); + await fs.copyFile(result.backups[0], dirs.furnitureData); + expect(await snapshot(dirs.furnitureData)).toBe(before); + }); + }); +}); diff --git a/src/lib/furni/gamedata-dedupe.ts b/src/lib/furni/gamedata-dedupe.ts new file mode 100644 index 00000000..fa2664f6 --- /dev/null +++ b/src/lib/furni/gamedata-dedupe.ts @@ -0,0 +1,879 @@ +import { existsSync, promises as fs } from "node:fs"; +import path from "node:path"; +import { getFigureDataPath, readFigureData } from "@/lib/services/figuredata"; +import { + getFigureMapPath, + listFigureLibraries, +} from "@/lib/services/figuremap"; +import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs"; +import { + getFurnitureDataWritePaths, + readFurniData, + withFurniDataLock, + writeFurniData, +} from "@/lib/services/furni-data"; +import { + readGamedataJson, + withGamedataLock, + writeGamedataJsonAtomic, +} from "@/lib/services/import/core/gamedata-json"; + +/** + * Dedupe and orphan-cleaning for the gamedata JSON files. + * + * These files drift apart over time. A failed import leaves a FurnitureData + * entry without its `.nitro`, so the client renders nothing for that item and + * a clean re-import of the classname is blocked. A half-finished figure + * import leaves a FigureMap part pointing at a set that never made it into + * FigureData. Re-importing the same furniture appends a second entry with an + * identical classname. + * + * The figure passes are independent of the furniture ones, but they are + * order-dependent: `cleanFigureMap` must run before `cleanFigureData`, + * because removing a dangling part is exactly what makes a set in FigureData + * become unreferenced. Running it the other way round deletes a set that the + * map still names. + * + * Every operation here is deliberately conservative: + * - only an exact `classname` collision counts as a duplicate, never a fuzzy + * match, and only within a single section; + * - the survivor is the entry with the highest `revision`, i.e. the freshest + * import. Two rows with one classname also point at one bundle, so bundle + * presence cannot pick between them; it is reported per classname instead; + * - only orphans are removed. A row is never dropped for having empty or odd + * fields, because that is a content decision rather than a cleanup one; + * - a classname present in both `roomitemtypes` and `wallitemtypes` is + * reported but never auto-removed, since that would change an item's type. + */ + +export const FURNITYPE_SECTIONS = ["roomitemtypes", "wallitemtypes"] as const; +export type FurnitypeSection = (typeof FURNITYPE_SECTIONS)[number]; + +interface FurniEntryLike { + classname?: unknown; + [key: string]: unknown; +} +/** + * FurnitureData as this module sees it: an open record of `furnitype` + * sections. Every other key is preserved untouched, so a write never invents + * or drops a section the caller did not ask about. + */ +export type FurnitureDataLike = Record; + +function entriesOf(data: FurnitureDataLike, section: string): FurniEntryLike[] { + const block = data[section]; + if (!block || typeof block !== "object") return []; + const furnitype = (block as { furnitype?: unknown }).furnitype; + return Array.isArray(furnitype) ? (furnitype as FurniEntryLike[]) : []; +} + +/** Normalise a classname for grouping so `Chair` and `chair` cannot slip past. */ +function classnameKey(value: unknown): string | null { + if (typeof value !== "string") return null; + const trimmed = value.trim(); + return trimmed.length ? trimmed.toLowerCase() : null; +} + +function setSection( + data: FurnitureDataLike, + section: string, + kept: FurniEntryLike[], +) { + data[section] = { ...(data[section] as object), furnitype: kept }; +} + +/** Classname of an entry, falling back to the normalised grouping key. */ +function classnameOf(entry: FurniEntryLike, key: string): string { + return typeof entry.classname === "string" ? entry.classname : key; +} + +/** + * `revision` is written by the Nitro bundle metadata and rises on every + * re-import, so it identifies the freshest of two rows for one classname. + * Missing or non-numeric values sort lowest rather than throwing. + */ +function revisionOf(entry: FurniEntryLike): number { + const value = entry.revision; + return typeof value === "number" && Number.isFinite(value) ? value : -1; +} + +/** Key-sorted JSON, so two rows compare field-by-field instead of by luck. */ +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`; + if (value && typeof value === "object") { + const record = value as Record; + const body = Object.keys(record) + .sort() + .map((key) => `${JSON.stringify(key)}:${canonical(record[key])}`) + .join(","); + return `{${body}}`; + } + return JSON.stringify(value ?? null); +} + +// ── Asset presence ───────────────────────────────────────────────── + +/** + * Resolved asset directories, cached so a scan over a large FurnitureData + * does not re-read site settings once per entry. + */ +let assetDirsCache: { nitroDirs: string[]; iconDirs: string[] } | null = null; + +async function assetDirs(): Promise<{ + nitroDirs: string[]; + iconDirs: string[]; +}> { + if (assetDirsCache) return assetDirsCache; + const targets = await getFurniAssetWriteTargets(); + // A file may live only in a mirror directory, so presence is checked + // across the primary plus every mirror. + const nitroDirs = [ + targets.nitroDir, + ...targets.mirrorDirs.map((d) => d.nitroDir), + ]; + const iconDirs = [ + targets.iconDir, + ...targets.mirrorDirs.map((d) => d.iconDir), + ]; + assetDirsCache = { + nitroDirs: [...new Set(nitroDirs)], + iconDirs: [...new Set(iconDirs)], + }; + return assetDirsCache; +} + +/** Drop the cached directory lookup; used by tests and after a settings change. */ +export function __resetGamedataCleanupCache(): void { + assetDirsCache = null; +} + +function existsIn(dirs: string[], fileName: string): boolean { + return dirs.some((dir) => existsSync(path.join(dir, fileName))); +} + +/** True when a `.nitro` bundle for this classname exists in any asset dir. */ +export async function hasNitroBundle(classname: string): Promise { + const { nitroDirs } = await assetDirs(); + return existsIn(nitroDirs, `${classname}.nitro`); +} + +/** True when a furniture icon for this classname exists in any asset dir. */ +export async function hasFurniIcon(classname: string): Promise { + const { iconDirs } = await assetDirs(); + return existsIn(iconDirs, `${classname}_icon.png`); +} + +// ── Safety guards ────────────────────────────────────────────────── + +/** + * How many rows one pass may delete before it stops and asks. + * + * A wrong `furni_nitro_dir` setting makes every entry look like an orphan, and + * a wrong FigureMap makes every set look unreferenced. Without a ceiling one + * misconfigured path turns a cleanup into a full wipe of a file that holds tens + * of thousands of rows, so a pass that would exceed this throws instead of + * writing. The caller has to come back with a higher `maxRemovals`. + */ +export const DEFAULT_MAX_REMOVALS = 500; + +export class GamedataCleanupLimitError extends Error { + constructor( + readonly pass: string, + readonly wouldRemove: number, + readonly maxRemovals: number, + ) { + super( + `${pass} would remove ${wouldRemove} rows, above the limit of ${maxRemovals}. Check the asset paths or re-run with a higher maxRemovals.`, + ); + this.name = "GamedataCleanupLimitError"; + } +} + +export interface CleanupOptions { + /** Rows this pass may delete before it refuses. */ + maxRemovals?: number; +} + +/** Throws before any write, which is the only moment a limit is useful. */ +function assertWithinLimit( + pass: string, + wouldRemove: number, + options?: CleanupOptions, +): void { + const max = options?.maxRemovals ?? DEFAULT_MAX_REMOVALS; + if (wouldRemove > max) + throw new GamedataCleanupLimitError(pass, wouldRemove, max); +} + +/** + * Copy a file aside before it is rewritten, so a pass that turns out to be + * wrong can be undone by hand. Returns the backup path, or `null` when there + * was nothing to copy. Only called once a pass has decided to write, so a + * no-op run leaves no litter. + */ +async function backupBeforeWrite(file: string): Promise { + if (!existsSync(file)) return null; + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const backup = `${file}.bak-${stamp}`; + await fs.copyFile(file, backup); + return backup; +} + +/** + * `writeFurniData` rewrites the primary file and every mirror, so all of them + * need a copy or restoring one file would still leave the others rewritten. + */ +async function backupWriteTargets(): Promise { + const targets = await getFurnitureDataWritePaths(); + const backups: string[] = []; + for (const target of targets) { + const backup = await backupBeforeWrite(target); + if (backup) backups.push(backup); + } + return backups; +} + +// ── Reporting types ──────────────────────────────────────────────── + +export interface DuplicateFurniGroup { + classname: string; + section: FurnitypeSection; + /** Indexes into that section's `furnitype` array that will be removed. */ + removeIndexes: number[]; + /** Index that survives. */ + keepIndex: number; + /** Bundle/icon presence for the classname. Classname-scoped, so shared. */ + hasNitro: boolean; + hasIcon: boolean; + /** + * True when a removed entry was not an exact copy of the survivor. A pure + * copy is safe to drop; a differing one means the duplicate carries data + * the survivor does not, so the operator should look before applying. + */ + differsFromSurvivor: boolean; +} + +export interface CrossSectionClash { + classname: string; + sections: FurnitypeSection[]; +} + +export interface FurniDataOrphan { + classname: string; + section: FurnitypeSection; +} + +export interface FigureMapPartIssue { + libraryId: string; + reason: "missing-set" | "duplicate-part"; + partId: number | null; + partType: string; +} + +export interface FigureSetIssue { + reason: "unreferenced-set" | "unreferenced-palette"; + id: number; + setType?: string; +} + +/** What every pass did, and how to undo it. */ +export interface CleanupOutcome { + /** Timestamp copies of the files this run was about to rewrite. */ + backups: string[]; + /** + * Set when the pass refused to act because its input was empty. An empty + * map or an empty set list carries no information, and treating it as + * "everything is unused" would delete the whole file. + */ + skipped?: string; +} + +export interface FurnitureDataDedupeResult extends CleanupOutcome { + removedDuplicates: number; + /** Groups where a removed entry differed from the survivor. */ + differingGroups: number; +} + +export interface FurnitureDataOrphanResult extends CleanupOutcome { + removedOrphans: number; + /** Share of all FurnitureData rows this run removed, 0–1. */ + removedFraction: number; +} + +export interface FigureMapCleanupResult extends CleanupOutcome { + removedParts: number; + /** Libraries that had parts and were left with none. */ + removedLibraries: number; +} + +export interface FigureDataCleanupResult extends CleanupOutcome { + removedSets: number; + removedPalettes: number; +} + +export interface GamedataCleanupPlan { + duplicates: DuplicateFurniGroup[]; + crossSection: CrossSectionClash[]; + orphans: FurniDataOrphan[]; + figureMapIssues: FigureMapPartIssue[]; + figureDataIssues: FigureSetIssue[]; + /** + * Things a human should look at before running this. A pass refuses to + * act on a warning it can prove is dangerous, so these are not decoration. + */ + warnings: string[]; +} + +// ── FurnitureData: duplicates ────────────────────────────────────── + +/** + * Find classnames that appear more than once inside a single section, plus + * classnames that appear in both sections (reported only — see above). + * + * `preloaded` lets a caller that already holds the parsed file hand it over. + * FurnitureData is tens of megabytes in a real deployment, so parsing it + * twice in one pass is the difference between one read and two. + */ +export async function scanFurnitureDataDuplicates( + preloaded?: FurnitureDataLike, +): Promise<{ + duplicates: DuplicateFurniGroup[]; + crossSection: CrossSectionClash[]; +}> { + const data = preloaded ?? ((await readFurniData()) as FurnitureDataLike); + const duplicates: DuplicateFurniGroup[] = []; + const sectionsByKey = new Map(); + + for (const section of FURNITYPE_SECTIONS) { + const entries = entriesOf(data, section); + const byKey = new Map(); + entries.forEach((entry, index) => { + const key = classnameKey(entry.classname); + if (!key) return; + const bucket = byKey.get(key); + if (bucket) bucket.push(index); + else byKey.set(key, [index]); + const sections = sectionsByKey.get(key) ?? []; + if (!sections.includes(section)) sections.push(section); + sectionsByKey.set(key, sections); + }); + + for (const [key, indexes] of byKey) { + if (indexes.length < 2) continue; + + // Both rows name the same bundle, so a file check cannot tell them + // apart — it is the same answer for every index in the group. The + // survivor is therefore the newest import (highest `revision`), + // with the first occurrence breaking a tie so the outcome is + // stable across runs. + let keepIndex = indexes[0]; + for (const index of indexes.slice(1)) + if (revisionOf(entries[index]) > revisionOf(entries[keepIndex])) + keepIndex = index; + + const classname = classnameOf(entries[keepIndex], key); + const survivor = entries[keepIndex]; + duplicates.push({ + classname, + section, + removeIndexes: indexes.filter((index) => index !== keepIndex), + keepIndex, + hasNitro: await hasNitroBundle(classname), + hasIcon: await hasFurniIcon(classname), + differsFromSurvivor: indexes.some( + (index) => + index !== keepIndex && + canonical(entries[index]) !== canonical(survivor), + ), + }); + } + } + + const crossSection: CrossSectionClash[] = []; + for (const [key, sections] of sectionsByKey) + if (sections.length > 1) crossSection.push({ classname: key, sections }); + + return { duplicates, crossSection }; +} + +/** + * Remove duplicate FurnitureData entries, keeping the one with the highest + * revision. Re-reads under the lock, so a plan made minutes ago is never + * applied to a file that changed in the meantime, and refuses to write past + * `options.maxRemovals`. + */ +export async function dedupeFurnitureData( + options?: CleanupOptions, +): Promise { + return withFurniDataLock(async () => { + const data = (await readFurniData()) as FurnitureDataLike; + const { duplicates } = await scanFurnitureDataDuplicates(data); + if (duplicates.length === 0) + return { + removedDuplicates: 0, + differingGroups: 0, + backups: [], + }; + + const doomed = new Map>(); + for (const group of duplicates) { + const set = doomed.get(group.section) ?? new Set(); + for (const index of group.removeIndexes) set.add(index); + doomed.set(group.section, set); + } + + let removed = 0; + for (const [section, indexes] of doomed) { + const entries = entriesOf(data, section); + if (!entries.length) continue; + const kept = entries.filter((_, index) => !indexes.has(index)); + removed += entries.length - kept.length; + setSection(data, section, kept); + } + assertWithinLimit("dedupeFurnitureData", removed, options); + if (removed === 0) + return { + removedDuplicates: 0, + differingGroups: 0, + backups: [], + }; + + // A pass can only ever remove part of a group, so the ceiling is + // applied to the group count too: a file where most rows collide + // points at a different problem, not at a cleanup. + const backups = await backupWriteTargets(); + await writeFurniData(data); + return { + removedDuplicates: removed, + differingGroups: duplicates.filter((g) => g.differsFromSurvivor).length, + backups, + }; + }); +} + +// ── FurnitureData: orphans ───────────────────────────────────────── + +/** Find FurnitureData entries whose `.nitro` bundle is missing everywhere. */ +export async function scanFurnitureDataOrphans( + preloaded?: FurnitureDataLike, +): Promise { + const data = preloaded ?? ((await readFurniData()) as FurnitureDataLike); + const orphans: FurniDataOrphan[] = []; + for (const section of FURNITYPE_SECTIONS) { + for (const entry of entriesOf(data, section)) { + const key = classnameKey(entry.classname); + if (!key) continue; + const classname = + typeof entry.classname === "string" ? entry.classname : key; + if (!(await hasNitroBundle(classname))) + orphans.push({ classname, section }); + } + } + return orphans; +} + +/** + * Remove FurnitureData entries that have no `.nitro` bundle on disk. + * + * The client cannot render an entry without its bundle, so a leftover row is + * dead weight that also blocks a clean re-import of that classname. + */ +/** + * Remove FurnitureData entries that have no `.nitro` bundle on disk. + * + * The client cannot render an entry without its bundle, so a leftover row is + * dead weight that also blocks a clean re-import of that classname. + * + * This is the pass that can destroy a whole catalog: with a wrong + * `furni_nitro_dir` every entry looks orphaned at once. It therefore refuses + * to write when the removals outnumber `options.maxRemovals`, and reports how + * much of the file it would touch so the caller can put a human on it. + */ +export async function cleanFurnitureData( + options?: CleanupOptions, +): Promise { + return withFurniDataLock(async () => { + const data = (await readFurniData()) as FurnitureDataLike; + const total = FURNITYPE_SECTIONS.reduce( + (sum, section) => sum + entriesOf(data, section).length, + 0, + ); + const orphans = await scanFurnitureDataOrphans(data); + if (orphans.length === 0) + return { removedOrphans: 0, removedFraction: 0, backups: [] }; + + const doomed = new Map>(); + for (const orphan of orphans) { + const set = doomed.get(orphan.section) ?? new Set(); + const key = classnameKey(orphan.classname); + if (key) set.add(key); + doomed.set(orphan.section, set); + } + + let removed = 0; + for (const [section, keys] of doomed) { + const entries = entriesOf(data, section); + if (!entries.length) continue; + const kept = entries.filter((entry) => { + const key = classnameKey(entry.classname); + return !key || !keys.has(key); + }); + removed += entries.length - kept.length; + setSection(data, section, kept); + } + assertWithinLimit("cleanFurnitureData", removed, options); + if (removed === 0) + return { removedOrphans: 0, removedFraction: 0, backups: [] }; + + const backups = await backupWriteTargets(); + await writeFurniData(data); + return { + removedOrphans: removed, + removedFraction: total > 0 ? removed / total : 0, + backups, + }; + }); +} + +// ── FigureMap ────────────────────────────────────────────────────── + +interface FigurePartLike { + id?: unknown; + type?: unknown; +} + +interface FigureLibraryLike { + id: string; + parts?: unknown; + [key: string]: unknown; +} + +/** + * Index every FigureData set as `type:id`, which is how a FigureMap part names + * one. The scan and the cleanup both build this, so a part is judged by the + * same rule in both — no matching on positions that could drift. + */ +async function figureSetIndex(): Promise>> { + const figureData = await readFigureData(); + const index = new Map>(); + for (const setType of figureData.setTypes ?? []) { + const type = String(setType.type ?? ""); + const ids = index.get(type) ?? new Set(); + for (const set of setType.sets ?? []) + if (typeof set.id === "number") ids.add(set.id); + index.set(type, ids); + } + return index; +} + +function partIdAndType(part: unknown): { id: number | null; type: string } { + const typed = part as FigurePartLike; + return { + id: typeof typed?.id === "number" ? typed.id : null, + type: typeof typed?.type === "string" ? typed.type : "", + }; +} + +/** Parts that point at a missing set, are malformed, or exactly repeat one. */ +export async function scanFigureMapIssues(): Promise { + const [libraries, setsByType] = await Promise.all([ + listFigureLibraries(), + figureSetIndex(), + ]); + + const issues: FigureMapPartIssue[] = []; + for (const library of libraries) { + const parts = Array.isArray(library.parts) ? library.parts : []; + const seen = new Set(); + for (const part of parts) { + const { id, type } = partIdAndType(part); + if (id === null) { + issues.push({ + libraryId: library.id, + reason: "missing-set", + partId: null, + partType: type, + }); + continue; + } + const fingerprint = `${type}:${id}`; + if (seen.has(fingerprint)) { + issues.push({ + libraryId: library.id, + reason: "duplicate-part", + partId: id, + partType: type, + }); + continue; + } + seen.add(fingerprint); + if (!setsByType.get(type)?.has(id)) + issues.push({ + libraryId: library.id, + reason: "missing-set", + partId: id, + partType: type, + }); + } + } + return issues; +} + +/** + * Remove FigureMap parts that reference a set missing from FigureData, plus + * exact duplicate parts, and drop libraries left with no parts at all. + * + * A library that never had a `parts` array is left untouched: that is a + * "no clothing" figure, not a broken one. + * + * An empty FigureData is treated as "no information", not as "every part is + * broken". That state is what a wrong `figuredata_url` or an interrupted import + * looks like, and acting on it would strip every library down to nothing. + */ +export async function cleanFigureMap( + options?: CleanupOptions, +): Promise { + const setsByType = await figureSetIndex(); + const file = await getFigureMapPath(); + + return withGamedataLock(file, async () => { + const data = await readGamedataJson<{ + libraries?: FigureLibraryLike[]; + }>(file); + const libraries = Array.isArray(data.libraries) ? data.libraries : []; + const totalParts = libraries.reduce( + (sum, library) => + sum + (Array.isArray(library.parts) ? library.parts.length : 0), + 0, + ); + if (setsByType.size === 0 && totalParts > 0) + return { + removedParts: 0, + removedLibraries: 0, + backups: [], + skipped: "FigureData holds no sets, so every part would look broken", + }; + + const keepPart = (part: unknown, seen: Set): boolean => { + const { id, type } = partIdAndType(part); + if (id === null) return false; + const fingerprint = `${type}:${id}`; + if (seen.has(fingerprint)) return false; + seen.add(fingerprint); + return setsByType.get(type)?.has(id) ?? false; + }; + + let removedParts = 0; + let removedLibraries = 0; + const next: FigureLibraryLike[] = []; + for (const library of libraries) { + if (!Array.isArray(library.parts)) { + next.push(library); + continue; + } + const seen = new Set(); + const kept = library.parts.filter((part) => keepPart(part, seen)); + removedParts += library.parts.length - kept.length; + if (kept.length === 0) { + removedLibraries++; + continue; + } + next.push({ ...library, parts: kept }); + } + + assertWithinLimit("cleanFigureMap", removedParts, options); + if (removedParts === 0 && removedLibraries === 0) + return { removedParts: 0, removedLibraries: 0, backups: [] }; + + const backups = await backupBeforeWrite(file); + await writeGamedataJsonAtomic(file, { ...data, libraries: next }); + return { + removedParts, + removedLibraries, + backups: backups ? [backups] : [], + }; + }); +} + +// ── FigureData ───────────────────────────────────────────────────── + +/** `type:id` fingerprints of every set a FigureMap part actually references. */ +async function referencedSetFingerprints(): Promise> { + const libraries = await listFigureLibraries(); + const referenced = new Set(); + for (const library of libraries) + for (const part of Array.isArray(library.parts) ? library.parts : []) { + const { id, type } = partIdAndType(part); + if (id !== null) referenced.add(`${type}:${id}`); + } + return referenced; +} + +/** Sets in FigureData no FigureMap part references, and unused palettes. */ +export async function scanFigureDataIssues(): Promise { + const [referenced, figureData] = await Promise.all([ + referencedSetFingerprints(), + readFigureData(), + ]); + + const issues: FigureSetIssue[] = []; + const usedPaletteIds = new Set(); + for (const setType of figureData.setTypes ?? []) { + const type = String(setType.type ?? ""); + if (typeof setType.paletteId === "number") + usedPaletteIds.add(setType.paletteId); + for (const set of setType.sets ?? []) + if (typeof set.id === "number" && !referenced.has(`${type}:${set.id}`)) + issues.push({ reason: "unreferenced-set", id: set.id, setType: type }); + } + for (const palette of figureData.palettes ?? []) + if (typeof palette.id === "number" && !usedPaletteIds.has(palette.id)) + issues.push({ reason: "unreferenced-palette", id: palette.id }); + return issues; +} + +/** + * Remove FigureData sets that no FigureMap part references, then remove palettes + * no set type points at any more. + * + * Must run after `cleanFigureMap`; see the note at the top of this file. + * + * A FigureMap without any part is the dangerous case: nothing is referenced, so + * a literal reading deletes every set in the file. That state means the map is + * missing or empty rather than that the sets are unused, so the pass reports it + * and leaves the file alone. + */ +export async function cleanFigureData( + options?: CleanupOptions, +): Promise { + const file = await getFigureDataPath(); + + return withGamedataLock(file, async () => { + const [referenced, data] = await Promise.all([ + referencedSetFingerprints(), + readGamedataJson<{ + palettes?: Array<{ id: number }>; + setTypes?: Array<{ + type?: string; + paletteId?: number; + sets?: Array<{ id: number }>; + }>; + }>(file), + ]); + + const setTypes = Array.isArray(data.setTypes) ? data.setTypes : []; + const totalSets = setTypes.reduce( + (sum, setType) => + sum + (Array.isArray(setType.sets) ? setType.sets.length : 0), + 0, + ); + if (referenced.size === 0 && totalSets > 0) + return { + removedSets: 0, + removedPalettes: 0, + backups: [], + skipped: "FigureMap references no sets, so every set would look unused", + }; + + let removedSets = 0; + const usedPaletteIds = new Set(); + for (const setType of setTypes) { + const type = String(setType.type ?? ""); + const sets = Array.isArray(setType.sets) ? setType.sets : []; + const kept = sets.filter((set) => { + if (typeof set?.id !== "number") return true; + return referenced.has(`${type}:${set.id}`); + }); + removedSets += sets.length - kept.length; + setType.sets = kept; + if (typeof setType.paletteId === "number") + usedPaletteIds.add(setType.paletteId); + } + + const palettes = Array.isArray(data.palettes) ? data.palettes : []; + const keptPalettes = palettes.filter( + (palette) => + typeof palette?.id !== "number" || usedPaletteIds.has(palette.id), + ); + const removedPalettes = palettes.length - keptPalettes.length; + + assertWithinLimit( + "cleanFigureData", + removedSets + removedPalettes, + options, + ); + if (removedSets === 0 && removedPalettes === 0) + return { removedSets: 0, removedPalettes: 0, backups: [] }; + + const backup = await backupBeforeWrite(file); + await writeGamedataJsonAtomic(file, { + ...data, + palettes: keptPalettes, + setTypes, + }); + return { + removedSets, + removedPalettes, + backups: backup ? [backup] : [], + }; + }); +} + +// ── Combined plan ────────────────────────────────────────────────── + +/** + * Read-only preview of everything the cleanup would do. The apply functions + * re-derive their own decisions under their locks, so this is a report for the + * operator, not something that is replayed later. + */ +export async function planGamedataCleanup(): Promise { + // FurnitureData is read once and shared by both furniture passes. Reading + // it inside each pass would parse the whole file twice and hold two copies + // in memory at the same time. + const furniData = (await readFurniData()) as FurnitureDataLike; + const [duplicates, orphans, figureMapIssues, figureDataIssues] = + await Promise.all([ + scanFurnitureDataDuplicates(furniData), + scanFurnitureDataOrphans(furniData), + scanFigureMapIssues(), + scanFigureDataIssues(), + ]); + + const totalRows = FURNITYPE_SECTIONS.reduce( + (sum, section) => sum + entriesOf(furniData, section).length, + 0, + ); + const warnings: string[] = []; + if (orphans.length > 0 && totalRows > 0) { + const fraction = Math.round((orphans.length / totalRows) * 100); + if (fraction >= 50) + warnings.push( + `${orphans.length} of ${totalRows} FurnitureData rows have no .nitro bundle (${fraction}%). That usually means the furni_nitro_dir setting points at the wrong directory, not that the catalog is full of dead rows.`, + ); + } + if (orphans.length > DEFAULT_MAX_REMOVALS) + warnings.push( + `Removing ${orphans.length} orphans is above the limit of ${DEFAULT_MAX_REMOVALS}; the run will stop unless the limit is raised on purpose.`, + ); + const libraries = (await listFigureLibraries()).length; + if (libraries === 0 && figureDataIssues.length > 0) + warnings.push( + "FigureMap holds no libraries while FigureData reports unused entries; check figuremap_url before cleaning FigureData, or that pass will refuse to run.", + ); + const differing = duplicates.duplicates.filter( + (g) => g.differsFromSurvivor, + ).length; + if (differing > 0) + warnings.push( + `${differing} duplicate group(s) are not exact copies of the row that would be kept, so the removed row held data the survivor does not have.`, + ); + + return { + duplicates: duplicates.duplicates, + crossSection: duplicates.crossSection, + orphans, + figureMapIssues, + figureDataIssues, + warnings, + }; +} diff --git a/src/messages/en.json b/src/messages/en.json index c6f90e55..43a31968 100644 --- a/src/messages/en.json +++ b/src/messages/en.json @@ -6645,6 +6645,38 @@ "toastAutoCleanDone": "Auto-cleaned {deleted} old fake .nitro file(s) ({copies} copies). Skipped {recent} recent + {broken} broken.", "toastAutoCleanNothing": "Nothing old enough to remove (skipped {recent} recent, {broken} broken)", "toastManifestDownload": "Scan manifest downloaded", + "gamedataTitle": "Gamedata cleanup", + "gamedataHint": "Preview first: this rewrites FurnitureData.json, FigureMap.json and FigureData.json. Unlike the asset tools above it removes rows that reference nothing — review the counts before running it.", + "gamedataScan": "Preview gamedata", + "gamedataScanning": "Scanning…", + "gamedataRun": "Run selected", + "gamedataRunning": "Cleaning…", + "gamedataNoResult": "No preview yet — scan the gamedata JSON files first.", + "gamedataCount_duplicates": "Duplicate FurnitureData", + "gamedataCount_orphans": "FurnitureData orphans", + "gamedataCount_figureMap": "FigureMap issues", + "gamedataCount_figureData": "FigureData unused", + "gamedataPass_dedupeFurnitureData": "Dedupe FurnitureData", + "gamedataPassDesc_dedupeFurnitureData": "Removes repeated classnames inside one section, keeping the entry with the highest revision.", + "gamedataPass_cleanFurnitureData": "Clean FurnitureData", + "gamedataPassDesc_cleanFurnitureData": "Removes entries whose .nitro bundle is missing from every directory.", + "gamedataPass_cleanFigureMap": "Clean FigureMap", + "gamedataPassDesc_cleanFigureMap": "Removes parts pointing at a set missing from FigureData, plus exact duplicate parts.", + "gamedataPass_cleanFigureData": "Clean FigureData", + "gamedataPassDesc_cleanFigureData": "Removes sets no FigureMap part references, then palettes no set type uses. Must run after Clean FigureMap.", + "gamedataNothingSelected": "Select at least one cleanup pass", + "gamedataConfirmTitle": "Rewrite gamedata JSON files?", + "gamedataConfirmDescription": "{passes} pass(es) will run against {duplicates} duplicate group(s), {orphans} orphan row(s), {figureMap} FigureMap issue(s) and {figureData} unused FigureData entr(ies). Rows are removed without a backup — this cannot be undone.", + "gamedataConfirmLabel": "Rewrite files", + "gamedataDifferingWarning": "At least one duplicate is not an exact copy of the entry that is kept, so the removed row held data the survivor does not have.", + "gamedataCrossSectionNote": "{count} classname(s) exist in both roomitemtypes and wallitemtypes. Those are reported only — removing one would change an item's type.", + "gamedataToastDone": "Gamedata cleanup removed {removed} row(s)", + "gamedataToastNothing": "Gamedata was already clean", + "gamedataMaxRemovalsLabel": "Max rows per pass", + "gamedataMaxRemovalsHint": "0 uses the default of 500. A pass that would remove more stops and asks instead of writing.", + "gamedataLimitTitle": "{pass} would remove {wouldRemove} rows, above the limit of {maxRemovals}.", + "gamedataLimitDescription": "Nothing was written by that pass, so the files are unchanged. Check the asset paths first — a wrong directory makes every row look orphaned. Raising the limit is a separate, deliberate run.", + "gamedataLimitOverride": "Remove {wouldRemove} rows anyway", "historyTitle": "Auto-clean history", "historyScheduled": "scheduled", "historyManual": "manual", diff --git a/src/messages/nl.json b/src/messages/nl.json index 9bdc3ca8..1dfcdc38 100644 --- a/src/messages/nl.json +++ b/src/messages/nl.json @@ -6651,6 +6651,38 @@ "toastAutoCleanDone": "{deleted} oude fake .nitro-bestand(en) opgeruimd ({copies} kopieën). {recent} recente en {broken} kapotte overgeslagen.", "toastAutoCleanNothing": "Niets oud genoeg om te verwijderen ({recent} recente en {broken} kapotte overgeslagen)", "toastManifestDownload": "Scanmanifest gedownload", + "gamedataTitle": "Gamedata opruimen", + "gamedataHint": "Eerst een voorbeeld bekijken: dit herschrijft FurnitureData.json, FigureMap.json en FigureData.json. Anders dan de tools hierboven verwijdert dit ook rijen die naar niets verwijzen — controleer de tellingen voordat je het uitvoert.", + "gamedataScan": "Gamedata scannen", + "gamedataScanning": "Scannen…", + "gamedataRun": "Geselecteerde uitvoeren", + "gamedataRunning": "Opruimen…", + "gamedataNoResult": "Nog geen voorbeeld — scan eerst de gamedata JSON-bestanden.", + "gamedataCount_duplicates": "Dubbele FurnitureData", + "gamedataCount_orphans": "FurnitureData-wezen", + "gamedataCount_figureMap": "FigureMap-problemen", + "gamedataCount_figureData": "Ongebruikte FigureData", + "gamedataPass_dedupeFurnitureData": "FurnitureData dedupliceren", + "gamedataPassDesc_dedupeFurnitureData": "Verwijdert herhaalde classnames binnen één sectie en houdt de entry met de hoogste revision.", + "gamedataPass_cleanFurnitureData": "FurnitureData opschonen", + "gamedataPassDesc_cleanFurnitureData": "Verwijdert entries waarvan het .nitro-bundle in geen enkele map meer staat.", + "gamedataPass_cleanFigureMap": "FigureMap opschonen", + "gamedataPassDesc_cleanFigureMap": "Verwijdert parts die naar een ontbrekende set in FigureData wijzen, plus exact dubbele parts.", + "gamedataPass_cleanFigureData": "FigureData opschonen", + "gamedataPassDesc_cleanFigureData": "Verwijdert sets waarnaar geen FigureMap-part verwijst en daarna palettes die geen enkele settype gebruikt. Moet na FigureMap opschonen draaien.", + "gamedataNothingSelected": "Selecteer minstens één opruimstap", + "gamedataConfirmTitle": "Gamedata JSON-bestanden herschrijven?", + "gamedataConfirmDescription": "{passes} stap(pen) draait tegen {duplicates} dubbele groep(en), {orphans} wezen-rij(en), {figureMap} FigureMap-probleem(en) en {figureData} ongebruikte FigureData-entry('s). Rijen worden zonder back-up verwijderd — dit is niet ongedaan te maken.", + "gamedataConfirmLabel": "Bestanden herschrijven", + "gamedataDifferingWarning": "Minstens één duplicaat is geen exacte kopie van de entry die behouden blijft; de verwijderde rij bevatte dus data die de overblijvende mist.", + "gamedataCrossSectionNote": "{count} classname(s) bestaan zowel in roomitemtypes als wallitemtypes. Die worden alleen gerapporteerd — één verwijderen zou het type van een item veranderen.", + "gamedataToastDone": "Gamedata-opruiming verwijderde {removed} rij(en)", + "gamedataToastNothing": "Gamedata was al schoon", + "gamedataMaxRemovalsLabel": "Max. rijen per stap", + "gamedataMaxRemovalsHint": "0 gebruikt de standaard van 500. Een stap die meer zou verwijderen stopt en vraagt om bevestiging in plaats van te schrijven.", + "gamedataLimitTitle": "{pass} zou {wouldRemove} rij(en) verwijderen, meer dan de limiet van {maxRemovals}.", + "gamedataLimitDescription": "Die stap heeft niets geschreven, dus de bestanden zijn ongewijzigd. Controleer eerst de asset-paden — een verkeerde map laat elke rij als wezen zien. De limiet verhogen is een aparte, bewuste uitvoering.", + "gamedataLimitOverride": "{wouldRemove} rij(en) toch verwijderen", "historyTitle": "Opruimgeschiedenis", "historyScheduled": "gepland", "historyManual": "handmatig",