From 6fc14b9b847b26c94329beb1945e9fc246ad0586 Mon Sep 17 00:00:00 2001
From: openhands <openhands@all-hands.dev>
Date: Mon, 3 Aug 2026 18:08:45 +0200
Subject: [PATCH] feat: catalog audit can repair orphaned refs and duplicate
 classnames

- add repairOrphanedCatalog: remove catalog_items rows whose item_ids only
  reference missing items_base entries, strip orphaned ids from mixed rows
- add repairDuplicateClassnames: merge items_base duplicates into one
  canonical row per classname, remap references, delete duplicate rows
- add 'repair structural issues' checkbox + result display in audit UI
---
 src/app/admin/import/audit/audit-client.tsx |  86 ++++++++-
 src/app/api/admin/import/audit/route.ts     |   4 +
 src/lib/services/catalog-audit.ts           |  96 ++++++++++
 src/lib/services/catalog-repair.ts          | 199 ++++++++++++++++++++
 4 files changed, 384 insertions(+), 1 deletion(-)

diff --git a/src/app/admin/import/audit/audit-client.tsx b/src/app/admin/import/audit/audit-client.tsx
index 9d896fb1..d2bf0b17 100644
--- a/src/app/admin/import/audit/audit-client.tsx
+++ b/src/app/admin/import/audit/audit-client.tsx
@@ -48,6 +48,11 @@ interface AuditSummary {
 	catalogSqlApplied: number;
 	furniDataAdded: number;
 	furniDataDuplicatesRemoved: number;
+	orphanedRemoved: number;
+	orphanedCleaned: number;
+	duplicatesMerged: number;
+	duplicateRowsRemoved: number;
+	remappedReferences: number;
 }
 
 interface RepairStats {
@@ -73,6 +78,13 @@ interface AuditEvent {
 	applied?: number;
 	sqlFailed?: number;
 	furniData?: { added: number; removedDuplicates: number };
+	structure?: {
+		orphanedRemoved: number;
+		orphanedCleaned: number;
+		duplicatesMerged: number;
+		duplicateRowsRemoved: number;
+		remappedReferences: number;
+	};
 }
 
 const TYPE_LABELS: Record<string, string> = {
@@ -131,6 +143,7 @@ export function AuditClient() {
 	const [generateSql, setGenerateSql] = useState(false);
 	const [applySql, setApplySql] = useState(false);
 	const [repairFurniData, setRepairFurniData] = useState(false);
+	const [repairStructure, setRepairStructure] = useState(false);
 
 	const [issues, setIssues] = useState<AuditIssue[]>([]);
 	const [missingFromSources, setMissingFromSources] = useState<
@@ -150,6 +163,13 @@ export function AuditClient() {
 		added: number;
 		removedDuplicates: number;
 	} | null>(null);
+	const [structureResult, setStructureResult] = useState<{
+		orphanedRemoved: number;
+		orphanedCleaned: number;
+		duplicatesMerged: number;
+		duplicateRowsRemoved: number;
+		remappedReferences: number;
+	} | null>(null);
 
 	const [checkProgress, setCheckProgress] = useState<{
 		section: string;
@@ -170,6 +190,7 @@ export function AuditClient() {
 		setSqlText(null);
 		setSqlApplied(null);
 		setFurniDataResult(null);
+		setStructureResult(null);
 		setCheckProgress(null);
 		setActiveTab("issues");
 
@@ -187,6 +208,7 @@ export function AuditClient() {
 					sql: generateSql || applySql,
 					applySql,
 					repairFurniData,
+					repairStructure,
 				}),
 			});
 
@@ -267,6 +289,10 @@ export function AuditClient() {
 							if (evt.furniData) setFurniDataResult(evt.furniData);
 						}
 
+						if (evt.type === "structure_repair_complete") {
+							if (evt.structure) setStructureResult(evt.structure);
+						}
+
 						if (evt.type === "batch_complete") {
 							setIssues(evt.issues ?? []);
 							setMissingFromSources(evt.missingFromSources ?? []);
@@ -309,7 +335,14 @@ export function AuditClient() {
 			setLoading(false);
 			abortRef.current = null;
 		}
-	}, [repairEnabled, repairNitros, generateSql, applySql, repairFurniData]);
+	}, [
+		repairEnabled,
+		repairNitros,
+		generateSql,
+		applySql,
+		repairFurniData,
+		repairStructure,
+	]);
 
 	const errors = issues.filter((i) => i.severity === "error");
 	const warnings = issues.filter((i) => i.severity === "warning");
@@ -372,6 +405,13 @@ export function AuditClient() {
 					>
 						Repair FurnitureData.json (add missing + dedupe)
 					</AuditCheckbox>
+					<AuditCheckbox
+						checked={repairStructure}
+						disabled={loading}
+						onChange={setRepairStructure}
+					>
+						Repair structural issues (orphaned refs + duplicates)
+					</AuditCheckbox>
 					{loading && (
 						<Button
 							variant="outline"
@@ -601,6 +641,49 @@ export function AuditClient() {
 								</span>
 							</div>
 						)}
+						{structureResult && (
+							<div className="flex items-center gap-2 col-span-full">
+								<span className="text-xs text-muted-foreground">
+									Structural repair:{" "}
+									{structureResult.orphanedRemoved > 0 && (
+										<>
+											<span className="text-[var(--admin-success)] font-medium">
+												{structureResult.orphanedRemoved} orphaned rows removed
+											</span>
+											{" · "}
+										</>
+									)}
+									{structureResult.orphanedCleaned > 0 && (
+										<>
+											<span className="font-medium">
+												{structureResult.orphanedCleaned} cleaned
+											</span>
+											{" · "}
+										</>
+									)}
+									{structureResult.duplicatesMerged > 0 && (
+										<>
+											<span className="text-[var(--admin-success)] font-medium">
+												{structureResult.duplicatesMerged} classname(s) merged
+											</span>
+											{" · "}
+										</>
+									)}
+									{structureResult.duplicateRowsRemoved > 0 && (
+										<>
+											<span className="text-[var(--admin-warning)] font-medium">
+												{structureResult.duplicateRowsRemoved} duplicate rows
+												removed
+											</span>
+											{" · "}
+										</>
+									)}
+									<span className="text-[var(--admin-text-muted)]">
+										{structureResult.remappedReferences} reference(s) remapped
+									</span>
+								</span>
+							</div>
+						)}
 					</div>
 
 					{/* Tabs */}
@@ -733,6 +816,7 @@ export function AuditClient() {
 							setSqlText(null);
 							setSqlApplied(null);
 							setFurniDataResult(null);
+							setStructureResult(null);
 							setProgress("");
 							setCheckProgress(null);
 						}}
diff --git a/src/app/api/admin/import/audit/route.ts b/src/app/api/admin/import/audit/route.ts
index cc720542..3f137e9d 100644
--- a/src/app/api/admin/import/audit/route.ts
+++ b/src/app/api/admin/import/audit/route.ts
@@ -16,6 +16,7 @@ export const POST = withAdmin(
 		let sql = false;
 		let applySql = false;
 		let repairFurniData = false;
+		let repairStructure = false;
 		try {
 			const body = (await request.json().catch(() => ({}))) as {
 				repair?: unknown;
@@ -23,12 +24,14 @@ export const POST = withAdmin(
 				sql?: unknown;
 				applySql?: unknown;
 				repairFurniData?: unknown;
+				repairStructure?: unknown;
 			};
 			repair = body.repair === true;
 			repairNitros = body.repairNitros === true;
 			sql = body.sql === true;
 			applySql = body.applySql === true;
 			repairFurniData = body.repairFurniData === true;
+			repairStructure = body.repairStructure === true;
 		} catch {
 			/* no body */
 		}
@@ -50,6 +53,7 @@ export const POST = withAdmin(
 						sql,
 						applySql,
 						repairFurniData,
+						repairStructure,
 					});
 				} catch (err) {
 					send({
diff --git a/src/lib/services/catalog-audit.ts b/src/lib/services/catalog-audit.ts
index b9c4231e..99e2a751 100644
--- a/src/lib/services/catalog-audit.ts
+++ b/src/lib/services/catalog-audit.ts
@@ -9,7 +9,9 @@ import {
 	applyCatalogSql,
 	generateCatalogSql,
 	type MissingCatalogEntry,
+	repairDuplicateClassnames,
 	repairFurniData,
+	repairOrphanedCatalog,
 } from "./catalog-repair";
 import { ensureDirectories } from "./furni-import";
 import { repairMissingIcons } from "./repair-icons";
@@ -25,6 +27,7 @@ export interface AuditEvent {
 		| "sql_ready"
 		| "sql_applied"
 		| "furnidata_repair_complete"
+		| "structure_repair_complete"
 		| "batch_complete"
 		| "error";
 	message?: string;
@@ -41,6 +44,13 @@ export interface AuditEvent {
 	applied?: number;
 	sqlFailed?: number;
 	furniData?: { added: number; removedDuplicates: number };
+	structure?: {
+		orphanedRemoved: number;
+		orphanedCleaned: number;
+		duplicatesMerged: number;
+		duplicateRowsRemoved: number;
+		remappedReferences: number;
+	};
 }
 
 export interface RepairStats {
@@ -84,6 +94,11 @@ export interface AuditSummary {
 	catalogSqlApplied: number;
 	furniDataAdded: number;
 	furniDataDuplicatesRemoved: number;
+	orphanedRemoved: number;
+	orphanedCleaned: number;
+	duplicatesMerged: number;
+	duplicateRowsRemoved: number;
+	remappedReferences: number;
 }
 
 export interface CatalogAuditOptions {
@@ -92,6 +107,7 @@ export interface CatalogAuditOptions {
 	sql?: boolean;
 	applySql?: boolean;
 	repairFurniData?: boolean;
+	repairStructure?: boolean;
 }
 
 export async function runCatalogAudit(
@@ -450,6 +466,81 @@ export async function runCatalogAudit(
 		}
 	}
 
+	// ── Structural repair: orphaned catalog refs + duplicate classnames ──
+	let orphanedRemoved = 0;
+	let orphanedCleaned = 0;
+	let duplicatesMerged = 0;
+	let duplicateRowsRemoved = 0;
+	let remappedReferences = 0;
+	if (options?.repairStructure) {
+		onEvent?.({
+			type: "progress",
+			message: "Repairing structural issues (orphaned refs + duplicates)…",
+		});
+		try {
+			const orphanResult = await repairOrphanedCatalog();
+			orphanedRemoved = orphanResult.removed;
+			orphanedCleaned = orphanResult.cleaned;
+
+			// Re-derive orphaned_catalog issues from the fixed data.
+			const [afterCatalog] = (await db.execute(sql`
+				SELECT id, item_ids, catalog_name FROM catalog_items ORDER BY id
+			`)) as unknown as [
+				Array<{ id: number; item_ids: string; catalog_name: string }>,
+				unknown,
+			];
+			const orphanedCatalogIds = new Set<number>();
+			for (const ci of afterCatalog) {
+				const ids = ci.item_ids.split(";").map(Number).filter(Boolean);
+				for (const id of ids) {
+					if (!itemById.has(id)) orphanedCatalogIds.add(ci.id);
+				}
+			}
+			issues = issues.filter(
+				(i) =>
+					i.type !== "orphaned_catalog" ||
+					orphanedCatalogIds.has(i.catalogId ?? -1),
+			);
+
+			const duplicateResult = await repairDuplicateClassnames();
+			duplicatesMerged = duplicateResult.merged;
+			duplicateRowsRemoved = duplicateResult.rowsRemoved;
+			remappedReferences = duplicateResult.remapped;
+
+			// Re-derive duplicate_classname issues from the fixed data.
+			const [afterItems] = (await db.execute(sql`
+				SELECT item_name FROM items_base
+			`)) as unknown as [Array<{ item_name: string }>, unknown];
+			const afterCount = new Map<string, number>();
+			for (const row of afterItems) {
+				afterCount.set(row.item_name, (afterCount.get(row.item_name) ?? 0) + 1);
+			}
+			const dupNames = new Set(
+				[...afterCount].filter(([, count]) => count > 1).map(([name]) => name),
+			);
+			issues = issues.filter(
+				(i) =>
+					i.type !== "duplicate_classname" || dupNames.has(i.classname ?? ""),
+			);
+
+			onEvent?.({
+				type: "structure_repair_complete",
+				structure: {
+					orphanedRemoved,
+					orphanedCleaned,
+					duplicatesMerged,
+					duplicateRowsRemoved,
+					remappedReferences,
+				},
+			});
+		} catch (err) {
+			onEvent?.({
+				type: "error",
+				message: `Structural repair failed: ${(err as Error).message}`,
+			});
+		}
+	}
+
 	// ── Catalog SQL generation / apply ────────────────────────────────
 	let catalogSqlCount = 0;
 	let catalogSqlApplied = 0;
@@ -557,6 +648,11 @@ export async function runCatalogAudit(
 		catalogSqlApplied,
 		furniDataAdded,
 		furniDataDuplicatesRemoved,
+		orphanedRemoved,
+		orphanedCleaned,
+		duplicatesMerged,
+		duplicateRowsRemoved,
+		remappedReferences,
 	};
 
 	onEvent?.({
diff --git a/src/lib/services/catalog-repair.ts b/src/lib/services/catalog-repair.ts
index 916947a2..ccca31e3 100644
--- a/src/lib/services/catalog-repair.ts
+++ b/src/lib/services/catalog-repair.ts
@@ -96,6 +96,205 @@ export async function applyCatalogSql(
 	return { applied, failed };
 }
 
+export interface OrphanedCatalogResult {
+	removed: number;
+	cleaned: number;
+}
+
+export interface DuplicateClassnameResult {
+	merged: number;
+	rowsRemoved: number;
+	remapped: number;
+}
+
+/**
+ * Remove catalog_items rows whose item_ids only reference non-existent
+ * items_base entries, and strip orphaned ids from mixed rows. Also deletes
+ * catalog_items_limited rows that pointed at the removed catalog entries so no
+ * new orphans are left behind.
+ */
+export async function repairOrphanedCatalog(): Promise<OrphanedCatalogResult> {
+	const [itemRows] = (await db.execute(sql`
+		SELECT id FROM items_base
+	`)) as unknown as [Array<{ id: number }>, unknown];
+	const valid = new Set<number>();
+	for (const row of itemRows) valid.add(Number(row.id));
+
+	const [catalogRows] = (await db.execute(sql`
+		SELECT id, item_ids FROM catalog_items
+	`)) as unknown as [Array<{ id: number; item_ids: string }>, unknown];
+
+	let removed = 0;
+	let cleaned = 0;
+
+	for (const row of catalogRows) {
+		const ids = (row.item_ids ?? "")
+			.split(";")
+			.map((s) => Number(s.trim()))
+			.filter(Boolean);
+		if (ids.length === 0) continue;
+		const kept = ids.filter((id) => valid.has(id));
+		if (kept.length === ids.length) continue;
+
+		if (kept.length === 0) {
+			await db.execute(sql`
+				DELETE FROM catalog_items_limited WHERE catalog_item_id = ${row.id}
+			`);
+			await db.execute(sql`DELETE FROM catalog_items WHERE id = ${row.id}`);
+			removed++;
+		} else {
+			await db.execute(sql`
+				UPDATE catalog_items SET item_ids = ${kept.join(";")} WHERE id = ${row.id}
+			`);
+			cleaned++;
+		}
+	}
+
+	return { removed, cleaned };
+}
+
+/**
+ * Merge duplicate classnames in items_base into a single canonical row per
+ * classname, remapping every reference (catalog item id lists, player items,
+ * presents, marketplace, etc.) to the canonical id, then delete the duplicate
+ * rows. The canonical id is the one players actually purchase through
+ * catalog_items when present, otherwise the lowest id.
+ */
+export async function repairDuplicateClassnames(): Promise<DuplicateClassnameResult> {
+	const [itemRows] = (await db.execute(sql`
+		SELECT id, item_name FROM items_base
+	`)) as unknown as [Array<{ id: number; item_name: string }>, unknown];
+
+	const byName = new Map<string, number[]>();
+	for (const row of itemRows) {
+		const arr = byName.get(row.item_name) ?? [];
+		arr.push(Number(row.id));
+		byName.set(row.item_name, arr);
+	}
+
+	const [catalogRows] = (await db.execute(sql`
+		SELECT item_ids FROM catalog_items
+	`)) as unknown as [Array<{ item_ids: string }>, unknown];
+	const referenced = new Set<number>();
+	for (const row of catalogRows) {
+		for (const part of (row.item_ids ?? "").split(";")) {
+			const id = Number(part.trim());
+			if (id) referenced.add(id);
+		}
+	}
+
+	const remap = new Map<number, number>();
+	let merged = 0;
+	for (const ids of byName.values()) {
+		if (ids.length <= 1) continue;
+		merged++;
+		const sorted = [...ids].sort((a, b) => a - b);
+		const canonical = sorted.find((id) => referenced.has(id)) ?? sorted[0];
+		for (const id of ids) {
+			if (id !== canonical) remap.set(id, canonical);
+		}
+	}
+
+	if (remap.size === 0) return { merged: 0, rowsRemoved: 0, remapped: 0 };
+
+	let remapped = 0;
+
+	// Semicolon-separated item_ids columns.
+	const listTables: Array<{ table: string; idCol: string }> = [
+		{ table: "catalog_items", idCol: "id" },
+		{ table: "catalog_items_bc", idCol: "id" },
+		{ table: "logs_shop_purchases", idCol: "id" },
+	];
+	for (const { table, idCol } of listTables) {
+		const [rows] = (await db.execute(
+			sql.raw(`SELECT ${idCol} AS pk, item_ids AS v FROM ${table}`),
+		)) as unknown as [Array<{ pk: number; v: string | null }>, unknown];
+		for (const row of rows) {
+			if (!row.v) continue;
+			const parts = row.v
+				.split(";")
+				.map((s) => s.trim())
+				.filter(Boolean);
+			let changed = false;
+			const out: number[] = [];
+			const seen = new Set<number>();
+			for (const part of parts) {
+				const id = Number(part);
+				const mapped = remap.get(id) ?? id;
+				if (mapped !== id) changed = true;
+				if (seen.has(mapped)) continue;
+				seen.add(mapped);
+				out.push(mapped);
+			}
+			if (!changed) continue;
+			await db.execute(
+				sql.raw(
+					`UPDATE ${table} SET item_ids = '${out.join(";")}' WHERE ${idCol} = ${row.pk}`,
+				),
+			);
+			remapped++;
+		}
+	}
+
+	// Single item_id columns that reference items_base.id. Remap in chunks via
+	// a single CASE statement per table to keep the query count low even with
+	// thousands of duplicate rows.
+	const singleTables = [
+		"catalog_items_limited",
+		"items",
+		"items_presents",
+		"pet_items",
+		"pet_foods",
+		"pet_drinks",
+		"items_hoppers",
+		"marketplace_items",
+		"items_highscore_data",
+		"items_crackable",
+		"builders_club_items",
+		"recycler_prizes",
+		"gift_wrappers",
+		"youtube_playlists",
+		"room_trade_log_items",
+		"calendar_rewards",
+		"room_trax_playlist",
+		"trax_playlist",
+		"website_event_prizes",
+		"room_templates_items",
+		"crafting_recipes_ingredients",
+		"website_rare_values",
+		"logs_economy",
+	];
+
+	const entries = [...remap.entries()];
+	const CHUNK = 400;
+	for (const table of singleTables) {
+		for (let i = 0; i < entries.length; i += CHUNK) {
+			const chunk = entries.slice(i, i + CHUNK);
+			const cases = chunk
+				.map(([dup, canonical]) => `WHEN ${dup} THEN ${canonical}`)
+				.join(" ");
+			const dupList = chunk.map(([dup]) => `${dup}`).join(",");
+			try {
+				const [result] = (await db.execute(
+					sql.raw(
+						`UPDATE ${table} SET item_id = CASE item_id ${cases} ELSE item_id END WHERE item_id IN (${dupList})`,
+					),
+				)) as unknown as [Record<string, unknown>, unknown];
+				remapped += Number(result.affectedRows ?? 0);
+			} catch {
+				// Table may not exist on some hotel schemas — skip it.
+			}
+		}
+	}
+
+	const dupIds = entries.map(([dup]) => dup);
+	await db.execute(
+		sql.raw(`DELETE FROM items_base WHERE id IN (${dupIds.join(",")})`),
+	);
+
+	return { merged, rowsRemoved: dupIds.length, remapped };
+}
+
 export interface FurniDataRepairResult {
 	added: number;
 	removedDuplicates: number;