Add catalog maintenance page with sprite-id, dedup and FurnitureData id-alignment repairs

src/lib/admin-nav.ts

@@ -475,6 +475,13 @@ export const ADMIN_NAV_GROUPS: AdminNavGroup[] = [
 				permission: PERMS.CATALOG_VIEW,
 				matchPrefixes: ["/admin/catalog"],
 			},
+			{
+				href: "/admin/catalog/maintenance",
+				labelKey: "catalogMaintenance",
+				icon: Wrench,
+				permission: PERMS.CATALOG_EDIT,
+				matchPrefixes: ["/admin/catalog/maintenance"],
+			},
 			{
 				href: "/admin/items",
 				labelKey: "itemsBase",

src/lib/services/furni-maintenance.ts

@@ -0,0 +1,501 @@
+import { sql } from "drizzle-orm";
+import { db } from "@/lib/db";
+import {
+	type CatalogOfferRebuildResult,
+	type FurniDataReconcileResult,
+	readFurniData,
+	rebuildCatalogOfferIds,
+	reconcileFurniDataWithItemsBase,
+	type SpriteIdVerifyResult,
+	verifyAndFixSpriteIds,
+	withFurniDataLock,
+} from "@/lib/services/furni-data";
+
+/**
+ * Catalog maintenance toolkit.
+ *
+ * The Catalog studio (and the raw furnidata import) already keep ids consistent
+ * on the happy path, but legacy rows / manual edits can drift. This module
+ * exposes safe, transaction-wrapped operations that the admin "Catalog
+ * maintenance" page triggers. Every mutating call has a `preview` (count-only)
+ * mode and an `apply` mode, and all writes run inside a single InnoDB
+ * transaction so a failure rolls back completely.
+ */
+
+/** Integer foreign-key columns that store a base id (item_id / sprite_id). */
+const INT_COLS: Array<[string, string]> = [
+	["items", "item_id"],
+	["room_templates_items", "item_id"],
+	["catalog_items_limited", "item_id"],
+	["crafting_recipes_ingredients", "item_id"],
+	["items_crackable", "item_id"],
+	["gift_wrappers", "sprite_id"],
+	["gift_wrappers", "item_id"],
+	["trax_playlist", "item_id"],
+	["pet_drinks", "item_id"],
+	["pet_foods", "item_id"],
+	["pet_items", "item_id"],
+	["marketplace_items", "item_id"],
+	["calendar_rewards", "item_id"],
+	["builders_club_items", "item_id"],
+	["youtube_playlists", "item_id"],
+	["room_trax_playlist", "item_id"],
+	["recycler_prizes", "item_id"],
+	["website_event_prizes", "item_id"],
+	["website_rare_values", "item_id"],
+	["catalog_products", "item_id"],
+	["room_trade_log_items", "item_id"],
+	["logs_economy", "item_id"],
+];
+
+/** String columns that hold a single numeric base id. */
+const STR_COLS: Array<[string, string]> = [
+	["catalog_items", "item_ids"],
+	["catalog_items_bc", "item_ids"],
+	["logs_shop_purchases", "item_ids"],
+	["catalog_version_offers", "item_ids"],
+];
+
+type Rows<T> = [T[], unknown];
+
+async function query<T>(q: unknown): Promise<T[]> {
+	const [rows] = (await db.execute(q as never)) as unknown as Rows<T>;
+	return rows ?? [];
+}
+
+async function exists(table: string): Promise<boolean> {
+	const rows = await query<{ c: number }>(sql`SELECT COUNT(*) c
+		FROM information_schema.TABLES
+		WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ${table}`);
+	return Number(rows[0]?.c ?? 0) > 0;
+}
+
+/** Pre-resolve each string column's charset/collation so JOINs never mix them. */
+async function loadCollationMap(): Promise<Map<string, [string, string]>> {
+	const map = new Map<string, [string, string]>();
+	for (const [table, col] of STR_COLS) {
+		if (!(await exists(table))) continue;
+		const rows = await query<{
+			CHARACTER_SET_NAME: string | null;
+			COLLATION_NAME: string | null;
+		}>(sql`
+			SELECT CHARACTER_SET_NAME, COLLATION_NAME
+			FROM information_schema.COLUMNS
+			WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ${table} AND COLUMN_NAME = ${col}`);
+		const r = rows[0];
+		if (r?.CHARACTER_SET_NAME)
+			map.set(`${table}.${col}`, [
+				r.CHARACTER_SET_NAME,
+				r.COLLATION_NAME ?? r.CHARACTER_SET_NAME,
+			]);
+	}
+	return map;
+}
+
+/** Convert a numeric expression to CHAR in the column's own charset/collation. */
+function numToStr(
+	map: Map<string, [string, string]>,
+	table: string,
+	col: string,
+	expr: string,
+): string {
+	const info = map.get(`${table}.${col}`);
+	if (!info) return `CAST(${expr} AS CHAR)`;
+	return `CONVERT(${expr}, CHAR CHARACTER SET ${info[0]}) COLLATE ${info[1]}`;
+}
+
+export interface FurniHealth {
+	itemsBaseTotal: number;
+	spriteIdMismatch: number;
+	catalogOfferMismatch: number;
+	duplicateClassnames: number;
+	idMismatchVsFurnidata: number;
+}
+
+/** Read-only snapshot of how far the furniture tables have drifted. */
+export async function getFurniHealth(): Promise<FurniHealth> {
+	const [ib] = await query<{ c: number }>(
+		sql`SELECT COUNT(*) c FROM items_base`,
+	);
+	const [sp] = await query<{ c: number }>(
+		sql`SELECT COUNT(*) c FROM items_base WHERE sprite_id <> id`,
+	);
+	const [co] = await query<{ c: number }>(
+		sql`SELECT COUNT(*) c FROM catalog_items WHERE offer_id <> id`,
+	);
+	const [dc] = await query<{ c: number }>(sql`
+		SELECT COUNT(*) c FROM (
+			SELECT item_name FROM items_base GROUP BY item_name HAVING COUNT(*) > 1
+		) g`);
+
+	const furniData = (await readFurniData()) as Record<
+		string,
+		{ furnitype?: Array<{ id?: unknown; classname?: unknown }> }
+	>;
+	const desired = new Map<string, number>();
+	for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
+		for (const e of furniData[section]?.furnitype ?? []) {
+			const cn = typeof e?.classname === "string" ? e.classname : "";
+			const id = Number(e?.id);
+			if (cn && Number.isFinite(id) && id > 0 && !desired.has(cn)) {
+				desired.set(cn, id);
+			}
+		}
+	}
+	const rows = await query<{ id: number; item_name: string }>(
+		sql`SELECT id, item_name FROM items_base`,
+	);
+	let idMismatch = 0;
+	for (const r of rows) {
+		const target = desired.get(r.item_name);
+		if (target !== undefined && target !== r.id) idMismatch++;
+	}
+
+	return {
+		itemsBaseTotal: Number(ib?.c ?? 0),
+		spriteIdMismatch: Number(sp?.c ?? 0),
+		catalogOfferMismatch: Number(co?.c ?? 0),
+		duplicateClassnames: Number(dc?.c ?? 0),
+		idMismatchVsFurnidata: idMismatch,
+	};
+}
+
+export interface FixSpriteResult extends SpriteIdVerifyResult {}
+export interface FixOfferResult extends CatalogOfferRebuildResult {}
+export interface ReconcileResult extends FurniDataReconcileResult {}
+
+/** Fix items_base.sprite_id -> id (only where furnidata agrees). */
+export async function fixSpriteIds(): Promise<FixSpriteResult> {
+	return verifyAndFixSpriteIds();
+}
+
+/** Force catalog_items.offer_id -> id (emulator offer lookup stays correct). */
+export async function fixCatalogOffers(): Promise<FixOfferResult> {
+	return rebuildCatalogOfferIds();
+}
+
+/** Make FurnitureData.json ids/offerids agree with items_base (items_base authoritative). */
+export async function reconcileIds(): Promise<ReconcileResult> {
+	return reconcileFurniDataWithItemsBase();
+}
+
+export interface DedupResult {
+	groups: number;
+	removed: number;
+	remapped: number;
+}
+
+/**
+ * Remove duplicate items_base rows that share the same item_name (classname).
+ * The lowest id in each group is kept; every referencing int/string column is
+ * remapped from the duplicate ids onto the keeper id, then the duplicates are
+ * deleted. All inside one transaction.
+ */
+export async function removeDuplicateItemsBase(): Promise<DedupResult> {
+	return withFurniDataLock(async () => {
+		// Fetch every row whose classname is duplicated (no GROUP_CONCAT, so it
+		// stays correct even for very large duplicate groups).
+		const dupRows = await query<{ id: number; item_name: string }>(sql`
+			SELECT b.id, b.item_name FROM items_base b
+			WHERE b.item_name IN (
+				SELECT item_name FROM items_base GROUP BY item_name HAVING COUNT(*) > 1
+			)
+			ORDER BY b.item_name, b.id`);
+
+		const groups = new Map<string, number[]>();
+		for (const r of dupRows) {
+			const arr = groups.get(r.item_name) ?? [];
+			arr.push(r.id);
+			groups.set(r.item_name, arr);
+		}
+		const groupList = [...groups.entries()].map(([item_name, ids]) => ({
+			item_name,
+			keeper: ids[0],
+			ids,
+		}));
+
+		const activeInt = (
+			await Promise.all(
+				INT_COLS.map(async ([t, c]) =>
+					(await exists(t)) ? ([t, c] as [string, string]) : null,
+				),
+			)
+		).filter(Boolean) as [string, string][];
+		const activeStr = (
+			await Promise.all(
+				STR_COLS.map(async ([t, c]) =>
+					(await exists(t)) ? ([t, c] as [string, string]) : null,
+				),
+			)
+		).filter(Boolean) as [string, string][];
+		const collation = await loadCollationMap();
+
+		let removed = 0;
+		let remapped = 0;
+
+		await db.transaction(async (tx) => {
+			await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 0`);
+			for (const g of groupList) {
+				const ids = g.ids.filter((x) => x !== g.keeper);
+				for (const dupId of ids) {
+					for (const [table, col] of activeInt) {
+						const r = (await tx.execute(
+							sql`UPDATE ${sql.raw(table)} t SET ${sql.raw(col)} = ${g.keeper} WHERE ${sql.raw(col)} = ${dupId}`,
+						)) as unknown as [Record<string, unknown>, unknown];
+						remapped += Number(r[0]?.affectedRows ?? 0);
+					}
+					for (const [table, col] of activeStr) {
+						await tx.execute(
+							sql`UPDATE ${sql.raw(table)} t SET ${sql.raw(col)} = ${g.keeper} WHERE ${sql.raw(col)} = ${sql.raw(numToStr(collation, table, col, String(dupId)))}`,
+						);
+					}
+					await tx.execute(sql`DELETE FROM items_base WHERE id = ${dupId}`);
+					removed++;
+				}
+			}
+			await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 1`);
+		});
+
+		return { groups: groupList.length, removed, remapped };
+	});
+}
+
+export interface AlignResult {
+	moves: number;
+	inserted: number;
+	skipped: string[];
+}
+
+/**
+ * Force items_base.id to equal the FurnitureData.json sprite id for every
+ * classname (FurnitureData is the source of truth). New classnames are inserted
+ * with their furnidata id; existing rows whose id differs are moved via a
+ * multi-pass primary-key swap that rewrites every referencing column so no
+ * foreign key / shop reference breaks. Mirrors scripts/align-db-ids-with-furnidata.ts.
+ */
+export async function forceItemsBaseIdsToFurnidata(
+	apply: boolean,
+): Promise<AlignResult> {
+	const furniData = (await readFurniData()) as Record<
+		string,
+		{ furnitype?: Array<{ id?: unknown; classname?: unknown }> }
+	>;
+
+	const all: Array<{ classname: string; id: number }> = [];
+	for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
+		for (const e of furniData[section]?.furnitype ?? []) {
+			const cn = typeof e?.classname === "string" ? e.classname : "";
+			const id = Number(e?.id);
+			if (cn && Number.isFinite(id) && id > 0) all.push({ classname: cn, id });
+		}
+	}
+
+	// Global iterative duplicate-id resolution (keep DB occupant, drop losers).
+	const desired = new Map<string, number>();
+	for (const e of all) {
+		if (!desired.has(e.classname)) desired.set(e.classname, e.id);
+	}
+	const rows = await query<{ id: number; item_name: string }>(
+		sql`SELECT id, item_name FROM items_base`,
+	);
+	const nameById = new Map<number, string>();
+	const idByName = new Map<string, number>();
+	let maxId = 0;
+	for (const r of rows) {
+		nameById.set(r.id, r.item_name);
+		idByName.set(r.item_name, r.id);
+		maxId = Math.max(maxId, r.id);
+	}
+	for (;;) {
+		const claims = new Map<number, string[]>();
+		for (const [cn, id] of desired) {
+			const arr = claims.get(id) ?? [];
+			arr.push(cn);
+			claims.set(id, arr);
+		}
+		let found = false;
+		for (const [id, claimants] of [...claims].sort((a, b) => a[0] - b[0])) {
+			if (claimants.length < 2) continue;
+			const live = claimants.filter((cn) => desired.get(cn) === id);
+			if (live.length < 2) continue;
+			found = true;
+			const dbHolder = nameById.get(id);
+			const winner =
+				dbHolder !== undefined && live.includes(dbHolder) ? dbHolder : live[0];
+			for (const cn of live) if (cn !== winner) desired.delete(cn);
+		}
+		if (!found) break;
+	}
+
+	let fresh = Math.max(maxId, ...desired.values()) + 1000;
+	const moves = new Map<number, number>();
+	for (const r of rows) {
+		const target = desired.get(r.item_name);
+		if (target !== undefined && target !== r.id) moves.set(r.id, target);
+	}
+	const occupied = new Map<number, true>(nameById.keys().map((k) => [k, true]));
+	for (;;) {
+		const dest = new Map<number, number>();
+		for (const t of moves.values()) dest.set(t, (dest.get(t) ?? 0) + 1);
+		let changed = false;
+		for (const [, target] of moves) {
+			if (!occupied.has(target) || moves.has(target)) continue;
+			const holder = nameById.get(target);
+			if (holder !== undefined && desired.get(holder) === target) continue;
+			moves.set(target, fresh++);
+			changed = true;
+			break;
+		}
+		if (!changed) break;
+	}
+	for (;;) {
+		let changed = false;
+		for (const [oldId, target] of moves) {
+			if (occupied.has(target) || moves.has(target)) continue;
+			const holder = nameById.get(target);
+			if (holder !== undefined && desired.get(holder) === target) {
+				moves.delete(oldId);
+				changed = true;
+				break;
+			}
+		}
+		if (!changed) break;
+	}
+
+	if (!apply) {
+		return { moves: moves.size, inserted: 0, skipped: [] };
+	}
+
+	const activeInt = (
+		await Promise.all(
+			INT_COLS.map(async ([t, c]) =>
+				(await exists(t)) ? ([t, c] as [string, string]) : null,
+			),
+		)
+	).filter(Boolean) as [string, string][];
+	const activeStr = (
+		await Promise.all(
+			STR_COLS.map(async ([t, c]) =>
+				(await exists(t)) ? ([t, c] as [string, string]) : null,
+			),
+		)
+	).filter(Boolean) as [string, string][];
+	const collation = await loadCollationMap();
+
+	await withFurniDataLock(async () => {
+		await db.transaction(async (tx) => {
+			await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 0`);
+			await tx.execute(sql`DROP TEMPORARY TABLE IF EXISTS _mm_id_map`);
+			await tx.execute(sql`CREATE TEMPORARY TABLE _mm_id_map (
+				old_id INT PRIMARY KEY, new_id INT NOT NULL, UNIQUE KEY uniq_new (new_id)
+			) ENGINE=InnoDB`);
+
+			const occupiedIds = new Map<number, true>(
+				nameById.keys().map((k) => [k, true]),
+			);
+			const pending = new Map(moves);
+			let scratch = fresh;
+			const applyPass = async (pairs: Array<[number, number]>) => {
+				if (pairs.length === 0) return;
+				await tx.execute(sql`TRUNCATE TABLE _mm_id_map`);
+				for (let i = 0; i < pairs.length; i += 500) {
+					const chunk = pairs.slice(i, i + 500);
+					await tx.execute(
+						sql`INSERT INTO _mm_id_map (old_id, new_id) VALUES ${sql.join(
+							chunk.map(([o, nw]) => sql`(${o}, ${nw})`),
+							sql`, `,
+						)}`,
+					);
+				}
+				await tx.execute(
+					sql`UPDATE items_base b JOIN _mm_id_map m ON b.id = m.old_id SET b.id = m.new_id`,
+				);
+				for (const [table, col] of activeInt) {
+					await tx.execute(
+						sql`UPDATE ${sql.raw(table)} t JOIN _mm_id_map m ON t.${sql.raw(col)} = m.old_id SET t.${sql.raw(col)} = m.new_id`,
+					);
+				}
+				for (const [table, col] of activeStr) {
+					await tx.execute(
+						sql`UPDATE ${sql.raw(table)} t JOIN _mm_id_map m ON t.${sql.raw(col)} = ${sql.raw(numToStr(collation, table, col, "m.old_id"))} SET t.${sql.raw(col)} = CAST(m.new_id AS CHAR)`,
+					);
+				}
+				for (const [o, nw] of pairs) {
+					occupiedIds.delete(o);
+					occupiedIds.set(nw, true);
+				}
+			};
+
+			while (pending.size > 0) {
+				const safe: Array<[number, number]> = [];
+				for (const [oldId, target] of pending) {
+					if (!occupiedIds.has(target)) safe.push([oldId, target]);
+				}
+				if (safe.length === 0) {
+					const oldest = Math.min(...pending.keys());
+					const target = pending.get(oldest);
+					if (target === undefined) break;
+					pending.delete(oldest);
+					await applyPass([[oldest, scratch]]);
+					pending.set(scratch, target);
+					scratch++;
+					continue;
+				}
+				await applyPass(safe);
+				for (const [oldId] of safe) pending.delete(oldId);
+			}
+
+			await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 1`);
+			await tx.execute(sql`DROP TEMPORARY TABLE IF EXISTS _mm_id_map`);
+		});
+	});
+
+	// Insert missing classnames with their exact furnidata id.
+	let inserted = 0;
+	const skipped: string[] = [];
+	const currentIds = new Set(
+		(await query<{ id: number }>(sql`SELECT id FROM items_base`)).map(
+			(r) => r.id,
+		),
+	);
+	for (const [cn, id] of desired) {
+		if (idByName.has(cn)) continue;
+		if (currentIds.has(id)) {
+			skipped.push(`${cn} (id ${id} occupied)`);
+			continue;
+		}
+		await db.execute(
+			sql`INSERT INTO items_base (id, item_name, sprite_id) VALUES (${id}, ${cn}, ${id})`,
+		);
+		currentIds.add(id);
+		inserted++;
+	}
+
+	await db.execute(
+		sql`UPDATE items_base SET sprite_id = id WHERE sprite_id <> id`,
+	);
+
+	return { moves: moves.size, inserted, skipped };
+}
+
+export interface FixAllResult {
+	healthBefore: FurniHealth;
+	sprite: FixSpriteResult;
+	offers: FixOfferResult;
+	reconcile: ReconcileResult;
+	dedup: DedupResult;
+	align: AlignResult;
+	healthAfter: FurniHealth;
+}
+
+/** Run every repair in a sensible order, then return before/after health. */
+export async function fixEverything(): Promise<FixAllResult> {
+	const healthBefore = await getFurniHealth();
+	const sprite = await fixSpriteIds();
+	const offers = await fixCatalogOffers();
+	const reconcile = await reconcileIds();
+	const dedup = await removeDuplicateItemsBase();
+	const align = await forceItemsBaseIdsToFurnidata(true);
+	const healthAfter = await getFurniHealth();
+	return { healthBefore, sprite, offers, reconcile, dedup, align, healthAfter };
+}