From 34475e9bc21f8e7151e2bc510b16c9fa2d43f545 Mon Sep 17 00:00:00 2001
From: openhands <openhands@all-hands.dev>
Date: Sun, 23 Aug 2026 15:30:52 +0200
Subject: [PATCH] Fix dedup to also remove duplicate
 catalog_items/catalog_items_bc rows (comma-list-aware item_ids remap)

---
 src/actions/furni-maintenance.ts           |  2 +-
 src/app/admin/catalog/maintenance/page.tsx | 11 ++-
 src/lib/services/furni-maintenance.ts      | 80 ++++++++++++++++++++--
 3 files changed, 86 insertions(+), 7 deletions(-)

diff --git a/src/actions/furni-maintenance.ts b/src/actions/furni-maintenance.ts
index f835f225..26b34079 100644
--- a/src/actions/furni-maintenance.ts
+++ b/src/actions/furni-maintenance.ts
@@ -71,7 +71,7 @@ export async function removeDuplicateItemsBaseAction(): Promise<
 > {
 	try {
 		await guard();
-		return actionOk(await maintenance.removeDuplicateItemsBase());
+		return actionOk(await maintenance.removeDuplicates());
 	} catch (e) {
 		return handleActionError(e);
 	}
diff --git a/src/app/admin/catalog/maintenance/page.tsx b/src/app/admin/catalog/maintenance/page.tsx
index f288873f..d2ed63c0 100644
--- a/src/app/admin/catalog/maintenance/page.tsx
+++ b/src/app/admin/catalog/maintenance/page.tsx
@@ -29,6 +29,8 @@ interface Health {
 	catalogOfferMismatch: number;
 	duplicateClassnames: number;
 	idMismatchVsFurnidata: number;
+	catalogDuplicates: number;
+	catalogBcDuplicates: number;
 }
 
 export default function CatalogMaintenancePage() {
@@ -150,6 +152,11 @@ export default function CatalogMaintenancePage() {
 								value={health.duplicateClassnames}
 								hint="items_base item_name repeats"
 							/>
+							<Stat
+								label="Duplicate catalog rows"
+								value={health.catalogDuplicates}
+								hint="same product listed twice on a page"
+							/>
 							<Stat
 								label="ID ≠ FurnitureData"
 								value={health.idMismatchVsFurnidata}
@@ -183,11 +190,11 @@ export default function CatalogMaintenancePage() {
 						void run(
 							"dup",
 							removeDuplicateItemsBaseAction,
-							"Duplicates removed",
+							"Duplicates removed (items + catalog)",
 						)
 					}
 				>
-					Remove duplicate furniture
+					Remove duplicates (items + catalog)
 				</Button>
 				<Button
 					disabled={busy !== null}
diff --git a/src/lib/services/furni-maintenance.ts b/src/lib/services/furni-maintenance.ts
index f9e82cdd..b7041a1a 100644
--- a/src/lib/services/furni-maintenance.ts
+++ b/src/lib/services/furni-maintenance.ts
@@ -110,6 +110,8 @@ export interface FurniHealth {
 	catalogOfferMismatch: number;
 	duplicateClassnames: number;
 	idMismatchVsFurnidata: number;
+	catalogDuplicates: number;
+	catalogBcDuplicates: number;
 }
 
 /** Read-only snapshot of how far the furniture tables have drifted. */
@@ -128,6 +130,19 @@ export async function getFurniHealth(): Promise<FurniHealth> {
 			SELECT item_name FROM items_base GROUP BY item_name HAVING COUNT(*) > 1
 		) g`);
 
+	const DUP_GRP =
+		"page_id,item_ids,catalog_name,cost_credits,cost_points,points_type,amount,limited_sells,limited_stack,club_only,song_id,have_offer,extradata,badge";
+	const [cd] = await query<{ c: number }>(sql`
+		SELECT COUNT(*) c FROM (
+			SELECT ${sql.raw(DUP_GRP)} FROM catalog_items
+			GROUP BY ${sql.raw(DUP_GRP)} HAVING COUNT(*) > 1
+		) g`);
+	const [cdb] = await query<{ c: number }>(sql`
+		SELECT COUNT(*) c FROM (
+			SELECT ${sql.raw(DUP_GRP)} FROM catalog_items_bc
+			GROUP BY ${sql.raw(DUP_GRP)} HAVING COUNT(*) > 1
+		) g`);
+
 	const furniData = (await readFurniData()) as Record<
 		string,
 		{ furnitype?: Array<{ id?: unknown; classname?: unknown }> }
@@ -157,6 +172,8 @@ export async function getFurniHealth(): Promise<FurniHealth> {
 		catalogOfferMismatch: Number(co?.c ?? 0),
 		duplicateClassnames: Number(dc?.c ?? 0),
 		idMismatchVsFurnidata: idMismatch,
+		catalogDuplicates: Number(cd?.c ?? 0),
+		catalogBcDuplicates: Number(cdb?.c ?? 0),
 	};
 }
 
@@ -183,6 +200,8 @@ export interface DedupResult {
 	groups: number;
 	removed: number;
 	remapped: number;
+	catalogRemoved: number;
+	catalogBcRemoved: number;
 }
 
 /**
@@ -228,7 +247,6 @@ export async function removeDuplicateItemsBase(): Promise<DedupResult> {
 				),
 			)
 		).filter(Boolean) as [string, string][];
-		const collation = await loadCollationMap();
 
 		let removed = 0;
 		let remapped = 0;
@@ -245,8 +263,12 @@ export async function removeDuplicateItemsBase(): Promise<DedupResult> {
 						remapped += Number(r[0]?.affectedRows ?? 0);
 					}
 					for (const [table, col] of activeStr) {
+						const from = String(dupId);
+						const to = String(g.keeper);
 						await tx.execute(
-							sql`UPDATE ${sql.raw(table)} t SET ${sql.raw(col)} = ${g.keeper} WHERE ${sql.raw(col)} = ${sql.raw(numToStr(collation, table, col, String(dupId)))}`,
+							sql.raw(
+								`UPDATE ${table} t SET ${col} = TRIM(BOTH ',' FROM REPLACE(CONCAT(',', ${col}, ','), CONCAT(',', '${from}', ','), CONCAT(',', '${to}', ','))) WHERE CONCAT(',', ${col}, ',') LIKE '%,${from},%'`,
+							),
 						);
 					}
 					await tx.execute(sql`DELETE FROM items_base WHERE id = ${dupId}`);
@@ -256,10 +278,60 @@ export async function removeDuplicateItemsBase(): Promise<DedupResult> {
 			await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 1`);
 		});
 
-		return { groups: groupList.length, removed, remapped };
+		return {
+			groups: groupList.length,
+			removed,
+			remapped,
+			catalogRemoved: 0,
+			catalogBcRemoved: 0,
+		};
 	});
 }
 
+/**
+ * Remove duplicate rows from a catalog table. Two rows are duplicates when they
+ * are the exact same product on the same page (identical page_id, item_ids,
+ * catalog_name and every cost/limit/extra flag). The lowest id is kept, the rest
+ * deleted. No foreign keys reference catalog_items.id, so this is safe.
+ */
+async function dedupCatalogTable(table: string): Promise<number> {
+	if (!(await exists(table))) return 0;
+	const grp = [
+		"page_id",
+		"item_ids",
+		"catalog_name",
+		"cost_credits",
+		"cost_points",
+		"points_type",
+		"amount",
+		"limited_sells",
+		"limited_stack",
+		"club_only",
+		"song_id",
+		"have_offer",
+		"extradata",
+		"badge",
+	];
+	const on = grp.map((c) => `t.${c} <=> d.${c}`).join(" AND ");
+	const r = (await db.execute(
+		sql.raw(
+			`DELETE t FROM ${table} t JOIN (SELECT ${grp.join(",")}, MIN(id) AS _keep FROM ${table} GROUP BY ${grp.join(",")} HAVING COUNT(*) > 1) d ON ${on} WHERE t.id <> d._keep`,
+		),
+	)) as unknown as [Record<string, unknown>, unknown];
+	return Number(r[0]?.affectedRows ?? 0);
+}
+
+/**
+ * Full duplicate cleanup: items_base (by classname) + catalog_items +
+ * catalog_items_bc (by identical product on the same page).
+ */
+export async function removeDuplicates(): Promise<DedupResult> {
+	const ib = await removeDuplicateItemsBase();
+	const catalogRemoved = await dedupCatalogTable("catalog_items");
+	const catalogBcRemoved = await dedupCatalogTable("catalog_items_bc");
+	return { ...ib, catalogRemoved, catalogBcRemoved };
+}
+
 export interface AlignResult {
 	moves: number;
 	inserted: number;
@@ -494,7 +566,7 @@ export async function fixEverything(): Promise<FixAllResult> {
 	const sprite = await fixSpriteIds();
 	const offers = await fixCatalogOffers();
 	const reconcile = await reconcileIds();
-	const dedup = await removeDuplicateItemsBase();
+	const dedup = await removeDuplicates();
 	const align = await forceItemsBaseIdsToFurnidata(true);
 	const healthAfter = await getFurniHealth();
 	return { healthBefore, sprite, offers, reconcile, dedup, align, healthAfter };