feat: catalog audit can repair orphaned refs and duplicate classnames

- add repairOrphanedCatalog: remove catalog_items rows whose item_ids only
  reference missing items_base entries, strip orphaned ids from mixed rows
- add repairDuplicateClassnames: merge items_base duplicates into one
  canonical row per classname, remap references, delete duplicate rows
- add 'repair structural issues' checkbox + result display in audit UI
This commit is contained in:
openhands committed 2026-08-03 18:08:45 +02:00
1 parent bee55e1fd4
commit 6fc14b9b84
4 files changed
+384 -1

No files matched your search

+96
View File
@@ -9,7 +9,9 @@ import {
applyCatalogSql,
generateCatalogSql,
type MissingCatalogEntry,
repairDuplicateClassnames,
repairFurniData,
repairOrphanedCatalog,
} from "./catalog-repair";
import { ensureDirectories } from "./furni-import";
import { repairMissingIcons } from "./repair-icons";
@@ -25,6 +27,7 @@ export interface AuditEvent {
| "sql_ready"
| "sql_applied"
| "furnidata_repair_complete"
| "structure_repair_complete"
| "batch_complete"
| "error";
message?: string;
@@ -41,6 +44,13 @@ export interface AuditEvent {
applied?: number;
sqlFailed?: number;
furniData?: { added: number; removedDuplicates: number };
structure?: {
orphanedRemoved: number;
orphanedCleaned: number;
duplicatesMerged: number;
duplicateRowsRemoved: number;
remappedReferences: number;
};
}
export interface RepairStats {
@@ -84,6 +94,11 @@ export interface AuditSummary {
catalogSqlApplied: number;
furniDataAdded: number;
furniDataDuplicatesRemoved: number;
orphanedRemoved: number;
orphanedCleaned: number;
duplicatesMerged: number;
duplicateRowsRemoved: number;
remappedReferences: number;
}
export interface CatalogAuditOptions {
@@ -92,6 +107,7 @@ export interface CatalogAuditOptions {
sql?: boolean;
applySql?: boolean;
repairFurniData?: boolean;
repairStructure?: boolean;
}
export async function runCatalogAudit(
@@ -450,6 +466,81 @@ export async function runCatalogAudit(
}
}
// ── Structural repair: orphaned catalog refs + duplicate classnames ──
let orphanedRemoved = 0;
let orphanedCleaned = 0;
let duplicatesMerged = 0;
let duplicateRowsRemoved = 0;
let remappedReferences = 0;
if (options?.repairStructure) {
onEvent?.({
type: "progress",
message: "Repairing structural issues (orphaned refs + duplicates)…",
});
try {
const orphanResult = await repairOrphanedCatalog();
orphanedRemoved = orphanResult.removed;
orphanedCleaned = orphanResult.cleaned;
// Re-derive orphaned_catalog issues from the fixed data.
const [afterCatalog] = (await db.execute(sql`
SELECT id, item_ids, catalog_name FROM catalog_items ORDER BY id
`)) as unknown as [
Array<{ id: number; item_ids: string; catalog_name: string }>,
unknown,
];
const orphanedCatalogIds = new Set<number>();
for (const ci of afterCatalog) {
const ids = ci.item_ids.split(";").map(Number).filter(Boolean);
for (const id of ids) {
if (!itemById.has(id)) orphanedCatalogIds.add(ci.id);
}
}
issues = issues.filter(
(i) =>
i.type !== "orphaned_catalog" ||
orphanedCatalogIds.has(i.catalogId ?? -1),
);
const duplicateResult = await repairDuplicateClassnames();
duplicatesMerged = duplicateResult.merged;
duplicateRowsRemoved = duplicateResult.rowsRemoved;
remappedReferences = duplicateResult.remapped;
// Re-derive duplicate_classname issues from the fixed data.
const [afterItems] = (await db.execute(sql`
SELECT item_name FROM items_base
`)) as unknown as [Array<{ item_name: string }>, unknown];
const afterCount = new Map<string, number>();
for (const row of afterItems) {
afterCount.set(row.item_name, (afterCount.get(row.item_name) ?? 0) + 1);
}
const dupNames = new Set(
[...afterCount].filter(([, count]) => count > 1).map(([name]) => name),
);
issues = issues.filter(
(i) =>
i.type !== "duplicate_classname" || dupNames.has(i.classname ?? ""),
);
onEvent?.({
type: "structure_repair_complete",
structure: {
orphanedRemoved,
orphanedCleaned,
duplicatesMerged,
duplicateRowsRemoved,
remappedReferences,
},
});
} catch (err) {
onEvent?.({
type: "error",
message: `Structural repair failed: ${(err as Error).message}`,
});
}
}
// ── Catalog SQL generation / apply ────────────────────────────────
let catalogSqlCount = 0;
let catalogSqlApplied = 0;
@@ -557,6 +648,11 @@ export async function runCatalogAudit(
catalogSqlApplied,
furniDataAdded,
furniDataDuplicatesRemoved,
orphanedRemoved,
orphanedCleaned,
duplicatesMerged,
duplicateRowsRemoved,
remappedReferences,
};
onEvent?.({
+199
View File
@@ -96,6 +96,205 @@ export async function applyCatalogSql(
return { applied, failed };
}
export interface OrphanedCatalogResult {
removed: number;
cleaned: number;
}
export interface DuplicateClassnameResult {
merged: number;
rowsRemoved: number;
remapped: number;
}
/**
* Remove catalog_items rows whose item_ids only reference non-existent
* items_base entries, and strip orphaned ids from mixed rows. Also deletes
* catalog_items_limited rows that pointed at the removed catalog entries so no
* new orphans are left behind.
*/
export async function repairOrphanedCatalog(): Promise<OrphanedCatalogResult> {
const [itemRows] = (await db.execute(sql`
SELECT id FROM items_base
`)) as unknown as [Array<{ id: number }>, unknown];
const valid = new Set<number>();
for (const row of itemRows) valid.add(Number(row.id));
const [catalogRows] = (await db.execute(sql`
SELECT id, item_ids FROM catalog_items
`)) as unknown as [Array<{ id: number; item_ids: string }>, unknown];
let removed = 0;
let cleaned = 0;
for (const row of catalogRows) {
const ids = (row.item_ids ?? "")
.split(";")
.map((s) => Number(s.trim()))
.filter(Boolean);
if (ids.length === 0) continue;
const kept = ids.filter((id) => valid.has(id));
if (kept.length === ids.length) continue;
if (kept.length === 0) {
await db.execute(sql`
DELETE FROM catalog_items_limited WHERE catalog_item_id = ${row.id}
`);
await db.execute(sql`DELETE FROM catalog_items WHERE id = ${row.id}`);
removed++;
} else {
await db.execute(sql`
UPDATE catalog_items SET item_ids = ${kept.join(";")} WHERE id = ${row.id}
`);
cleaned++;
}
}
return { removed, cleaned };
}
/**
* Merge duplicate classnames in items_base into a single canonical row per
* classname, remapping every reference (catalog item id lists, player items,
* presents, marketplace, etc.) to the canonical id, then delete the duplicate
* rows. The canonical id is the one players actually purchase through
* catalog_items when present, otherwise the lowest id.
*/
export async function repairDuplicateClassnames(): Promise<DuplicateClassnameResult> {
const [itemRows] = (await db.execute(sql`
SELECT id, item_name FROM items_base
`)) as unknown as [Array<{ id: number; item_name: string }>, unknown];
const byName = new Map<string, number[]>();
for (const row of itemRows) {
const arr = byName.get(row.item_name) ?? [];
arr.push(Number(row.id));
byName.set(row.item_name, arr);
}
const [catalogRows] = (await db.execute(sql`
SELECT item_ids FROM catalog_items
`)) as unknown as [Array<{ item_ids: string }>, unknown];
const referenced = new Set<number>();
for (const row of catalogRows) {
for (const part of (row.item_ids ?? "").split(";")) {
const id = Number(part.trim());
if (id) referenced.add(id);
}
}
const remap = new Map<number, number>();
let merged = 0;
for (const ids of byName.values()) {
if (ids.length <= 1) continue;
merged++;
const sorted = [...ids].sort((a, b) => a - b);
const canonical = sorted.find((id) => referenced.has(id)) ?? sorted[0];
for (const id of ids) {
if (id !== canonical) remap.set(id, canonical);
}
}
if (remap.size === 0) return { merged: 0, rowsRemoved: 0, remapped: 0 };
let remapped = 0;
// Semicolon-separated item_ids columns.
const listTables: Array<{ table: string; idCol: string }> = [
{ table: "catalog_items", idCol: "id" },
{ table: "catalog_items_bc", idCol: "id" },
{ table: "logs_shop_purchases", idCol: "id" },
];
for (const { table, idCol } of listTables) {
const [rows] = (await db.execute(
sql.raw(`SELECT ${idCol} AS pk, item_ids AS v FROM ${table}`),
)) as unknown as [Array<{ pk: number; v: string | null }>, unknown];
for (const row of rows) {
if (!row.v) continue;
const parts = row.v
.split(";")
.map((s) => s.trim())
.filter(Boolean);
let changed = false;
const out: number[] = [];
const seen = new Set<number>();
for (const part of parts) {
const id = Number(part);
const mapped = remap.get(id) ?? id;
if (mapped !== id) changed = true;
if (seen.has(mapped)) continue;
seen.add(mapped);
out.push(mapped);
}
if (!changed) continue;
await db.execute(
sql.raw(
`UPDATE ${table} SET item_ids = '${out.join(";")}' WHERE ${idCol} = ${row.pk}`,
),
);
remapped++;
}
}
// Single item_id columns that reference items_base.id. Remap in chunks via
// a single CASE statement per table to keep the query count low even with
// thousands of duplicate rows.
const singleTables = [
"catalog_items_limited",
"items",
"items_presents",
"pet_items",
"pet_foods",
"pet_drinks",
"items_hoppers",
"marketplace_items",
"items_highscore_data",
"items_crackable",
"builders_club_items",
"recycler_prizes",
"gift_wrappers",
"youtube_playlists",
"room_trade_log_items",
"calendar_rewards",
"room_trax_playlist",
"trax_playlist",
"website_event_prizes",
"room_templates_items",
"crafting_recipes_ingredients",
"website_rare_values",
"logs_economy",
];
const entries = [...remap.entries()];
const CHUNK = 400;
for (const table of singleTables) {
for (let i = 0; i < entries.length; i += CHUNK) {
const chunk = entries.slice(i, i + CHUNK);
const cases = chunk
.map(([dup, canonical]) => `WHEN ${dup} THEN ${canonical}`)
.join(" ");
const dupList = chunk.map(([dup]) => `${dup}`).join(",");
try {
const [result] = (await db.execute(
sql.raw(
`UPDATE ${table} SET item_id = CASE item_id ${cases} ELSE item_id END WHERE item_id IN (${dupList})`,
),
)) as unknown as [Record<string, unknown>, unknown];
remapped += Number(result.affectedRows ?? 0);
} catch {
// Table may not exist on some hotel schemas — skip it.
}
}
}
const dupIds = entries.map(([dup]) => dup);
await db.execute(
sql.raw(`DELETE FROM items_base WHERE id IN (${dupIds.join(",")})`),
);
return { merged, rowsRemoved: dupIds.length, remapped };
}
export interface FurniDataRepairResult {
added: number;
removedDuplicates: number;