feat: catalog audit can repair orphaned refs and duplicate classnames
- add repairOrphanedCatalog: remove catalog_items rows whose item_ids only reference missing items_base entries, strip orphaned ids from mixed rows - add repairDuplicateClassnames: merge items_base duplicates into one canonical row per classname, remap references, delete duplicate rows - add 'repair structural issues' checkbox + result display in audit UI
This commit is contained in:
openhands committed
1 parent
bee55e1fd4
commit
6fc14b9b84
4 files changed
+384
-1
No files matched your search
@@ -96,6 +96,205 @@ export async function applyCatalogSql(
|
||||
return { applied, failed };
|
||||
}
|
||||
|
||||
export interface OrphanedCatalogResult {
|
||||
removed: number;
|
||||
cleaned: number;
|
||||
}
|
||||
|
||||
export interface DuplicateClassnameResult {
|
||||
merged: number;
|
||||
rowsRemoved: number;
|
||||
remapped: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove catalog_items rows whose item_ids only reference non-existent
|
||||
* items_base entries, and strip orphaned ids from mixed rows. Also deletes
|
||||
* catalog_items_limited rows that pointed at the removed catalog entries so no
|
||||
* new orphans are left behind.
|
||||
*/
|
||||
export async function repairOrphanedCatalog(): Promise<OrphanedCatalogResult> {
|
||||
const [itemRows] = (await db.execute(sql`
|
||||
SELECT id FROM items_base
|
||||
`)) as unknown as [Array<{ id: number }>, unknown];
|
||||
const valid = new Set<number>();
|
||||
for (const row of itemRows) valid.add(Number(row.id));
|
||||
|
||||
const [catalogRows] = (await db.execute(sql`
|
||||
SELECT id, item_ids FROM catalog_items
|
||||
`)) as unknown as [Array<{ id: number; item_ids: string }>, unknown];
|
||||
|
||||
let removed = 0;
|
||||
let cleaned = 0;
|
||||
|
||||
for (const row of catalogRows) {
|
||||
const ids = (row.item_ids ?? "")
|
||||
.split(";")
|
||||
.map((s) => Number(s.trim()))
|
||||
.filter(Boolean);
|
||||
if (ids.length === 0) continue;
|
||||
const kept = ids.filter((id) => valid.has(id));
|
||||
if (kept.length === ids.length) continue;
|
||||
|
||||
if (kept.length === 0) {
|
||||
await db.execute(sql`
|
||||
DELETE FROM catalog_items_limited WHERE catalog_item_id = ${row.id}
|
||||
`);
|
||||
await db.execute(sql`DELETE FROM catalog_items WHERE id = ${row.id}`);
|
||||
removed++;
|
||||
} else {
|
||||
await db.execute(sql`
|
||||
UPDATE catalog_items SET item_ids = ${kept.join(";")} WHERE id = ${row.id}
|
||||
`);
|
||||
cleaned++;
|
||||
}
|
||||
}
|
||||
|
||||
return { removed, cleaned };
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge duplicate classnames in items_base into a single canonical row per
|
||||
* classname, remapping every reference (catalog item id lists, player items,
|
||||
* presents, marketplace, etc.) to the canonical id, then delete the duplicate
|
||||
* rows. The canonical id is the one players actually purchase through
|
||||
* catalog_items when present, otherwise the lowest id.
|
||||
*/
|
||||
export async function repairDuplicateClassnames(): Promise<DuplicateClassnameResult> {
|
||||
const [itemRows] = (await db.execute(sql`
|
||||
SELECT id, item_name FROM items_base
|
||||
`)) as unknown as [Array<{ id: number; item_name: string }>, unknown];
|
||||
|
||||
const byName = new Map<string, number[]>();
|
||||
for (const row of itemRows) {
|
||||
const arr = byName.get(row.item_name) ?? [];
|
||||
arr.push(Number(row.id));
|
||||
byName.set(row.item_name, arr);
|
||||
}
|
||||
|
||||
const [catalogRows] = (await db.execute(sql`
|
||||
SELECT item_ids FROM catalog_items
|
||||
`)) as unknown as [Array<{ item_ids: string }>, unknown];
|
||||
const referenced = new Set<number>();
|
||||
for (const row of catalogRows) {
|
||||
for (const part of (row.item_ids ?? "").split(";")) {
|
||||
const id = Number(part.trim());
|
||||
if (id) referenced.add(id);
|
||||
}
|
||||
}
|
||||
|
||||
const remap = new Map<number, number>();
|
||||
let merged = 0;
|
||||
for (const ids of byName.values()) {
|
||||
if (ids.length <= 1) continue;
|
||||
merged++;
|
||||
const sorted = [...ids].sort((a, b) => a - b);
|
||||
const canonical = sorted.find((id) => referenced.has(id)) ?? sorted[0];
|
||||
for (const id of ids) {
|
||||
if (id !== canonical) remap.set(id, canonical);
|
||||
}
|
||||
}
|
||||
|
||||
if (remap.size === 0) return { merged: 0, rowsRemoved: 0, remapped: 0 };
|
||||
|
||||
let remapped = 0;
|
||||
|
||||
// Semicolon-separated item_ids columns.
|
||||
const listTables: Array<{ table: string; idCol: string }> = [
|
||||
{ table: "catalog_items", idCol: "id" },
|
||||
{ table: "catalog_items_bc", idCol: "id" },
|
||||
{ table: "logs_shop_purchases", idCol: "id" },
|
||||
];
|
||||
for (const { table, idCol } of listTables) {
|
||||
const [rows] = (await db.execute(
|
||||
sql.raw(`SELECT ${idCol} AS pk, item_ids AS v FROM ${table}`),
|
||||
)) as unknown as [Array<{ pk: number; v: string | null }>, unknown];
|
||||
for (const row of rows) {
|
||||
if (!row.v) continue;
|
||||
const parts = row.v
|
||||
.split(";")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
let changed = false;
|
||||
const out: number[] = [];
|
||||
const seen = new Set<number>();
|
||||
for (const part of parts) {
|
||||
const id = Number(part);
|
||||
const mapped = remap.get(id) ?? id;
|
||||
if (mapped !== id) changed = true;
|
||||
if (seen.has(mapped)) continue;
|
||||
seen.add(mapped);
|
||||
out.push(mapped);
|
||||
}
|
||||
if (!changed) continue;
|
||||
await db.execute(
|
||||
sql.raw(
|
||||
`UPDATE ${table} SET item_ids = '${out.join(";")}' WHERE ${idCol} = ${row.pk}`,
|
||||
),
|
||||
);
|
||||
remapped++;
|
||||
}
|
||||
}
|
||||
|
||||
// Single item_id columns that reference items_base.id. Remap in chunks via
|
||||
// a single CASE statement per table to keep the query count low even with
|
||||
// thousands of duplicate rows.
|
||||
const singleTables = [
|
||||
"catalog_items_limited",
|
||||
"items",
|
||||
"items_presents",
|
||||
"pet_items",
|
||||
"pet_foods",
|
||||
"pet_drinks",
|
||||
"items_hoppers",
|
||||
"marketplace_items",
|
||||
"items_highscore_data",
|
||||
"items_crackable",
|
||||
"builders_club_items",
|
||||
"recycler_prizes",
|
||||
"gift_wrappers",
|
||||
"youtube_playlists",
|
||||
"room_trade_log_items",
|
||||
"calendar_rewards",
|
||||
"room_trax_playlist",
|
||||
"trax_playlist",
|
||||
"website_event_prizes",
|
||||
"room_templates_items",
|
||||
"crafting_recipes_ingredients",
|
||||
"website_rare_values",
|
||||
"logs_economy",
|
||||
];
|
||||
|
||||
const entries = [...remap.entries()];
|
||||
const CHUNK = 400;
|
||||
for (const table of singleTables) {
|
||||
for (let i = 0; i < entries.length; i += CHUNK) {
|
||||
const chunk = entries.slice(i, i + CHUNK);
|
||||
const cases = chunk
|
||||
.map(([dup, canonical]) => `WHEN ${dup} THEN ${canonical}`)
|
||||
.join(" ");
|
||||
const dupList = chunk.map(([dup]) => `${dup}`).join(",");
|
||||
try {
|
||||
const [result] = (await db.execute(
|
||||
sql.raw(
|
||||
`UPDATE ${table} SET item_id = CASE item_id ${cases} ELSE item_id END WHERE item_id IN (${dupList})`,
|
||||
),
|
||||
)) as unknown as [Record<string, unknown>, unknown];
|
||||
remapped += Number(result.affectedRows ?? 0);
|
||||
} catch {
|
||||
// Table may not exist on some hotel schemas — skip it.
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const dupIds = entries.map(([dup]) => dup);
|
||||
await db.execute(
|
||||
sql.raw(`DELETE FROM items_base WHERE id IN (${dupIds.join(",")})`),
|
||||
);
|
||||
|
||||
return { merged, rowsRemoved: dupIds.length, remapped };
|
||||
}
|
||||
|
||||
export interface FurniDataRepairResult {
|
||||
added: number;
|
||||
removedDuplicates: number;
|
||||
|
||||
Reference in new issue
Block a user