import { readdir } from "node:fs/promises"; import { sql } from "drizzle-orm"; import { db } from "@/lib/db"; import { fetchSourceFurnidata } from "@/lib/services/clone-import"; import { listSources } from "@/lib/services/clone-sources"; import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs"; import { applyCatalogSql, generateCatalogSql, type MissingCatalogEntry, repairDuplicateClassnames, repairFurniData, repairOrphanedCatalog, } from "./catalog-repair"; import { ensureDirectories } from "./furni-import"; import { repairMissingIcons } from "./repair-icons"; import { repairMissingNitros } from "./repair-nitros"; export interface AuditEvent { type: | "progress" | "items_loaded" | "catalog_loaded" | "checking" | "repair_complete" | "sql_ready" | "sql_applied" | "furnidata_repair_complete" | "structure_repair_complete" | "batch_complete" | "error"; message?: string; current?: number; total?: number; section?: string; issues?: AuditIssue[]; missingFromSources?: MissingFromSource[]; summary?: AuditSummary; repair?: RepairStats; repairNitro?: RepairStats; sql?: string; sqlCount?: number; applied?: number; sqlFailed?: number; furniData?: { added: number; removedDuplicates: number }; structure?: { orphanedRemoved: number; orphanedCleaned: number; duplicatesMerged: number; duplicateRowsRemoved: number; remappedReferences: number; }; } export interface RepairStats { succeeded: number; failed: number; skipped: number; stillMissing: number; } export interface AuditIssue { type: string; severity: "error" | "warning" | "info"; classname?: string; itemId?: number; catalogId?: number; message: string; sourceName?: string; } export interface MissingFromSource { sourceName: string; sourceId: string; classname: string; itemName: string; } export interface AuditSummary { totalItemsBase: number; totalCatalogItems: number; missingCatalog: number; orphanedCatalog: number; missingNitro: number; missingIcon: number; duplicates: number; missingFromSourcesTotal: number; iconsRepaired: number; iconsStillMissing: number; nitrosRepaired: number; nitrosStillMissing: number; catalogSqlCount: number; catalogSqlApplied: number; furniDataAdded: number; furniDataDuplicatesRemoved: number; orphanedRemoved: number; orphanedCleaned: number; duplicatesMerged: number; duplicateRowsRemoved: number; remappedReferences: number; } export interface CatalogAuditOptions { repair?: boolean; repairNitros?: boolean; sql?: boolean; applySql?: boolean; repairFurniData?: boolean; repairStructure?: boolean; } export async function runCatalogAudit( onEvent?: (evt: AuditEvent) => void, options?: CatalogAuditOptions, ): Promise { let issues: AuditIssue[] = []; const missingFromSources: MissingFromSource[] = []; try { await ensureDirectories(); } catch (err) { onEvent?.({ type: "error", message: `ensureDirectories failed: ${(err as Error).message}`, }); return; } let iconDir: string; let nitroDir: string; try { const dirs = await getFurniAssetDirs(); iconDir = dirs.iconDir; nitroDir = dirs.nitroDir; } catch (err) { onEvent?.({ type: "error", message: `getFurniAssetDirs failed: ${(err as Error).message}`, }); return; } const [iconFiles, nitroFiles] = await Promise.all([ readdir(iconDir).catch(() => [] as string[]), readdir(nitroDir).catch(() => [] as string[]), ]); const iconSet = new Set(iconFiles); const nitroSet = new Set(nitroFiles); onEvent?.({ type: "progress", message: "Loading items_base…" }); let items: Array<{ id: number; item_name: string; public_name: string; type: string; }>; try { const [rows] = (await db.execute(sql` SELECT id, item_name, public_name, type FROM items_base ORDER BY id `)) as unknown as [ Array<{ id: number; item_name: string; public_name: string; type: string; }>, unknown, ]; items = rows; } catch (err) { onEvent?.({ type: "error", message: `items_base query failed: ${(err as Error).message}`, }); return; } onEvent?.({ type: "items_loaded", total: items.length }); onEvent?.({ type: "progress", message: "Loading catalog_items…" }); let catalogItems: Array<{ id: number; item_ids: string; catalog_name: string; }>; try { const [rows] = (await db.execute(sql` SELECT id, item_ids, catalog_name FROM catalog_items ORDER BY id `)) as unknown as [ Array<{ id: number; item_ids: string; catalog_name: string }>, unknown, ]; catalogItems = rows; } catch (err) { onEvent?.({ type: "error", message: `catalog_items query failed: ${(err as Error).message}`, }); return; } onEvent?.({ type: "catalog_loaded", total: catalogItems.length }); const itemIdsWithCatalog = new Set(); for (const ci of catalogItems) { const ids = ci.item_ids.split(";").map(Number).filter(Boolean); for (const id of ids) { itemIdsWithCatalog.add(id); } } const itemById = new Map(); const itemByName = new Map(); const classnameCount = new Map(); for (const item of items) { itemById.set(item.id, item); itemByName.set(item.item_name, item); classnameCount.set( item.item_name, (classnameCount.get(item.item_name) ?? 0) + 1, ); } onEvent?.({ type: "checking", section: "missing_catalog", total: items.length, }); const missingCatalogEntries: MissingCatalogEntry[] = []; for (let i = 0; i < items.length; i++) { const item = items[i]; if (!itemIdsWithCatalog.has(item.id)) { issues.push({ type: "missing_catalog", severity: "warning", classname: item.item_name, itemId: item.id, message: `"${item.item_name}" (ID ${item.id}) has no catalog_items entry — not purchasable`, }); // Badge items use .gif files, not .nitro bundles. if ( !item.item_name.startsWith("badge_") && nitroSet.has(`${item.item_name}.nitro`) ) { missingCatalogEntries.push({ classname: item.item_name, itemId: item.id, itemType: item.type, }); } } if (i % 100 === 0) { onEvent?.({ type: "checking", section: "missing_catalog", current: i, total: items.length, }); } } onEvent?.({ type: "checking", section: "duplicates" }); for (const [name, count] of classnameCount) { if (count > 1) { issues.push({ type: "duplicate_classname", severity: "error", classname: name, message: `"${name}" appears ${count} times in items_base`, }); } } onEvent?.({ type: "checking", section: "orphaned_catalog", total: catalogItems.length, }); for (let i = 0; i < catalogItems.length; i++) { const ci = catalogItems[i]; const ids = ci.item_ids.split(";").map(Number).filter(Boolean); for (const id of ids) { if (!itemById.has(id)) { issues.push({ type: "orphaned_catalog", severity: "error", catalogId: ci.id, message: `catalog_items ID ${ci.id} ("${ci.catalog_name}") references items_base ID ${id} which does not exist`, }); } } if (i % 200 === 0) { onEvent?.({ type: "checking", section: "orphaned_catalog", current: i, total: catalogItems.length, }); } } onEvent?.({ type: "checking", section: "missing_nitro", total: items.length, }); const missingNitroClassnames: string[] = []; for (let i = 0; i < items.length; i++) { const item = items[i]; // Badge items use .gif files, not .nitro bundles. if (item.item_name.startsWith("badge_")) continue; if (!nitroSet.has(`${item.item_name}.nitro`)) { missingNitroClassnames.push(item.item_name); issues.push({ type: "missing_nitro", severity: "error", classname: item.item_name, itemId: item.id, message: `"${item.item_name}" (ID ${item.id}) has no .nitro file`, }); } if (i % 100 === 0) { onEvent?.({ type: "checking", section: "missing_nitro", current: i, total: items.length, }); } } onEvent?.({ type: "checking", section: "missing_icon", total: items.length }); const missingIconClassnames: string[] = []; for (let i = 0; i < items.length; i++) { const item = items[i]; // Badge items use .gif icons stored as badge__icon.png, // not _icon.png like regular furni. const iconFileName = `${item.item_name}_icon.png`; const badgeIconFileName = `badge_${item.item_name}_icon.png`; if (!iconSet.has(iconFileName) && !iconSet.has(badgeIconFileName)) { missingIconClassnames.push(item.item_name); issues.push({ type: "missing_icon", severity: "warning", classname: item.item_name, itemId: item.id, message: `"${item.item_name}" (ID ${item.id}) has no icon file`, }); } if (i % 100 === 0) { onEvent?.({ type: "checking", section: "missing_icon", current: i, total: items.length, }); } } let repairStats: RepairStats | null = null; if (options?.repair && missingIconClassnames.length > 0) { onEvent?.({ type: "progress", message: `Repairing ${missingIconClassnames.length} missing icons…`, }); try { const result = await repairMissingIcons((evt) => { if (evt.type === "progress") { onEvent?.({ type: "checking", section: "repair_icons", current: evt.index, total: evt.total, message: evt.status === "done" ? `Repaired ${evt.classname}` : evt.status === "failed" ? `Failed: ${evt.classname}` : undefined, }); } else if (evt.type === "error") { onEvent?.({ type: "error", message: evt.message }); } }); // Re-check which of the previously-missing icons are still missing // and drop the ones the repair fixed from the issue list. const stillMissing: string[] = []; for (const classname of missingIconClassnames) { if (!iconSet.has(`${classname}_icon.png`)) { stillMissing.push(classname); } } issues = issues.filter( (i) => i.type !== "missing_icon" || stillMissing.includes(i.classname ?? ""), ); repairStats = { succeeded: result.succeeded, failed: result.failed, skipped: result.skipped, stillMissing: stillMissing.length, }; onEvent?.({ type: "repair_complete", repair: repairStats }); } catch (err) { onEvent?.({ type: "error", message: `Icon repair failed: ${(err as Error).message}`, }); } } let nitroRepairStats: RepairStats | null = null; if (options?.repairNitros && missingNitroClassnames.length > 0) { onEvent?.({ type: "progress", message: `Repairing ${missingNitroClassnames.length} missing .nitro files…`, }); try { const result = await repairMissingNitros((evt) => { if (evt.type === "progress") { onEvent?.({ type: "checking", section: "repair_nitros", current: evt.index, total: evt.total, message: evt.status === "done" ? `Repaired ${evt.classname}` : evt.status === "failed" ? `Failed: ${evt.classname}` : undefined, }); } else if (evt.type === "error") { onEvent?.({ type: "error", message: evt.message }); } }); // Re-check which of the previously-missing nitros are still missing // and drop the ones the repair fixed from the issue list. const stillMissing: string[] = []; for (const classname of missingNitroClassnames) { if (!nitroSet.has(`${classname}.nitro`)) { stillMissing.push(classname); } } issues = issues.filter( (i) => i.type !== "missing_nitro" || stillMissing.includes(i.classname ?? ""), ); nitroRepairStats = { succeeded: result.succeeded, failed: result.failed, skipped: result.skipped, stillMissing: stillMissing.length, }; onEvent?.({ type: "repair_complete", repairNitro: nitroRepairStats, }); } catch (err) { onEvent?.({ type: "error", message: `.nitro repair failed: ${(err as Error).message}`, }); } } // ── Structural repair: orphaned catalog refs + duplicate classnames ── let orphanedRemoved = 0; let orphanedCleaned = 0; let duplicatesMerged = 0; let duplicateRowsRemoved = 0; let remappedReferences = 0; if (options?.repairStructure) { onEvent?.({ type: "progress", message: "Repairing structural issues (orphaned refs + duplicates)…", }); try { const orphanResult = await repairOrphanedCatalog(); orphanedRemoved = orphanResult.removed; orphanedCleaned = orphanResult.cleaned; // Re-derive orphaned_catalog issues from the fixed data. const [afterCatalog] = (await db.execute(sql` SELECT id, item_ids, catalog_name FROM catalog_items ORDER BY id `)) as unknown as [ Array<{ id: number; item_ids: string; catalog_name: string }>, unknown, ]; const orphanedCatalogIds = new Set(); for (const ci of afterCatalog) { const ids = ci.item_ids.split(";").map(Number).filter(Boolean); for (const id of ids) { if (!itemById.has(id)) orphanedCatalogIds.add(ci.id); } } issues = issues.filter( (i) => i.type !== "orphaned_catalog" || orphanedCatalogIds.has(i.catalogId ?? -1), ); const duplicateResult = await repairDuplicateClassnames(); duplicatesMerged = duplicateResult.merged; duplicateRowsRemoved = duplicateResult.rowsRemoved; remappedReferences = duplicateResult.remapped; // Re-derive duplicate_classname issues from the fixed data. const [afterItems] = (await db.execute(sql` SELECT item_name FROM items_base `)) as unknown as [Array<{ item_name: string }>, unknown]; const afterCount = new Map(); for (const row of afterItems) { afterCount.set(row.item_name, (afterCount.get(row.item_name) ?? 0) + 1); } const dupNames = new Set( [...afterCount].filter(([, count]) => count > 1).map(([name]) => name), ); issues = issues.filter( (i) => i.type !== "duplicate_classname" || dupNames.has(i.classname ?? ""), ); onEvent?.({ type: "structure_repair_complete", structure: { orphanedRemoved, orphanedCleaned, duplicatesMerged, duplicateRowsRemoved, remappedReferences, }, }); } catch (err) { onEvent?.({ type: "error", message: `Structural repair failed: ${(err as Error).message}`, }); } } // ── Catalog SQL generation / apply ──────────────────────────────── let catalogSqlCount = 0; let catalogSqlApplied = 0; if ((options?.sql || options?.applySql) && missingCatalogEntries.length > 0) { onEvent?.({ type: "progress", message: `Generating catalog SQL for ${missingCatalogEntries.length} items…`, }); try { const result = await generateCatalogSql(missingCatalogEntries); catalogSqlCount = result.count; onEvent?.({ type: "sql_ready", sql: result.sql, sqlCount: result.count, }); if (options.applySql) { const applyResult = await applyCatalogSql(result.statements); catalogSqlApplied = applyResult.applied; onEvent?.({ type: "sql_applied", applied: applyResult.applied, sqlFailed: applyResult.failed, }); } } catch (err) { onEvent?.({ type: "error", message: `Catalog SQL failed: ${(err as Error).message}`, }); } } // ── FurnitureData.json repair (add missing + dedupe) ────────────── let furniDataAdded = 0; let furniDataDuplicatesRemoved = 0; if (options?.repairFurniData) { onEvent?.({ type: "progress", message: "Repairing FurnitureData.json…" }); try { const result = await repairFurniData(items); furniDataAdded = result.added; furniDataDuplicatesRemoved = result.removedDuplicates; onEvent?.({ type: "furnidata_repair_complete", furniData: { added: result.added, removedDuplicates: result.removedDuplicates, }, }); } catch (err) { onEvent?.({ type: "error", message: `FurnitureData repair failed: ${(err as Error).message}`, }); } } onEvent?.({ type: "progress", message: "Comparing with clone sources…" }); const sources = await listSources(); const sourceResults = await Promise.all( sources .filter((s) => s.furnidataUrl) .map(async (source) => { try { const entries = await fetchSourceFurnidata(source.furnidataUrl); return { source, entries, error: null, } as const; } catch { return { source, entries: [] as Array<{ classname: string; name: string }>, error: true, } as const; } }), ); for (const result of sourceResults) { if (result.error) { issues.push({ type: "source_fetch_failed", severity: "error", sourceName: result.source.name, message: `Failed to fetch furnidata from "${result.source.name}"`, }); continue; } for (const entry of result.entries) { if (!itemByName.has(entry.classname)) { missingFromSources.push({ sourceName: result.source.name, sourceId: result.source.id, classname: entry.classname, itemName: entry.name, }); } } } issues.sort((a, b) => { const sev = { error: 0, warning: 1, info: 2 }; return (sev[a.severity] ?? 0) - (sev[b.severity] ?? 0); }); const summary: AuditSummary = { totalItemsBase: items.length, totalCatalogItems: catalogItems.length, missingCatalog: issues.filter((i) => i.type === "missing_catalog").length, orphanedCatalog: issues.filter((i) => i.type === "orphaned_catalog").length, missingNitro: issues.filter((i) => i.type === "missing_nitro").length, missingIcon: issues.filter((i) => i.type === "missing_icon").length, duplicates: issues.filter((i) => i.type === "duplicate_classname").length, missingFromSourcesTotal: missingFromSources.length, iconsRepaired: repairStats?.succeeded ?? 0, iconsStillMissing: repairStats?.stillMissing ?? missingIconClassnames.length, nitrosRepaired: nitroRepairStats?.succeeded ?? 0, nitrosStillMissing: nitroRepairStats?.stillMissing ?? missingNitroClassnames.length, catalogSqlCount, catalogSqlApplied, furniDataAdded, furniDataDuplicatesRemoved, orphanedRemoved, orphanedCleaned, duplicatesMerged, duplicateRowsRemoved, remappedReferences, }; onEvent?.({ type: "batch_complete", issues, missingFromSources, summary, }); }