From 5308ce6a1227c4380caa6bfd67c25f422557dd85 Mon Sep 17 00:00:00 2001 From: openhands Date: Sun, 2 Aug 2026 19:12:28 +0200 Subject: [PATCH] feat: parallelize audit repair and add nitro/SQL repair options - Parallelize icon and nitro downloads in the audit repair with a sliding-window worker pool (6 concurrent) to speed up large catalogs - Add repairMissingNitros: fetch missing .nitro bundles from configured nitro sources (Wibbo default), validating each bundle before writing - Add catalog-repair service: generate/apply catalog_items SQL for furni missing a catalog entry and repair FurnitureData.json (add missing + dedupe classnames) - Wire all options through the audit API and client UI with live progress and result stats (icons, nitros, SQL, furnidata) - Add Wibbo as default nitro source alongside existing icon sources - Ignore runtime furni assets downloaded into public/ during repair --- .gitignore | 5 + src/app/admin/import/audit/audit-client.tsx | 280 +++++++++++++++++- src/app/api/admin/import/audit/route.ts | 97 ++++--- src/lib/services/catalog-audit.ts | 240 ++++++++++++++- src/lib/services/catalog-repair.ts | 205 +++++++++++++ src/lib/services/clone-sources.ts | 10 + src/lib/services/repair-icons.ts | 82 ++++-- src/lib/services/repair-nitros.ts | 305 ++++++++++++++++++++ 8 files changed, 1166 insertions(+), 58 deletions(-) create mode 100644 src/lib/services/catalog-repair.ts create mode 100644 src/lib/services/repair-nitros.ts diff --git a/.gitignore b/.gitignore index 1f9e9026..a2005ee7 100644 --- a/.gitignore +++ b/.gitignore @@ -24,5 +24,10 @@ gitea # Runtime uploaded media (persistent, outside public/) storage/ +# Runtime downloaded furni assets (mirrored from gamedata / audit repair) +public/nitro-assets/bundled/furniture/ +public/swf/dcr/ +public/tmp/ + # Test coverage reports coverage/ diff --git a/src/app/admin/import/audit/audit-client.tsx b/src/app/admin/import/audit/audit-client.tsx index bc62f77a..9d896fb1 100644 --- a/src/app/admin/import/audit/audit-client.tsx +++ b/src/app/admin/import/audit/audit-client.tsx @@ -2,6 +2,7 @@ import { AlertCircle, + Check, CheckCircle2, ClipboardList, ExternalLink, @@ -39,6 +40,21 @@ interface AuditSummary { missingIcon: number; duplicates: number; missingFromSourcesTotal: number; + iconsRepaired: number; + iconsStillMissing: number; + nitrosRepaired: number; + nitrosStillMissing: number; + catalogSqlCount: number; + catalogSqlApplied: number; + furniDataAdded: number; + furniDataDuplicatesRemoved: number; +} + +interface RepairStats { + succeeded: number; + failed: number; + skipped: number; + stillMissing: number; } interface AuditEvent { @@ -50,6 +66,13 @@ interface AuditEvent { issues?: AuditIssue[]; missingFromSources?: MissingFromSource[]; summary?: AuditSummary; + repair?: RepairStats; + repairNitro?: RepairStats; + sql?: string; + sqlCount?: number; + applied?: number; + sqlFailed?: number; + furniData?: { added: number; removedDuplicates: number }; } const TYPE_LABELS: Record = { @@ -61,16 +84,72 @@ const TYPE_LABELS: Record = { source_fetch_failed: "Source fetch failed", }; +function AuditCheckbox({ + checked, + disabled, + onChange, + children, +}: { + checked: boolean; + disabled?: boolean; + onChange: (value: boolean) => void; + children: React.ReactNode; +}) { + return ( + // biome-ignore lint/a11y/useSemanticElements: custom checkbox so the checked state shows a visible checkmark on every admin theme + + ); +} + export function AuditClient() { const [loading, setLoading] = useState(false); const [progress, setProgress] = useState(""); const [activeTab, setActiveTab] = useState<"issues" | "sources">("issues"); + const [repairEnabled, setRepairEnabled] = useState(true); + const [repairNitros, setRepairNitros] = useState(true); + const [generateSql, setGenerateSql] = useState(false); + const [applySql, setApplySql] = useState(false); + const [repairFurniData, setRepairFurniData] = useState(false); const [issues, setIssues] = useState([]); const [missingFromSources, setMissingFromSources] = useState< MissingFromSource[] >([]); const [summary, setSummary] = useState(null); + const [repairStats, setRepairStats] = useState(null); + const [nitroRepairStats, setNitroRepairStats] = useState( + null, + ); + const [sqlText, setSqlText] = useState(null); + const [sqlApplied, setSqlApplied] = useState<{ + applied: number; + failed: number; + } | null>(null); + const [furniDataResult, setFurniDataResult] = useState<{ + added: number; + removedDuplicates: number; + } | null>(null); const [checkProgress, setCheckProgress] = useState<{ section: string; @@ -86,6 +165,11 @@ export function AuditClient() { setIssues([]); setMissingFromSources([]); setSummary(null); + setRepairStats(null); + setNitroRepairStats(null); + setSqlText(null); + setSqlApplied(null); + setFurniDataResult(null); setCheckProgress(null); setActiveTab("issues"); @@ -96,6 +180,14 @@ export function AuditClient() { const res = await adminFetch("/api/admin/import/audit", { method: "POST", signal: abort.signal, + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + repair: repairEnabled, + repairNitros, + sql: generateSql || applySql, + applySql, + repairFurniData, + }), }); if (!res.body) { @@ -155,6 +247,26 @@ export function AuditClient() { } } + if (evt.type === "repair_complete") { + if (evt.repair) setRepairStats(evt.repair); + if (evt.repairNitro) setNitroRepairStats(evt.repairNitro); + } + + if (evt.type === "sql_ready") { + setSqlText(evt.sql ?? null); + } + + if (evt.type === "sql_applied") { + setSqlApplied({ + applied: evt.applied ?? 0, + failed: evt.sqlFailed ?? 0, + }); + } + + if (evt.type === "furnidata_repair_complete") { + if (evt.furniData) setFurniDataResult(evt.furniData); + } + if (evt.type === "batch_complete") { setIssues(evt.issues ?? []); setMissingFromSources(evt.missingFromSources ?? []); @@ -197,11 +309,22 @@ export function AuditClient() { setLoading(false); abortRef.current = null; } - }, []); + }, [repairEnabled, repairNitros, generateSql, applySql, repairFurniData]); const errors = issues.filter((i) => i.severity === "error"); const warnings = issues.filter((i) => i.severity === "warning"); + const downloadSql = () => { + if (!sqlText) return; + const blob = new Blob([sqlText], { type: "text/sql" }); + const url = URL.createObjectURL(blob); + const a = document.createElement("a"); + a.href = url; + a.download = `catalog-repair-${new Date().toISOString().slice(0, 10)}.sql`; + a.click(); + URL.revokeObjectURL(url); + }; + return (
{!summary && ( @@ -214,6 +337,41 @@ export function AuditClient() { )} {loading ? progress || "Running…" : "Run Full Catalog Audit"} + + Repair missing icons + + + Repair missing .nitro + + + Generate catalog SQL + + + Apply SQL to database + + + Repair FurnitureData.json (add missing + dedupe) + {loading && (
+ {repairStats && ( +
+ + Icon repair:{" "} + + {repairStats.succeeded} fixed + + {repairStats.failed > 0 && ( + <> + {" · "} + + {repairStats.failed} failed + + + )} + {repairStats.skipped > 0 && ( + <> + {" · "} + + {repairStats.skipped} skipped + + + )} + {repairStats.stillMissing > 0 && ( + <> + {" · "} + + {repairStats.stillMissing} still missing + + + )} + +
+ )} + {nitroRepairStats && ( +
+ + Nitro repair:{" "} + + {nitroRepairStats.succeeded} fixed + + {nitroRepairStats.failed > 0 && ( + <> + {" · "} + + {nitroRepairStats.failed} failed + + + )} + {nitroRepairStats.skipped > 0 && ( + <> + {" · "} + + {nitroRepairStats.skipped} skipped + + + )} + {nitroRepairStats.stillMissing > 0 && ( + <> + {" · "} + + {nitroRepairStats.stillMissing} still missing + + + )} + +
+ )} + {sqlText && ( +
+ + Catalog SQL:{" "} + + {summary.catalogSqlCount} statement(s) generated + + {sqlApplied && ( + <> + {" · "} + + {sqlApplied.applied} applied + + {sqlApplied.failed > 0 && ( + <> + {" · "} + + {sqlApplied.failed} failed + + + )} + + )} + + +
+ )} + {furniDataResult && ( +
+ + FurnitureData repair:{" "} + + {furniDataResult.added} added + + {furniDataResult.removedDuplicates > 0 && ( + <> + {" · "} + + {furniDataResult.removedDuplicates} duplicates removed + + + )} + +
+ )} {/* Tabs */} @@ -455,6 +728,11 @@ export function AuditClient() { setSummary(null); setIssues([]); setMissingFromSources([]); + setRepairStats(null); + setNitroRepairStats(null); + setSqlText(null); + setSqlApplied(null); + setFurniDataResult(null); setProgress(""); setCheckProgress(null); }} diff --git a/src/app/api/admin/import/audit/route.ts b/src/app/api/admin/import/audit/route.ts index ef28994d..cc720542 100644 --- a/src/app/api/admin/import/audit/route.ts +++ b/src/app/api/admin/import/audit/route.ts @@ -6,41 +6,72 @@ function sseEvent(data: unknown): string { return `data: ${JSON.stringify(data)}\n\n`; } -export const POST = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => { - const encoder = new TextEncoder(); +export const POST = withAdmin( + { permission: PERMS.ASSETS_IMPORT }, + async (request) => { + const encoder = new TextEncoder(); - const stream = new ReadableStream({ - async start(controller) { - const send = (data: unknown) => { - try { - controller.enqueue(encoder.encode(sseEvent(data))); - } catch { - /* stream closed */ - } + let repair = false; + let repairNitros = false; + let sql = false; + let applySql = false; + let repairFurniData = false; + try { + const body = (await request.json().catch(() => ({}))) as { + repair?: unknown; + repairNitros?: unknown; + sql?: unknown; + applySql?: unknown; + repairFurniData?: unknown; }; + repair = body.repair === true; + repairNitros = body.repairNitros === true; + sql = body.sql === true; + applySql = body.applySql === true; + repairFurniData = body.repairFurniData === true; + } catch { + /* no body */ + } - try { - await runCatalogAudit(send); - } catch (err) { - send({ - type: "error", - message: `Audit threw: ${(err as Error).message}`, - }); - } + const stream = new ReadableStream({ + async start(controller) { + const send = (data: unknown) => { + try { + controller.enqueue(encoder.encode(sseEvent(data))); + } catch { + /* stream closed */ + } + }; - try { - controller.close(); - } catch { - /* ignore */ - } - }, - }); + try { + await runCatalogAudit(send, { + repair, + repairNitros, + sql, + applySql, + repairFurniData, + }); + } catch (err) { + send({ + type: "error", + message: `Audit threw: ${(err as Error).message}`, + }); + } - return new Response(stream, { - headers: { - "Content-Type": "text/event-stream", - "Cache-Control": "no-cache", - Connection: "keep-alive", - }, - }); -}); + try { + controller.close(); + } catch { + /* ignore */ + } + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }, + }); + }, +); diff --git a/src/lib/services/catalog-audit.ts b/src/lib/services/catalog-audit.ts index 85b26c28..b9c4231e 100644 --- a/src/lib/services/catalog-audit.ts +++ b/src/lib/services/catalog-audit.ts @@ -5,7 +5,15 @@ import { db } from "@/lib/db"; import { fetchSourceFurnidata } from "@/lib/services/clone-import"; import { listSources } from "@/lib/services/clone-sources"; import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs"; +import { + applyCatalogSql, + generateCatalogSql, + type MissingCatalogEntry, + repairFurniData, +} from "./catalog-repair"; import { ensureDirectories } from "./furni-import"; +import { repairMissingIcons } from "./repair-icons"; +import { repairMissingNitros } from "./repair-nitros"; export interface AuditEvent { type: @@ -13,6 +21,10 @@ export interface AuditEvent { | "items_loaded" | "catalog_loaded" | "checking" + | "repair_complete" + | "sql_ready" + | "sql_applied" + | "furnidata_repair_complete" | "batch_complete" | "error"; message?: string; @@ -22,6 +34,20 @@ export interface AuditEvent { issues?: AuditIssue[]; missingFromSources?: MissingFromSource[]; summary?: AuditSummary; + repair?: RepairStats; + repairNitro?: RepairStats; + sql?: string; + sqlCount?: number; + applied?: number; + sqlFailed?: number; + furniData?: { added: number; removedDuplicates: number }; +} + +export interface RepairStats { + succeeded: number; + failed: number; + skipped: number; + stillMissing: number; } export interface AuditIssue { @@ -50,12 +76,29 @@ export interface AuditSummary { missingIcon: number; duplicates: number; missingFromSourcesTotal: number; + iconsRepaired: number; + iconsStillMissing: number; + nitrosRepaired: number; + nitrosStillMissing: number; + catalogSqlCount: number; + catalogSqlApplied: number; + furniDataAdded: number; + furniDataDuplicatesRemoved: number; +} + +export interface CatalogAuditOptions { + repair?: boolean; + repairNitros?: boolean; + sql?: boolean; + applySql?: boolean; + repairFurniData?: boolean; } export async function runCatalogAudit( onEvent?: (evt: AuditEvent) => void, + options?: CatalogAuditOptions, ): Promise { - const issues: AuditIssue[] = []; + let issues: AuditIssue[] = []; const missingFromSources: MissingFromSource[] = []; try { @@ -164,6 +207,7 @@ export async function runCatalogAudit( total: items.length, }); + const missingCatalogEntries: MissingCatalogEntry[] = []; for (let i = 0; i < items.length; i++) { const item = items[i]; if (!itemIdsWithCatalog.has(item.id)) { @@ -174,6 +218,15 @@ export async function runCatalogAudit( itemId: item.id, message: `"${item.item_name}" (ID ${item.id}) has no catalog_items entry — not purchasable`, }); + // Only include items that actually have a .nitro asset — a catalog + // entry for a furni without a bundle would be un-placeable in-game. + if (existsSync(path.join(nitroDir, `${item.item_name}.nitro`))) { + missingCatalogEntries.push({ + classname: item.item_name, + itemId: item.id, + itemType: item.type, + }); + } } if (i % 100 === 0) { onEvent?.({ @@ -233,10 +286,12 @@ export async function runCatalogAudit( total: items.length, }); + const missingNitroClassnames: string[] = []; for (let i = 0; i < items.length; i++) { const item = items[i]; const nitroPath = path.join(nitroDir, `${item.item_name}.nitro`); if (!existsSync(nitroPath)) { + missingNitroClassnames.push(item.item_name); issues.push({ type: "missing_nitro", severity: "error", @@ -257,10 +312,12 @@ export async function runCatalogAudit( onEvent?.({ type: "checking", section: "missing_icon", total: items.length }); + const missingIconClassnames: string[] = []; for (let i = 0; i < items.length; i++) { const item = items[i]; const iconPath = path.join(iconDir, `${item.item_name}_icon.png`); if (!existsSync(iconPath)) { + missingIconClassnames.push(item.item_name); issues.push({ type: "missing_icon", severity: "warning", @@ -279,6 +336,177 @@ export async function runCatalogAudit( } } + let repairStats: RepairStats | null = null; + if (options?.repair && missingIconClassnames.length > 0) { + onEvent?.({ + type: "progress", + message: `Repairing ${missingIconClassnames.length} missing icons…`, + }); + + try { + const result = await repairMissingIcons((evt) => { + if (evt.type === "progress") { + onEvent?.({ + type: "checking", + section: "repair_icons", + current: evt.index, + total: evt.total, + message: + evt.status === "done" + ? `Repaired ${evt.classname}` + : evt.status === "failed" + ? `Failed: ${evt.classname}` + : undefined, + }); + } else if (evt.type === "error") { + onEvent?.({ type: "error", message: evt.message }); + } + }); + + // Re-check which of the previously-missing icons are still missing + // and drop the ones the repair fixed from the issue list. + const stillMissing: string[] = []; + for (const classname of missingIconClassnames) { + if (!existsSync(path.join(iconDir, `${classname}_icon.png`))) { + stillMissing.push(classname); + } + } + issues = issues.filter( + (i) => + i.type !== "missing_icon" || stillMissing.includes(i.classname ?? ""), + ); + + repairStats = { + succeeded: result.succeeded, + failed: result.failed, + skipped: result.skipped, + stillMissing: stillMissing.length, + }; + onEvent?.({ type: "repair_complete", repair: repairStats }); + } catch (err) { + onEvent?.({ + type: "error", + message: `Icon repair failed: ${(err as Error).message}`, + }); + } + } + + let nitroRepairStats: RepairStats | null = null; + if (options?.repairNitros && missingNitroClassnames.length > 0) { + onEvent?.({ + type: "progress", + message: `Repairing ${missingNitroClassnames.length} missing .nitro files…`, + }); + + try { + const result = await repairMissingNitros((evt) => { + if (evt.type === "progress") { + onEvent?.({ + type: "checking", + section: "repair_nitros", + current: evt.index, + total: evt.total, + message: + evt.status === "done" + ? `Repaired ${evt.classname}` + : evt.status === "failed" + ? `Failed: ${evt.classname}` + : undefined, + }); + } else if (evt.type === "error") { + onEvent?.({ type: "error", message: evt.message }); + } + }); + + // Re-check which of the previously-missing nitros are still missing + // and drop the ones the repair fixed from the issue list. + const stillMissing: string[] = []; + for (const classname of missingNitroClassnames) { + if (!existsSync(path.join(nitroDir, `${classname}.nitro`))) { + stillMissing.push(classname); + } + } + issues = issues.filter( + (i) => + i.type !== "missing_nitro" || + stillMissing.includes(i.classname ?? ""), + ); + + nitroRepairStats = { + succeeded: result.succeeded, + failed: result.failed, + skipped: result.skipped, + stillMissing: stillMissing.length, + }; + onEvent?.({ + type: "repair_complete", + repairNitro: nitroRepairStats, + }); + } catch (err) { + onEvent?.({ + type: "error", + message: `.nitro repair failed: ${(err as Error).message}`, + }); + } + } + + // ── Catalog SQL generation / apply ──────────────────────────────── + let catalogSqlCount = 0; + let catalogSqlApplied = 0; + if ((options?.sql || options?.applySql) && missingCatalogEntries.length > 0) { + onEvent?.({ + type: "progress", + message: `Generating catalog SQL for ${missingCatalogEntries.length} items…`, + }); + try { + const result = await generateCatalogSql(missingCatalogEntries); + catalogSqlCount = result.count; + onEvent?.({ + type: "sql_ready", + sql: result.sql, + sqlCount: result.count, + }); + if (options.applySql) { + const applyResult = await applyCatalogSql(result.statements); + catalogSqlApplied = applyResult.applied; + onEvent?.({ + type: "sql_applied", + applied: applyResult.applied, + sqlFailed: applyResult.failed, + }); + } + } catch (err) { + onEvent?.({ + type: "error", + message: `Catalog SQL failed: ${(err as Error).message}`, + }); + } + } + + // ── FurnitureData.json repair (add missing + dedupe) ────────────── + let furniDataAdded = 0; + let furniDataDuplicatesRemoved = 0; + if (options?.repairFurniData) { + onEvent?.({ type: "progress", message: "Repairing FurnitureData.json…" }); + try { + const result = await repairFurniData(items); + furniDataAdded = result.added; + furniDataDuplicatesRemoved = result.removedDuplicates; + onEvent?.({ + type: "furnidata_repair_complete", + furniData: { + added: result.added, + removedDuplicates: result.removedDuplicates, + }, + }); + } catch (err) { + onEvent?.({ + type: "error", + message: `FurnitureData repair failed: ${(err as Error).message}`, + }); + } + } + onEvent?.({ type: "progress", message: "Comparing with clone sources…" }); const sources = await listSources(); @@ -319,6 +547,16 @@ export async function runCatalogAudit( missingIcon: issues.filter((i) => i.type === "missing_icon").length, duplicates: issues.filter((i) => i.type === "duplicate_classname").length, missingFromSourcesTotal: missingFromSources.length, + iconsRepaired: repairStats?.succeeded ?? 0, + iconsStillMissing: + repairStats?.stillMissing ?? missingIconClassnames.length, + nitrosRepaired: nitroRepairStats?.succeeded ?? 0, + nitrosStillMissing: + nitroRepairStats?.stillMissing ?? missingNitroClassnames.length, + catalogSqlCount, + catalogSqlApplied, + furniDataAdded, + furniDataDuplicatesRemoved, }; onEvent?.({ diff --git a/src/lib/services/catalog-repair.ts b/src/lib/services/catalog-repair.ts new file mode 100644 index 00000000..70241615 --- /dev/null +++ b/src/lib/services/catalog-repair.ts @@ -0,0 +1,205 @@ +import { sql } from "drizzle-orm"; +import { db } from "@/lib/db"; +import { + readFurniData, + withFurniDataLock, + writeFurniData, +} from "@/lib/services/furni-data"; +import { + allocateCatalogItemId, + autoPriceFurni, + getOrCreateCategoryPage, +} from "@/lib/services/furni-import"; + +export interface MissingCatalogEntry { + classname: string; + itemId: number; + itemType: string; +} + +export interface CatalogSqlResult { + statements: string[]; + sql: string; + count: number; +} + +function escSqlLiteral(value: string): string { + return value.replace(/'/g, "''"); +} + +/** + * Build standalone `catalog_items` INSERT statements for items_base entries + * that exist (with assets) but have no catalog entry yet. Ids and prices are + * resolved the same way the app's own imports do it (autoPriceFurni + category + * page + next catalog id), so running the SQL matches an app-managed import. + */ +export async function generateCatalogSql( + entries: MissingCatalogEntry[], +): Promise { + const statements: string[] = [ + "-- AtomCMS catalog repair", + `-- Generated ${new Date().toISOString()}`, + `-- ${entries.length} item(s) to add`, + "START TRANSACTION;", + ]; + + for (const entry of entries) { + let pageId: number; + try { + pageId = await getOrCreateCategoryPage(entry.classname, entry.itemType); + } catch { + continue; + } + const price = autoPriceFurni(entry.classname); + let nextId: number; + try { + nextId = await allocateCatalogItemId(async (id) => id); + } catch { + continue; + } + statements.push( + `INSERT INTO catalog_items (id, page_id, item_ids, catalog_name, cost_credits, cost_points, points_type, amount, order_number, offer_id, extradata) VALUES (${nextId}, ${pageId}, ${entry.itemId}, '${escSqlLiteral(entry.classname)}', ${price.credits}, ${price.points}, ${price.pointsType}, 1, 1, '-1', '');`, + ); + } + + statements.push("COMMIT;"); + return { statements, sql: statements.join("\n"), count: entries.length }; +} + +/** Execute generated INSERT statements directly against the database. */ +export async function applyCatalogSql( + statements: string[], +): Promise<{ applied: number; failed: number }> { + let applied = 0; + let failed = 0; + for (const statement of statements) { + const trimmed = statement.trim(); + if ( + !trimmed || + trimmed.startsWith("--") || + trimmed === "START TRANSACTION;" || + trimmed === "COMMIT;" + ) { + continue; + } + try { + await db.execute(sql.raw(trimmed)); + applied++; + } catch { + failed++; + } + } + return { applied, failed }; +} + +export interface FurniDataRepairResult { + added: number; + removedDuplicates: number; +} + +function buildMinimalFurniEntry(item: { + id: number; + item_name: string; + public_name: string; +}): Record { + return { + id: item.id, + classname: item.item_name, + revision: 0, + category: "unknown", + defaultdir: 0, + xdim: 1, + ydim: 1, + partcolors: { color: [] }, + name: item.public_name || item.item_name, + description: "", + adurl: "", + offerid: item.id, + buyout: false, + rentofferid: -1, + rentbuyout: false, + bc: false, + excludeddynamic: false, + customparams: "", + specialtype: 0, + canstandon: false, + cansiton: false, + canlayon: false, + furniline: "", + environment: "", + rare: false, + }; +} + +/** + * Add FurnitureData.json entries for items_base rows that are missing from the + * file, and remove duplicate classnames (keeping the first occurrence). + */ +export async function repairFurniData( + items: Array<{ + id: number; + item_name: string; + public_name: string; + type: string; + }>, +): Promise { + if (items.length === 0) return { added: 0, removedDuplicates: 0 }; + + return withFurniDataLock(async () => { + const furniData = (await readFurniData()) as Record< + string, + { furnitype: Array> } + >; + + // 1) Deduplicate classnames per section (keep the first occurrence). + let removedDuplicates = 0; + for (const section of ["roomitemtypes", "wallitemtypes"] as const) { + const sec = furniData[section]; + if (!sec?.furnitype) continue; + const seen = new Set(); + const deduped = sec.furnitype.filter((e) => { + const classname = typeof e.classname === "string" ? e.classname : ""; + if (!classname) return true; + if (seen.has(classname)) { + removedDuplicates++; + return false; + } + seen.add(classname); + return true; + }); + if (deduped.length !== sec.furnitype.length) { + furniData[section].furnitype = deduped; + } + } + + // 2) Add entries for items_base rows missing from the file. + const existing = new Set(); + for (const section of ["roomitemtypes", "wallitemtypes"] as const) { + for (const e of furniData[section]?.furnitype ?? []) { + if (typeof e.classname === "string") existing.add(e.classname); + } + } + const added: Array<{ entry: Record; itemType: string }> = + []; + for (const item of items) { + if (existing.has(item.item_name)) continue; + added.push({ + entry: buildMinimalFurniEntry(item), + itemType: item.type, + }); + } + + // 3) Write once if anything changed. + if (removedDuplicates > 0 || added.length > 0) { + for (const { entry, itemType } of added) { + const sectionKey = itemType === "i" ? "wallitemtypes" : "roomitemtypes"; + if (!furniData[sectionKey]) { + furniData[sectionKey] = { furnitype: [] }; + } + furniData[sectionKey].furnitype.push(entry); + } + await writeFurniData(furniData as Record); + } + return { added: added.length, removedDuplicates }; + }); +} diff --git a/src/lib/services/clone-sources.ts b/src/lib/services/clone-sources.ts index c73e0400..b37be36b 100644 --- a/src/lib/services/clone-sources.ts +++ b/src/lib/services/clone-sources.ts @@ -35,6 +35,16 @@ export const DEFAULT_ICON_SOURCES: CloneSource[] = [ }, ]; +export const DEFAULT_NITRO_SOURCES: CloneSource[] = [ + { + id: "default-wibbo", + name: "Wibbo", + furnidataUrl: "", + nitroBaseUrl: "https://assets.wibbo.org/bundled/furniture", + iconBaseUrl: "", + }, +]; + export async function listSources(): Promise { const raw = (await siteSettings.get(KEY, "[]")) ?? "[]"; try { diff --git a/src/lib/services/repair-icons.ts b/src/lib/services/repair-icons.ts index ae55b380..fa81ad13 100644 --- a/src/lib/services/repair-icons.ts +++ b/src/lib/services/repair-icons.ts @@ -16,6 +16,28 @@ import { import { downloadFile } from "@/lib/services/import/core/download"; import { ensureDirectories } from "./furni-import"; +const REPAIR_CONCURRENCY = 6; + +/** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */ +async function runPool( + items: T[], + concurrency: number, + worker: (item: T, index: number) => Promise, +): Promise { + const limit = Math.min(Math.max(concurrency, 1), items.length); + if (limit === 0) return; + let next = 0; + await Promise.allSettled( + Array.from({ length: limit }, async () => { + while (true) { + const i = next++; + if (i >= items.length) break; + await worker(items[i], i); + } + }), + ); +} + function uniqueDirs(dirs: string[]): string[] { const seen = new Set(); const out: string[] = []; @@ -135,8 +157,7 @@ export async function repairMissingIcons( const total = items.length; onEvent?.({ type: "started", total }); - for (let i = 0; i < items.length; i++) { - const { item_name: classname } = items[i]; + const processItem = async (classname: string) => { const fileName = `${classname}_icon.png`; // A dir may already have the icon (e.g. the full gamedata) while the @@ -147,15 +168,7 @@ export async function repairMissingIcons( const missingDirs = iconDirs.filter((dir) => !presentDirs.includes(dir)); if (missingDirs.length === 0) { - skipped++; - onEvent?.({ - type: "progress", - classname, - status: "skipped", - index: i, - total, - }); - continue; + return { status: "skipped" } as const; } let ok = false; @@ -303,6 +316,29 @@ export async function repairMissingIcons( } if (ok) { + return { status: "done", warnings } as const; + } + return { + status: "failed", + message: warnings[0] || "could not obtain icon", + } as const; + }; + + await runPool(items, REPAIR_CONCURRENCY, async (item, i) => { + const classname = item.item_name; + const result = await processItem(classname); + if (result.status === "skipped") { + skipped++; + onEvent?.({ + type: "progress", + classname, + status: "skipped", + index: i, + total, + }); + return; + } + if (result.status === "done") { succeeded++; onEvent?.({ type: "progress", @@ -310,20 +346,20 @@ export async function repairMissingIcons( status: "done", index: i, total, - warnings: warnings.length ? warnings : undefined, - }); - } else { - failed++; - onEvent?.({ - type: "progress", - classname, - status: "failed", - index: i, - total, - message: warnings[0] || "could not obtain icon", + warnings: result.warnings.length ? result.warnings : undefined, }); + return; } - } + failed++; + onEvent?.({ + type: "progress", + classname, + status: "failed", + index: i, + total, + message: result.message, + }); + }); onEvent?.({ type: "batch_complete", diff --git a/src/lib/services/repair-nitros.ts b/src/lib/services/repair-nitros.ts new file mode 100644 index 00000000..8717402d --- /dev/null +++ b/src/lib/services/repair-nitros.ts @@ -0,0 +1,305 @@ +import { existsSync, promises as fs } from "node:fs"; +import os from "node:os"; +import path from "node:path"; +import { sql } from "drizzle-orm"; +import { db } from "@/lib/db"; +import { + type CloneSource, + DEFAULT_NITRO_SOURCES, + listSources, +} from "@/lib/services/clone-sources"; +import { + type FurniAssetWriteTargets, + getFurniAssetWriteTargets, +} from "@/lib/services/furni-asset-dirs"; +import { downloadFile } from "@/lib/services/import/core/download"; +import { parseNitroBundle } from "@/lib/services/swf/nitro-builder"; +import { ensureDirectories } from "./furni-import"; + +const REPAIR_CONCURRENCY = 6; + +/** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */ +async function runPool( + items: T[], + concurrency: number, + worker: (item: T, index: number) => Promise, +): Promise { + const limit = Math.min(Math.max(concurrency, 1), items.length); + if (limit === 0) return; + let next = 0; + await Promise.allSettled( + Array.from({ length: limit }, async () => { + while (true) { + const i = next++; + if (i >= items.length) break; + await worker(items[i], i); + } + }), + ); +} + +function uniqueDirs(dirs: string[]): string[] { + const seen = new Set(); + const out: string[] = []; + for (const dir of dirs) { + const key = process.platform === "win32" ? dir.toLowerCase() : dir; + if (seen.has(key)) continue; + seen.add(key); + out.push(dir); + } + return out; +} + +async function writeNitroToDirs( + nitro: Buffer, + dirs: string[], + fileName: string, +): Promise { + const errors: string[] = []; + for (const dir of dirs) { + try { + await fs.mkdir(dir, { recursive: true }); + await fs.writeFile(path.join(dir, fileName), nitro); + } catch (err) { + errors.push(`write .nitro to ${dir} failed: ${(err as Error).message}`); + } + } + return errors; +} + +async function copyNitroAcrossDirs( + srcDir: string, + destDirs: string[], + fileName: string, +): Promise { + const errors: string[] = []; + const src = path.join(srcDir, fileName); + for (const dir of destDirs) { + if (dir === srcDir) continue; + try { + await fs.mkdir(dir, { recursive: true }); + await fs.copyFile(src, path.join(dir, fileName)); + } catch (err) { + errors.push(`copy .nitro to ${dir} failed: ${(err as Error).message}`); + } + } + return errors; +} + +export interface RepairNitroEvent { + type: "started" | "progress" | "done" | "batch_complete" | "error"; + classname?: string; + status?: string; + index?: number; + total?: number; + message?: string; + warnings?: string[]; + succeeded?: number; + failed?: number; + skipped?: number; + duration?: number; +} + +export async function repairMissingNitros( + onEvent?: (evt: RepairNitroEvent) => void, +): Promise<{ succeeded: number; failed: number; skipped: number }> { + let succeeded = 0; + let failed = 0; + let skipped = 0; + + try { + await ensureDirectories(); + } catch (err) { + onEvent?.({ + type: "error", + message: `ensureDirectories failed: ${(err as Error).message}`, + }); + return { succeeded: 0, failed: 0, skipped: 0 }; + } + + let targets: FurniAssetWriteTargets; + try { + targets = await getFurniAssetWriteTargets(); + } catch (err) { + onEvent?.({ + type: "error", + message: `getFurniAssetWriteTargets failed: ${(err as Error).message}`, + }); + return { succeeded: 0, failed: 0, skipped: 0 }; + } + + // Every location the app mirrors furni assets into — the webroot (public/) + // plus the live gamedata (e.g. /var/www/Gamedata). Repair writes to all of + // them so both the CMS and the client see the .nitro. + const nitroDirs = uniqueDirs([ + targets.nitroDir, + ...targets.mirrorDirs.map((d) => d.nitroDir), + ]); + + let items: Array<{ id: number; item_name: string }>; + try { + const [rows] = (await db.execute(sql` + SELECT id, item_name FROM items_base ORDER BY id + `)) as unknown as [Array<{ id: number; item_name: string }>, unknown]; + items = rows; + } catch (err) { + onEvent?.({ + type: "error", + message: `Database query failed: ${(err as Error).message}`, + }); + return { succeeded: 0, failed: 0, skipped: 0 }; + } + + const total = items.length; + onEvent?.({ type: "started", total }); + + const processItem = async (classname: string) => { + const fileName = `${classname}.nitro`; + + const presentDirs = nitroDirs.filter((dir) => + existsSync(path.join(dir, fileName)), + ); + const missingDirs = nitroDirs.filter((dir) => !presentDirs.includes(dir)); + + if (missingDirs.length === 0) { + return { status: "skipped" } as const; + } + + let ok = false; + const warnings: string[] = []; + + // 1) Copy across mirrors — the .nitro already exists in one target (e.g. + // gamedata), so just fill the ones that are missing it. + if (presentDirs.length > 0) { + try { + const copyErrors = await copyNitroAcrossDirs( + presentDirs[0], + missingDirs, + fileName, + ); + warnings.push(...copyErrors); + ok = copyErrors.length === 0; + } catch (err) { + warnings.push( + `copy across .nitro dirs failed: ${(err as Error).message}`, + ); + } + } + + // 2) Download from configured or default nitro sources. + if (!ok) { + let sources: CloneSource[] = []; + try { + sources = (await listSources()).filter((s) => s.nitroBaseUrl); + } catch (err) { + warnings.push(`listSources failed: ${(err as Error).message}`); + } + if (sources.length === 0) sources = DEFAULT_NITRO_SOURCES; + + const nitroCandidates = [ + `${classname}.nitro`, + // Variant classnames (e.g. `table_silo_med*10`) often only exist + // on sources under their base name (e.g. `table_silo_med.nitro`) + // or with the `*` replaced by `_` (e.g. `table_silo_med_10.nitro`). + ...(classname.includes("*") + ? [ + `${classname.split("*")[0]}.nitro`, + `${classname.replace(/\*/g, "_")}.nitro`, + ] + : []), + ]; + + for (const source of sources) { + for (const nitroName of nitroCandidates) { + const tmpPath = path.join( + os.tmpdir(), + `.repair_nitro_${classname}_${Math.random().toString(36).slice(2, 8)}.nitro`, + ); + try { + const dl = await downloadFile( + `${source.nitroBaseUrl}/${nitroName}`, + tmpPath, + { maxRetries: 1 }, + ); + if (dl.ok) { + // Validate it is a real Nitro bundle before writing it + // anywhere (mirrors clone-import.ts). + const buffer = await fs.readFile(tmpPath); + parseNitroBundle(buffer); + const writeErrors = await writeNitroToDirs( + buffer, + missingDirs, + fileName, + ); + warnings.push(...writeErrors); + ok = writeErrors.length === 0; + break; + } + warnings.push(`${nitroName} not found on ${source.name}`); + } catch (err) { + warnings.push( + `download from ${source.name} failed: ${(err as Error).message}`, + ); + } finally { + await fs.unlink(tmpPath).catch(() => {}); + } + } + if (ok) break; + } + } + + if (ok) { + return { status: "done", warnings } as const; + } + return { + status: "failed", + message: warnings[0] || "could not obtain .nitro", + } as const; + }; + + await runPool(items, REPAIR_CONCURRENCY, async (item, i) => { + const classname = item.item_name; + const result = await processItem(classname); + if (result.status === "skipped") { + skipped++; + onEvent?.({ + type: "progress", + classname, + status: "skipped", + index: i, + total, + }); + return; + } + if (result.status === "done") { + succeeded++; + onEvent?.({ + type: "progress", + classname, + status: "done", + index: i, + total, + warnings: result.warnings.length ? result.warnings : undefined, + }); + return; + } + failed++; + onEvent?.({ + type: "progress", + classname, + status: "failed", + index: i, + total, + message: result.message, + }); + }); + + onEvent?.({ + type: "batch_complete", + succeeded, + failed, + skipped, + }); + + return { succeeded, failed, skipped }; +}