feat: parallelize audit repair and add nitro/SQL repair options
- Parallelize icon and nitro downloads in the audit repair with a sliding-window worker pool (6 concurrent) to speed up large catalogs - Add repairMissingNitros: fetch missing .nitro bundles from configured nitro sources (Wibbo default), validating each bundle before writing - Add catalog-repair service: generate/apply catalog_items SQL for furni missing a catalog entry and repair FurnitureData.json (add missing + dedupe classnames) - Wire all options through the audit API and client UI with live progress and result stats (icons, nitros, SQL, furnidata) - Add Wibbo as default nitro source alongside existing icon sources - Ignore runtime furni assets downloaded into public/ during repair
This commit is contained in:
openhands committed
1 parent
0c8e62357f
commit
5308ce6a12
8 files changed
+1166
-58
No files matched your search
@@ -5,7 +5,15 @@ import { db } from "@/lib/db";
|
||||
import { fetchSourceFurnidata } from "@/lib/services/clone-import";
|
||||
import { listSources } from "@/lib/services/clone-sources";
|
||||
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
||||
import {
|
||||
applyCatalogSql,
|
||||
generateCatalogSql,
|
||||
type MissingCatalogEntry,
|
||||
repairFurniData,
|
||||
} from "./catalog-repair";
|
||||
import { ensureDirectories } from "./furni-import";
|
||||
import { repairMissingIcons } from "./repair-icons";
|
||||
import { repairMissingNitros } from "./repair-nitros";
|
||||
|
||||
export interface AuditEvent {
|
||||
type:
|
||||
@@ -13,6 +21,10 @@ export interface AuditEvent {
|
||||
| "items_loaded"
|
||||
| "catalog_loaded"
|
||||
| "checking"
|
||||
| "repair_complete"
|
||||
| "sql_ready"
|
||||
| "sql_applied"
|
||||
| "furnidata_repair_complete"
|
||||
| "batch_complete"
|
||||
| "error";
|
||||
message?: string;
|
||||
@@ -22,6 +34,20 @@ export interface AuditEvent {
|
||||
issues?: AuditIssue[];
|
||||
missingFromSources?: MissingFromSource[];
|
||||
summary?: AuditSummary;
|
||||
repair?: RepairStats;
|
||||
repairNitro?: RepairStats;
|
||||
sql?: string;
|
||||
sqlCount?: number;
|
||||
applied?: number;
|
||||
sqlFailed?: number;
|
||||
furniData?: { added: number; removedDuplicates: number };
|
||||
}
|
||||
|
||||
export interface RepairStats {
|
||||
succeeded: number;
|
||||
failed: number;
|
||||
skipped: number;
|
||||
stillMissing: number;
|
||||
}
|
||||
|
||||
export interface AuditIssue {
|
||||
@@ -50,12 +76,29 @@ export interface AuditSummary {
|
||||
missingIcon: number;
|
||||
duplicates: number;
|
||||
missingFromSourcesTotal: number;
|
||||
iconsRepaired: number;
|
||||
iconsStillMissing: number;
|
||||
nitrosRepaired: number;
|
||||
nitrosStillMissing: number;
|
||||
catalogSqlCount: number;
|
||||
catalogSqlApplied: number;
|
||||
furniDataAdded: number;
|
||||
furniDataDuplicatesRemoved: number;
|
||||
}
|
||||
|
||||
export interface CatalogAuditOptions {
|
||||
repair?: boolean;
|
||||
repairNitros?: boolean;
|
||||
sql?: boolean;
|
||||
applySql?: boolean;
|
||||
repairFurniData?: boolean;
|
||||
}
|
||||
|
||||
export async function runCatalogAudit(
|
||||
onEvent?: (evt: AuditEvent) => void,
|
||||
options?: CatalogAuditOptions,
|
||||
): Promise<void> {
|
||||
const issues: AuditIssue[] = [];
|
||||
let issues: AuditIssue[] = [];
|
||||
const missingFromSources: MissingFromSource[] = [];
|
||||
|
||||
try {
|
||||
@@ -164,6 +207,7 @@ export async function runCatalogAudit(
|
||||
total: items.length,
|
||||
});
|
||||
|
||||
const missingCatalogEntries: MissingCatalogEntry[] = [];
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const item = items[i];
|
||||
if (!itemIdsWithCatalog.has(item.id)) {
|
||||
@@ -174,6 +218,15 @@ export async function runCatalogAudit(
|
||||
itemId: item.id,
|
||||
message: `"${item.item_name}" (ID ${item.id}) has no catalog_items entry — not purchasable`,
|
||||
});
|
||||
// Only include items that actually have a .nitro asset — a catalog
|
||||
// entry for a furni without a bundle would be un-placeable in-game.
|
||||
if (existsSync(path.join(nitroDir, `${item.item_name}.nitro`))) {
|
||||
missingCatalogEntries.push({
|
||||
classname: item.item_name,
|
||||
itemId: item.id,
|
||||
itemType: item.type,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (i % 100 === 0) {
|
||||
onEvent?.({
|
||||
@@ -233,10 +286,12 @@ export async function runCatalogAudit(
|
||||
total: items.length,
|
||||
});
|
||||
|
||||
const missingNitroClassnames: string[] = [];
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const item = items[i];
|
||||
const nitroPath = path.join(nitroDir, `${item.item_name}.nitro`);
|
||||
if (!existsSync(nitroPath)) {
|
||||
missingNitroClassnames.push(item.item_name);
|
||||
issues.push({
|
||||
type: "missing_nitro",
|
||||
severity: "error",
|
||||
@@ -257,10 +312,12 @@ export async function runCatalogAudit(
|
||||
|
||||
onEvent?.({ type: "checking", section: "missing_icon", total: items.length });
|
||||
|
||||
const missingIconClassnames: string[] = [];
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const item = items[i];
|
||||
const iconPath = path.join(iconDir, `${item.item_name}_icon.png`);
|
||||
if (!existsSync(iconPath)) {
|
||||
missingIconClassnames.push(item.item_name);
|
||||
issues.push({
|
||||
type: "missing_icon",
|
||||
severity: "warning",
|
||||
@@ -279,6 +336,177 @@ export async function runCatalogAudit(
|
||||
}
|
||||
}
|
||||
|
||||
let repairStats: RepairStats | null = null;
|
||||
if (options?.repair && missingIconClassnames.length > 0) {
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
message: `Repairing ${missingIconClassnames.length} missing icons…`,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await repairMissingIcons((evt) => {
|
||||
if (evt.type === "progress") {
|
||||
onEvent?.({
|
||||
type: "checking",
|
||||
section: "repair_icons",
|
||||
current: evt.index,
|
||||
total: evt.total,
|
||||
message:
|
||||
evt.status === "done"
|
||||
? `Repaired ${evt.classname}`
|
||||
: evt.status === "failed"
|
||||
? `Failed: ${evt.classname}`
|
||||
: undefined,
|
||||
});
|
||||
} else if (evt.type === "error") {
|
||||
onEvent?.({ type: "error", message: evt.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Re-check which of the previously-missing icons are still missing
|
||||
// and drop the ones the repair fixed from the issue list.
|
||||
const stillMissing: string[] = [];
|
||||
for (const classname of missingIconClassnames) {
|
||||
if (!existsSync(path.join(iconDir, `${classname}_icon.png`))) {
|
||||
stillMissing.push(classname);
|
||||
}
|
||||
}
|
||||
issues = issues.filter(
|
||||
(i) =>
|
||||
i.type !== "missing_icon" || stillMissing.includes(i.classname ?? ""),
|
||||
);
|
||||
|
||||
repairStats = {
|
||||
succeeded: result.succeeded,
|
||||
failed: result.failed,
|
||||
skipped: result.skipped,
|
||||
stillMissing: stillMissing.length,
|
||||
};
|
||||
onEvent?.({ type: "repair_complete", repair: repairStats });
|
||||
} catch (err) {
|
||||
onEvent?.({
|
||||
type: "error",
|
||||
message: `Icon repair failed: ${(err as Error).message}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let nitroRepairStats: RepairStats | null = null;
|
||||
if (options?.repairNitros && missingNitroClassnames.length > 0) {
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
message: `Repairing ${missingNitroClassnames.length} missing .nitro files…`,
|
||||
});
|
||||
|
||||
try {
|
||||
const result = await repairMissingNitros((evt) => {
|
||||
if (evt.type === "progress") {
|
||||
onEvent?.({
|
||||
type: "checking",
|
||||
section: "repair_nitros",
|
||||
current: evt.index,
|
||||
total: evt.total,
|
||||
message:
|
||||
evt.status === "done"
|
||||
? `Repaired ${evt.classname}`
|
||||
: evt.status === "failed"
|
||||
? `Failed: ${evt.classname}`
|
||||
: undefined,
|
||||
});
|
||||
} else if (evt.type === "error") {
|
||||
onEvent?.({ type: "error", message: evt.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Re-check which of the previously-missing nitros are still missing
|
||||
// and drop the ones the repair fixed from the issue list.
|
||||
const stillMissing: string[] = [];
|
||||
for (const classname of missingNitroClassnames) {
|
||||
if (!existsSync(path.join(nitroDir, `${classname}.nitro`))) {
|
||||
stillMissing.push(classname);
|
||||
}
|
||||
}
|
||||
issues = issues.filter(
|
||||
(i) =>
|
||||
i.type !== "missing_nitro" ||
|
||||
stillMissing.includes(i.classname ?? ""),
|
||||
);
|
||||
|
||||
nitroRepairStats = {
|
||||
succeeded: result.succeeded,
|
||||
failed: result.failed,
|
||||
skipped: result.skipped,
|
||||
stillMissing: stillMissing.length,
|
||||
};
|
||||
onEvent?.({
|
||||
type: "repair_complete",
|
||||
repairNitro: nitroRepairStats,
|
||||
});
|
||||
} catch (err) {
|
||||
onEvent?.({
|
||||
type: "error",
|
||||
message: `.nitro repair failed: ${(err as Error).message}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── Catalog SQL generation / apply ────────────────────────────────
|
||||
let catalogSqlCount = 0;
|
||||
let catalogSqlApplied = 0;
|
||||
if ((options?.sql || options?.applySql) && missingCatalogEntries.length > 0) {
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
message: `Generating catalog SQL for ${missingCatalogEntries.length} items…`,
|
||||
});
|
||||
try {
|
||||
const result = await generateCatalogSql(missingCatalogEntries);
|
||||
catalogSqlCount = result.count;
|
||||
onEvent?.({
|
||||
type: "sql_ready",
|
||||
sql: result.sql,
|
||||
sqlCount: result.count,
|
||||
});
|
||||
if (options.applySql) {
|
||||
const applyResult = await applyCatalogSql(result.statements);
|
||||
catalogSqlApplied = applyResult.applied;
|
||||
onEvent?.({
|
||||
type: "sql_applied",
|
||||
applied: applyResult.applied,
|
||||
sqlFailed: applyResult.failed,
|
||||
});
|
||||
}
|
||||
} catch (err) {
|
||||
onEvent?.({
|
||||
type: "error",
|
||||
message: `Catalog SQL failed: ${(err as Error).message}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ── FurnitureData.json repair (add missing + dedupe) ──────────────
|
||||
let furniDataAdded = 0;
|
||||
let furniDataDuplicatesRemoved = 0;
|
||||
if (options?.repairFurniData) {
|
||||
onEvent?.({ type: "progress", message: "Repairing FurnitureData.json…" });
|
||||
try {
|
||||
const result = await repairFurniData(items);
|
||||
furniDataAdded = result.added;
|
||||
furniDataDuplicatesRemoved = result.removedDuplicates;
|
||||
onEvent?.({
|
||||
type: "furnidata_repair_complete",
|
||||
furniData: {
|
||||
added: result.added,
|
||||
removedDuplicates: result.removedDuplicates,
|
||||
},
|
||||
});
|
||||
} catch (err) {
|
||||
onEvent?.({
|
||||
type: "error",
|
||||
message: `FurnitureData repair failed: ${(err as Error).message}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
onEvent?.({ type: "progress", message: "Comparing with clone sources…" });
|
||||
|
||||
const sources = await listSources();
|
||||
@@ -319,6 +547,16 @@ export async function runCatalogAudit(
|
||||
missingIcon: issues.filter((i) => i.type === "missing_icon").length,
|
||||
duplicates: issues.filter((i) => i.type === "duplicate_classname").length,
|
||||
missingFromSourcesTotal: missingFromSources.length,
|
||||
iconsRepaired: repairStats?.succeeded ?? 0,
|
||||
iconsStillMissing:
|
||||
repairStats?.stillMissing ?? missingIconClassnames.length,
|
||||
nitrosRepaired: nitroRepairStats?.succeeded ?? 0,
|
||||
nitrosStillMissing:
|
||||
nitroRepairStats?.stillMissing ?? missingNitroClassnames.length,
|
||||
catalogSqlCount,
|
||||
catalogSqlApplied,
|
||||
furniDataAdded,
|
||||
furniDataDuplicatesRemoved,
|
||||
};
|
||||
|
||||
onEvent?.({
|
||||
|
||||
Reference in new issue
Block a user