feat: parallelize audit repair and add nitro/SQL repair options
- Parallelize icon and nitro downloads in the audit repair with a sliding-window worker pool (6 concurrent) to speed up large catalogs - Add repairMissingNitros: fetch missing .nitro bundles from configured nitro sources (Wibbo default), validating each bundle before writing - Add catalog-repair service: generate/apply catalog_items SQL for furni missing a catalog entry and repair FurnitureData.json (add missing + dedupe classnames) - Wire all options through the audit API and client UI with live progress and result stats (icons, nitros, SQL, furnidata) - Add Wibbo as default nitro source alongside existing icon sources - Ignore runtime furni assets downloaded into public/ during repair
This commit is contained in:
1 parent
0c8e62357f
commit
5308ce6a12
8 files changed
+1166
-58
No files matched your search
@@ -16,6 +16,28 @@ import {
|
||||
import { downloadFile } from "@/lib/services/import/core/download";
|
||||
import { ensureDirectories } from "./furni-import";
|
||||
|
||||
const REPAIR_CONCURRENCY = 6;
|
||||
|
||||
/** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */
|
||||
async function runPool<T>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
worker: (item: T, index: number) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const limit = Math.min(Math.max(concurrency, 1), items.length);
|
||||
if (limit === 0) return;
|
||||
let next = 0;
|
||||
await Promise.allSettled(
|
||||
Array.from({ length: limit }, async () => {
|
||||
while (true) {
|
||||
const i = next++;
|
||||
if (i >= items.length) break;
|
||||
await worker(items[i], i);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function uniqueDirs(dirs: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
@@ -135,8 +157,7 @@ export async function repairMissingIcons(
|
||||
const total = items.length;
|
||||
onEvent?.({ type: "started", total });
|
||||
|
||||
for (let i = 0; i < items.length; i++) {
|
||||
const { item_name: classname } = items[i];
|
||||
const processItem = async (classname: string) => {
|
||||
const fileName = `${classname}_icon.png`;
|
||||
|
||||
// A dir may already have the icon (e.g. the full gamedata) while the
|
||||
@@ -147,15 +168,7 @@ export async function repairMissingIcons(
|
||||
const missingDirs = iconDirs.filter((dir) => !presentDirs.includes(dir));
|
||||
|
||||
if (missingDirs.length === 0) {
|
||||
skipped++;
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
classname,
|
||||
status: "skipped",
|
||||
index: i,
|
||||
total,
|
||||
});
|
||||
continue;
|
||||
return { status: "skipped" } as const;
|
||||
}
|
||||
|
||||
let ok = false;
|
||||
@@ -303,6 +316,29 @@ export async function repairMissingIcons(
|
||||
}
|
||||
|
||||
if (ok) {
|
||||
return { status: "done", warnings } as const;
|
||||
}
|
||||
return {
|
||||
status: "failed",
|
||||
message: warnings[0] || "could not obtain icon",
|
||||
} as const;
|
||||
};
|
||||
|
||||
await runPool(items, REPAIR_CONCURRENCY, async (item, i) => {
|
||||
const classname = item.item_name;
|
||||
const result = await processItem(classname);
|
||||
if (result.status === "skipped") {
|
||||
skipped++;
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
classname,
|
||||
status: "skipped",
|
||||
index: i,
|
||||
total,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (result.status === "done") {
|
||||
succeeded++;
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
@@ -310,20 +346,20 @@ export async function repairMissingIcons(
|
||||
status: "done",
|
||||
index: i,
|
||||
total,
|
||||
warnings: warnings.length ? warnings : undefined,
|
||||
});
|
||||
} else {
|
||||
failed++;
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
classname,
|
||||
status: "failed",
|
||||
index: i,
|
||||
total,
|
||||
message: warnings[0] || "could not obtain icon",
|
||||
warnings: result.warnings.length ? result.warnings : undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
failed++;
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
classname,
|
||||
status: "failed",
|
||||
index: i,
|
||||
total,
|
||||
message: result.message,
|
||||
});
|
||||
});
|
||||
|
||||
onEvent?.({
|
||||
type: "batch_complete",
|
||||
|
||||
Reference in new issue
Block a user