feat: parallelize audit repair and add nitro/SQL repair options
- Parallelize icon and nitro downloads in the audit repair with a sliding-window worker pool (6 concurrent) to speed up large catalogs - Add repairMissingNitros: fetch missing .nitro bundles from configured nitro sources (Wibbo default), validating each bundle before writing - Add catalog-repair service: generate/apply catalog_items SQL for furni missing a catalog entry and repair FurnitureData.json (add missing + dedupe classnames) - Wire all options through the audit API and client UI with live progress and result stats (icons, nitros, SQL, furnidata) - Add Wibbo as default nitro source alongside existing icon sources - Ignore runtime furni assets downloaded into public/ during repair
This commit is contained in:
openhands committed
1 parent
0c8e62357f
commit
5308ce6a12
8 files changed
+1166
-58
No files matched your search
@@ -0,0 +1,305 @@
|
||||
import { existsSync, promises as fs } from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { db } from "@/lib/db";
|
||||
import {
|
||||
type CloneSource,
|
||||
DEFAULT_NITRO_SOURCES,
|
||||
listSources,
|
||||
} from "@/lib/services/clone-sources";
|
||||
import {
|
||||
type FurniAssetWriteTargets,
|
||||
getFurniAssetWriteTargets,
|
||||
} from "@/lib/services/furni-asset-dirs";
|
||||
import { downloadFile } from "@/lib/services/import/core/download";
|
||||
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
|
||||
import { ensureDirectories } from "./furni-import";
|
||||
|
||||
const REPAIR_CONCURRENCY = 6;
|
||||
|
||||
/** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */
|
||||
async function runPool<T>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
worker: (item: T, index: number) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const limit = Math.min(Math.max(concurrency, 1), items.length);
|
||||
if (limit === 0) return;
|
||||
let next = 0;
|
||||
await Promise.allSettled(
|
||||
Array.from({ length: limit }, async () => {
|
||||
while (true) {
|
||||
const i = next++;
|
||||
if (i >= items.length) break;
|
||||
await worker(items[i], i);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
function uniqueDirs(dirs: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
for (const dir of dirs) {
|
||||
const key = process.platform === "win32" ? dir.toLowerCase() : dir;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(dir);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
async function writeNitroToDirs(
|
||||
nitro: Buffer,
|
||||
dirs: string[],
|
||||
fileName: string,
|
||||
): Promise<string[]> {
|
||||
const errors: string[] = [];
|
||||
for (const dir of dirs) {
|
||||
try {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.writeFile(path.join(dir, fileName), nitro);
|
||||
} catch (err) {
|
||||
errors.push(`write .nitro to ${dir} failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
async function copyNitroAcrossDirs(
|
||||
srcDir: string,
|
||||
destDirs: string[],
|
||||
fileName: string,
|
||||
): Promise<string[]> {
|
||||
const errors: string[] = [];
|
||||
const src = path.join(srcDir, fileName);
|
||||
for (const dir of destDirs) {
|
||||
if (dir === srcDir) continue;
|
||||
try {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.copyFile(src, path.join(dir, fileName));
|
||||
} catch (err) {
|
||||
errors.push(`copy .nitro to ${dir} failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
export interface RepairNitroEvent {
|
||||
type: "started" | "progress" | "done" | "batch_complete" | "error";
|
||||
classname?: string;
|
||||
status?: string;
|
||||
index?: number;
|
||||
total?: number;
|
||||
message?: string;
|
||||
warnings?: string[];
|
||||
succeeded?: number;
|
||||
failed?: number;
|
||||
skipped?: number;
|
||||
duration?: number;
|
||||
}
|
||||
|
||||
export async function repairMissingNitros(
|
||||
onEvent?: (evt: RepairNitroEvent) => void,
|
||||
): Promise<{ succeeded: number; failed: number; skipped: number }> {
|
||||
let succeeded = 0;
|
||||
let failed = 0;
|
||||
let skipped = 0;
|
||||
|
||||
try {
|
||||
await ensureDirectories();
|
||||
} catch (err) {
|
||||
onEvent?.({
|
||||
type: "error",
|
||||
message: `ensureDirectories failed: ${(err as Error).message}`,
|
||||
});
|
||||
return { succeeded: 0, failed: 0, skipped: 0 };
|
||||
}
|
||||
|
||||
let targets: FurniAssetWriteTargets;
|
||||
try {
|
||||
targets = await getFurniAssetWriteTargets();
|
||||
} catch (err) {
|
||||
onEvent?.({
|
||||
type: "error",
|
||||
message: `getFurniAssetWriteTargets failed: ${(err as Error).message}`,
|
||||
});
|
||||
return { succeeded: 0, failed: 0, skipped: 0 };
|
||||
}
|
||||
|
||||
// Every location the app mirrors furni assets into — the webroot (public/)
|
||||
// plus the live gamedata (e.g. /var/www/Gamedata). Repair writes to all of
|
||||
// them so both the CMS and the client see the .nitro.
|
||||
const nitroDirs = uniqueDirs([
|
||||
targets.nitroDir,
|
||||
...targets.mirrorDirs.map((d) => d.nitroDir),
|
||||
]);
|
||||
|
||||
let items: Array<{ id: number; item_name: string }>;
|
||||
try {
|
||||
const [rows] = (await db.execute(sql`
|
||||
SELECT id, item_name FROM items_base ORDER BY id
|
||||
`)) as unknown as [Array<{ id: number; item_name: string }>, unknown];
|
||||
items = rows;
|
||||
} catch (err) {
|
||||
onEvent?.({
|
||||
type: "error",
|
||||
message: `Database query failed: ${(err as Error).message}`,
|
||||
});
|
||||
return { succeeded: 0, failed: 0, skipped: 0 };
|
||||
}
|
||||
|
||||
const total = items.length;
|
||||
onEvent?.({ type: "started", total });
|
||||
|
||||
const processItem = async (classname: string) => {
|
||||
const fileName = `${classname}.nitro`;
|
||||
|
||||
const presentDirs = nitroDirs.filter((dir) =>
|
||||
existsSync(path.join(dir, fileName)),
|
||||
);
|
||||
const missingDirs = nitroDirs.filter((dir) => !presentDirs.includes(dir));
|
||||
|
||||
if (missingDirs.length === 0) {
|
||||
return { status: "skipped" } as const;
|
||||
}
|
||||
|
||||
let ok = false;
|
||||
const warnings: string[] = [];
|
||||
|
||||
// 1) Copy across mirrors — the .nitro already exists in one target (e.g.
|
||||
// gamedata), so just fill the ones that are missing it.
|
||||
if (presentDirs.length > 0) {
|
||||
try {
|
||||
const copyErrors = await copyNitroAcrossDirs(
|
||||
presentDirs[0],
|
||||
missingDirs,
|
||||
fileName,
|
||||
);
|
||||
warnings.push(...copyErrors);
|
||||
ok = copyErrors.length === 0;
|
||||
} catch (err) {
|
||||
warnings.push(
|
||||
`copy across .nitro dirs failed: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Download from configured or default nitro sources.
|
||||
if (!ok) {
|
||||
let sources: CloneSource[] = [];
|
||||
try {
|
||||
sources = (await listSources()).filter((s) => s.nitroBaseUrl);
|
||||
} catch (err) {
|
||||
warnings.push(`listSources failed: ${(err as Error).message}`);
|
||||
}
|
||||
if (sources.length === 0) sources = DEFAULT_NITRO_SOURCES;
|
||||
|
||||
const nitroCandidates = [
|
||||
`${classname}.nitro`,
|
||||
// Variant classnames (e.g. `table_silo_med*10`) often only exist
|
||||
// on sources under their base name (e.g. `table_silo_med.nitro`)
|
||||
// or with the `*` replaced by `_` (e.g. `table_silo_med_10.nitro`).
|
||||
...(classname.includes("*")
|
||||
? [
|
||||
`${classname.split("*")[0]}.nitro`,
|
||||
`${classname.replace(/\*/g, "_")}.nitro`,
|
||||
]
|
||||
: []),
|
||||
];
|
||||
|
||||
for (const source of sources) {
|
||||
for (const nitroName of nitroCandidates) {
|
||||
const tmpPath = path.join(
|
||||
os.tmpdir(),
|
||||
`.repair_nitro_${classname}_${Math.random().toString(36).slice(2, 8)}.nitro`,
|
||||
);
|
||||
try {
|
||||
const dl = await downloadFile(
|
||||
`${source.nitroBaseUrl}/${nitroName}`,
|
||||
tmpPath,
|
||||
{ maxRetries: 1 },
|
||||
);
|
||||
if (dl.ok) {
|
||||
// Validate it is a real Nitro bundle before writing it
|
||||
// anywhere (mirrors clone-import.ts).
|
||||
const buffer = await fs.readFile(tmpPath);
|
||||
parseNitroBundle(buffer);
|
||||
const writeErrors = await writeNitroToDirs(
|
||||
buffer,
|
||||
missingDirs,
|
||||
fileName,
|
||||
);
|
||||
warnings.push(...writeErrors);
|
||||
ok = writeErrors.length === 0;
|
||||
break;
|
||||
}
|
||||
warnings.push(`${nitroName} not found on ${source.name}`);
|
||||
} catch (err) {
|
||||
warnings.push(
|
||||
`download from ${source.name} failed: ${(err as Error).message}`,
|
||||
);
|
||||
} finally {
|
||||
await fs.unlink(tmpPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
if (ok) break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ok) {
|
||||
return { status: "done", warnings } as const;
|
||||
}
|
||||
return {
|
||||
status: "failed",
|
||||
message: warnings[0] || "could not obtain .nitro",
|
||||
} as const;
|
||||
};
|
||||
|
||||
await runPool(items, REPAIR_CONCURRENCY, async (item, i) => {
|
||||
const classname = item.item_name;
|
||||
const result = await processItem(classname);
|
||||
if (result.status === "skipped") {
|
||||
skipped++;
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
classname,
|
||||
status: "skipped",
|
||||
index: i,
|
||||
total,
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (result.status === "done") {
|
||||
succeeded++;
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
classname,
|
||||
status: "done",
|
||||
index: i,
|
||||
total,
|
||||
warnings: result.warnings.length ? result.warnings : undefined,
|
||||
});
|
||||
return;
|
||||
}
|
||||
failed++;
|
||||
onEvent?.({
|
||||
type: "progress",
|
||||
classname,
|
||||
status: "failed",
|
||||
index: i,
|
||||
total,
|
||||
message: result.message,
|
||||
});
|
||||
});
|
||||
|
||||
onEvent?.({
|
||||
type: "batch_complete",
|
||||
succeeded,
|
||||
failed,
|
||||
skipped,
|
||||
});
|
||||
|
||||
return { succeeded, failed, skipped };
|
||||
}
|
||||
Reference in new issue
Block a user