perf: speed up batch furniture import

- Run batch imports with bounded concurrency (IMPORT_BATCH_CONCURRENCY = 8)
  instead of fully sequentially, parallelizing network downloads and DB writes.
- Collect FurnitureData.json entries and write the whole file once per batch
  (appendFurniEntriesBulk) instead of rewriting it on every single item.

Catalog id allocation stays serialized and FurnitureData writes remain
lock-guarded, so concurrent imports are safe.
This commit is contained in:
openhands committed 2026-08-22 12:40:43 +02:00
1 parent 488f40919d
commit 6843af235e
2 files changed
+95 -18

No files matched your search

+64 -18
View File
@@ -10,6 +10,7 @@ import { getCloneList } from "@/lib/services/clone-import";
import { getSource } from "@/lib/services/clone-sources";
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
import {
appendFurniEntriesBulk,
rebuildCatalogOfferIds,
reconcileFurniDataWithItemsBase,
verifyAndFixSpriteIds,
@@ -47,6 +48,34 @@ export {
} from "@/lib/services/furni-import";
export type { ImportSingleResult } from "@/types/furni";
import type { ImportSingleResult } from "@/types/furni";
// How many items to import concurrently in a batch. The per-item work is
// network-bound (asset downloads) and DB-bound, so a modest pool gives a large
// speedup over the old fully sequential loop without overwhelming the DB.
const IMPORT_BATCH_CONCURRENCY = 8;
async function runWithConcurrency<T, R>(
items: T[],
limit: number,
fn: (item: T) => Promise<R>,
): Promise<R[]> {
const out = new Array<R>(items.length);
let cursor = 0;
const workers = Array.from(
{ length: Math.min(limit, items.length) },
async () => {
for (;;) {
const i = cursor++;
if (i >= items.length) break;
out[i] = await fn(items[i]);
}
},
);
await Promise.all(workers);
return out;
}
// ── Delete imported item or clean up SWF files ──────────────────────
export const DELETE = withAdmin(
{ permission: PERMS.ASSETS_IMPORT },
@@ -382,26 +411,18 @@ export const POST = withAdmin(
let source: Awaited<ReturnType<typeof getSource>> | null = null;
if (sourceId) source = await getSource(sourceId);
const results: Array<{
classname: string;
ok: boolean;
itemId?: number;
catalogItemId?: number | null;
dimensions?: { x: number; y: number; z: number };
spriteCount?: number;
warnings: string[];
error?: string;
}> = [];
type BatchResultItem = ImportSingleResult & { classname: string };
for (const item of body.items) {
const runOne = async (
item: (typeof body.items)[number],
): Promise<BatchResultItem> => {
if (!item.classname || !item.name) {
results.push({
return {
classname: item.classname || "unknown",
ok: false,
warnings: [],
error: "Missing classname or name",
});
continue;
};
}
try {
@@ -417,10 +438,9 @@ export const POST = withAdmin(
sourceSwfBaseUrl: source?.sourceSwfBaseUrl,
nitroBaseUrl: source?.nitroBaseUrl || undefined,
iconBaseUrl: source?.iconBaseUrl || undefined,
skipFurniDataWrite: true,
});
results.push({ classname: item.classname, ...result });
if (result.ok) {
logAudit({
userId: ctx.session.user.id,
@@ -439,13 +459,39 @@ export const POST = withAdmin(
},
});
}
return { classname: item.classname, ...result };
} catch (err) {
results.push({
return {
classname: item.classname,
ok: false,
warnings: [],
error: (err as Error).message,
});
};
}
};
const results = await runWithConcurrency<
(typeof body.items)[number],
BatchResultItem
>(body.items, IMPORT_BATCH_CONCURRENCY, runOne);
// Collect FurnitureData entries and write the whole file once for the
// batch instead of once per item (each per-item write rewrites it fully).
const collectedFurniEntries: Array<{
entry: Record<string, unknown>;
itemType: string;
}> = [];
for (const r of results) {
if (r.furniDataEntry) collectedFurniEntries.push(r.furniDataEntry);
}
if (collectedFurniEntries.length > 0) {
try {
await appendFurniEntriesBulk(collectedFurniEntries);
} catch (err) {
console.warn(
"[import-furni] Bulk FurnitureData.json write failed:",
(err as Error).message,
);
}
}
+31
View File
@@ -317,6 +317,37 @@ export async function appendFurniEntry(
});
}
/**
* Append many FurnitureData entries in a single read/parse/write cycle
* (under the furnidata lock). Used by batch imports so the whole JSON file
* is rewritten once instead of once per item.
*/
export async function appendFurniEntriesBulk(
entries: Array<{ entry: Record<string, unknown>; itemType: string }>,
): Promise<void> {
if (entries.length === 0) return;
await withFurniDataLock(async () => {
const furniData = (await readFurniData()) as Record<
string,
{ furnitype: Array<Record<string, unknown>> }
>;
const existingOwners = furniDataIdOwners(furniData);
assertNoFurniDataIdConflicts(existingOwners, entries);
for (const { entry, itemType } of entries) {
if (itemType === "i") {
if (!furniData.wallitemtypes)
furniData.wallitemtypes = { furnitype: [] };
upsertEntryInSection(furniData.wallitemtypes, entry);
} else {
if (!furniData.roomitemtypes)
furniData.roomitemtypes = { furnitype: [] };
upsertEntryInSection(furniData.roomitemtypes, entry);
}
}
await writeFurniData(furniData as Record<string, unknown>);
});
}
export async function removeFurniEntry(classname: string): Promise<boolean> {
return withFurniDataLock(async () => {
const furniData = (await readFurniData()) as Record<