perf: speed up batch furniture import
- Run batch imports with bounded concurrency (IMPORT_BATCH_CONCURRENCY = 8) instead of fully sequentially, parallelizing network downloads and DB writes. - Collect FurnitureData.json entries and write the whole file once per batch (appendFurniEntriesBulk) instead of rewriting it on every single item. Catalog id allocation stays serialized and FurnitureData writes remain lock-guarded, so concurrent imports are safe.
This commit is contained in:
1 parent
488f40919d
commit
6843af235e
2 files changed
+95
-18
No files matched your search
@@ -10,6 +10,7 @@ import { getCloneList } from "@/lib/services/clone-import";
|
||||
import { getSource } from "@/lib/services/clone-sources";
|
||||
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
||||
import {
|
||||
appendFurniEntriesBulk,
|
||||
rebuildCatalogOfferIds,
|
||||
reconcileFurniDataWithItemsBase,
|
||||
verifyAndFixSpriteIds,
|
||||
@@ -47,6 +48,34 @@ export {
|
||||
} from "@/lib/services/furni-import";
|
||||
export type { ImportSingleResult } from "@/types/furni";
|
||||
|
||||
import type { ImportSingleResult } from "@/types/furni";
|
||||
|
||||
// How many items to import concurrently in a batch. The per-item work is
|
||||
// network-bound (asset downloads) and DB-bound, so a modest pool gives a large
|
||||
// speedup over the old fully sequential loop without overwhelming the DB.
|
||||
const IMPORT_BATCH_CONCURRENCY = 8;
|
||||
|
||||
async function runWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
fn: (item: T) => Promise<R>,
|
||||
): Promise<R[]> {
|
||||
const out = new Array<R>(items.length);
|
||||
let cursor = 0;
|
||||
const workers = Array.from(
|
||||
{ length: Math.min(limit, items.length) },
|
||||
async () => {
|
||||
for (;;) {
|
||||
const i = cursor++;
|
||||
if (i >= items.length) break;
|
||||
out[i] = await fn(items[i]);
|
||||
}
|
||||
},
|
||||
);
|
||||
await Promise.all(workers);
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Delete imported item or clean up SWF files ──────────────────────
|
||||
export const DELETE = withAdmin(
|
||||
{ permission: PERMS.ASSETS_IMPORT },
|
||||
@@ -382,26 +411,18 @@ export const POST = withAdmin(
|
||||
let source: Awaited<ReturnType<typeof getSource>> | null = null;
|
||||
if (sourceId) source = await getSource(sourceId);
|
||||
|
||||
const results: Array<{
|
||||
classname: string;
|
||||
ok: boolean;
|
||||
itemId?: number;
|
||||
catalogItemId?: number | null;
|
||||
dimensions?: { x: number; y: number; z: number };
|
||||
spriteCount?: number;
|
||||
warnings: string[];
|
||||
error?: string;
|
||||
}> = [];
|
||||
type BatchResultItem = ImportSingleResult & { classname: string };
|
||||
|
||||
for (const item of body.items) {
|
||||
const runOne = async (
|
||||
item: (typeof body.items)[number],
|
||||
): Promise<BatchResultItem> => {
|
||||
if (!item.classname || !item.name) {
|
||||
results.push({
|
||||
return {
|
||||
classname: item.classname || "unknown",
|
||||
ok: false,
|
||||
warnings: [],
|
||||
error: "Missing classname or name",
|
||||
});
|
||||
continue;
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
@@ -417,10 +438,9 @@ export const POST = withAdmin(
|
||||
sourceSwfBaseUrl: source?.sourceSwfBaseUrl,
|
||||
nitroBaseUrl: source?.nitroBaseUrl || undefined,
|
||||
iconBaseUrl: source?.iconBaseUrl || undefined,
|
||||
skipFurniDataWrite: true,
|
||||
});
|
||||
|
||||
results.push({ classname: item.classname, ...result });
|
||||
|
||||
if (result.ok) {
|
||||
logAudit({
|
||||
userId: ctx.session.user.id,
|
||||
@@ -439,13 +459,39 @@ export const POST = withAdmin(
|
||||
},
|
||||
});
|
||||
}
|
||||
return { classname: item.classname, ...result };
|
||||
} catch (err) {
|
||||
results.push({
|
||||
return {
|
||||
classname: item.classname,
|
||||
ok: false,
|
||||
warnings: [],
|
||||
error: (err as Error).message,
|
||||
});
|
||||
};
|
||||
}
|
||||
};
|
||||
|
||||
const results = await runWithConcurrency<
|
||||
(typeof body.items)[number],
|
||||
BatchResultItem
|
||||
>(body.items, IMPORT_BATCH_CONCURRENCY, runOne);
|
||||
|
||||
// Collect FurnitureData entries and write the whole file once for the
|
||||
// batch instead of once per item (each per-item write rewrites it fully).
|
||||
const collectedFurniEntries: Array<{
|
||||
entry: Record<string, unknown>;
|
||||
itemType: string;
|
||||
}> = [];
|
||||
for (const r of results) {
|
||||
if (r.furniDataEntry) collectedFurniEntries.push(r.furniDataEntry);
|
||||
}
|
||||
if (collectedFurniEntries.length > 0) {
|
||||
try {
|
||||
await appendFurniEntriesBulk(collectedFurniEntries);
|
||||
} catch (err) {
|
||||
console.warn(
|
||||
"[import-furni] Bulk FurnitureData.json write failed:",
|
||||
(err as Error).message,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -317,6 +317,37 @@ export async function appendFurniEntry(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Append many FurnitureData entries in a single read/parse/write cycle
|
||||
* (under the furnidata lock). Used by batch imports so the whole JSON file
|
||||
* is rewritten once instead of once per item.
|
||||
*/
|
||||
export async function appendFurniEntriesBulk(
|
||||
entries: Array<{ entry: Record<string, unknown>; itemType: string }>,
|
||||
): Promise<void> {
|
||||
if (entries.length === 0) return;
|
||||
await withFurniDataLock(async () => {
|
||||
const furniData = (await readFurniData()) as Record<
|
||||
string,
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
const existingOwners = furniDataIdOwners(furniData);
|
||||
assertNoFurniDataIdConflicts(existingOwners, entries);
|
||||
for (const { entry, itemType } of entries) {
|
||||
if (itemType === "i") {
|
||||
if (!furniData.wallitemtypes)
|
||||
furniData.wallitemtypes = { furnitype: [] };
|
||||
upsertEntryInSection(furniData.wallitemtypes, entry);
|
||||
} else {
|
||||
if (!furniData.roomitemtypes)
|
||||
furniData.roomitemtypes = { furnitype: [] };
|
||||
upsertEntryInSection(furniData.roomitemtypes, entry);
|
||||
}
|
||||
}
|
||||
await writeFurniData(furniData as Record<string, unknown>);
|
||||
});
|
||||
}
|
||||
|
||||
export async function removeFurniEntry(classname: string): Promise<boolean> {
|
||||
return withFurniDataLock(async () => {
|
||||
const furniData = (await readFurniData()) as Record<
|
||||
|
||||
Reference in new issue
Block a user