perf: speed up batch furniture import
- Run batch imports with bounded concurrency (IMPORT_BATCH_CONCURRENCY = 8) instead of fully sequentially, parallelizing network downloads and DB writes. - Collect FurnitureData.json entries and write the whole file once per batch (appendFurniEntriesBulk) instead of rewriting it on every single item. Catalog id allocation stays serialized and FurnitureData writes remain lock-guarded, so concurrent imports are safe.
This commit is contained in:
1 parent
488f40919d
commit
6843af235e
2 files changed
+95
-18
No files matched your search
@@ -10,6 +10,7 @@ import { getCloneList } from "@/lib/services/clone-import";
|
|||||||
import { getSource } from "@/lib/services/clone-sources";
|
import { getSource } from "@/lib/services/clone-sources";
|
||||||
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
||||||
import {
|
import {
|
||||||
|
appendFurniEntriesBulk,
|
||||||
rebuildCatalogOfferIds,
|
rebuildCatalogOfferIds,
|
||||||
reconcileFurniDataWithItemsBase,
|
reconcileFurniDataWithItemsBase,
|
||||||
verifyAndFixSpriteIds,
|
verifyAndFixSpriteIds,
|
||||||
@@ -47,6 +48,34 @@ export {
|
|||||||
} from "@/lib/services/furni-import";
|
} from "@/lib/services/furni-import";
|
||||||
export type { ImportSingleResult } from "@/types/furni";
|
export type { ImportSingleResult } from "@/types/furni";
|
||||||
|
|
||||||
|
import type { ImportSingleResult } from "@/types/furni";
|
||||||
|
|
||||||
|
// How many items to import concurrently in a batch. The per-item work is
|
||||||
|
// network-bound (asset downloads) and DB-bound, so a modest pool gives a large
|
||||||
|
// speedup over the old fully sequential loop without overwhelming the DB.
|
||||||
|
const IMPORT_BATCH_CONCURRENCY = 8;
|
||||||
|
|
||||||
|
async function runWithConcurrency<T, R>(
|
||||||
|
items: T[],
|
||||||
|
limit: number,
|
||||||
|
fn: (item: T) => Promise<R>,
|
||||||
|
): Promise<R[]> {
|
||||||
|
const out = new Array<R>(items.length);
|
||||||
|
let cursor = 0;
|
||||||
|
const workers = Array.from(
|
||||||
|
{ length: Math.min(limit, items.length) },
|
||||||
|
async () => {
|
||||||
|
for (;;) {
|
||||||
|
const i = cursor++;
|
||||||
|
if (i >= items.length) break;
|
||||||
|
out[i] = await fn(items[i]);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
);
|
||||||
|
await Promise.all(workers);
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
// ── Delete imported item or clean up SWF files ──────────────────────
|
// ── Delete imported item or clean up SWF files ──────────────────────
|
||||||
export const DELETE = withAdmin(
|
export const DELETE = withAdmin(
|
||||||
{ permission: PERMS.ASSETS_IMPORT },
|
{ permission: PERMS.ASSETS_IMPORT },
|
||||||
@@ -382,26 +411,18 @@ export const POST = withAdmin(
|
|||||||
let source: Awaited<ReturnType<typeof getSource>> | null = null;
|
let source: Awaited<ReturnType<typeof getSource>> | null = null;
|
||||||
if (sourceId) source = await getSource(sourceId);
|
if (sourceId) source = await getSource(sourceId);
|
||||||
|
|
||||||
const results: Array<{
|
type BatchResultItem = ImportSingleResult & { classname: string };
|
||||||
classname: string;
|
|
||||||
ok: boolean;
|
|
||||||
itemId?: number;
|
|
||||||
catalogItemId?: number | null;
|
|
||||||
dimensions?: { x: number; y: number; z: number };
|
|
||||||
spriteCount?: number;
|
|
||||||
warnings: string[];
|
|
||||||
error?: string;
|
|
||||||
}> = [];
|
|
||||||
|
|
||||||
for (const item of body.items) {
|
const runOne = async (
|
||||||
|
item: (typeof body.items)[number],
|
||||||
|
): Promise<BatchResultItem> => {
|
||||||
if (!item.classname || !item.name) {
|
if (!item.classname || !item.name) {
|
||||||
results.push({
|
return {
|
||||||
classname: item.classname || "unknown",
|
classname: item.classname || "unknown",
|
||||||
ok: false,
|
ok: false,
|
||||||
warnings: [],
|
warnings: [],
|
||||||
error: "Missing classname or name",
|
error: "Missing classname or name",
|
||||||
});
|
};
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -417,10 +438,9 @@ export const POST = withAdmin(
|
|||||||
sourceSwfBaseUrl: source?.sourceSwfBaseUrl,
|
sourceSwfBaseUrl: source?.sourceSwfBaseUrl,
|
||||||
nitroBaseUrl: source?.nitroBaseUrl || undefined,
|
nitroBaseUrl: source?.nitroBaseUrl || undefined,
|
||||||
iconBaseUrl: source?.iconBaseUrl || undefined,
|
iconBaseUrl: source?.iconBaseUrl || undefined,
|
||||||
|
skipFurniDataWrite: true,
|
||||||
});
|
});
|
||||||
|
|
||||||
results.push({ classname: item.classname, ...result });
|
|
||||||
|
|
||||||
if (result.ok) {
|
if (result.ok) {
|
||||||
logAudit({
|
logAudit({
|
||||||
userId: ctx.session.user.id,
|
userId: ctx.session.user.id,
|
||||||
@@ -439,13 +459,39 @@ export const POST = withAdmin(
|
|||||||
},
|
},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
return { classname: item.classname, ...result };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
results.push({
|
return {
|
||||||
classname: item.classname,
|
classname: item.classname,
|
||||||
ok: false,
|
ok: false,
|
||||||
warnings: [],
|
warnings: [],
|
||||||
error: (err as Error).message,
|
error: (err as Error).message,
|
||||||
});
|
};
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const results = await runWithConcurrency<
|
||||||
|
(typeof body.items)[number],
|
||||||
|
BatchResultItem
|
||||||
|
>(body.items, IMPORT_BATCH_CONCURRENCY, runOne);
|
||||||
|
|
||||||
|
// Collect FurnitureData entries and write the whole file once for the
|
||||||
|
// batch instead of once per item (each per-item write rewrites it fully).
|
||||||
|
const collectedFurniEntries: Array<{
|
||||||
|
entry: Record<string, unknown>;
|
||||||
|
itemType: string;
|
||||||
|
}> = [];
|
||||||
|
for (const r of results) {
|
||||||
|
if (r.furniDataEntry) collectedFurniEntries.push(r.furniDataEntry);
|
||||||
|
}
|
||||||
|
if (collectedFurniEntries.length > 0) {
|
||||||
|
try {
|
||||||
|
await appendFurniEntriesBulk(collectedFurniEntries);
|
||||||
|
} catch (err) {
|
||||||
|
console.warn(
|
||||||
|
"[import-furni] Bulk FurnitureData.json write failed:",
|
||||||
|
(err as Error).message,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -317,6 +317,37 @@ export async function appendFurniEntry(
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Append many FurnitureData entries in a single read/parse/write cycle
|
||||||
|
* (under the furnidata lock). Used by batch imports so the whole JSON file
|
||||||
|
* is rewritten once instead of once per item.
|
||||||
|
*/
|
||||||
|
export async function appendFurniEntriesBulk(
|
||||||
|
entries: Array<{ entry: Record<string, unknown>; itemType: string }>,
|
||||||
|
): Promise<void> {
|
||||||
|
if (entries.length === 0) return;
|
||||||
|
await withFurniDataLock(async () => {
|
||||||
|
const furniData = (await readFurniData()) as Record<
|
||||||
|
string,
|
||||||
|
{ furnitype: Array<Record<string, unknown>> }
|
||||||
|
>;
|
||||||
|
const existingOwners = furniDataIdOwners(furniData);
|
||||||
|
assertNoFurniDataIdConflicts(existingOwners, entries);
|
||||||
|
for (const { entry, itemType } of entries) {
|
||||||
|
if (itemType === "i") {
|
||||||
|
if (!furniData.wallitemtypes)
|
||||||
|
furniData.wallitemtypes = { furnitype: [] };
|
||||||
|
upsertEntryInSection(furniData.wallitemtypes, entry);
|
||||||
|
} else {
|
||||||
|
if (!furniData.roomitemtypes)
|
||||||
|
furniData.roomitemtypes = { furnitype: [] };
|
||||||
|
upsertEntryInSection(furniData.roomitemtypes, entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await writeFurniData(furniData as Record<string, unknown>);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
export async function removeFurniEntry(classname: string): Promise<boolean> {
|
export async function removeFurniEntry(classname: string): Promise<boolean> {
|
||||||
return withFurniDataLock(async () => {
|
return withFurniDataLock(async () => {
|
||||||
const furniData = (await readFurniData()) as Record<
|
const furniData = (await readFurniData()) as Record<
|
||||||
|
|||||||
Reference in new issue
Block a user