perf(import): parallelize ID allocation and raise clone concurrency to 10
CI / check (push) Successful in 27s
CI / release (push) Skipped
CI / deploy (push) Successful in 41s

Replace the serialized SELECT MAX + INSERT id-allocation chains for
items_base and catalog_items with a lazy-seeded in-process counter so
concurrent clone workers no longer queue on a global lock per item.
Re-seeds after 60s idle to avoid colliding with externally added rows.
Raise the clone import concurrency default from 6 to the batch cap of 10.
This commit is contained in:
openhands committed 2026-08-05 11:37:17 +02:00
1 parent 742536820a
commit 172941c542
8 files changed
+144 -52

No files matched your search

+43 -15
View File
@@ -137,26 +137,54 @@ export async function fetchSourceFurnidata(
* concurrent clones in the same Node process cannot race on MAX(id)+1.
* Mirrors the `allocateCatalogItemId` pattern from furni-import.ts.
*/
let itemsBaseIdAllocChain: Promise<unknown> = Promise.resolve();
let itemsBaseNextId: number | null = null;
let itemsBaseIdLastUsed = 0;
let itemsBaseIdSeedChain: Promise<void> = Promise.resolve();
const ITEMS_BASE_ID_REFRESH_MS = 60_000;
/** Allocate a unique items_base id. Seeded once with MAX(id)+1, then handed
* out from an in-process counter so concurrent clones don't serialize on a
* SELECT MAX + INSERT round-trip per item. Re-seeds when idle for a while so
* rows added by the emulator or other processes don't collide. */
async function allocateItemsBaseId<T>(
insertFn: (nextId: number) => Promise<T>,
): Promise<T> {
const prev = itemsBaseIdAllocChain;
let settle!: () => void;
itemsBaseIdAllocChain = new Promise<void>((r) => {
settle = r;
});
await prev.catch(() => {});
try {
const [idRows] = (await db.execute(sql`
SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base
`)) as unknown as [Array<{ next: number }>, unknown];
const nextId = Number(idRows[0]?.next ?? 1);
return await insertFn(nextId);
} finally {
settle();
const now = Date.now();
if (
itemsBaseNextId === null ||
now - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS
) {
let settle!: () => void;
const prev = itemsBaseIdSeedChain;
itemsBaseIdSeedChain = new Promise<void>((r) => {
settle = r;
});
await prev.catch(() => {});
try {
if (
itemsBaseNextId === null ||
Date.now() - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS
) {
const [idRows] = (await db.execute(sql`
SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base
`)) as unknown as [Array<{ next: number }>, unknown];
itemsBaseNextId = Number(idRows[0]?.next ?? 1);
}
} finally {
settle();
}
}
itemsBaseIdLastUsed = Date.now();
const nextId = itemsBaseNextId;
itemsBaseNextId += 1;
return await insertFn(nextId);
}
/** Test seam: drop the cached id counter between tests. */
export function __resetItemsBaseAllocatorForTests(): void {
itemsBaseNextId = null;
itemsBaseIdLastUsed = 0;
itemsBaseIdSeedChain = Promise.resolve();
}
export async function cloneSingleFurni(params: {