perf(import): parallelize ID allocation and raise clone concurrency to 10
Replace the serialized SELECT MAX + INSERT id-allocation chains for items_base and catalog_items with a lazy-seeded in-process counter so concurrent clone workers no longer queue on a global lock per item. Re-seeds after 60s idle to avoid colliding with externally added rows. Raise the clone import concurrency default from 6 to the batch cap of 10.
This commit is contained in:
1 parent
742536820a
commit
172941c542
8 files changed
+144
-52
No files matched your search
@@ -137,26 +137,54 @@ export async function fetchSourceFurnidata(
|
||||
* concurrent clones in the same Node process cannot race on MAX(id)+1.
|
||||
* Mirrors the `allocateCatalogItemId` pattern from furni-import.ts.
|
||||
*/
|
||||
let itemsBaseIdAllocChain: Promise<unknown> = Promise.resolve();
|
||||
let itemsBaseNextId: number | null = null;
|
||||
let itemsBaseIdLastUsed = 0;
|
||||
let itemsBaseIdSeedChain: Promise<void> = Promise.resolve();
|
||||
const ITEMS_BASE_ID_REFRESH_MS = 60_000;
|
||||
|
||||
/** Allocate a unique items_base id. Seeded once with MAX(id)+1, then handed
|
||||
* out from an in-process counter so concurrent clones don't serialize on a
|
||||
* SELECT MAX + INSERT round-trip per item. Re-seeds when idle for a while so
|
||||
* rows added by the emulator or other processes don't collide. */
|
||||
async function allocateItemsBaseId<T>(
|
||||
insertFn: (nextId: number) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const prev = itemsBaseIdAllocChain;
|
||||
let settle!: () => void;
|
||||
itemsBaseIdAllocChain = new Promise<void>((r) => {
|
||||
settle = r;
|
||||
});
|
||||
await prev.catch(() => {});
|
||||
try {
|
||||
const [idRows] = (await db.execute(sql`
|
||||
SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base
|
||||
`)) as unknown as [Array<{ next: number }>, unknown];
|
||||
const nextId = Number(idRows[0]?.next ?? 1);
|
||||
return await insertFn(nextId);
|
||||
} finally {
|
||||
settle();
|
||||
const now = Date.now();
|
||||
if (
|
||||
itemsBaseNextId === null ||
|
||||
now - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS
|
||||
) {
|
||||
let settle!: () => void;
|
||||
const prev = itemsBaseIdSeedChain;
|
||||
itemsBaseIdSeedChain = new Promise<void>((r) => {
|
||||
settle = r;
|
||||
});
|
||||
await prev.catch(() => {});
|
||||
try {
|
||||
if (
|
||||
itemsBaseNextId === null ||
|
||||
Date.now() - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS
|
||||
) {
|
||||
const [idRows] = (await db.execute(sql`
|
||||
SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base
|
||||
`)) as unknown as [Array<{ next: number }>, unknown];
|
||||
itemsBaseNextId = Number(idRows[0]?.next ?? 1);
|
||||
}
|
||||
} finally {
|
||||
settle();
|
||||
}
|
||||
}
|
||||
itemsBaseIdLastUsed = Date.now();
|
||||
const nextId = itemsBaseNextId;
|
||||
itemsBaseNextId += 1;
|
||||
return await insertFn(nextId);
|
||||
}
|
||||
|
||||
/** Test seam: drop the cached id counter between tests. */
|
||||
export function __resetItemsBaseAllocatorForTests(): void {
|
||||
itemsBaseNextId = null;
|
||||
itemsBaseIdLastUsed = 0;
|
||||
itemsBaseIdSeedChain = Promise.resolve();
|
||||
}
|
||||
|
||||
export async function cloneSingleFurni(params: {
|
||||
|
||||
Reference in new issue
Block a user