perf(import): parallelize ID allocation and raise clone concurrency to 10
CI / check (push) Successful in 27s
CI / release (push) Skipped
CI / deploy (push) Successful in 41s

Replace the serialized SELECT MAX + INSERT id-allocation chains for
items_base and catalog_items with a lazy-seeded in-process counter so
concurrent clone workers no longer queue on a global lock per item.
Re-seeds after 60s idle to avoid colliding with externally added rows.
Raise the clone import concurrency default from 6 to the batch cap of 10.
This commit is contained in:
openhands committed 2026-08-05 11:37:17 +02:00
1 parent 742536820a
commit 172941c542
8 files changed
+144 -52

No files matched your search

+43 -15
View File
@@ -276,26 +276,54 @@ export function autoPriceFurni(classname: string): {
* connection for the whole operation; the in-process mutex is enough
* for the single-server deployment and avoids pool-connection gymnastics.
*/
let catalogIdAllocChain: Promise<unknown> = Promise.resolve();
let catalogNextId: number | null = null;
let catalogIdLastUsed = 0;
let catalogIdSeedChain: Promise<void> = Promise.resolve();
const CATALOG_ID_REFRESH_MS = 60_000;
/** Allocate a unique catalog_items id. Seeded once with MAX(id)+1, then handed
* out from an in-process counter so concurrent imports don't serialize on a
* SELECT MAX + INSERT round-trip per item. Re-seeds when idle so rows added
* externally don't collide. */
export async function allocateCatalogItemId<T>(
insertFn: (nextId: number) => Promise<T>,
): Promise<T> {
const prev = catalogIdAllocChain;
let settle!: () => void;
catalogIdAllocChain = new Promise<void>((r) => {
settle = r;
});
await prev.catch(() => {});
try {
const [maxIdResult] = (await db.execute(sql`
SELECT MAX(id) as maxId FROM catalog_items
`)) as unknown as [Array<{ maxId: number | bigint | null }>, unknown];
const nextId = Number(maxIdResult[0]?.maxId ?? 0) + 1;
return await insertFn(nextId);
} finally {
settle();
const now = Date.now();
if (
catalogNextId === null ||
now - catalogIdLastUsed > CATALOG_ID_REFRESH_MS
) {
let settle!: () => void;
const prev = catalogIdSeedChain;
catalogIdSeedChain = new Promise<void>((r) => {
settle = r;
});
await prev.catch(() => {});
try {
if (
catalogNextId === null ||
Date.now() - catalogIdLastUsed > CATALOG_ID_REFRESH_MS
) {
const [maxIdResult] = (await db.execute(sql`
SELECT MAX(id) as maxId FROM catalog_items
`)) as unknown as [Array<{ maxId: number | bigint | null }>, unknown];
catalogNextId = Number(maxIdResult[0]?.maxId ?? 0) + 1;
}
} finally {
settle();
}
}
catalogIdLastUsed = Date.now();
const nextId = catalogNextId;
catalogNextId += 1;
return await insertFn(nextId);
}
/** Test seam: drop the cached id counter between tests. */
export function __resetCatalogIdAllocatorForTests(): void {
catalogNextId = null;
catalogIdLastUsed = 0;
catalogIdSeedChain = Promise.resolve();
}
/** Get or create a category sub-page under the imported parent page. */