perf(import): parallelize ID allocation and raise clone concurrency to 10
Replace the serialized SELECT MAX + INSERT id-allocation chains for items_base and catalog_items with a lazy-seeded in-process counter so concurrent clone workers no longer queue on a global lock per item. Re-seeds after 60s idle to avoid colliding with externally added rows. Raise the clone import concurrency default from 6 to the batch cap of 10.
This commit is contained in:
1 parent
742536820a
commit
172941c542
8 files changed
+144
-52
No files matched your search
@@ -276,26 +276,54 @@ export function autoPriceFurni(classname: string): {
|
||||
* connection for the whole operation; the in-process mutex is enough
|
||||
* for the single-server deployment and avoids pool-connection gymnastics.
|
||||
*/
|
||||
let catalogIdAllocChain: Promise<unknown> = Promise.resolve();
|
||||
let catalogNextId: number | null = null;
|
||||
let catalogIdLastUsed = 0;
|
||||
let catalogIdSeedChain: Promise<void> = Promise.resolve();
|
||||
const CATALOG_ID_REFRESH_MS = 60_000;
|
||||
|
||||
/** Allocate a unique catalog_items id. Seeded once with MAX(id)+1, then handed
|
||||
* out from an in-process counter so concurrent imports don't serialize on a
|
||||
* SELECT MAX + INSERT round-trip per item. Re-seeds when idle so rows added
|
||||
* externally don't collide. */
|
||||
export async function allocateCatalogItemId<T>(
|
||||
insertFn: (nextId: number) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const prev = catalogIdAllocChain;
|
||||
let settle!: () => void;
|
||||
catalogIdAllocChain = new Promise<void>((r) => {
|
||||
settle = r;
|
||||
});
|
||||
await prev.catch(() => {});
|
||||
try {
|
||||
const [maxIdResult] = (await db.execute(sql`
|
||||
SELECT MAX(id) as maxId FROM catalog_items
|
||||
`)) as unknown as [Array<{ maxId: number | bigint | null }>, unknown];
|
||||
const nextId = Number(maxIdResult[0]?.maxId ?? 0) + 1;
|
||||
return await insertFn(nextId);
|
||||
} finally {
|
||||
settle();
|
||||
const now = Date.now();
|
||||
if (
|
||||
catalogNextId === null ||
|
||||
now - catalogIdLastUsed > CATALOG_ID_REFRESH_MS
|
||||
) {
|
||||
let settle!: () => void;
|
||||
const prev = catalogIdSeedChain;
|
||||
catalogIdSeedChain = new Promise<void>((r) => {
|
||||
settle = r;
|
||||
});
|
||||
await prev.catch(() => {});
|
||||
try {
|
||||
if (
|
||||
catalogNextId === null ||
|
||||
Date.now() - catalogIdLastUsed > CATALOG_ID_REFRESH_MS
|
||||
) {
|
||||
const [maxIdResult] = (await db.execute(sql`
|
||||
SELECT MAX(id) as maxId FROM catalog_items
|
||||
`)) as unknown as [Array<{ maxId: number | bigint | null }>, unknown];
|
||||
catalogNextId = Number(maxIdResult[0]?.maxId ?? 0) + 1;
|
||||
}
|
||||
} finally {
|
||||
settle();
|
||||
}
|
||||
}
|
||||
catalogIdLastUsed = Date.now();
|
||||
const nextId = catalogNextId;
|
||||
catalogNextId += 1;
|
||||
return await insertFn(nextId);
|
||||
}
|
||||
|
||||
/** Test seam: drop the cached id counter between tests. */
|
||||
export function __resetCatalogIdAllocatorForTests(): void {
|
||||
catalogNextId = null;
|
||||
catalogIdLastUsed = 0;
|
||||
catalogIdSeedChain = Promise.resolve();
|
||||
}
|
||||
|
||||
/** Get or create a category sub-page under the imported parent page. */
|
||||
|
||||
Reference in new issue
Block a user