perf(import): parallelize ID allocation and raise clone concurrency to 10
CI / check (push) Successful in 27s
CI / release (push) Skipped
CI / deploy (push) Successful in 41s

Replace the serialized SELECT MAX + INSERT id-allocation chains for
items_base and catalog_items with a lazy-seeded in-process counter so
concurrent clone workers no longer queue on a global lock per item.
Re-seeds after 60s idle to avoid colliding with externally added rows.
Raise the clone import concurrency default from 6 to the batch cap of 10.
This commit is contained in:
openhands committed 2026-08-05 11:37:17 +02:00
1 parent 742536820a
commit 172941c542
8 files changed
+144 -52

No files matched your search

+39 -15
View File
@@ -41,7 +41,10 @@ const MIGRATIONS_DIR = path.resolve(
"drizzle/migrations",
);
let itemsBaseIdAllocChain: Promise<unknown> = Promise.resolve();
let itemsBaseNextId: number | null = null;
let itemsBaseIdLastUsed = 0;
let itemsBaseIdSeedChain: Promise<void> = Promise.resolve();
const ITEMS_BASE_ID_REFRESH_MS = 60_000;
function pathKey(value: string): string {
const normalized = path.normalize(value);
@@ -76,21 +79,42 @@ async function mirrorUploadedAsset(
async function allocateItemsBaseId<T>(
insertFn: (nextId: number) => Promise<T>,
): Promise<T> {
const prev = itemsBaseIdAllocChain;
let settle!: () => void;
itemsBaseIdAllocChain = new Promise<void>((r) => {
settle = r;
});
await prev.catch(() => {});
try {
const [idRows] = (await db.execute(sql`
SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base
`)) as unknown as [Array<{ next: number }>, unknown];
const nextId = Number(idRows[0]?.next ?? 1);
return await insertFn(nextId);
} finally {
settle();
const now = Date.now();
if (
itemsBaseNextId === null ||
now - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS
) {
let settle!: () => void;
const prev = itemsBaseIdSeedChain;
itemsBaseIdSeedChain = new Promise<void>((r) => {
settle = r;
});
await prev.catch(() => {});
try {
if (
itemsBaseNextId === null ||
Date.now() - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS
) {
const [idRows] = (await db.execute(sql`
SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base
`)) as unknown as [Array<{ next: number }>, unknown];
itemsBaseNextId = Number(idRows[0]?.next ?? 1);
}
} finally {
settle();
}
}
itemsBaseIdLastUsed = Date.now();
const nextId = itemsBaseNextId;
itemsBaseNextId += 1;
return await insertFn(nextId);
}
/** Test seam: drop the cached id counter between tests. */
export function __resetUploadItemsBaseAllocatorForTests(): void {
itemsBaseNextId = null;
itemsBaseIdLastUsed = 0;
itemsBaseIdSeedChain = Promise.resolve();
}
function escapeSql(val: string | number): string {