diff --git a/src/app/admin/import/clone/import-clone-client.tsx b/src/app/admin/import/clone/import-clone-client.tsx index 463a237a..095fb34e 100644 --- a/src/app/admin/import/clone/import-clone-client.tsx +++ b/src/app/admin/import/clone/import-clone-client.tsx @@ -454,7 +454,7 @@ function FurniGrid({ source }: FurniGridProps) { { sourceId: source.id, classnames: toClone.map((it) => it.classname), - concurrency: 6, + concurrency: 10, }, (classname) => { done++; @@ -545,7 +545,7 @@ function FurniGrid({ source }: FurniGridProps) { const chunk = names.slice(i, i + CHUNK); await runSseImport( "/api/admin/import/clone/batch", - { sourceId: source.id, classnames: chunk, concurrency: 6 }, + { sourceId: source.id, classnames: chunk, concurrency: 10 }, (classname) => { done++; markDone(classname); diff --git a/src/app/api/admin/import/clone/batch/route.ts b/src/app/api/admin/import/clone/batch/route.ts index a6c9ab71..85abca89 100644 --- a/src/app/api/admin/import/clone/batch/route.ts +++ b/src/app/api/admin/import/clone/batch/route.ts @@ -67,7 +67,7 @@ export const POST = withAdmin( return runSseBatch({ items, - concurrency: body.concurrency || 6, + concurrency: body.concurrency || 10, signal: request.signal, labelOf: (it) => it.classname, worker: async (it, _index, report) => { diff --git a/src/app/api/admin/import/clone/sync-all/route.ts b/src/app/api/admin/import/clone/sync-all/route.ts index 50ed2adb..733cfdf7 100644 --- a/src/app/api/admin/import/clone/sync-all/route.ts +++ b/src/app/api/admin/import/clone/sync-all/route.ts @@ -108,7 +108,7 @@ export const POST = withAdmin( return runSseBatch({ items: allItems, - concurrency: 6, + concurrency: 10, signal: request.signal, labelOf: (it) => `${it.sourceName}/${it.classname}`, worker: async (it, _index, report) => { diff --git a/src/lib/services/clone-import.test.ts b/src/lib/services/clone-import.test.ts index 0953ccf5..525930a1 100644 --- a/src/lib/services/clone-import.test.ts +++ b/src/lib/services/clone-import.test.ts @@ -1,5 +1,5 @@ // @vitest-environment node -import { describe, expect, it, vi } from "vitest"; +import { beforeEach, describe, expect, it, vi } from "vitest"; // biome-ignore lint/suspicious/noExplicitAny: test helper for mocking with arbitrary signatures type AnyFn = (...args: any[]) => any; @@ -69,7 +69,11 @@ vi.mock("@/lib/db", () => ({ ItemsBase: { id: "id", itemName: "itemName" }, })); -import { cloneSingleFurni, parseFurnidata } from "./clone-import"; +import { + __resetItemsBaseAllocatorForTests, + cloneSingleFurni, + parseFurnidata, +} from "./clone-import"; const SOURCE = { id: "s", @@ -92,6 +96,10 @@ const ENTRY = { }; describe("clone-import", () => { + beforeEach(() => { + __resetItemsBaseAllocatorForTests(); + }); + it("parseFurnidata normalizes room + wall items with itemType", () => { const list = parseFurnidata({ roomitemtypes: { furnitype: [ENTRY] }, diff --git a/src/lib/services/clone-import.ts b/src/lib/services/clone-import.ts index c43f3aae..45e67196 100644 --- a/src/lib/services/clone-import.ts +++ b/src/lib/services/clone-import.ts @@ -137,26 +137,54 @@ export async function fetchSourceFurnidata( * concurrent clones in the same Node process cannot race on MAX(id)+1. * Mirrors the `allocateCatalogItemId` pattern from furni-import.ts. */ -let itemsBaseIdAllocChain: Promise = Promise.resolve(); +let itemsBaseNextId: number | null = null; +let itemsBaseIdLastUsed = 0; +let itemsBaseIdSeedChain: Promise = Promise.resolve(); +const ITEMS_BASE_ID_REFRESH_MS = 60_000; +/** Allocate a unique items_base id. Seeded once with MAX(id)+1, then handed + * out from an in-process counter so concurrent clones don't serialize on a + * SELECT MAX + INSERT round-trip per item. Re-seeds when idle for a while so + * rows added by the emulator or other processes don't collide. */ async function allocateItemsBaseId( insertFn: (nextId: number) => Promise, ): Promise { - const prev = itemsBaseIdAllocChain; - let settle!: () => void; - itemsBaseIdAllocChain = new Promise((r) => { - settle = r; - }); - await prev.catch(() => {}); - try { - const [idRows] = (await db.execute(sql` - SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base - `)) as unknown as [Array<{ next: number }>, unknown]; - const nextId = Number(idRows[0]?.next ?? 1); - return await insertFn(nextId); - } finally { - settle(); + const now = Date.now(); + if ( + itemsBaseNextId === null || + now - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS + ) { + let settle!: () => void; + const prev = itemsBaseIdSeedChain; + itemsBaseIdSeedChain = new Promise((r) => { + settle = r; + }); + await prev.catch(() => {}); + try { + if ( + itemsBaseNextId === null || + Date.now() - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS + ) { + const [idRows] = (await db.execute(sql` + SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base + `)) as unknown as [Array<{ next: number }>, unknown]; + itemsBaseNextId = Number(idRows[0]?.next ?? 1); + } + } finally { + settle(); + } } + itemsBaseIdLastUsed = Date.now(); + const nextId = itemsBaseNextId; + itemsBaseNextId += 1; + return await insertFn(nextId); +} + +/** Test seam: drop the cached id counter between tests. */ +export function __resetItemsBaseAllocatorForTests(): void { + itemsBaseNextId = null; + itemsBaseIdLastUsed = 0; + itemsBaseIdSeedChain = Promise.resolve(); } export async function cloneSingleFurni(params: { diff --git a/src/lib/services/furni-import.ts b/src/lib/services/furni-import.ts index ab3f626d..2ba47186 100644 --- a/src/lib/services/furni-import.ts +++ b/src/lib/services/furni-import.ts @@ -276,26 +276,54 @@ export function autoPriceFurni(classname: string): { * connection for the whole operation; the in-process mutex is enough * for the single-server deployment and avoids pool-connection gymnastics. */ -let catalogIdAllocChain: Promise = Promise.resolve(); +let catalogNextId: number | null = null; +let catalogIdLastUsed = 0; +let catalogIdSeedChain: Promise = Promise.resolve(); +const CATALOG_ID_REFRESH_MS = 60_000; +/** Allocate a unique catalog_items id. Seeded once with MAX(id)+1, then handed + * out from an in-process counter so concurrent imports don't serialize on a + * SELECT MAX + INSERT round-trip per item. Re-seeds when idle so rows added + * externally don't collide. */ export async function allocateCatalogItemId( insertFn: (nextId: number) => Promise, ): Promise { - const prev = catalogIdAllocChain; - let settle!: () => void; - catalogIdAllocChain = new Promise((r) => { - settle = r; - }); - await prev.catch(() => {}); - try { - const [maxIdResult] = (await db.execute(sql` - SELECT MAX(id) as maxId FROM catalog_items - `)) as unknown as [Array<{ maxId: number | bigint | null }>, unknown]; - const nextId = Number(maxIdResult[0]?.maxId ?? 0) + 1; - return await insertFn(nextId); - } finally { - settle(); + const now = Date.now(); + if ( + catalogNextId === null || + now - catalogIdLastUsed > CATALOG_ID_REFRESH_MS + ) { + let settle!: () => void; + const prev = catalogIdSeedChain; + catalogIdSeedChain = new Promise((r) => { + settle = r; + }); + await prev.catch(() => {}); + try { + if ( + catalogNextId === null || + Date.now() - catalogIdLastUsed > CATALOG_ID_REFRESH_MS + ) { + const [maxIdResult] = (await db.execute(sql` + SELECT MAX(id) as maxId FROM catalog_items + `)) as unknown as [Array<{ maxId: number | bigint | null }>, unknown]; + catalogNextId = Number(maxIdResult[0]?.maxId ?? 0) + 1; + } + } finally { + settle(); + } } + catalogIdLastUsed = Date.now(); + const nextId = catalogNextId; + catalogNextId += 1; + return await insertFn(nextId); +} + +/** Test seam: drop the cached id counter between tests. */ +export function __resetCatalogIdAllocatorForTests(): void { + catalogNextId = null; + catalogIdLastUsed = 0; + catalogIdSeedChain = Promise.resolve(); } /** Get or create a category sub-page under the imported parent page. */ diff --git a/src/lib/services/upload-import.test.ts b/src/lib/services/upload-import.test.ts index 5f12e342..5a1d7f01 100644 --- a/src/lib/services/upload-import.test.ts +++ b/src/lib/services/upload-import.test.ts @@ -82,12 +82,16 @@ vi.mock("@/lib/services/swf/nitro-builder", () => ({ parseNitroBundle: vi.fn(() => ({ json: { xdim: 1, ydim: 1 } })), })); -import { uploadSingleFurni } from "./upload-import"; +import { + __resetUploadItemsBaseAllocatorForTests, + uploadSingleFurni, +} from "./upload-import"; describe("uploadSingleFurni live asset mirrors", () => { let tempDir: string; beforeEach(async () => { + __resetUploadItemsBaseAllocatorForTests(); tempDir = await fs.mkdtemp(path.join(os.tmpdir(), "upload-furni-")); assetTargets.swfDir = path.join(tempDir, "cms", "swf"); assetTargets.iconDir = path.join(tempDir, "cms", "icons"); diff --git a/src/lib/services/upload-import.ts b/src/lib/services/upload-import.ts index 0f019874..9aa50a2f 100644 --- a/src/lib/services/upload-import.ts +++ b/src/lib/services/upload-import.ts @@ -41,7 +41,10 @@ const MIGRATIONS_DIR = path.resolve( "drizzle/migrations", ); -let itemsBaseIdAllocChain: Promise = Promise.resolve(); +let itemsBaseNextId: number | null = null; +let itemsBaseIdLastUsed = 0; +let itemsBaseIdSeedChain: Promise = Promise.resolve(); +const ITEMS_BASE_ID_REFRESH_MS = 60_000; function pathKey(value: string): string { const normalized = path.normalize(value); @@ -76,21 +79,42 @@ async function mirrorUploadedAsset( async function allocateItemsBaseId( insertFn: (nextId: number) => Promise, ): Promise { - const prev = itemsBaseIdAllocChain; - let settle!: () => void; - itemsBaseIdAllocChain = new Promise((r) => { - settle = r; - }); - await prev.catch(() => {}); - try { - const [idRows] = (await db.execute(sql` - SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base - `)) as unknown as [Array<{ next: number }>, unknown]; - const nextId = Number(idRows[0]?.next ?? 1); - return await insertFn(nextId); - } finally { - settle(); + const now = Date.now(); + if ( + itemsBaseNextId === null || + now - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS + ) { + let settle!: () => void; + const prev = itemsBaseIdSeedChain; + itemsBaseIdSeedChain = new Promise((r) => { + settle = r; + }); + await prev.catch(() => {}); + try { + if ( + itemsBaseNextId === null || + Date.now() - itemsBaseIdLastUsed > ITEMS_BASE_ID_REFRESH_MS + ) { + const [idRows] = (await db.execute(sql` + SELECT COALESCE(MAX(id), 0) + 1 AS next FROM items_base + `)) as unknown as [Array<{ next: number }>, unknown]; + itemsBaseNextId = Number(idRows[0]?.next ?? 1); + } + } finally { + settle(); + } } + itemsBaseIdLastUsed = Date.now(); + const nextId = itemsBaseNextId; + itemsBaseNextId += 1; + return await insertFn(nextId); +} + +/** Test seam: drop the cached id counter between tests. */ +export function __resetUploadItemsBaseAllocatorForTests(): void { + itemsBaseNextId = null; + itemsBaseIdLastUsed = 0; + itemsBaseIdSeedChain = Promise.resolve(); } function escapeSql(val: string | number): string {