feat: speed up admin clone import and enable Cache Components
- Clone import: defer FurnitureData.json writes and append all entries in a single batched write instead of one read-modify-write per item, removing the main serialization bottleneck for large batches. - Clone import: raise SSE batch concurrency cap from 5 to 10 and bump the clone client/route default from 2 to 6. - Add a flush hook to runSseBatch so callers can batch deferred work before batch_complete is emitted, and surface flush errors as an error event. - Enable Next.js Cache Components (instant: false opt-out) and silence the related build warnings in next.config.ts. - Switch isomorphic-dompurify to dompurify and refresh dependencies.
This commit is contained in:
1 parent
ac9b3e3cb5
commit
dc8fb8a6ed
254 files changed
+910
-551
No files matched your search
@@ -7,6 +7,10 @@ import {
|
||||
fetchSourceFurnidata,
|
||||
} from "@/lib/services/clone-import";
|
||||
import { getSource } from "@/lib/services/clone-sources";
|
||||
import {
|
||||
appendFurniEntriesBatch,
|
||||
appendFurniEntry,
|
||||
} from "@/lib/services/furni-data";
|
||||
import { runSseBatch } from "@/lib/services/import/core/sse-batch";
|
||||
|
||||
interface BatchItem {
|
||||
@@ -53,9 +57,17 @@ export const POST = withAdmin(
|
||||
if (items.length > 500)
|
||||
return apiError("Too many classnames (max 500)", 400);
|
||||
|
||||
// FurnitureData entries for successfully cloned items are appended once
|
||||
// at the end (single read-modify-write instead of one per item), which
|
||||
// removes the main serialization bottleneck when cloning hundreds of items.
|
||||
const deferredEntries: Array<{
|
||||
entry: Record<string, unknown>;
|
||||
itemType: string;
|
||||
}> = [];
|
||||
|
||||
return runSseBatch<BatchItem>({
|
||||
items,
|
||||
concurrency: body.concurrency || 2,
|
||||
concurrency: body.concurrency || 6,
|
||||
signal: request.signal,
|
||||
labelOf: (it) => it.classname,
|
||||
worker: async (it, _index, report) => {
|
||||
@@ -71,9 +83,13 @@ export const POST = withAdmin(
|
||||
source,
|
||||
entry,
|
||||
onProgress: report,
|
||||
deferFurniData: true,
|
||||
});
|
||||
|
||||
if (result.ok) {
|
||||
if (result.furniDataEntry) {
|
||||
deferredEntries.push(result.furniDataEntry);
|
||||
}
|
||||
logAudit({
|
||||
userId: ctx.session.user.id,
|
||||
action: "clone_furni",
|
||||
@@ -93,6 +109,24 @@ export const POST = withAdmin(
|
||||
error: result.error,
|
||||
};
|
||||
},
|
||||
flush: async () => {
|
||||
if (deferredEntries.length === 0) return;
|
||||
try {
|
||||
await appendFurniEntriesBatch(deferredEntries);
|
||||
} catch (err) {
|
||||
// Fallback so partial data survives: write each entry individually.
|
||||
for (const { entry, itemType } of deferredEntries) {
|
||||
try {
|
||||
await appendFurniEntry(entry, itemType);
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`FurnitureData batch write failed: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -8,6 +8,10 @@ import {
|
||||
fetchSourceFurnidata,
|
||||
} from "@/lib/services/clone-import";
|
||||
import { listSources } from "@/lib/services/clone-sources";
|
||||
import {
|
||||
appendFurniEntriesBatch,
|
||||
appendFurniEntry,
|
||||
} from "@/lib/services/furni-data";
|
||||
import { runSseBatch } from "@/lib/services/import/core/sse-batch";
|
||||
|
||||
function sseEvent(data: unknown): string {
|
||||
@@ -95,9 +99,16 @@ export const POST = withAdmin(
|
||||
);
|
||||
}
|
||||
|
||||
// FurnitureData entries are appended once at the end (single write) rather
|
||||
// than once per item, removing the main serialization bottleneck.
|
||||
const deferredEntries: Array<{
|
||||
entry: Record<string, unknown>;
|
||||
itemType: string;
|
||||
}> = [];
|
||||
|
||||
return runSseBatch({
|
||||
items: allItems,
|
||||
concurrency: 2,
|
||||
concurrency: 6,
|
||||
signal: request.signal,
|
||||
labelOf: (it) => `${it.sourceName}/${it.classname}`,
|
||||
worker: async (it, _index, report) => {
|
||||
@@ -129,9 +140,13 @@ export const POST = withAdmin(
|
||||
source,
|
||||
entry,
|
||||
onProgress: report,
|
||||
deferFurniData: true,
|
||||
});
|
||||
|
||||
if (result.ok) {
|
||||
if (result.furniDataEntry) {
|
||||
deferredEntries.push(result.furniDataEntry);
|
||||
}
|
||||
logAudit({
|
||||
userId: ctx.session.user.id,
|
||||
action: "auto_sync_furni",
|
||||
@@ -151,6 +166,23 @@ export const POST = withAdmin(
|
||||
error: result.error,
|
||||
};
|
||||
},
|
||||
flush: async () => {
|
||||
if (deferredEntries.length === 0) return;
|
||||
try {
|
||||
await appendFurniEntriesBatch(deferredEntries);
|
||||
} catch (err) {
|
||||
for (const { entry, itemType } of deferredEntries) {
|
||||
try {
|
||||
await appendFurniEntry(entry, itemType);
|
||||
} catch {
|
||||
/* best effort */
|
||||
}
|
||||
}
|
||||
throw new Error(
|
||||
`FurnitureData batch write failed: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
},
|
||||
});
|
||||
},
|
||||
);
|
||||
Reference in new issue
Block a user