fix: prevent import hanging by adding abort signals and idle timeouts
- Furni batch: wire request.signal to abort controller, send post_import progress events, skip post-import steps when aborted - Clone sync-all: pre-fetch furnidata once per source instead of per item (eliminates 2000+ redundant DB reads + HTTP requests) - SSE client: add 60s idle timeout to prevent infinite hangs when server stops responding
This commit is contained in:
1 parent
96c33efced
commit
f70d96b81c
3 files changed
+179
-85
No files matched your search
@@ -6,6 +6,7 @@ import { logAudit } from "@/lib/services/audit";
|
||||
import {
|
||||
cloneSingleFurni,
|
||||
fetchSourceFurnidata,
|
||||
type SourceFurni,
|
||||
} from "@/lib/services/clone-import";
|
||||
import { listSources } from "@/lib/services/clone-sources";
|
||||
import {
|
||||
@@ -39,6 +40,20 @@ export const POST = withAdmin(
|
||||
);
|
||||
}
|
||||
|
||||
// Pre-fetch furnidata ONCE per source and build a lookup map.
|
||||
// The old code re-fetched inside the worker loop for every item,
|
||||
// causing 2000+ redundant DB reads + HTTP requests.
|
||||
const sourceFurniDataMap = new Map<string, Map<string, SourceFurni>>();
|
||||
|
||||
for (const source of sources) {
|
||||
try {
|
||||
const entries = await fetchSourceFurnidata(source.furnidataUrl);
|
||||
const byClassname = new Map(entries.map((e) => [e.classname, e]));
|
||||
sourceFurniDataMap.set(source.id, byClassname);
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Collect all missing items using pre-fetched data
|
||||
const allItems: Array<{
|
||||
classname: string;
|
||||
sourceName: string;
|
||||
@@ -46,14 +61,10 @@ export const POST = withAdmin(
|
||||
}> = [];
|
||||
|
||||
for (const source of sources) {
|
||||
let entries: Awaited<ReturnType<typeof fetchSourceFurnidata>>;
|
||||
try {
|
||||
entries = await fetchSourceFurnidata(source.furnidataUrl);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const byClassname = sourceFurniDataMap.get(source.id);
|
||||
if (!byClassname) continue;
|
||||
|
||||
const classnames = entries.map((e) => e.classname);
|
||||
const classnames = [...byClassname.keys()];
|
||||
if (classnames.length === 0) continue;
|
||||
|
||||
const rows = await db
|
||||
@@ -62,10 +73,10 @@ export const POST = withAdmin(
|
||||
.where(inArray(ItemsBase.itemName, classnames));
|
||||
const have = new Set(rows.map((r) => r.item_name));
|
||||
|
||||
for (const entry of entries) {
|
||||
if (!have.has(entry.classname)) {
|
||||
for (const classname of classnames) {
|
||||
if (!have.has(classname)) {
|
||||
allItems.push({
|
||||
classname: entry.classname,
|
||||
classname,
|
||||
sourceName: source.name,
|
||||
sourceId: source.id,
|
||||
});
|
||||
@@ -106,29 +117,30 @@ export const POST = withAdmin(
|
||||
itemType: string;
|
||||
}> = [];
|
||||
|
||||
// Source lookup cache — pre-fetched above, no need to re-fetch per item
|
||||
const sourceMap = new Map(sources.map((s) => [s.id, s]));
|
||||
|
||||
return runSseBatch({
|
||||
items: allItems,
|
||||
concurrency: 10,
|
||||
signal: request.signal,
|
||||
labelOf: (it) => `${it.sourceName}/${it.classname}`,
|
||||
worker: async (it, _index, report) => {
|
||||
const allSrc = await listSources();
|
||||
const source = allSrc.find((s) => s.id === it.sourceId);
|
||||
const source = sourceMap.get(it.sourceId);
|
||||
if (!source) {
|
||||
return { ok: false, error: "source not found" };
|
||||
}
|
||||
|
||||
let allEntries: Awaited<ReturnType<typeof fetchSourceFurnidata>>;
|
||||
try {
|
||||
allEntries = await fetchSourceFurnidata(source.furnidataUrl);
|
||||
} catch (err) {
|
||||
// Use pre-fetched furnidata instead of re-fetching
|
||||
const byClassname = sourceFurniDataMap.get(it.sourceId);
|
||||
if (!byClassname) {
|
||||
return {
|
||||
ok: false,
|
||||
error: `furnidata fetch failed: ${(err as Error).message}`,
|
||||
error: "furnidata not available for this source",
|
||||
};
|
||||
}
|
||||
|
||||
const entry = allEntries.find((e) => e.classname === it.classname);
|
||||
const entry = byClassname.get(it.classname);
|
||||
if (!entry) {
|
||||
return {
|
||||
ok: false,
|
||||
|
||||
Reference in new issue
Block a user