fix: prevent import hanging by adding abort signals and idle timeouts
CI / check (push) Successful in 28s
CI / release (push) Skipped
CI / deploy (push) Successful in 59s

- Furni batch: wire request.signal to abort controller, send post_import
  progress events, skip post-import steps when aborted
- Clone sync-all: pre-fetch furnidata once per source instead of per item
  (eliminates 2000+ redundant DB reads + HTTP requests)
- SSE client: add 60s idle timeout to prevent infinite hangs when server
  stops responding
This commit is contained in:
openhands committed 2026-08-31 18:11:54 +02:00
1 parent 96c33efced
commit f70d96b81c
3 files changed
+179 -85

No files matched your search

@@ -6,6 +6,7 @@ import { logAudit } from "@/lib/services/audit";
import {
cloneSingleFurni,
fetchSourceFurnidata,
type SourceFurni,
} from "@/lib/services/clone-import";
import { listSources } from "@/lib/services/clone-sources";
import {
@@ -39,6 +40,20 @@ export const POST = withAdmin(
);
}
// Pre-fetch furnidata ONCE per source and build a lookup map.
// The old code re-fetched inside the worker loop for every item,
// causing 2000+ redundant DB reads + HTTP requests.
const sourceFurniDataMap = new Map<string, Map<string, SourceFurni>>();
for (const source of sources) {
try {
const entries = await fetchSourceFurnidata(source.furnidataUrl);
const byClassname = new Map(entries.map((e) => [e.classname, e]));
sourceFurniDataMap.set(source.id, byClassname);
} catch {}
}
// Collect all missing items using pre-fetched data
const allItems: Array<{
classname: string;
sourceName: string;
@@ -46,14 +61,10 @@ export const POST = withAdmin(
}> = [];
for (const source of sources) {
let entries: Awaited<ReturnType<typeof fetchSourceFurnidata>>;
try {
entries = await fetchSourceFurnidata(source.furnidataUrl);
} catch {
continue;
}
const byClassname = sourceFurniDataMap.get(source.id);
if (!byClassname) continue;
const classnames = entries.map((e) => e.classname);
const classnames = [...byClassname.keys()];
if (classnames.length === 0) continue;
const rows = await db
@@ -62,10 +73,10 @@ export const POST = withAdmin(
.where(inArray(ItemsBase.itemName, classnames));
const have = new Set(rows.map((r) => r.item_name));
for (const entry of entries) {
if (!have.has(entry.classname)) {
for (const classname of classnames) {
if (!have.has(classname)) {
allItems.push({
classname: entry.classname,
classname,
sourceName: source.name,
sourceId: source.id,
});
@@ -106,29 +117,30 @@ export const POST = withAdmin(
itemType: string;
}> = [];
// Source lookup cache — pre-fetched above, no need to re-fetch per item
const sourceMap = new Map(sources.map((s) => [s.id, s]));
return runSseBatch({
items: allItems,
concurrency: 10,
signal: request.signal,
labelOf: (it) => `${it.sourceName}/${it.classname}`,
worker: async (it, _index, report) => {
const allSrc = await listSources();
const source = allSrc.find((s) => s.id === it.sourceId);
const source = sourceMap.get(it.sourceId);
if (!source) {
return { ok: false, error: "source not found" };
}
let allEntries: Awaited<ReturnType<typeof fetchSourceFurnidata>>;
try {
allEntries = await fetchSourceFurnidata(source.furnidataUrl);
} catch (err) {
// Use pre-fetched furnidata instead of re-fetching
const byClassname = sourceFurniDataMap.get(it.sourceId);
if (!byClassname) {
return {
ok: false,
error: `furnidata fetch failed: ${(err as Error).message}`,
error: "furnidata not available for this source",
};
}
const entry = allEntries.find((e) => e.classname === it.classname);
const entry = byClassname.get(it.classname);
if (!entry) {
return {
ok: false,