perf: race furniture asset download candidates instead of trying sequentially
tryDownloadCandidates now fires every candidate URL concurrently and keeps the first valid response, removing the old 15s×retry sequential waits on slow/unreachable sources that made import speed uneven. Batch concurrency raised 8 -> 12 to absorb the now-faster downloads.
This commit is contained in:
1 parent
ac7c427c3d
commit
1d8672ccc5
2 files changed
+32
-7
No files matched your search
@@ -53,7 +53,7 @@ import type { ImportSingleResult } from "@/types/furni";
|
||||
// How many items to import concurrently in a batch. The per-item work is
|
||||
// network-bound (asset downloads) and DB-bound, so a modest pool gives a large
|
||||
// speedup over the old fully sequential loop without overwhelming the DB.
|
||||
const IMPORT_BATCH_CONCURRENCY = 8;
|
||||
const IMPORT_BATCH_CONCURRENCY = 12;
|
||||
|
||||
async function runWithConcurrency<T, R>(
|
||||
items: T[],
|
||||
|
||||
@@ -26,6 +26,7 @@ import {
|
||||
type OfficialHabboFurniEntry,
|
||||
} from "@/lib/services/habbo-furnidata-cache";
|
||||
import { getHabboGamedataHotel } from "@/lib/services/habbo-gamedata-hotel";
|
||||
import { browserHeaders } from "@/lib/services/import/core/browser-headers";
|
||||
import {
|
||||
downloadFile,
|
||||
validatePngBytes,
|
||||
@@ -502,17 +503,41 @@ export function __resetCatalogIdAllocatorForTests(): void {
|
||||
|
||||
// ── Import Single Furni ─────────────────────────────────────────────
|
||||
|
||||
// Try each candidate URL in order until one downloads successfully.
|
||||
// Race every candidate URL concurrently and keep the first one that downloads
|
||||
// and validates successfully. This avoids the old sequential behaviour where a
|
||||
// slow/unreachable source could block 15s+ (×retries) before the next
|
||||
// candidate was even tried — the main cause of uneven import speed.
|
||||
async function tryDownloadCandidates(
|
||||
urls: string[],
|
||||
dest: string,
|
||||
validate: "png" | "swf",
|
||||
): Promise<boolean> {
|
||||
for (const url of urls) {
|
||||
const dl = await downloadFile(url, dest, { validate });
|
||||
if (dl.ok) return true;
|
||||
}
|
||||
return false;
|
||||
if (urls.length === 0) return false;
|
||||
const tasks = urls.map((url) =>
|
||||
(async () => {
|
||||
try {
|
||||
const res = await fetch(url, {
|
||||
signal: AbortSignal.timeout(12000),
|
||||
headers: browserHeaders("image", {
|
||||
Accept: "image/png,image/*,*/*;q=0.8",
|
||||
}),
|
||||
});
|
||||
if (!res.ok) return null;
|
||||
const buffer = Buffer.from(await res.arrayBuffer());
|
||||
if (buffer.length < 8) return null;
|
||||
if (validate === "swf" && !validateSwfBytes(buffer)) return null;
|
||||
if (validate === "png" && !validatePngBytes(buffer)) return null;
|
||||
return buffer;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
})(),
|
||||
);
|
||||
const results = await Promise.all(tasks);
|
||||
const first = results.find((r) => r);
|
||||
if (!first) return false;
|
||||
await fs.writeFile(/*turbopackIgnore: true*/ dest, first);
|
||||
return true;
|
||||
}
|
||||
|
||||
export async function importSingleFurni(params: {
|
||||
|
||||
Reference in new issue
Block a user