perf: race furniture asset download candidates instead of trying sequentially
tryDownloadCandidates now fires every candidate URL concurrently and keeps the first valid response, removing the old 15s×retry sequential waits on slow/unreachable sources that made import speed uneven. Batch concurrency raised 8 -> 12 to absorb the now-faster downloads.
This commit is contained in:
1 parent
ac7c427c3d
commit
1d8672ccc5
2 files changed
+31
-6
No files matched your search
@@ -53,7 +53,7 @@ import type { ImportSingleResult } from "@/types/furni";
|
|||||||
// How many items to import concurrently in a batch. The per-item work is
|
// How many items to import concurrently in a batch. The per-item work is
|
||||||
// network-bound (asset downloads) and DB-bound, so a modest pool gives a large
|
// network-bound (asset downloads) and DB-bound, so a modest pool gives a large
|
||||||
// speedup over the old fully sequential loop without overwhelming the DB.
|
// speedup over the old fully sequential loop without overwhelming the DB.
|
||||||
const IMPORT_BATCH_CONCURRENCY = 8;
|
const IMPORT_BATCH_CONCURRENCY = 12;
|
||||||
|
|
||||||
async function runWithConcurrency<T, R>(
|
async function runWithConcurrency<T, R>(
|
||||||
items: T[],
|
items: T[],
|
||||||
|
|||||||
@@ -26,6 +26,7 @@ import {
|
|||||||
type OfficialHabboFurniEntry,
|
type OfficialHabboFurniEntry,
|
||||||
} from "@/lib/services/habbo-furnidata-cache";
|
} from "@/lib/services/habbo-furnidata-cache";
|
||||||
import { getHabboGamedataHotel } from "@/lib/services/habbo-gamedata-hotel";
|
import { getHabboGamedataHotel } from "@/lib/services/habbo-gamedata-hotel";
|
||||||
|
import { browserHeaders } from "@/lib/services/import/core/browser-headers";
|
||||||
import {
|
import {
|
||||||
downloadFile,
|
downloadFile,
|
||||||
validatePngBytes,
|
validatePngBytes,
|
||||||
@@ -502,17 +503,41 @@ export function __resetCatalogIdAllocatorForTests(): void {
|
|||||||
|
|
||||||
// ── Import Single Furni ─────────────────────────────────────────────
|
// ── Import Single Furni ─────────────────────────────────────────────
|
||||||
|
|
||||||
// Try each candidate URL in order until one downloads successfully.
|
// Race every candidate URL concurrently and keep the first one that downloads
|
||||||
|
// and validates successfully. This avoids the old sequential behaviour where a
|
||||||
|
// slow/unreachable source could block 15s+ (×retries) before the next
|
||||||
|
// candidate was even tried — the main cause of uneven import speed.
|
||||||
async function tryDownloadCandidates(
|
async function tryDownloadCandidates(
|
||||||
urls: string[],
|
urls: string[],
|
||||||
dest: string,
|
dest: string,
|
||||||
validate: "png" | "swf",
|
validate: "png" | "swf",
|
||||||
): Promise<boolean> {
|
): Promise<boolean> {
|
||||||
for (const url of urls) {
|
if (urls.length === 0) return false;
|
||||||
const dl = await downloadFile(url, dest, { validate });
|
const tasks = urls.map((url) =>
|
||||||
if (dl.ok) return true;
|
(async () => {
|
||||||
|
try {
|
||||||
|
const res = await fetch(url, {
|
||||||
|
signal: AbortSignal.timeout(12000),
|
||||||
|
headers: browserHeaders("image", {
|
||||||
|
Accept: "image/png,image/*,*/*;q=0.8",
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
if (!res.ok) return null;
|
||||||
|
const buffer = Buffer.from(await res.arrayBuffer());
|
||||||
|
if (buffer.length < 8) return null;
|
||||||
|
if (validate === "swf" && !validateSwfBytes(buffer)) return null;
|
||||||
|
if (validate === "png" && !validatePngBytes(buffer)) return null;
|
||||||
|
return buffer;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
}
|
}
|
||||||
return false;
|
})(),
|
||||||
|
);
|
||||||
|
const results = await Promise.all(tasks);
|
||||||
|
const first = results.find((r) => r);
|
||||||
|
if (!first) return false;
|
||||||
|
await fs.writeFile(/*turbopackIgnore: true*/ dest, first);
|
||||||
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function importSingleFurni(params: {
|
export async function importSingleFurni(params: {
|
||||||
|
|||||||
Reference in new issue
Block a user