From 121dda7249765857917b7bd54fd637fe1c43c8b5 Mon Sep 17 00:00:00 2001 From: openhands Date: Wed, 19 Aug 2026 19:39:32 +0200 Subject: [PATCH] fix: clone sources blocked by content-type check and TLS fingerprint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - fetchSourceFurnidata now parses JSON regardless of content-type, so GitHub raw mirrors (Kyzegs, sphynxkitten) that serve JSON as text/plain work again instead of forcing a FlareSolverr round-trip. - Add curl subprocess fallback (curl-fetch.ts) for CDNs that block Node's fetch by TLS fingerprint (Leet.city) — used for furnidata, nitro bundle and icon downloads before giving up. - Merge verified default clone source presets with stored user sources so the admin always offers many working sources. --- src/lib/services/clone-import.test.ts | 26 ++++++- src/lib/services/clone-import.ts | 71 +++++++++++++++---- src/lib/services/clone-sources.ts | 13 +++- src/lib/services/import/core/curl-fetch.ts | 81 ++++++++++++++++++++++ 4 files changed, 173 insertions(+), 18 deletions(-) create mode 100644 src/lib/services/import/core/curl-fetch.ts diff --git a/src/lib/services/clone-import.test.ts b/src/lib/services/clone-import.test.ts index dd5859755f..43e0fd274e 100644 --- a/src/lib/services/clone-import.test.ts +++ b/src/lib/services/clone-import.test.ts @@ -13,6 +13,8 @@ const { fsUnlink, fsReadFile, fetchWithFlareSolver, + curlFetchText, + curlDownload, } = vi.hoisted(() => ({ downloadFile: vi.fn(async () => ({ ok: true, size: 200 })), appendFurniEntry: vi.fn(async () => {}), @@ -22,9 +24,15 @@ const { fsUnlink: vi.fn(async () => {}), fsReadFile: vi.fn(async () => Buffer.from("NITRO")), fetchWithFlareSolver: vi.fn(async () => ""), + curlFetchText: vi.fn(async () => ""), + curlDownload: vi.fn(async () => ({ ok: true, size: 200 })), })); vi.mock("@/lib/services/import/core/download", () => ({ downloadFile })); +vi.mock("@/lib/services/import/core/curl-fetch", () => ({ + curlDownload, + curlFetchText, +})); vi.mock("@/lib/services/flare-solver", () => ({ fetchWithFlareSolver })); vi.mock("@/lib/services/furni-data", () => ({ appendFurniEntry })); vi.mock("@/lib/services/furni-import", () => ({ @@ -247,7 +255,7 @@ describe("fetchSourceFurnidata", () => { expect(fetchWithFlareSolver).toHaveBeenCalledWith("https://b.test/fd.json"); }); - it("falls back to FlareSolverr on non-JSON content-type even when status is 200", async () => { + it("falls back to FlareSolverr on HTML body even when status is 200", async () => { vi.stubGlobal( "fetch", vi.fn( @@ -263,6 +271,22 @@ describe("fetchSourceFurnidata", () => { expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]); }); + it("parses JSON served with text/plain content-type directly (e.g. GitHub raw)", async () => { + vi.stubGlobal( + "fetch", + vi.fn( + async () => + new Response(JSON_BODY, { + status: 200, + headers: { "content-type": "text/plain; charset=utf-8" }, + }), + ), + ); + const list = await fetchSourceFurnidata("https://g.test/fd.json", 1); + expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]); + expect(fetchWithFlareSolver).not.toHaveBeenCalled(); + }); + it("throws when HTML has no embedded JSON payload", async () => { vi.stubGlobal( "fetch", diff --git a/src/lib/services/clone-import.ts b/src/lib/services/clone-import.ts index 6bfcf57cff..6ce2f147d6 100644 --- a/src/lib/services/clone-import.ts +++ b/src/lib/services/clone-import.ts @@ -17,6 +17,10 @@ import { getOrCreateCategoryPage, } from "@/lib/services/furni-import"; import { browserHeaders } from "@/lib/services/import/core/browser-headers"; +import { + curlDownload, + curlFetchText, +} from "@/lib/services/import/core/curl-fetch"; import { downloadFile } from "@/lib/services/import/core/download"; import { siteSettings } from "@/lib/services/site-settings"; import { parseNitroBundle } from "@/lib/services/swf/nitro-builder"; @@ -106,6 +110,26 @@ function extractJsonFromHtml(body: string): string | null { } } +/** + * Fetch a source's furnidata via FlareSolverr, falling back to a curl + * subprocess. Some CDNs (e.g. Leet.city) block Node's TLS fingerprint while + * the same request succeeds from curl — so curl is a useful second fallback + * when FlareSolverr is unavailable. + */ +async function fetchWithFallback(url: string): Promise { + try { + return await fetchWithFlareSolver(url); + } catch { + try { + return await curlFetchText(url); + } catch (err) { + throw new Error( + `FlareSolverr + curl fallback failed for ${url}: ${(err as Error).message}`, + ); + } + } +} + export async function fetchSourceFurnidata( url: string, now = Date.now(), @@ -118,22 +142,24 @@ export async function fetchSourceFurnidata( signal: AbortSignal.timeout(30000), headers: browserHeaders(), }); - if ( - res.ok && - res.headers.get("content-type")?.includes("application/json") - ) { - body = await res.text(); + if (res.ok) { + // Some sources (e.g. GitHub raw) serve JSON with a text/plain + // content-type — only route to FlareSolverr when the body is + // actually HTML, otherwise parse the JSON directly. + const text = await res.text(); + if ( + text.trimStart().startsWith("{") || + text.trimStart().startsWith("[") + ) { + body = text; + } else { + body = await fetchWithFallback(url); + } } else { - body = await fetchWithFlareSolver(url); + body = await fetchWithFallback(url); } } catch { - try { - body = await fetchWithFlareSolver(url); - } catch (err) { - throw new Error( - `FlareSolverr fallback failed for ${url}: ${(err as Error).message}`, - ); - } + body = await fetchWithFallback(url); } if (body.trimStart().startsWith("<")) { const extracted = extractJsonFromHtml(body); @@ -276,8 +302,16 @@ export async function cloneSingleFurni(params: { }, ); if (!dl.ok) { - logger.warn("[clone-import] nitro download failed for", { classname }); - return { ok: false, classname, warnings, error: "nitro download failed" }; + // Node's fetch may be TLS-fingerprint-blocked by the source CDN + // (e.g. Leet.city) while curl still succeeds — try that before giving up. + const curlDl = await curlDownload( + `${source.nitroBaseUrl}/${classname}.nitro`, + nitroPath, + ); + if (!curlDl.ok) { + logger.warn("[clone-import] nitro download failed for", { classname }); + return { ok: false, classname, warnings, error: "nitro download failed" }; + } } // Validate it is a real Nitro bundle. try { @@ -297,6 +331,13 @@ export async function cloneSingleFurni(params: { validate: "png", }, ); + if (!iconDl.ok) { + // Same TLS-fingerprint fallback as the nitro download. + iconDl = await curlDownload( + `${source.iconBaseUrl}/${classname}_icon.png`, + iconPath, + ); + } } else { // No standalone icons on this source (e.g. Habbo/Hubba) — fall through to // extracting the catalog icon from the .nitro bundle we just downloaded. diff --git a/src/lib/services/clone-sources.ts b/src/lib/services/clone-sources.ts index f9f6e7c220..222b5a8dce 100644 --- a/src/lib/services/clone-sources.ts +++ b/src/lib/services/clone-sources.ts @@ -160,12 +160,21 @@ export const DEFAULT_NITRO_SOURCES: CloneSource[] = DEFAULT_SOURCES.filter( export async function listSources(): Promise { const raw = (await siteSettings.get(KEY, "[]")) ?? "[]"; + let stored: CloneSource[] = []; try { const arr = JSON.parse(raw); - return Array.isArray(arr) && arr.length > 0 ? arr : DEFAULT_SOURCES; + if (Array.isArray(arr)) stored = arr; } catch { - return DEFAULT_SOURCES; + // fall through to defaults-only } + // Merge the verified default presets with any user-added/edited sources so + // the admin always has many sources to pick from. User entries win on id. + const merged = new Map(); + for (const src of DEFAULT_SOURCES) merged.set(src.id, src); + for (const src of stored) { + if (src.id && src.furnidataUrl) merged.set(src.id, src); + } + return Array.from(merged.values()); } export async function upsertSource( diff --git a/src/lib/services/import/core/curl-fetch.ts b/src/lib/services/import/core/curl-fetch.ts new file mode 100644 index 0000000000..1f0cac82f5 --- /dev/null +++ b/src/lib/services/import/core/curl-fetch.ts @@ -0,0 +1,81 @@ +import { execFile } from "node:child_process"; +import { promises as fs } from "node:fs"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +/** + * Some retro CDNs (e.g. Leet.city) block Node's fetch by TLS fingerprint + * (JA3/JA4) even though the same request succeeds from a real browser or + * curl. This helper shells out to `curl` as a fallback for those hosts. + */ + +const CURL_USER_AGENT = + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"; + +function curlArgs(url: string, extra: string[]): string[] { + return [ + "-sSL", + "--compressed", + "--max-time", + "30", + "--connect-timeout", + "10", + "-A", + CURL_USER_AGENT, + "-H", + "Accept: application/json,text/plain,*/*;q=0.9", + "-H", + "Accept-Language: en-US,en;q=0.9", + ...extra, + url, + ]; +} + +/** Fetch a URL's body via curl. Throws on non-zero exit / timeout. */ +export async function curlFetchText( + url: string, + timeoutMs = 30_000, +): Promise { + const { stdout } = await execFileAsync("curl", curlArgs(url, []), { + timeout: timeoutMs, + maxBuffer: 512 * 1024 * 1024, + }); + return stdout; +} + +/** Download a URL to a file via curl. Returns ok=false on failure (curl -f). */ +export async function curlDownload( + url: string, + destPath: string, + timeoutMs = 30_000, +): Promise<{ ok: boolean; size: number }> { + try { + await execFileAsync( + "curl", + [ + "-sSL", + "--compressed", + "--fail", + "--max-time", + "30", + "--connect-timeout", + "10", + "-A", + CURL_USER_AGENT, + "-H", + "Accept: application/octet-stream,*/*;q=0.9", + "-H", + "Accept-Language: en-US,en;q=0.9", + "-o", + destPath, + url, + ], + { timeout: timeoutMs }, + ); + const stat = await fs.stat(destPath); + return { ok: stat.size > 0, size: stat.size }; + } catch { + return { ok: false, size: 0 }; + } +}