fix: clone sources blocked by content-type check and TLS fingerprint
CI / check (push) Successful in 28s
CI / release (push) Skipped
CI / deploy (push) Successful in 1m0s

- fetchSourceFurnidata now parses JSON regardless of content-type,
  so GitHub raw mirrors (Kyzegs, sphynxkitten) that serve JSON as
  text/plain work again instead of forcing a FlareSolverr round-trip.
- Add curl subprocess fallback (curl-fetch.ts) for CDNs that block
  Node's fetch by TLS fingerprint (Leet.city) — used for furnidata,
  nitro bundle and icon downloads before giving up.
- Merge verified default clone source presets with stored user sources
  so the admin always offers many working sources.
This commit is contained in:
openhands committed 2026-08-19 19:39:32 +02:00
1 parent 7911a25ced
commit 121dda7249
4 files changed
+173 -18

No files matched your search

+56 -15
View File
@@ -17,6 +17,10 @@ import {
getOrCreateCategoryPage,
} from "@/lib/services/furni-import";
import { browserHeaders } from "@/lib/services/import/core/browser-headers";
import {
curlDownload,
curlFetchText,
} from "@/lib/services/import/core/curl-fetch";
import { downloadFile } from "@/lib/services/import/core/download";
import { siteSettings } from "@/lib/services/site-settings";
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
@@ -106,6 +110,26 @@ function extractJsonFromHtml(body: string): string | null {
}
}
/**
* Fetch a source's furnidata via FlareSolverr, falling back to a curl
* subprocess. Some CDNs (e.g. Leet.city) block Node's TLS fingerprint while
* the same request succeeds from curl — so curl is a useful second fallback
* when FlareSolverr is unavailable.
*/
async function fetchWithFallback(url: string): Promise<string> {
try {
return await fetchWithFlareSolver(url);
} catch {
try {
return await curlFetchText(url);
} catch (err) {
throw new Error(
`FlareSolverr + curl fallback failed for ${url}: ${(err as Error).message}`,
);
}
}
}
export async function fetchSourceFurnidata(
url: string,
now = Date.now(),
@@ -118,22 +142,24 @@ export async function fetchSourceFurnidata(
signal: AbortSignal.timeout(30000),
headers: browserHeaders(),
});
if (
res.ok &&
res.headers.get("content-type")?.includes("application/json")
) {
body = await res.text();
if (res.ok) {
// Some sources (e.g. GitHub raw) serve JSON with a text/plain
// content-type — only route to FlareSolverr when the body is
// actually HTML, otherwise parse the JSON directly.
const text = await res.text();
if (
text.trimStart().startsWith("{") ||
text.trimStart().startsWith("[")
) {
body = text;
} else {
body = await fetchWithFallback(url);
}
} else {
body = await fetchWithFlareSolver(url);
body = await fetchWithFallback(url);
}
} catch {
try {
body = await fetchWithFlareSolver(url);
} catch (err) {
throw new Error(
`FlareSolverr fallback failed for ${url}: ${(err as Error).message}`,
);
}
body = await fetchWithFallback(url);
}
if (body.trimStart().startsWith("<")) {
const extracted = extractJsonFromHtml(body);
@@ -276,8 +302,16 @@ export async function cloneSingleFurni(params: {
},
);
if (!dl.ok) {
logger.warn("[clone-import] nitro download failed for", { classname });
return { ok: false, classname, warnings, error: "nitro download failed" };
// Node's fetch may be TLS-fingerprint-blocked by the source CDN
// (e.g. Leet.city) while curl still succeeds — try that before giving up.
const curlDl = await curlDownload(
`${source.nitroBaseUrl}/${classname}.nitro`,
nitroPath,
);
if (!curlDl.ok) {
logger.warn("[clone-import] nitro download failed for", { classname });
return { ok: false, classname, warnings, error: "nitro download failed" };
}
}
// Validate it is a real Nitro bundle.
try {
@@ -297,6 +331,13 @@ export async function cloneSingleFurni(params: {
validate: "png",
},
);
if (!iconDl.ok) {
// Same TLS-fingerprint fallback as the nitro download.
iconDl = await curlDownload(
`${source.iconBaseUrl}/${classname}_icon.png`,
iconPath,
);
}
} else {
// No standalone icons on this source (e.g. Habbo/Hubba) — fall through to
// extracting the catalog icon from the .nitro bundle we just downloaded.