fix: clone sources blocked by content-type check and TLS fingerprint
- fetchSourceFurnidata now parses JSON regardless of content-type, so GitHub raw mirrors (Kyzegs, sphynxkitten) that serve JSON as text/plain work again instead of forcing a FlareSolverr round-trip. - Add curl subprocess fallback (curl-fetch.ts) for CDNs that block Node's fetch by TLS fingerprint (Leet.city) — used for furnidata, nitro bundle and icon downloads before giving up. - Merge verified default clone source presets with stored user sources so the admin always offers many working sources.
This commit is contained in:
1 parent
7911a25ced
commit
121dda7249
4 files changed
+173
-18
No files matched your search
@@ -17,6 +17,10 @@ import {
|
||||
getOrCreateCategoryPage,
|
||||
} from "@/lib/services/furni-import";
|
||||
import { browserHeaders } from "@/lib/services/import/core/browser-headers";
|
||||
import {
|
||||
curlDownload,
|
||||
curlFetchText,
|
||||
} from "@/lib/services/import/core/curl-fetch";
|
||||
import { downloadFile } from "@/lib/services/import/core/download";
|
||||
import { siteSettings } from "@/lib/services/site-settings";
|
||||
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
|
||||
@@ -106,6 +110,26 @@ function extractJsonFromHtml(body: string): string | null {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a source's furnidata via FlareSolverr, falling back to a curl
|
||||
* subprocess. Some CDNs (e.g. Leet.city) block Node's TLS fingerprint while
|
||||
* the same request succeeds from curl — so curl is a useful second fallback
|
||||
* when FlareSolverr is unavailable.
|
||||
*/
|
||||
async function fetchWithFallback(url: string): Promise<string> {
|
||||
try {
|
||||
return await fetchWithFlareSolver(url);
|
||||
} catch {
|
||||
try {
|
||||
return await curlFetchText(url);
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`FlareSolverr + curl fallback failed for ${url}: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchSourceFurnidata(
|
||||
url: string,
|
||||
now = Date.now(),
|
||||
@@ -118,22 +142,24 @@ export async function fetchSourceFurnidata(
|
||||
signal: AbortSignal.timeout(30000),
|
||||
headers: browserHeaders(),
|
||||
});
|
||||
if (
|
||||
res.ok &&
|
||||
res.headers.get("content-type")?.includes("application/json")
|
||||
) {
|
||||
body = await res.text();
|
||||
if (res.ok) {
|
||||
// Some sources (e.g. GitHub raw) serve JSON with a text/plain
|
||||
// content-type — only route to FlareSolverr when the body is
|
||||
// actually HTML, otherwise parse the JSON directly.
|
||||
const text = await res.text();
|
||||
if (
|
||||
text.trimStart().startsWith("{") ||
|
||||
text.trimStart().startsWith("[")
|
||||
) {
|
||||
body = text;
|
||||
} else {
|
||||
body = await fetchWithFallback(url);
|
||||
}
|
||||
} else {
|
||||
body = await fetchWithFlareSolver(url);
|
||||
body = await fetchWithFallback(url);
|
||||
}
|
||||
} catch {
|
||||
try {
|
||||
body = await fetchWithFlareSolver(url);
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`FlareSolverr fallback failed for ${url}: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
body = await fetchWithFallback(url);
|
||||
}
|
||||
if (body.trimStart().startsWith("<")) {
|
||||
const extracted = extractJsonFromHtml(body);
|
||||
@@ -276,8 +302,16 @@ export async function cloneSingleFurni(params: {
|
||||
},
|
||||
);
|
||||
if (!dl.ok) {
|
||||
logger.warn("[clone-import] nitro download failed for", { classname });
|
||||
return { ok: false, classname, warnings, error: "nitro download failed" };
|
||||
// Node's fetch may be TLS-fingerprint-blocked by the source CDN
|
||||
// (e.g. Leet.city) while curl still succeeds — try that before giving up.
|
||||
const curlDl = await curlDownload(
|
||||
`${source.nitroBaseUrl}/${classname}.nitro`,
|
||||
nitroPath,
|
||||
);
|
||||
if (!curlDl.ok) {
|
||||
logger.warn("[clone-import] nitro download failed for", { classname });
|
||||
return { ok: false, classname, warnings, error: "nitro download failed" };
|
||||
}
|
||||
}
|
||||
// Validate it is a real Nitro bundle.
|
||||
try {
|
||||
@@ -297,6 +331,13 @@ export async function cloneSingleFurni(params: {
|
||||
validate: "png",
|
||||
},
|
||||
);
|
||||
if (!iconDl.ok) {
|
||||
// Same TLS-fingerprint fallback as the nitro download.
|
||||
iconDl = await curlDownload(
|
||||
`${source.iconBaseUrl}/${classname}_icon.png`,
|
||||
iconPath,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// No standalone icons on this source (e.g. Habbo/Hubba) — fall through to
|
||||
// extracting the catalog icon from the .nitro bundle we just downloaded.
|
||||
|
||||
Reference in new issue
Block a user