fix: clone sources blocked by content-type check and TLS fingerprint
CI / check (push) Successful in 28s
CI / release (push) Skipped
CI / deploy (push) Successful in 1m0s

- fetchSourceFurnidata now parses JSON regardless of content-type,
  so GitHub raw mirrors (Kyzegs, sphynxkitten) that serve JSON as
  text/plain work again instead of forcing a FlareSolverr round-trip.
- Add curl subprocess fallback (curl-fetch.ts) for CDNs that block
  Node's fetch by TLS fingerprint (Leet.city) — used for furnidata,
  nitro bundle and icon downloads before giving up.
- Merge verified default clone source presets with stored user sources
  so the admin always offers many working sources.
This commit is contained in:
openhands committed 2026-08-19 19:39:32 +02:00
1 parent 7911a25ced
commit 121dda7249
4 files changed
+173 -18

No files matched your search

@@ -0,0 +1,81 @@
import { execFile } from "node:child_process";
import { promises as fs } from "node:fs";
import { promisify } from "node:util";
const execFileAsync = promisify(execFile);
/**
* Some retro CDNs (e.g. Leet.city) block Node's fetch by TLS fingerprint
* (JA3/JA4) even though the same request succeeds from a real browser or
* curl. This helper shells out to `curl` as a fallback for those hosts.
*/
const CURL_USER_AGENT =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
function curlArgs(url: string, extra: string[]): string[] {
return [
"-sSL",
"--compressed",
"--max-time",
"30",
"--connect-timeout",
"10",
"-A",
CURL_USER_AGENT,
"-H",
"Accept: application/json,text/plain,*/*;q=0.9",
"-H",
"Accept-Language: en-US,en;q=0.9",
...extra,
url,
];
}
/** Fetch a URL's body via curl. Throws on non-zero exit / timeout. */
export async function curlFetchText(
url: string,
timeoutMs = 30_000,
): Promise<string> {
const { stdout } = await execFileAsync("curl", curlArgs(url, []), {
timeout: timeoutMs,
maxBuffer: 512 * 1024 * 1024,
});
return stdout;
}
/** Download a URL to a file via curl. Returns ok=false on failure (curl -f). */
export async function curlDownload(
url: string,
destPath: string,
timeoutMs = 30_000,
): Promise<{ ok: boolean; size: number }> {
try {
await execFileAsync(
"curl",
[
"-sSL",
"--compressed",
"--fail",
"--max-time",
"30",
"--connect-timeout",
"10",
"-A",
CURL_USER_AGENT,
"-H",
"Accept: application/octet-stream,*/*;q=0.9",
"-H",
"Accept-Language: en-US,en;q=0.9",
"-o",
destPath,
url,
],
{ timeout: timeoutMs },
);
const stat = await fs.stat(destPath);
return { ok: stat.size > 0, size: stat.size };
} catch {
return { ok: false, size: 0 };
}
}