fix: clone sources blocked by content-type check and TLS fingerprint
- fetchSourceFurnidata now parses JSON regardless of content-type, so GitHub raw mirrors (Kyzegs, sphynxkitten) that serve JSON as text/plain work again instead of forcing a FlareSolverr round-trip. - Add curl subprocess fallback (curl-fetch.ts) for CDNs that block Node's fetch by TLS fingerprint (Leet.city) — used for furnidata, nitro bundle and icon downloads before giving up. - Merge verified default clone source presets with stored user sources so the admin always offers many working sources.
This commit is contained in:
1 parent
7911a25ced
commit
121dda7249
4 files changed
+173
-18
No files matched your search
@@ -13,6 +13,8 @@ const {
|
||||
fsUnlink,
|
||||
fsReadFile,
|
||||
fetchWithFlareSolver,
|
||||
curlFetchText,
|
||||
curlDownload,
|
||||
} = vi.hoisted(() => ({
|
||||
downloadFile: vi.fn<AnyFn>(async () => ({ ok: true, size: 200 })),
|
||||
appendFurniEntry: vi.fn<AnyFn>(async () => {}),
|
||||
@@ -22,9 +24,15 @@ const {
|
||||
fsUnlink: vi.fn<AnyFn>(async () => {}),
|
||||
fsReadFile: vi.fn<AnyFn>(async () => Buffer.from("NITRO")),
|
||||
fetchWithFlareSolver: vi.fn<AnyFn>(async () => ""),
|
||||
curlFetchText: vi.fn<AnyFn>(async () => ""),
|
||||
curlDownload: vi.fn<AnyFn>(async () => ({ ok: true, size: 200 })),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/services/import/core/download", () => ({ downloadFile }));
|
||||
vi.mock("@/lib/services/import/core/curl-fetch", () => ({
|
||||
curlDownload,
|
||||
curlFetchText,
|
||||
}));
|
||||
vi.mock("@/lib/services/flare-solver", () => ({ fetchWithFlareSolver }));
|
||||
vi.mock("@/lib/services/furni-data", () => ({ appendFurniEntry }));
|
||||
vi.mock("@/lib/services/furni-import", () => ({
|
||||
@@ -247,7 +255,7 @@ describe("fetchSourceFurnidata", () => {
|
||||
expect(fetchWithFlareSolver).toHaveBeenCalledWith("https://b.test/fd.json");
|
||||
});
|
||||
|
||||
it("falls back to FlareSolverr on non-JSON content-type even when status is 200", async () => {
|
||||
it("falls back to FlareSolverr on HTML body even when status is 200", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(
|
||||
@@ -263,6 +271,22 @@ describe("fetchSourceFurnidata", () => {
|
||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||
});
|
||||
|
||||
it("parses JSON served with text/plain content-type directly (e.g. GitHub raw)", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(
|
||||
async () =>
|
||||
new Response(JSON_BODY, {
|
||||
status: 200,
|
||||
headers: { "content-type": "text/plain; charset=utf-8" },
|
||||
}),
|
||||
),
|
||||
);
|
||||
const list = await fetchSourceFurnidata("https://g.test/fd.json", 1);
|
||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||
expect(fetchWithFlareSolver).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("throws when HTML has no embedded JSON payload", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
@@ -160,12 +160,21 @@ export const DEFAULT_NITRO_SOURCES: CloneSource[] = DEFAULT_SOURCES.filter(
|
||||
|
||||
export async function listSources(): Promise<CloneSource[]> {
|
||||
const raw = (await siteSettings.get(KEY, "[]")) ?? "[]";
|
||||
let stored: CloneSource[] = [];
|
||||
try {
|
||||
const arr = JSON.parse(raw);
|
||||
return Array.isArray(arr) && arr.length > 0 ? arr : DEFAULT_SOURCES;
|
||||
if (Array.isArray(arr)) stored = arr;
|
||||
} catch {
|
||||
return DEFAULT_SOURCES;
|
||||
// fall through to defaults-only
|
||||
}
|
||||
// Merge the verified default presets with any user-added/edited sources so
|
||||
// the admin always has many sources to pick from. User entries win on id.
|
||||
const merged = new Map<string, CloneSource>();
|
||||
for (const src of DEFAULT_SOURCES) merged.set(src.id, src);
|
||||
for (const src of stored) {
|
||||
if (src.id && src.furnidataUrl) merged.set(src.id, src);
|
||||
}
|
||||
return Array.from(merged.values());
|
||||
}
|
||||
|
||||
export async function upsertSource(
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
import { execFile } from "node:child_process";
|
||||
import { promises as fs } from "node:fs";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
|
||||
/**
|
||||
* Some retro CDNs (e.g. Leet.city) block Node's fetch by TLS fingerprint
|
||||
* (JA3/JA4) even though the same request succeeds from a real browser or
|
||||
* curl. This helper shells out to `curl` as a fallback for those hosts.
|
||||
*/
|
||||
|
||||
const CURL_USER_AGENT =
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
|
||||
|
||||
function curlArgs(url: string, extra: string[]): string[] {
|
||||
return [
|
||||
"-sSL",
|
||||
"--compressed",
|
||||
"--max-time",
|
||||
"30",
|
||||
"--connect-timeout",
|
||||
"10",
|
||||
"-A",
|
||||
CURL_USER_AGENT,
|
||||
"-H",
|
||||
"Accept: application/json,text/plain,*/*;q=0.9",
|
||||
"-H",
|
||||
"Accept-Language: en-US,en;q=0.9",
|
||||
...extra,
|
||||
url,
|
||||
];
|
||||
}
|
||||
|
||||
/** Fetch a URL's body via curl. Throws on non-zero exit / timeout. */
|
||||
export async function curlFetchText(
|
||||
url: string,
|
||||
timeoutMs = 30_000,
|
||||
): Promise<string> {
|
||||
const { stdout } = await execFileAsync("curl", curlArgs(url, []), {
|
||||
timeout: timeoutMs,
|
||||
maxBuffer: 512 * 1024 * 1024,
|
||||
});
|
||||
return stdout;
|
||||
}
|
||||
|
||||
/** Download a URL to a file via curl. Returns ok=false on failure (curl -f). */
|
||||
export async function curlDownload(
|
||||
url: string,
|
||||
destPath: string,
|
||||
timeoutMs = 30_000,
|
||||
): Promise<{ ok: boolean; size: number }> {
|
||||
try {
|
||||
await execFileAsync(
|
||||
"curl",
|
||||
[
|
||||
"-sSL",
|
||||
"--compressed",
|
||||
"--fail",
|
||||
"--max-time",
|
||||
"30",
|
||||
"--connect-timeout",
|
||||
"10",
|
||||
"-A",
|
||||
CURL_USER_AGENT,
|
||||
"-H",
|
||||
"Accept: application/octet-stream,*/*;q=0.9",
|
||||
"-H",
|
||||
"Accept-Language: en-US,en;q=0.9",
|
||||
"-o",
|
||||
destPath,
|
||||
url,
|
||||
],
|
||||
{ timeout: timeoutMs },
|
||||
);
|
||||
const stat = await fs.stat(destPath);
|
||||
return { ok: stat.size > 0, size: stat.size };
|
||||
} catch {
|
||||
return { ok: false, size: 0 };
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user