feat: browser headers on audit fetches + verified clone furnidata sources

src/lib/services/import/core/browser-headers.ts

@@ -0,0 +1,49 @@
+export type BrowserFetchDest =
+	| "document"
+	| "image"
+	| "script"
+	| "style"
+	| "empty";
+
+const CHROME_UA =
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36";
+
+const SEC_CH_UA =
+	'"Google Chrome";v="125", "Chromium";v="125", "Not.A/Brand";v="24"';
+
+/**
+ * Realistic browser request headers so CDNs (Cloudflare, etc.) treat the
+ * server-side fetches as a normal browser instead of a bot/curl and block us.
+ */
+export function browserHeaders(
+	dest: BrowserFetchDest = "empty",
+	extra: Record<string, string> = {},
+): Record<string, string> {
+	const accept =
+		dest === "document"
+			? "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"
+			: dest === "image"
+				? "image/avif,image/webp,image/apng,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5"
+				: dest === "script"
+					? "*/*"
+					: "application/json,text/plain,*/*;q=0.9";
+
+	const headers: Record<string, string> = {
+		"User-Agent": CHROME_UA,
+		"sec-ch-ua": SEC_CH_UA,
+		"sec-ch-ua-mobile": "?0",
+		"sec-ch-ua-platform": '"Windows"',
+		Accept: accept,
+		"Accept-Language": "en-US,en;q=0.9",
+		"Sec-Fetch-Site": "cross-site",
+		"Sec-Fetch-Mode":
+			dest === "document" ? "navigate" : dest === "image" ? "no-cors" : "cors",
+		"Sec-Fetch-Dest": dest,
+		"Cache-Control": "no-cache",
+		Pragma: "no-cache",
+	};
+
+	if (dest === "document") headers["Upgrade-Insecure-Requests"] = "1";
+
+	return { ...headers, ...extra };
+}

src/lib/services/import/core/download.ts

@@ -1,5 +1,7 @@
 import { promises as fs } from "node:fs";
 
+import { browserHeaders } from "./browser-headers";
+
 export function validateSwfBytes(buffer: Buffer): boolean {
 	if (buffer.length < 8) return false;
 	const sig = buffer.toString("ascii", 0, 3);
@@ -31,11 +33,9 @@ export async function downloadFile(
 			}
 			const res = await fetch(url, {
 				signal: AbortSignal.timeout(15000),
-				headers: {
-					"User-Agent":
-						"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0 Safari/537.36",
+				headers: browserHeaders("image", {
 					Accept: "image/png,image/*,*/*;q=0.8",
-				},
+				}),
 			});
 			if (!res.ok) {
 				const deterministic =

src/lib/services/import/core/gordon.ts

@@ -1,3 +1,5 @@
+import { browserHeaders } from "./browser-headers";
+
 const EXTERNAL_VARIABLES_URL =
 	"https://www.habbo.com/gamedata/external_variables/1";
 const CACHE_TTL = 30 * 60 * 1000;
@@ -14,6 +16,7 @@ export async function resolveGordonBuildUrl(): Promise<string> {
 	if (gordonCache && now - gordonCache.ts < CACHE_TTL) return gordonCache.url;
 	const res = await fetch(EXTERNAL_VARIABLES_URL, {
 		signal: AbortSignal.timeout(20000),
+		headers: browserHeaders("document"),
 	});
 	if (!res.ok)
 		throw new Error(`external_variables fetch failed: ${res.status}`);