feat: add Cloudflare bypass to fetchSourceFurnidata using puppeteer

- Add cf-fetch.ts with puppeteer-extra + stealth plugin for CF bypass
- Modify fetchSourceFurnidata to detect CF challenge and retry with puppeteer
- Restore Leet, Hubbly, and Habblet City to clone sources (now CF-protected)

src/lib/services/cf-fetch.ts

@@ -0,0 +1,93 @@
+import puppeteer from "puppeteer-extra";
+import StealthPlugin from "puppeteer-extra-plugin-stealth";
+
+puppeteer.use(StealthPlugin());
+
+const UA =
+	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
+	"AppleWebKit/537.36 (KHTML, like Gecko) " +
+	"Chrome/125.0.0.0 Safari/537.36";
+
+let browser: Awaited<ReturnType<typeof puppeteer.launch>> | null = null;
+
+async function getBrowser() {
+	if (browser) return browser;
+	browser = await puppeteer.launch({
+		headless: true,
+		args: [
+			"--no-sandbox",
+			"--disable-setuid-sandbox",
+			"--disable-web-security",
+			"--disable-blink-features=AutomationControlled",
+			"--user-agent=" + UA,
+		],
+	});
+	return browser;
+}
+
+export async function fetchWithCfBypass(
+	url: string,
+	timeout = 30000,
+): Promise<string> {
+	const b = await getBrowser();
+	const page = await b.newPage();
+	await page.setUserAgent(UA);
+	await page.setDefaultTimeout(timeout);
+
+	try {
+		const response = await page.goto(url, {
+			waitUntil: "networkidle2",
+			timeout,
+		});
+
+		const status = response?.status() ?? 0;
+		const body = await page.content();
+
+		const isCfChallenge =
+			body.includes("Attention Required") ||
+			body.includes("Checking your browser") ||
+			body.includes("Just a moment") ||
+			body.includes("cf-challenge") ||
+			status === 403;
+
+		if (isCfChallenge) {
+			await page.waitForFunction(
+				() =>
+					!document.body.innerHTML.includes("Attention Required") &&
+					!document.body.innerHTML.includes("Checking your browser") &&
+					!document.body.innerHTML.includes("Just a moment"),
+				{ timeout: 10000 },
+			);
+		}
+
+		const finalBody = await page.content();
+		return finalBody;
+	} finally {
+		await page.close();
+	}
+}
+
+export async function fetchJsonWithCfBypass<T = unknown>(
+	url: string,
+	timeout = 30000,
+): Promise<T> {
+	const html = await fetchWithCfBypass(url, timeout);
+	const match = html.match(
+		/<pre[^>]*class="[^"]*json[^"]*"[^>]*>([\s\S]*?)<\/pre>/i,
+	);
+	if (match) {
+		return JSON.parse(match[1].trim()) as T;
+	}
+	try {
+		return JSON.parse(html) as T;
+	} catch {
+		throw new Error(`cf-fetch: could not parse JSON from ${url}`);
+	}
+}
+
+export async function closeCfBrowser(): Promise<void> {
+	if (browser) {
+		await browser.close();
+		browser = null;
+	}
+}

src/lib/services/clone-import.ts

@@ -3,6 +3,7 @@ import path from "node:path";
 import { eq, inArray, sql } from "drizzle-orm";
 import { db, ItemsBase } from "@/lib/db";
 import { logger } from "@/lib/logger";
+import { fetchWithCfBypass } from "@/lib/services/cf-fetch";
 import { extractFurniIconPng } from "@/lib/services/clone-icon";
 import type { CloneSource } from "@/lib/services/clone-sources";
 import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
@@ -88,14 +89,23 @@ export async function fetchSourceFurnidata(
 ): Promise<SourceFurni[]> {
 	const hit = cache.get(url);
 	if (hit && now && now - hit.ts < TTL) return hit.list;
-	const res = await fetch(url, {
-		signal: AbortSignal.timeout(30000),
-		headers: browserHeaders(),
-	});
-	if (!res.ok) throw new Error(`furnidata fetch failed: ${res.status} ${url}`);
+	let body: string;
+	try {
+		const res = await fetch(url, {
+			signal: AbortSignal.timeout(30000),
+			headers: browserHeaders(),
+		});
+		if (!res.ok) throw new Error(`HTTP ${res.status}`);
+		const text = await res.text();
+		if (typeof text === "string" && text.trim().startsWith("<"))
+			throw new Error("Cloudflare challenge detected");
+		body = text;
+	} catch {
+		body = await fetchWithCfBypass(url);
+	}
 	let list: SourceFurni[];
 	try {
-		const json = await res.json();
+		const json = JSON.parse(body);
 		list = parseFurnidata(json);
 	} catch (err) {
 		throw new Error(

src/lib/services/clone-sources.ts

@@ -38,6 +38,13 @@ export const DEFAULT_SOURCES: CloneSource[] = [
 		nitroBaseUrl: "https://nitro.hubba.cc/bundled/furniture",
 		iconBaseUrl: "",
 	},
+	{
+		id: "default-sodaho",
+		name: "Soda Ho",
+		furnidataUrl: "https://assets.sodaho.tel/gamedata/FurnitureData.json",
+		nitroBaseUrl: "",
+		iconBaseUrl: "",
+	},
 	{
 		id: "default-leet",
 		name: "Leet",
@@ -59,13 +66,6 @@ export const DEFAULT_SOURCES: CloneSource[] = [
 		nitroBaseUrl: "https://www.habblet.city/nitro/bundled/furniture",
 		iconBaseUrl: "https://www.habblet.city/nitro/icons",
 	},
-	{
-		id: "default-sodaho",
-		name: "Soda Ho",
-		furnidataUrl: "https://assets.sodaho.tel/gamedata/FurnitureData.json",
-		nitroBaseUrl: "",
-		iconBaseUrl: "",
-	},
 ];
 
 export const DEFAULT_ICON_SOURCES: CloneSource[] = DEFAULT_SOURCES.filter(