From 1851653afbcf1fcdb611f68a3bd8770d37b56d84 Mon Sep 17 00:00:00 2001
From: openhands <openhands@all-hands.dev>
Date: Wed, 5 Aug 2026 12:10:02 +0200
Subject: [PATCH] fix(import): support HTML-wrapped clone furnidata and skip
 empty icon sources

Leet serves its furnidata as HTML with the JSON embedded in a <pre>
block, and Cloudflare blocks Node's direct fetch. Extract the JSON
payload from the HTML (direct or via the FlareSolverr fallback) before
giving up on a Cloudflare challenge.

Also skip the standalone icon download when a clone source has no
iconBaseUrl configured (Habbo, Hubba, Fresh, Kyzegs, RidgeRP), which
previously produced invalid relative URLs like /xxx_icon.png and a
flood of download errors before falling back to extracting the icon
from the .nitro bundle.
---
 src/lib/services/clone-import.test.ts | 85 +++++++++++++++++++++++++++
 src/lib/services/clone-import.ts      | 51 ++++++++++++----
 2 files changed, 125 insertions(+), 11 deletions(-)

diff --git a/src/lib/services/clone-import.test.ts b/src/lib/services/clone-import.test.ts
index 525930a1..7b06f4cd 100644
--- a/src/lib/services/clone-import.test.ts
+++ b/src/lib/services/clone-import.test.ts
@@ -12,6 +12,7 @@ const {
 	selectLimit,
 	fsUnlink,
 	fsReadFile,
+	fetchWithFlareSolver,
 } = vi.hoisted(() => ({
 	downloadFile: vi.fn<AnyFn>(async () => ({ ok: true, size: 200 })),
 	appendFurniEntry: vi.fn<AnyFn>(async () => {}),
@@ -20,9 +21,11 @@ const {
 	selectLimit: vi.fn<AnyFn>(async () => []),
 	fsUnlink: vi.fn<AnyFn>(async () => {}),
 	fsReadFile: vi.fn<AnyFn>(async () => Buffer.from("NITRO")),
+	fetchWithFlareSolver: vi.fn<AnyFn>(async () => ""),
 }));
 
 vi.mock("@/lib/services/import/core/download", () => ({ downloadFile }));
+vi.mock("@/lib/services/flare-solver", () => ({ fetchWithFlareSolver }));
 vi.mock("@/lib/services/furni-data", () => ({ appendFurniEntry }));
 vi.mock("@/lib/services/furni-import", () => ({
 	ensureDirectories: vi.fn<AnyFn>(async () => {}),
@@ -72,6 +75,7 @@ vi.mock("@/lib/db", () => ({
 import {
 	__resetItemsBaseAllocatorForTests,
 	cloneSingleFurni,
+	fetchSourceFurnidata,
 	parseFurnidata,
 } from "./clone-import";
 
@@ -194,3 +198,84 @@ describe("clone-import", () => {
 		}
 	});
 });
+
+describe("fetchSourceFurnidata", () => {
+	const FD = {
+		roomitemtypes: { furnitype: [{ classname: "bc_sofa" }] },
+		wallitemtypes: { furnitype: [{ classname: "wall_x" }] },
+	};
+	const JSON_BODY = JSON.stringify(FD);
+	const HTML_WRAPPED = `<html><head><meta charset="utf-8"></head><body><pre>${JSON_BODY}</pre></body></html>`;
+
+	beforeEach(() => {
+		fetchWithFlareSolver.mockReset();
+		fetchWithFlareSolver.mockResolvedValue("");
+		vi.unstubAllGlobals();
+	});
+
+	it("parses a plain JSON furnidata directly", async () => {
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(
+				async () =>
+					new Response(JSON_BODY, {
+						status: 200,
+						headers: { "content-type": "application/json" },
+					}),
+			),
+		);
+		const list = await fetchSourceFurnidata("https://a.test/fd.json", 1);
+		expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
+		expect(fetchWithFlareSolver).not.toHaveBeenCalled();
+	});
+
+	it("extracts JSON from an HTML-wrapped furnidata (e.g. Leet via FlareSolverr)", async () => {
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(
+				async () =>
+					new Response("<html>challenge</html>", {
+						status: 403,
+						headers: { "content-type": "text/html" },
+					}),
+			),
+		);
+		fetchWithFlareSolver.mockResolvedValue(HTML_WRAPPED);
+		const list = await fetchSourceFurnidata("https://b.test/fd.json", 1);
+		expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
+		expect(fetchWithFlareSolver).toHaveBeenCalledWith("https://b.test/fd.json");
+	});
+
+	it("falls back to FlareSolverr on non-JSON content-type even when status is 200", async () => {
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(
+				async () =>
+					new Response(HTML_WRAPPED, {
+						status: 200,
+						headers: { "content-type": "text/html" },
+					}),
+			),
+		);
+		fetchWithFlareSolver.mockResolvedValue(HTML_WRAPPED);
+		const list = await fetchSourceFurnidata("https://c.test/fd.json", 1);
+		expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
+	});
+
+	it("throws when HTML has no embedded JSON payload", async () => {
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(
+				async () =>
+					new Response("<html>challenge</html>", {
+						status: 403,
+						headers: { "content-type": "text/html" },
+					}),
+			),
+		);
+		fetchWithFlareSolver.mockResolvedValue("<html>still a challenge</html>");
+		await expect(
+			fetchSourceFurnidata("https://d.test/fd.json", 1),
+		).rejects.toThrow(/Cloudflare challenge/);
+	});
+});
diff --git a/src/lib/services/clone-import.ts b/src/lib/services/clone-import.ts
index 45e67196..19f0f02e 100644
--- a/src/lib/services/clone-import.ts
+++ b/src/lib/services/clone-import.ts
@@ -85,6 +85,24 @@ export function parseFurnidata(data: RemoteFurnidata): SourceFurni[] {
 const cache = new Map<string, { list: SourceFurni[]; ts: number }>();
 const TTL = 5 * 60 * 1000;
 
+/**
+ * Some sources serve their furnidata wrapped in an HTML document (e.g. Leet's
+ * JSON is embedded in a `<pre>` block inside the page, often surfaced by the
+ * FlareSolverr fallback). Extract the JSON payload before giving up.
+ */
+function extractJsonFromHtml(body: string): string | null {
+	const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/);
+	const candidate = pre?.[1] ?? body;
+	const trimmed = candidate.trim();
+	if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return null;
+	try {
+		JSON.parse(trimmed);
+		return trimmed;
+	} catch {
+		return null;
+	}
+}
+
 export async function fetchSourceFurnidata(
 	url: string,
 	now = Date.now(),
@@ -115,9 +133,13 @@ export async function fetchSourceFurnidata(
 		}
 	}
 	if (body.trimStart().startsWith("<")) {
-		throw new Error(
-			`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`,
-		);
+		const extracted = extractJsonFromHtml(body);
+		if (!extracted) {
+			throw new Error(
+				`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`,
+			);
+		}
+		body = extracted;
 	}
 	let list: SourceFurni[];
 	try {
@@ -253,14 +275,21 @@ export async function cloneSingleFurni(params: {
 		logger.warn("[clone-import] invalid .nitro bundle for", { classname });
 		return { ok: false, classname, warnings, error: "invalid .nitro bundle" };
 	}
-	const iconDl = await downloadFile(
-		`${source.iconBaseUrl}/${classname}_icon.png`,
-		iconPath,
-		{
-			maxRetries: 1,
-			validate: "png",
-		},
-	);
+	let iconDl: { ok: boolean; size: number };
+	if (source.iconBaseUrl) {
+		iconDl = await downloadFile(
+			`${source.iconBaseUrl}/${classname}_icon.png`,
+			iconPath,
+			{
+				maxRetries: 1,
+				validate: "png",
+			},
+		);
+	} else {
+		// No standalone icons on this source (e.g. Habbo/Hubba) — fall through to
+		// extracting the catalog icon from the .nitro bundle we just downloaded.
+		iconDl = { ok: false, size: 0 };
+	}
 	if (!iconDl.ok) {
 		// Source serves no standalone icon (e.g. icons embedded in the .nitro) —
 		// extract the catalog icon from the bundle we just downloaded.