Update Dockerfile and Compose configuration

src/lib/services/catalog-audit.ts

@@ -50,6 +50,12 @@ export interface AuditEvent {
 	missingFromSources?: MissingFromSource[];
 	unrepairable?: UnrepairableItem[];
 	summary?: AuditSummary;
+	furniDataIdConflicts?: Array<{
+		classname: string;
+		itemId: number;
+		conflictingId: number;
+		existingClassname: string;
+	}>;
 	repair?: RepairStats;
 	repairNitro?: RepairStats;
 	sql?: string;

src/lib/services/flare-solver.ts

@@ -3,10 +3,28 @@ const FLARESOLVERR_URL =
 
 const FLARESOLVERR_API = `${FLARESOLVERR_URL}/v1`;
 
-export async function fetchWithFlareSolver(
+type FlareCookie = {
+	name: string;
+	value: string;
+	domain: string;
+	path: string;
+};
+
+type FlareSolution = {
+	response: string;
+	status: number;
+	cookies: FlareCookie[];
+};
+
+type FlareResult = {
+	solution?: FlareSolution;
+	error?: string;
+};
+
+async function flareRequest(
 	url: string,
 	timeout = 30000,
-): Promise<string> {
+): Promise<FlareSolution> {
 	const res = await fetch(FLARESOLVERR_API, {
 		method: "POST",
 		headers: { "Content-Type": "application/json" },
@@ -24,10 +42,7 @@ export async function fetchWithFlareSolver(
 		);
 	}
 
-	const data = (await res.json()) as {
-		solution?: { response: string };
-		error?: string;
-	};
+	const data = (await res.json()) as FlareResult;
 
 	if (data.error) {
 		throw new Error(`FlareSolverr error: ${data.error}`);
@@ -37,5 +52,31 @@ export async function fetchWithFlareSolver(
 		throw new Error("FlareSolverr: no solution in response");
 	}
 
-	return data.solution.response;
+	return data.solution;
+}
+
+export async function fetchWithFlareSolver(
+	url: string,
+	timeout = 30000,
+): Promise<string> {
+	const solution = await flareRequest(url, timeout);
+	return solution.response;
+}
+
+/**
+ * Solve a Cloudflare challenge for the given URL and return the clearance
+ * cookies as a `-b "name=value; ..."` argument string suitable for curl.
+ * Returns null if FlareSolverr is unavailable or no cookies were set.
+ */
+export async function getFlareSolverrCookies(
+	url: string,
+	timeout = 30000,
+): Promise<string | null> {
+	try {
+		const solution = await flareRequest(url, timeout);
+		if (!solution.cookies?.length) return null;
+		return solution.cookies.map((c) => `${c.name}=${c.value}`).join("; ");
+	} catch {
+		return null;
+	}
 }

src/lib/services/import/core/curl-fetch.ts

@@ -1,5 +1,5 @@
 import { execFile } from "node:child_process";
-import { promises as fs } from "node:fs";
+import { existsSync, promises as fs } from "node:fs";
 import { promisify } from "node:util";
 
 const execFileAsync = promisify(execFile);
@@ -8,8 +8,17 @@ const execFileAsync = promisify(execFile);
  * Some retro CDNs (e.g. Leet.city) block Node's fetch by TLS fingerprint
  * (JA3/JA4) even though the same request succeeds from a real browser or
  * curl. This helper shells out to `curl` as a fallback for those hosts.
+ *
+ * The runtime image ships `curl-impersonate` (a curl compiled with Chrome's
+ * exact TLS fingerprint, statically-bundled BoringSSL) at /opt/curl-impersonate
+ * because several sources enable Cloudflare `cf-mitigated: challenge` against
+ * the distro curl's fingerprint too. Prefer it when present, else fall back to
+ * the system curl.
  */
 
+const IMPERSONATE_CURL = "/opt/curl-impersonate/curl_chrome116";
+const CURL_BIN = existsSync(IMPERSONATE_CURL) ? IMPERSONATE_CURL : "curl";
+
 const CURL_USER_AGENT =
 	"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
 
@@ -37,7 +46,7 @@ export async function curlFetchText(
 	url: string,
 	timeoutMs = 30_000,
 ): Promise<string> {
-	const { stdout } = await execFileAsync("curl", curlArgs(url, []), {
+	const { stdout } = await execFileAsync(CURL_BIN, curlArgs(url, []), {
 		timeout: timeoutMs,
 		maxBuffer: 512 * 1024 * 1024,
 	});
@@ -49,10 +58,12 @@ export async function curlDownload(
 	url: string,
 	destPath: string,
 	timeoutMs = 30_000,
+	cookieHeader?: string,
 ): Promise<{ ok: boolean; size: number }> {
 	try {
+		const cookieArgs = cookieHeader ? ["-H", `Cookie: ${cookieHeader}`] : [];
 		await execFileAsync(
-			"curl",
+			CURL_BIN,
 			[
 				"-sSL",
 				"--compressed",
@@ -67,6 +78,7 @@ export async function curlDownload(
 				"Accept: application/octet-stream,*/*;q=0.9",
 				"-H",
 				"Accept-Language: en-US,en;q=0.9",
+				...cookieArgs,
 				"-o",
 				destPath,
 				url,

src/lib/services/import/core/download.ts

@@ -1,7 +1,8 @@
 import { promises as fs } from "node:fs";
+import { getFlareSolverrCookies } from "@/lib/services/flare-solver";
 import { parseNitroBundle } from "../../swf/nitro-builder";
-
 import { browserHeaders } from "./browser-headers";
+import { curlDownload } from "./curl-fetch";
 
 export function validateSwfBytes(buffer: Buffer): boolean {
 	if (buffer.length < 8) return false;
@@ -58,6 +59,38 @@ export async function downloadFile(
 	const maxRetries = options?.maxRetries ?? 3;
 	const baseDelay = 1000;
 
+	const curlFallback = async (): Promise<boolean> => {
+		let curlRes = await curlDownload(url, destPath);
+		if (!curlRes.ok) {
+			const cookieHeader = await getFlareSolverrCookies(url);
+			curlRes = await curlDownload(
+				url,
+				destPath,
+				30_000,
+				cookieHeader ?? undefined,
+			);
+		}
+		if (!curlRes.ok) return false;
+		const validate = options?.validate;
+		if (validate) {
+			let valid = false;
+			try {
+				const buffer = await fs.readFile(destPath);
+				valid =
+					(validate === "swf" && validateSwfBytes(buffer)) ||
+					(validate === "png" && validatePngBytes(buffer)) ||
+					(validate === "nitro" && validateNitroBytes(buffer));
+			} catch {
+				valid = false;
+			}
+			if (!valid) {
+				await fs.unlink(destPath).catch(() => {});
+				return false;
+			}
+		}
+		return true;
+	};
+
 	for (let attempt = 0; attempt <= maxRetries; attempt++) {
 		let stage: "download" | "write" = "download";
 		try {
@@ -74,6 +107,13 @@ export async function downloadFile(
 				const deterministic =
 					res.status === 404 || res.status === 410 || res.status === 403;
 				if (deterministic || attempt === maxRetries) {
+					// HTTP 403 is often a TLS-fingerprint / Cloudflare challenge
+					// block (e.g. Leet.city) — retry via curl, with FlareSolverr
+					// clearance cookies if the plain curl is also challenged.
+					if (res.status === 403 && (await curlFallback())) {
+						const stat = await fs.stat(destPath);
+						return { ok: true, size: stat.size };
+					}
 					console.warn(
 						`[import-download] Download failed ${url}: ${res.status}${deterministic ? " (deterministic, not retrying)" : ` after ${maxRetries + 1} attempts`}`,
 					);
@@ -108,6 +148,12 @@ export async function downloadFile(
 			return { ok: true, size: buffer.length };
 		} catch (err) {
 			if (attempt === maxRetries || stage === "write") {
+				// TLS fingerprint aborts land here too — try curl/FlareSolverr
+				// before reporting the network failure.
+				if (stage === "download" && (await curlFallback())) {
+					const stat = await fs.stat(destPath);
+					return { ok: true, size: stat.size };
+				}
 				console.warn(
 					`[import-download] Download error ${url}:`,
 					(err as Error).message,