From 3edc987281c6c7aa3aeec267828a4b93977a715a Mon Sep 17 00:00:00 2001 From: openhands Date: Tue, 4 Aug 2026 19:00:30 +0200 Subject: [PATCH] feat: integrate FlareSolverr for Cloudflare bypass on clone sources - Add FLARESOLVERR_URL env var to .env.example - Update fetchSourceFurnidata to fall back to FlareSolverr on CF challenges (403/HTML) - Add docker-compose.yml with FlareSolverr service - Add scripts/health-check.sh for FlareSolverr readiness check - Add health:check script to package.json - Document FlareSolverr setup in README --- .env.example | 3 ++ README.md | 20 +++++++++++++ docker-compose.yml | 11 +++++++ package.json | 1 + scripts/health-check.sh | 20 +++++++++++++ src/lib/services/clone-import.ts | 25 ++++++++++++---- src/lib/services/flare-solver.ts | 51 ++++++++++++++++++++++++++++++++ 7 files changed, 125 insertions(+), 6 deletions(-) create mode 100644 docker-compose.yml create mode 100755 scripts/health-check.sh create mode 100644 src/lib/services/flare-solver.ts diff --git a/.env.example b/.env.example index 06ac4b0e..346d985b 100644 --- a/.env.example +++ b/.env.example @@ -71,3 +71,6 @@ PAYPAL_API=https://api-m.sandbox.paypal.com # --- LOGGING --- LOG_LEVEL=error + +# --- FLARESOLVERR (Cloudflare bypass for clone sources) --- +FLARESOLVERR_URL=http://localhost:8191 diff --git a/README.md b/README.md index 2d0f7f33..41a31b79 100644 --- a/README.md +++ b/README.md @@ -514,6 +514,26 @@ Supports Cloudflare Turnstile and Google reCAPTCHA. Configure via CMS Settings. Optional OpenAI-powered moderation for comments and guestbook posts. Set `OPENAI_API_KEY`. +### FlareSolverr (Cloudflare Bypass) + +Some clone sources (e.g. Leet, Hubbly, Habblet City) are protected by Cloudflare and return challenge pages instead of JSON. FlareSolverr acts as a proxy that solves these challenges automatically. + +**Setup:** + +```bash +docker compose up -d flaresolverr +``` + +Set the URL in `.env`: + +``` +FLARESOLVERR_URL=http://localhost:8191 +``` + +When a source URL returns a 403 or HTML (CF challenge), the clone import automatically falls back to FlareSolverr. If FlareSolverr is not running or is unreachable, the source is skipped with a warning. + +See `docker-compose.yml` for the FlareSolverr service definition. + --- ## Development diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 00000000..b68cad52 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,11 @@ +version: "3.8" + +services: + flaresolverr: + image: flaresolverr/flaresolverr:latest + container_name: flaresolverr + ports: + - "8191:8191" + restart: unless-stopped + environment: + - LOG_LEVEL=info diff --git a/package.json b/package.json index 003c3348..d901f8fd 100644 --- a/package.json +++ b/package.json @@ -24,6 +24,7 @@ "db:studio": "drizzle-kit studio", "db:introspect": "drizzle-kit introspect", "jobs:worker": "tsx scripts/jobs-worker.ts", + "health:check": "bash scripts/health-check.sh", "prepare": "husky" }, "lint-staged": { diff --git a/scripts/health-check.sh b/scripts/health-check.sh new file mode 100755 index 00000000..b8bcb2bc --- /dev/null +++ b/scripts/health-check.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +set -euo pipefail + +URL="${FLARESOLVERR_URL:-http://localhost:8191}" +MAX_RETRIES="${FLARESOLVERR_MAX_RETRIES:-30}" +RETRY_INTERVAL="${FLARESOLVERR_RETRY_INTERVAL:-2}" + +echo "Waiting for FlareSolverr at ${URL}..." + +for i in $(seq 1 "$MAX_RETRIES"); do + if curl -sf "${URL}/health" >/dev/null 2>&1; then + echo "FlareSolverr is healthy." + exit 0 + fi + echo "Attempt ${i}/${MAX_RETRIES} - FlareSolverr not ready, retrying in ${RETRY_INTERVAL}s..." + sleep "$RETRY_INTERVAL" +done + +echo "ERROR: FlareSolverr did not become healthy after ${MAX_RETRIES} attempts." +exit 1 \ No newline at end of file diff --git a/src/lib/services/clone-import.ts b/src/lib/services/clone-import.ts index c5fce36b..bbc8021e 100644 --- a/src/lib/services/clone-import.ts +++ b/src/lib/services/clone-import.ts @@ -5,6 +5,7 @@ import { db, ItemsBase } from "@/lib/db"; import { logger } from "@/lib/logger"; import { extractFurniIconPng } from "@/lib/services/clone-icon"; import type { CloneSource } from "@/lib/services/clone-sources"; +import { fetchWithFlareSolver } from "@/lib/services/flare-solver"; import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs"; import { appendFurniEntry } from "@/lib/services/furni-data"; import { @@ -88,14 +89,26 @@ export async function fetchSourceFurnidata( ): Promise { const hit = cache.get(url); if (hit && now && now - hit.ts < TTL) return hit.list; - const res = await fetch(url, { - signal: AbortSignal.timeout(30000), - headers: browserHeaders(), - }); - if (!res.ok) throw new Error(`furnidata fetch failed: ${res.status} ${url}`); + let body: string; + try { + const res = await fetch(url, { + signal: AbortSignal.timeout(30000), + headers: browserHeaders(), + }); + if ( + res.ok && + res.headers.get("content-type")?.includes("application/json") + ) { + body = await res.text(); + } else { + body = await fetchWithFlareSolver(url); + } + } catch { + body = await fetchWithFlareSolver(url); + } let list: SourceFurni[]; try { - const json = await res.json(); + const json = JSON.parse(body); list = parseFurnidata(json); } catch (err) { throw new Error( diff --git a/src/lib/services/flare-solver.ts b/src/lib/services/flare-solver.ts new file mode 100644 index 00000000..e6fe796b --- /dev/null +++ b/src/lib/services/flare-solver.ts @@ -0,0 +1,51 @@ +const FLARESOLVERR_URL = + process.env.FLARESOLVERR_URL ?? "http://localhost:8191"; + +export async function fetchWithFlareSolver( + url: string, + timeout = 30000, +): Promise { + const res = await fetch(FLARESOLVERR_URL, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + cmd: "request.get", + url, + maxTimeout: timeout, + }), + signal: AbortSignal.timeout(timeout + 5000), + }); + + if (!res.ok) { + throw new Error( + `FlareSolverr request failed: ${res.status} ${res.statusText}`, + ); + } + + const data = (await res.json()) as { + solution?: { response: string }; + error?: string; + }; + + if (data.error) { + throw new Error(`FlareSolverr error: ${data.error}`); + } + + if (!data.solution) { + throw new Error("FlareSolverr: no solution in response"); + } + + return data.solution.response; +} + +export async function fetchJsonWithFlareSolver( + url: string, + timeout = 30000, +): Promise { + const body = await fetchWithFlareSolver(url, timeout); + try { + return JSON.parse(body) as T; + } catch { + throw new Error(`flare-solver: could not parse JSON from ${url}`); + } +}