feat(ci): switch Cloudflare bypass from FlareSolverr to Byparr
This commit is contained in:
1 parent
fabd160250
commit
25f4d74209
10 files changed
+138
-131
No files matched your search
+2
-2
@@ -76,8 +76,8 @@ PAYPAL_API=https://api-m.sandbox.paypal.com
|
||||
# --- LOGGING ---
|
||||
LOG_LEVEL=error
|
||||
|
||||
# --- FLARESOLVERR (Cloudflare bypass for clone sources) ---
|
||||
FLARESOLVERR_URL=http://localhost:8191
|
||||
# --- BYPARR (Cloudflare bypass for clone sources) ---
|
||||
BYPARR_URL=http://localhost:8191
|
||||
|
||||
# Catalog Studio export: dedicated clean clone on Beta-3 with Git push credentials.
|
||||
CATALOG_GIT_CHECKOUT=
|
||||
|
||||
@@ -808,16 +808,16 @@ pnpm jobs:worker # or: npm run jobs:worker / yarn jobs:worker
|
||||
|
||||
Optional OpenAI-powered moderation for comments and guestbook posts. Set `OPENAI_API_KEY`.
|
||||
|
||||
### FlareSolverr (Cloudflare Bypass)
|
||||
### Byparr (Cloudflare Bypass)
|
||||
|
||||
Some clone sources are protected by Cloudflare. FlareSolverr solves these challenges automatically.
|
||||
Some clone sources are protected by Cloudflare. Byparr (a FlareSolverr successor) solves these challenges automatically.
|
||||
|
||||
```bash
|
||||
docker compose up -d flaresolverr
|
||||
docker compose up -d byparr
|
||||
```
|
||||
|
||||
```dotenv
|
||||
FLARESOLVERR_URL=http://localhost:8191
|
||||
BYPARR_URL=http://localhost:8191
|
||||
```
|
||||
|
||||
### Furniture Import with Auto-Translation
|
||||
|
||||
+8
-9
@@ -58,19 +58,18 @@ services:
|
||||
start_period: 40s
|
||||
mem_limit: 2g
|
||||
|
||||
# ── FlareSolverr (Cloudflare bypass for clone sources) ──
|
||||
# ── Byparr (Cloudflare bypass for clone sources) ──
|
||||
# Solves Cloudflare/TLS-fingerprint blocks via a headless Chrome browser.
|
||||
# The CMS calls this on http://localhost:8191 for sources that block Node's
|
||||
# TLS fingerprint (e.g. Leet.city). Used by src/lib/services/flare-solver.ts.
|
||||
flaresolverr:
|
||||
image: ghcr.io/flaresolverr/flaresolverr:latest
|
||||
container_name: flaresolverr
|
||||
# Drop-in successor to FlareSolverr. The CMS calls this on
|
||||
# http://localhost:8191 for sources that block Node's TLS fingerprint
|
||||
# (e.g. Leet.city). Used by src/lib/services/byparr.ts.
|
||||
byparr:
|
||||
image: ghcr.io/thephaseless/byparr:latest
|
||||
container_name: byparr
|
||||
network_mode: host
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- LOG_LEVEL=info
|
||||
- BROWSER_TIMEOUT=60000
|
||||
- BROWSER_WORKERS=1
|
||||
- LOG_LEVEL=INFO
|
||||
mem_limit: 2g
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-sf", "http://localhost:8191/health"]
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
URL="${FLARESOLVERR_URL:-http://localhost:8191}"
|
||||
MAX_RETRIES="${FLARESOLVERR_MAX_RETRIES:-30}"
|
||||
RETRY_INTERVAL="${FLARESOLVERR_RETRY_INTERVAL:-2}"
|
||||
URL="${BYPARR_URL:-${FLARESOLVERR_URL:-http://localhost:8191}}"
|
||||
MAX_RETRIES="${BYPARR_MAX_RETRIES:-30}"
|
||||
RETRY_INTERVAL="${BYPARR_RETRY_INTERVAL:-2}"
|
||||
|
||||
echo "Waiting for FlareSolverr at ${URL}..."
|
||||
echo "Waiting for Byparr at ${URL}..."
|
||||
|
||||
for i in $(seq 1 "$MAX_RETRIES"); do
|
||||
if curl -sf "${URL}/health" >/dev/null 2>&1; then
|
||||
echo "FlareSolverr is healthy."
|
||||
echo "Byparr is healthy."
|
||||
exit 0
|
||||
fi
|
||||
echo "Attempt ${i}/${MAX_RETRIES} - FlareSolverr not ready, retrying in ${RETRY_INTERVAL}s..."
|
||||
echo "Attempt ${i}/${MAX_RETRIES} - Byparr not ready, retrying in ${RETRY_INTERVAL}s..."
|
||||
sleep "$RETRY_INTERVAL"
|
||||
done
|
||||
|
||||
echo "ERROR: FlareSolverr did not become healthy after ${MAX_RETRIES} attempts."
|
||||
echo "ERROR: Byparr did not become healthy after ${MAX_RETRIES} attempts."
|
||||
exit 1
|
||||
@@ -0,0 +1,90 @@
|
||||
const BYPARR_URL =
|
||||
process.env.BYPARR_URL ??
|
||||
process.env.FLARESOLVERR_URL ?? // legacy env fallback
|
||||
"http://localhost:8191";
|
||||
|
||||
const BYPARR_API = `${BYPARR_URL}/v1`;
|
||||
|
||||
type ByparrCookie = {
|
||||
name: string;
|
||||
value: string;
|
||||
domain?: string;
|
||||
path?: string;
|
||||
};
|
||||
|
||||
type ByparrSolution = {
|
||||
response: string;
|
||||
status: number;
|
||||
cookies: ByparrCookie[];
|
||||
};
|
||||
|
||||
type ByparrResult = {
|
||||
status?: "ok" | "error";
|
||||
message?: string;
|
||||
solution?: ByparrSolution;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
async function byparrRequest(
|
||||
url: string,
|
||||
timeout = 30000,
|
||||
): Promise<ByparrSolution> {
|
||||
const res = await fetch(BYPARR_API, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
cmd: "request.get",
|
||||
url,
|
||||
maxTimeout: timeout,
|
||||
}),
|
||||
signal: AbortSignal.timeout(timeout + 5000),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`Byparr request failed: ${res.status} ${res.statusText}`);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as ByparrResult;
|
||||
|
||||
if (data.error) {
|
||||
throw new Error(`Byparr error: ${data.error}`);
|
||||
}
|
||||
|
||||
// Byparr returns HTTP 200 with `status: "error"` for invalid requests
|
||||
// (FlareSolverr used a top-level `error` field instead).
|
||||
if (data.status && data.status !== "ok") {
|
||||
throw new Error(`Byparr error: ${data.message ?? "request failed"}`);
|
||||
}
|
||||
|
||||
if (!data.solution) {
|
||||
throw new Error("Byparr: no solution in response");
|
||||
}
|
||||
|
||||
return data.solution;
|
||||
}
|
||||
|
||||
export async function fetchWithByparr(
|
||||
url: string,
|
||||
timeout = 30000,
|
||||
): Promise<string> {
|
||||
const solution = await byparrRequest(url, timeout);
|
||||
return solution.response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Solve a Cloudflare challenge for the given URL and return the clearance
|
||||
* cookies as a `-b "name=value; ..."` argument string suitable for curl.
|
||||
* Returns null if Byparr is unavailable or no cookies were set.
|
||||
*/
|
||||
export async function getByparrCookies(
|
||||
url: string,
|
||||
timeout = 30000,
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
const solution = await byparrRequest(url, timeout);
|
||||
if (!solution.cookies?.length) return null;
|
||||
return solution.cookies.map((c) => `${c.name}=${c.value}`).join("; ");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@ const {
|
||||
selectLimit,
|
||||
fsUnlink,
|
||||
fsReadFile,
|
||||
fetchWithFlareSolver,
|
||||
fetchWithByparr,
|
||||
curlFetchText,
|
||||
curlDownload,
|
||||
} = vi.hoisted(() => ({
|
||||
@@ -23,7 +23,7 @@ const {
|
||||
selectLimit: vi.fn<AnyFn>(async () => []),
|
||||
fsUnlink: vi.fn<AnyFn>(async () => {}),
|
||||
fsReadFile: vi.fn<AnyFn>(async () => Buffer.from("NITRO")),
|
||||
fetchWithFlareSolver: vi.fn<AnyFn>(async () => ""),
|
||||
fetchWithByparr: vi.fn<AnyFn>(async () => ""),
|
||||
curlFetchText: vi.fn<AnyFn>(async () => ""),
|
||||
curlDownload: vi.fn<AnyFn>(async () => ({ ok: true, size: 200 })),
|
||||
}));
|
||||
@@ -33,7 +33,7 @@ vi.mock("@/lib/services/import/core/curl-fetch", () => ({
|
||||
curlDownload,
|
||||
curlFetchText,
|
||||
}));
|
||||
vi.mock("@/lib/services/flare-solver", () => ({ fetchWithFlareSolver }));
|
||||
vi.mock("@/lib/services/byparr", () => ({ fetchWithByparr }));
|
||||
vi.mock("@/lib/services/furni-data", () => ({ appendFurniEntry }));
|
||||
vi.mock("@/lib/services/furni-import", () => ({
|
||||
ensureDirectories: vi.fn<AnyFn>(async () => {}),
|
||||
@@ -217,8 +217,8 @@ describe("fetchSourceFurnidata", () => {
|
||||
const HTML_WRAPPED = `<html><head><meta charset="utf-8"></head><body><pre>${JSON_BODY}</pre></body></html>`;
|
||||
|
||||
beforeEach(() => {
|
||||
fetchWithFlareSolver.mockReset();
|
||||
fetchWithFlareSolver.mockResolvedValue("");
|
||||
fetchWithByparr.mockReset();
|
||||
fetchWithByparr.mockResolvedValue("");
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
@@ -235,10 +235,10 @@ describe("fetchSourceFurnidata", () => {
|
||||
);
|
||||
const list = await fetchSourceFurnidata("https://a.test/fd.json", 1);
|
||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||
expect(fetchWithFlareSolver).not.toHaveBeenCalled();
|
||||
expect(fetchWithByparr).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("extracts JSON from an HTML-wrapped furnidata (e.g. Leet via FlareSolverr)", async () => {
|
||||
it("extracts JSON from an HTML-wrapped furnidata (e.g. Leet via Byparr)", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(
|
||||
@@ -249,13 +249,13 @@ describe("fetchSourceFurnidata", () => {
|
||||
}),
|
||||
),
|
||||
);
|
||||
fetchWithFlareSolver.mockResolvedValue(HTML_WRAPPED);
|
||||
fetchWithByparr.mockResolvedValue(HTML_WRAPPED);
|
||||
const list = await fetchSourceFurnidata("https://b.test/fd.json", 1);
|
||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||
expect(fetchWithFlareSolver).toHaveBeenCalledWith("https://b.test/fd.json");
|
||||
expect(fetchWithByparr).toHaveBeenCalledWith("https://b.test/fd.json");
|
||||
});
|
||||
|
||||
it("falls back to FlareSolverr on HTML body even when status is 200", async () => {
|
||||
it("falls back to Byparr on HTML body even when status is 200", async () => {
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(
|
||||
@@ -266,7 +266,7 @@ describe("fetchSourceFurnidata", () => {
|
||||
}),
|
||||
),
|
||||
);
|
||||
fetchWithFlareSolver.mockResolvedValue(HTML_WRAPPED);
|
||||
fetchWithByparr.mockResolvedValue(HTML_WRAPPED);
|
||||
const list = await fetchSourceFurnidata("https://c.test/fd.json", 1);
|
||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||
});
|
||||
@@ -284,7 +284,7 @@ describe("fetchSourceFurnidata", () => {
|
||||
);
|
||||
const list = await fetchSourceFurnidata("https://g.test/fd.json", 1);
|
||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||
expect(fetchWithFlareSolver).not.toHaveBeenCalled();
|
||||
expect(fetchWithByparr).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("throws when HTML has no embedded JSON payload", async () => {
|
||||
@@ -298,7 +298,7 @@ describe("fetchSourceFurnidata", () => {
|
||||
}),
|
||||
),
|
||||
);
|
||||
fetchWithFlareSolver.mockResolvedValue("<html>still a challenge</html>");
|
||||
fetchWithByparr.mockResolvedValue("<html>still a challenge</html>");
|
||||
await expect(
|
||||
fetchSourceFurnidata("https://d.test/fd.json", 1),
|
||||
).rejects.toThrow(/Cloudflare challenge/);
|
||||
|
||||
@@ -5,9 +5,9 @@ import { autoDetectInteraction } from "@/lib/furni/auto-interaction";
|
||||
import { normalizeClassname } from "@/lib/furni/classname";
|
||||
import { HABBO_GAMEDATA_HOTEL_SETTING_KEY } from "@/lib/habbo-gamedata-hotel";
|
||||
import { logger } from "@/lib/logger";
|
||||
import { fetchWithByparr } from "@/lib/services/byparr";
|
||||
import { extractFurniIconPng } from "@/lib/services/clone-icon";
|
||||
import type { CloneSource } from "@/lib/services/clone-sources";
|
||||
import { fetchWithFlareSolver } from "@/lib/services/flare-solver";
|
||||
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
||||
import { appendFurniEntry } from "@/lib/services/furni-data";
|
||||
import {
|
||||
@@ -101,7 +101,7 @@ const TTL = 5 * 60 * 1000;
|
||||
/**
|
||||
* Some sources serve their furnidata wrapped in an HTML document (e.g. Leet's
|
||||
* JSON is embedded in a `<pre>` block inside the page, often surfaced by the
|
||||
* FlareSolverr fallback). Extract the JSON payload before giving up.
|
||||
* Byparr fallback). Extract the JSON payload before giving up.
|
||||
*/
|
||||
function extractJsonFromHtml(body: string): string | null {
|
||||
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/);
|
||||
@@ -117,20 +117,20 @@ function extractJsonFromHtml(body: string): string | null {
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch a source's furnidata via FlareSolverr, falling back to a curl
|
||||
* Fetch a source's furnidata via Byparr, falling back to a curl
|
||||
* subprocess. Some CDNs (e.g. Leet.city) block Node's TLS fingerprint while
|
||||
* the same request succeeds from curl — so curl is a useful second fallback
|
||||
* when FlareSolverr is unavailable.
|
||||
* when Byparr is unavailable.
|
||||
*/
|
||||
async function fetchWithFallback(url: string): Promise<string> {
|
||||
try {
|
||||
return await fetchWithFlareSolver(url);
|
||||
return await fetchWithByparr(url);
|
||||
} catch {
|
||||
try {
|
||||
return await curlFetchText(url);
|
||||
} catch (err) {
|
||||
throw new Error(
|
||||
`FlareSolverr + curl fallback failed for ${url}: ${(err as Error).message}`,
|
||||
`Byparr + curl fallback failed for ${url}: ${(err as Error).message}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -150,8 +150,8 @@ export async function fetchSourceFurnidata(
|
||||
});
|
||||
if (res.ok) {
|
||||
// Some sources (e.g. GitHub raw) serve JSON with a text/plain
|
||||
// content-type — only route to FlareSolverr when the body is
|
||||
// actually HTML, otherwise parse the JSON directly.
|
||||
// content-type — only route to Byparr when the body is actually
|
||||
// HTML, otherwise parse the JSON directly.
|
||||
const text = await res.text();
|
||||
if (
|
||||
text.trimStart().startsWith("{") ||
|
||||
@@ -171,7 +171,7 @@ export async function fetchSourceFurnidata(
|
||||
const extracted = extractJsonFromHtml(body);
|
||||
if (!extracted) {
|
||||
throw new Error(
|
||||
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`,
|
||||
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start Byparr and check its logs.`,
|
||||
);
|
||||
}
|
||||
body = extracted;
|
||||
|
||||
@@ -1,82 +0,0 @@
|
||||
const FLARESOLVERR_URL =
|
||||
process.env.FLARESOLVERR_URL ?? "http://localhost:8191";
|
||||
|
||||
const FLARESOLVERR_API = `${FLARESOLVERR_URL}/v1`;
|
||||
|
||||
type FlareCookie = {
|
||||
name: string;
|
||||
value: string;
|
||||
domain: string;
|
||||
path: string;
|
||||
};
|
||||
|
||||
type FlareSolution = {
|
||||
response: string;
|
||||
status: number;
|
||||
cookies: FlareCookie[];
|
||||
};
|
||||
|
||||
type FlareResult = {
|
||||
solution?: FlareSolution;
|
||||
error?: string;
|
||||
};
|
||||
|
||||
async function flareRequest(
|
||||
url: string,
|
||||
timeout = 30000,
|
||||
): Promise<FlareSolution> {
|
||||
const res = await fetch(FLARESOLVERR_API, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
cmd: "request.get",
|
||||
url,
|
||||
maxTimeout: timeout,
|
||||
}),
|
||||
signal: AbortSignal.timeout(timeout + 5000),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`FlareSolverr request failed: ${res.status} ${res.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as FlareResult;
|
||||
|
||||
if (data.error) {
|
||||
throw new Error(`FlareSolverr error: ${data.error}`);
|
||||
}
|
||||
|
||||
if (!data.solution) {
|
||||
throw new Error("FlareSolverr: no solution in response");
|
||||
}
|
||||
|
||||
return data.solution;
|
||||
}
|
||||
|
||||
export async function fetchWithFlareSolver(
|
||||
url: string,
|
||||
timeout = 30000,
|
||||
): Promise<string> {
|
||||
const solution = await flareRequest(url, timeout);
|
||||
return solution.response;
|
||||
}
|
||||
|
||||
/**
|
||||
* Solve a Cloudflare challenge for the given URL and return the clearance
|
||||
* cookies as a `-b "name=value; ..."` argument string suitable for curl.
|
||||
* Returns null if FlareSolverr is unavailable or no cookies were set.
|
||||
*/
|
||||
export async function getFlareSolverrCookies(
|
||||
url: string,
|
||||
timeout = 30000,
|
||||
): Promise<string | null> {
|
||||
try {
|
||||
const solution = await flareRequest(url, timeout);
|
||||
if (!solution.cookies?.length) return null;
|
||||
return solution.cookies.map((c) => `${c.name}=${c.value}`).join("; ");
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { promises as fs } from "node:fs";
|
||||
import { getFlareSolverrCookies } from "@/lib/services/flare-solver";
|
||||
import { getByparrCookies } from "@/lib/services/byparr";
|
||||
import { parseNitroBundle } from "../../swf/nitro-builder";
|
||||
import { browserHeaders } from "./browser-headers";
|
||||
import { curlDownload } from "./curl-fetch";
|
||||
@@ -62,7 +62,7 @@ export async function downloadFile(
|
||||
const curlFallback = async (): Promise<boolean> => {
|
||||
let curlRes = await curlDownload(url, destPath);
|
||||
if (!curlRes.ok) {
|
||||
const cookieHeader = await getFlareSolverrCookies(url);
|
||||
const cookieHeader = await getByparrCookies(url);
|
||||
curlRes = await curlDownload(
|
||||
url,
|
||||
destPath,
|
||||
@@ -108,7 +108,7 @@ export async function downloadFile(
|
||||
res.status === 404 || res.status === 410 || res.status === 403;
|
||||
if (deterministic || attempt === maxRetries) {
|
||||
// HTTP 403 is often a TLS-fingerprint / Cloudflare challenge
|
||||
// block (e.g. Leet.city) — retry via curl, with FlareSolverr
|
||||
// block (e.g. Leet.city) — retry via curl, with Byparr
|
||||
// clearance cookies if the plain curl is also challenged.
|
||||
if (res.status === 403 && (await curlFallback())) {
|
||||
const stat = await fs.stat(destPath);
|
||||
@@ -148,7 +148,7 @@ export async function downloadFile(
|
||||
return { ok: true, size: buffer.length };
|
||||
} catch (err) {
|
||||
if (attempt === maxRetries || stage === "write") {
|
||||
// TLS fingerprint aborts land here too — try curl/FlareSolverr
|
||||
// TLS fingerprint aborts land here too — try curl/Byparr
|
||||
// before reporting the network failure.
|
||||
if (stage === "download" && (await curlFallback())) {
|
||||
const stat = await fs.stat(destPath);
|
||||
|
||||
+1
-1
@@ -1282,7 +1282,7 @@ with open(out, "w", encoding="utf-8") as f:
|
||||
|
||||
# Sanity-check a .env file for the failure modes that actually break a deploy:
|
||||
# duplicate keys, and a value concatenated onto another key (missing newline),
|
||||
# e.g. FLARESOLVERR_URL=http://localhost:8191AUTH_TRUST_HOST="..."
|
||||
# e.g. BYPARR_URL=http://localhost:8191AUTH_TRUST_HOST="..."
|
||||
validate_env_file() {
|
||||
local base_dir="${1:-$SCRIPT_DIR}"
|
||||
local en="$base_dir/.env"
|
||||
|
||||
Reference in new issue
Block a user