feat(ci): switch Cloudflare bypass from FlareSolverr to Byparr
This commit is contained in:
1 parent
fabd160250
commit
25f4d74209
10 files changed
+138
-131
No files matched your search
+2
-2
@@ -76,8 +76,8 @@ PAYPAL_API=https://api-m.sandbox.paypal.com
|
|||||||
# --- LOGGING ---
|
# --- LOGGING ---
|
||||||
LOG_LEVEL=error
|
LOG_LEVEL=error
|
||||||
|
|
||||||
# --- FLARESOLVERR (Cloudflare bypass for clone sources) ---
|
# --- BYPARR (Cloudflare bypass for clone sources) ---
|
||||||
FLARESOLVERR_URL=http://localhost:8191
|
BYPARR_URL=http://localhost:8191
|
||||||
|
|
||||||
# Catalog Studio export: dedicated clean clone on Beta-3 with Git push credentials.
|
# Catalog Studio export: dedicated clean clone on Beta-3 with Git push credentials.
|
||||||
CATALOG_GIT_CHECKOUT=
|
CATALOG_GIT_CHECKOUT=
|
||||||
|
|||||||
@@ -808,16 +808,16 @@ pnpm jobs:worker # or: npm run jobs:worker / yarn jobs:worker
|
|||||||
|
|
||||||
Optional OpenAI-powered moderation for comments and guestbook posts. Set `OPENAI_API_KEY`.
|
Optional OpenAI-powered moderation for comments and guestbook posts. Set `OPENAI_API_KEY`.
|
||||||
|
|
||||||
### FlareSolverr (Cloudflare Bypass)
|
### Byparr (Cloudflare Bypass)
|
||||||
|
|
||||||
Some clone sources are protected by Cloudflare. FlareSolverr solves these challenges automatically.
|
Some clone sources are protected by Cloudflare. Byparr (a FlareSolverr successor) solves these challenges automatically.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
docker compose up -d flaresolverr
|
docker compose up -d byparr
|
||||||
```
|
```
|
||||||
|
|
||||||
```dotenv
|
```dotenv
|
||||||
FLARESOLVERR_URL=http://localhost:8191
|
BYPARR_URL=http://localhost:8191
|
||||||
```
|
```
|
||||||
|
|
||||||
### Furniture Import with Auto-Translation
|
### Furniture Import with Auto-Translation
|
||||||
|
|||||||
+8
-9
@@ -58,19 +58,18 @@ services:
|
|||||||
start_period: 40s
|
start_period: 40s
|
||||||
mem_limit: 2g
|
mem_limit: 2g
|
||||||
|
|
||||||
# ── FlareSolverr (Cloudflare bypass for clone sources) ──
|
# ── Byparr (Cloudflare bypass for clone sources) ──
|
||||||
# Solves Cloudflare/TLS-fingerprint blocks via a headless Chrome browser.
|
# Solves Cloudflare/TLS-fingerprint blocks via a headless Chrome browser.
|
||||||
# The CMS calls this on http://localhost:8191 for sources that block Node's
|
# Drop-in successor to FlareSolverr. The CMS calls this on
|
||||||
# TLS fingerprint (e.g. Leet.city). Used by src/lib/services/flare-solver.ts.
|
# http://localhost:8191 for sources that block Node's TLS fingerprint
|
||||||
flaresolverr:
|
# (e.g. Leet.city). Used by src/lib/services/byparr.ts.
|
||||||
image: ghcr.io/flaresolverr/flaresolverr:latest
|
byparr:
|
||||||
container_name: flaresolverr
|
image: ghcr.io/thephaseless/byparr:latest
|
||||||
|
container_name: byparr
|
||||||
network_mode: host
|
network_mode: host
|
||||||
restart: unless-stopped
|
restart: unless-stopped
|
||||||
environment:
|
environment:
|
||||||
- LOG_LEVEL=info
|
- LOG_LEVEL=INFO
|
||||||
- BROWSER_TIMEOUT=60000
|
|
||||||
- BROWSER_WORKERS=1
|
|
||||||
mem_limit: 2g
|
mem_limit: 2g
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "curl", "-sf", "http://localhost:8191/health"]
|
test: ["CMD", "curl", "-sf", "http://localhost:8191/health"]
|
||||||
|
|||||||
@@ -1,20 +1,20 @@
|
|||||||
#!/usr/bin/env bash
|
#!/usr/bin/env bash
|
||||||
set -euo pipefail
|
set -euo pipefail
|
||||||
|
|
||||||
URL="${FLARESOLVERR_URL:-http://localhost:8191}"
|
URL="${BYPARR_URL:-${FLARESOLVERR_URL:-http://localhost:8191}}"
|
||||||
MAX_RETRIES="${FLARESOLVERR_MAX_RETRIES:-30}"
|
MAX_RETRIES="${BYPARR_MAX_RETRIES:-30}"
|
||||||
RETRY_INTERVAL="${FLARESOLVERR_RETRY_INTERVAL:-2}"
|
RETRY_INTERVAL="${BYPARR_RETRY_INTERVAL:-2}"
|
||||||
|
|
||||||
echo "Waiting for FlareSolverr at ${URL}..."
|
echo "Waiting for Byparr at ${URL}..."
|
||||||
|
|
||||||
for i in $(seq 1 "$MAX_RETRIES"); do
|
for i in $(seq 1 "$MAX_RETRIES"); do
|
||||||
if curl -sf "${URL}/health" >/dev/null 2>&1; then
|
if curl -sf "${URL}/health" >/dev/null 2>&1; then
|
||||||
echo "FlareSolverr is healthy."
|
echo "Byparr is healthy."
|
||||||
exit 0
|
exit 0
|
||||||
fi
|
fi
|
||||||
echo "Attempt ${i}/${MAX_RETRIES} - FlareSolverr not ready, retrying in ${RETRY_INTERVAL}s..."
|
echo "Attempt ${i}/${MAX_RETRIES} - Byparr not ready, retrying in ${RETRY_INTERVAL}s..."
|
||||||
sleep "$RETRY_INTERVAL"
|
sleep "$RETRY_INTERVAL"
|
||||||
done
|
done
|
||||||
|
|
||||||
echo "ERROR: FlareSolverr did not become healthy after ${MAX_RETRIES} attempts."
|
echo "ERROR: Byparr did not become healthy after ${MAX_RETRIES} attempts."
|
||||||
exit 1
|
exit 1
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
const BYPARR_URL =
|
||||||
|
process.env.BYPARR_URL ??
|
||||||
|
process.env.FLARESOLVERR_URL ?? // legacy env fallback
|
||||||
|
"http://localhost:8191";
|
||||||
|
|
||||||
|
const BYPARR_API = `${BYPARR_URL}/v1`;
|
||||||
|
|
||||||
|
type ByparrCookie = {
|
||||||
|
name: string;
|
||||||
|
value: string;
|
||||||
|
domain?: string;
|
||||||
|
path?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
type ByparrSolution = {
|
||||||
|
response: string;
|
||||||
|
status: number;
|
||||||
|
cookies: ByparrCookie[];
|
||||||
|
};
|
||||||
|
|
||||||
|
type ByparrResult = {
|
||||||
|
status?: "ok" | "error";
|
||||||
|
message?: string;
|
||||||
|
solution?: ByparrSolution;
|
||||||
|
error?: string;
|
||||||
|
};
|
||||||
|
|
||||||
|
async function byparrRequest(
|
||||||
|
url: string,
|
||||||
|
timeout = 30000,
|
||||||
|
): Promise<ByparrSolution> {
|
||||||
|
const res = await fetch(BYPARR_API, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
cmd: "request.get",
|
||||||
|
url,
|
||||||
|
maxTimeout: timeout,
|
||||||
|
}),
|
||||||
|
signal: AbortSignal.timeout(timeout + 5000),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!res.ok) {
|
||||||
|
throw new Error(`Byparr request failed: ${res.status} ${res.statusText}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const data = (await res.json()) as ByparrResult;
|
||||||
|
|
||||||
|
if (data.error) {
|
||||||
|
throw new Error(`Byparr error: ${data.error}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Byparr returns HTTP 200 with `status: "error"` for invalid requests
|
||||||
|
// (FlareSolverr used a top-level `error` field instead).
|
||||||
|
if (data.status && data.status !== "ok") {
|
||||||
|
throw new Error(`Byparr error: ${data.message ?? "request failed"}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!data.solution) {
|
||||||
|
throw new Error("Byparr: no solution in response");
|
||||||
|
}
|
||||||
|
|
||||||
|
return data.solution;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function fetchWithByparr(
|
||||||
|
url: string,
|
||||||
|
timeout = 30000,
|
||||||
|
): Promise<string> {
|
||||||
|
const solution = await byparrRequest(url, timeout);
|
||||||
|
return solution.response;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Solve a Cloudflare challenge for the given URL and return the clearance
|
||||||
|
* cookies as a `-b "name=value; ..."` argument string suitable for curl.
|
||||||
|
* Returns null if Byparr is unavailable or no cookies were set.
|
||||||
|
*/
|
||||||
|
export async function getByparrCookies(
|
||||||
|
url: string,
|
||||||
|
timeout = 30000,
|
||||||
|
): Promise<string | null> {
|
||||||
|
try {
|
||||||
|
const solution = await byparrRequest(url, timeout);
|
||||||
|
if (!solution.cookies?.length) return null;
|
||||||
|
return solution.cookies.map((c) => `${c.name}=${c.value}`).join("; ");
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,7 +12,7 @@ const {
|
|||||||
selectLimit,
|
selectLimit,
|
||||||
fsUnlink,
|
fsUnlink,
|
||||||
fsReadFile,
|
fsReadFile,
|
||||||
fetchWithFlareSolver,
|
fetchWithByparr,
|
||||||
curlFetchText,
|
curlFetchText,
|
||||||
curlDownload,
|
curlDownload,
|
||||||
} = vi.hoisted(() => ({
|
} = vi.hoisted(() => ({
|
||||||
@@ -23,7 +23,7 @@ const {
|
|||||||
selectLimit: vi.fn<AnyFn>(async () => []),
|
selectLimit: vi.fn<AnyFn>(async () => []),
|
||||||
fsUnlink: vi.fn<AnyFn>(async () => {}),
|
fsUnlink: vi.fn<AnyFn>(async () => {}),
|
||||||
fsReadFile: vi.fn<AnyFn>(async () => Buffer.from("NITRO")),
|
fsReadFile: vi.fn<AnyFn>(async () => Buffer.from("NITRO")),
|
||||||
fetchWithFlareSolver: vi.fn<AnyFn>(async () => ""),
|
fetchWithByparr: vi.fn<AnyFn>(async () => ""),
|
||||||
curlFetchText: vi.fn<AnyFn>(async () => ""),
|
curlFetchText: vi.fn<AnyFn>(async () => ""),
|
||||||
curlDownload: vi.fn<AnyFn>(async () => ({ ok: true, size: 200 })),
|
curlDownload: vi.fn<AnyFn>(async () => ({ ok: true, size: 200 })),
|
||||||
}));
|
}));
|
||||||
@@ -33,7 +33,7 @@ vi.mock("@/lib/services/import/core/curl-fetch", () => ({
|
|||||||
curlDownload,
|
curlDownload,
|
||||||
curlFetchText,
|
curlFetchText,
|
||||||
}));
|
}));
|
||||||
vi.mock("@/lib/services/flare-solver", () => ({ fetchWithFlareSolver }));
|
vi.mock("@/lib/services/byparr", () => ({ fetchWithByparr }));
|
||||||
vi.mock("@/lib/services/furni-data", () => ({ appendFurniEntry }));
|
vi.mock("@/lib/services/furni-data", () => ({ appendFurniEntry }));
|
||||||
vi.mock("@/lib/services/furni-import", () => ({
|
vi.mock("@/lib/services/furni-import", () => ({
|
||||||
ensureDirectories: vi.fn<AnyFn>(async () => {}),
|
ensureDirectories: vi.fn<AnyFn>(async () => {}),
|
||||||
@@ -217,8 +217,8 @@ describe("fetchSourceFurnidata", () => {
|
|||||||
const HTML_WRAPPED = `<html><head><meta charset="utf-8"></head><body><pre>${JSON_BODY}</pre></body></html>`;
|
const HTML_WRAPPED = `<html><head><meta charset="utf-8"></head><body><pre>${JSON_BODY}</pre></body></html>`;
|
||||||
|
|
||||||
beforeEach(() => {
|
beforeEach(() => {
|
||||||
fetchWithFlareSolver.mockReset();
|
fetchWithByparr.mockReset();
|
||||||
fetchWithFlareSolver.mockResolvedValue("");
|
fetchWithByparr.mockResolvedValue("");
|
||||||
vi.unstubAllGlobals();
|
vi.unstubAllGlobals();
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -235,10 +235,10 @@ describe("fetchSourceFurnidata", () => {
|
|||||||
);
|
);
|
||||||
const list = await fetchSourceFurnidata("https://a.test/fd.json", 1);
|
const list = await fetchSourceFurnidata("https://a.test/fd.json", 1);
|
||||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||||
expect(fetchWithFlareSolver).not.toHaveBeenCalled();
|
expect(fetchWithByparr).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("extracts JSON from an HTML-wrapped furnidata (e.g. Leet via FlareSolverr)", async () => {
|
it("extracts JSON from an HTML-wrapped furnidata (e.g. Leet via Byparr)", async () => {
|
||||||
vi.stubGlobal(
|
vi.stubGlobal(
|
||||||
"fetch",
|
"fetch",
|
||||||
vi.fn(
|
vi.fn(
|
||||||
@@ -249,13 +249,13 @@ describe("fetchSourceFurnidata", () => {
|
|||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
fetchWithFlareSolver.mockResolvedValue(HTML_WRAPPED);
|
fetchWithByparr.mockResolvedValue(HTML_WRAPPED);
|
||||||
const list = await fetchSourceFurnidata("https://b.test/fd.json", 1);
|
const list = await fetchSourceFurnidata("https://b.test/fd.json", 1);
|
||||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||||
expect(fetchWithFlareSolver).toHaveBeenCalledWith("https://b.test/fd.json");
|
expect(fetchWithByparr).toHaveBeenCalledWith("https://b.test/fd.json");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("falls back to FlareSolverr on HTML body even when status is 200", async () => {
|
it("falls back to Byparr on HTML body even when status is 200", async () => {
|
||||||
vi.stubGlobal(
|
vi.stubGlobal(
|
||||||
"fetch",
|
"fetch",
|
||||||
vi.fn(
|
vi.fn(
|
||||||
@@ -266,7 +266,7 @@ describe("fetchSourceFurnidata", () => {
|
|||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
fetchWithFlareSolver.mockResolvedValue(HTML_WRAPPED);
|
fetchWithByparr.mockResolvedValue(HTML_WRAPPED);
|
||||||
const list = await fetchSourceFurnidata("https://c.test/fd.json", 1);
|
const list = await fetchSourceFurnidata("https://c.test/fd.json", 1);
|
||||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||||
});
|
});
|
||||||
@@ -284,7 +284,7 @@ describe("fetchSourceFurnidata", () => {
|
|||||||
);
|
);
|
||||||
const list = await fetchSourceFurnidata("https://g.test/fd.json", 1);
|
const list = await fetchSourceFurnidata("https://g.test/fd.json", 1);
|
||||||
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
expect(list.map((e) => e.classname)).toEqual(["bc_sofa", "wall_x"]);
|
||||||
expect(fetchWithFlareSolver).not.toHaveBeenCalled();
|
expect(fetchWithByparr).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it("throws when HTML has no embedded JSON payload", async () => {
|
it("throws when HTML has no embedded JSON payload", async () => {
|
||||||
@@ -298,7 +298,7 @@ describe("fetchSourceFurnidata", () => {
|
|||||||
}),
|
}),
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
fetchWithFlareSolver.mockResolvedValue("<html>still a challenge</html>");
|
fetchWithByparr.mockResolvedValue("<html>still a challenge</html>");
|
||||||
await expect(
|
await expect(
|
||||||
fetchSourceFurnidata("https://d.test/fd.json", 1),
|
fetchSourceFurnidata("https://d.test/fd.json", 1),
|
||||||
).rejects.toThrow(/Cloudflare challenge/);
|
).rejects.toThrow(/Cloudflare challenge/);
|
||||||
|
|||||||
@@ -5,9 +5,9 @@ import { autoDetectInteraction } from "@/lib/furni/auto-interaction";
|
|||||||
import { normalizeClassname } from "@/lib/furni/classname";
|
import { normalizeClassname } from "@/lib/furni/classname";
|
||||||
import { HABBO_GAMEDATA_HOTEL_SETTING_KEY } from "@/lib/habbo-gamedata-hotel";
|
import { HABBO_GAMEDATA_HOTEL_SETTING_KEY } from "@/lib/habbo-gamedata-hotel";
|
||||||
import { logger } from "@/lib/logger";
|
import { logger } from "@/lib/logger";
|
||||||
|
import { fetchWithByparr } from "@/lib/services/byparr";
|
||||||
import { extractFurniIconPng } from "@/lib/services/clone-icon";
|
import { extractFurniIconPng } from "@/lib/services/clone-icon";
|
||||||
import type { CloneSource } from "@/lib/services/clone-sources";
|
import type { CloneSource } from "@/lib/services/clone-sources";
|
||||||
import { fetchWithFlareSolver } from "@/lib/services/flare-solver";
|
|
||||||
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
||||||
import { appendFurniEntry } from "@/lib/services/furni-data";
|
import { appendFurniEntry } from "@/lib/services/furni-data";
|
||||||
import {
|
import {
|
||||||
@@ -101,7 +101,7 @@ const TTL = 5 * 60 * 1000;
|
|||||||
/**
|
/**
|
||||||
* Some sources serve their furnidata wrapped in an HTML document (e.g. Leet's
|
* Some sources serve their furnidata wrapped in an HTML document (e.g. Leet's
|
||||||
* JSON is embedded in a `<pre>` block inside the page, often surfaced by the
|
* JSON is embedded in a `<pre>` block inside the page, often surfaced by the
|
||||||
* FlareSolverr fallback). Extract the JSON payload before giving up.
|
* Byparr fallback). Extract the JSON payload before giving up.
|
||||||
*/
|
*/
|
||||||
function extractJsonFromHtml(body: string): string | null {
|
function extractJsonFromHtml(body: string): string | null {
|
||||||
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/);
|
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/);
|
||||||
@@ -117,20 +117,20 @@ function extractJsonFromHtml(body: string): string | null {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Fetch a source's furnidata via FlareSolverr, falling back to a curl
|
* Fetch a source's furnidata via Byparr, falling back to a curl
|
||||||
* subprocess. Some CDNs (e.g. Leet.city) block Node's TLS fingerprint while
|
* subprocess. Some CDNs (e.g. Leet.city) block Node's TLS fingerprint while
|
||||||
* the same request succeeds from curl — so curl is a useful second fallback
|
* the same request succeeds from curl — so curl is a useful second fallback
|
||||||
* when FlareSolverr is unavailable.
|
* when Byparr is unavailable.
|
||||||
*/
|
*/
|
||||||
async function fetchWithFallback(url: string): Promise<string> {
|
async function fetchWithFallback(url: string): Promise<string> {
|
||||||
try {
|
try {
|
||||||
return await fetchWithFlareSolver(url);
|
return await fetchWithByparr(url);
|
||||||
} catch {
|
} catch {
|
||||||
try {
|
try {
|
||||||
return await curlFetchText(url);
|
return await curlFetchText(url);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`FlareSolverr + curl fallback failed for ${url}: ${(err as Error).message}`,
|
`Byparr + curl fallback failed for ${url}: ${(err as Error).message}`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -150,8 +150,8 @@ export async function fetchSourceFurnidata(
|
|||||||
});
|
});
|
||||||
if (res.ok) {
|
if (res.ok) {
|
||||||
// Some sources (e.g. GitHub raw) serve JSON with a text/plain
|
// Some sources (e.g. GitHub raw) serve JSON with a text/plain
|
||||||
// content-type — only route to FlareSolverr when the body is
|
// content-type — only route to Byparr when the body is actually
|
||||||
// actually HTML, otherwise parse the JSON directly.
|
// HTML, otherwise parse the JSON directly.
|
||||||
const text = await res.text();
|
const text = await res.text();
|
||||||
if (
|
if (
|
||||||
text.trimStart().startsWith("{") ||
|
text.trimStart().startsWith("{") ||
|
||||||
@@ -171,7 +171,7 @@ export async function fetchSourceFurnidata(
|
|||||||
const extracted = extractJsonFromHtml(body);
|
const extracted = extractJsonFromHtml(body);
|
||||||
if (!extracted) {
|
if (!extracted) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`,
|
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start Byparr and check its logs.`,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
body = extracted;
|
body = extracted;
|
||||||
|
|||||||
@@ -1,82 +0,0 @@
|
|||||||
const FLARESOLVERR_URL =
|
|
||||||
process.env.FLARESOLVERR_URL ?? "http://localhost:8191";
|
|
||||||
|
|
||||||
const FLARESOLVERR_API = `${FLARESOLVERR_URL}/v1`;
|
|
||||||
|
|
||||||
type FlareCookie = {
|
|
||||||
name: string;
|
|
||||||
value: string;
|
|
||||||
domain: string;
|
|
||||||
path: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
type FlareSolution = {
|
|
||||||
response: string;
|
|
||||||
status: number;
|
|
||||||
cookies: FlareCookie[];
|
|
||||||
};
|
|
||||||
|
|
||||||
type FlareResult = {
|
|
||||||
solution?: FlareSolution;
|
|
||||||
error?: string;
|
|
||||||
};
|
|
||||||
|
|
||||||
async function flareRequest(
|
|
||||||
url: string,
|
|
||||||
timeout = 30000,
|
|
||||||
): Promise<FlareSolution> {
|
|
||||||
const res = await fetch(FLARESOLVERR_API, {
|
|
||||||
method: "POST",
|
|
||||||
headers: { "Content-Type": "application/json" },
|
|
||||||
body: JSON.stringify({
|
|
||||||
cmd: "request.get",
|
|
||||||
url,
|
|
||||||
maxTimeout: timeout,
|
|
||||||
}),
|
|
||||||
signal: AbortSignal.timeout(timeout + 5000),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!res.ok) {
|
|
||||||
throw new Error(
|
|
||||||
`FlareSolverr request failed: ${res.status} ${res.statusText}`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const data = (await res.json()) as FlareResult;
|
|
||||||
|
|
||||||
if (data.error) {
|
|
||||||
throw new Error(`FlareSolverr error: ${data.error}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!data.solution) {
|
|
||||||
throw new Error("FlareSolverr: no solution in response");
|
|
||||||
}
|
|
||||||
|
|
||||||
return data.solution;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function fetchWithFlareSolver(
|
|
||||||
url: string,
|
|
||||||
timeout = 30000,
|
|
||||||
): Promise<string> {
|
|
||||||
const solution = await flareRequest(url, timeout);
|
|
||||||
return solution.response;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Solve a Cloudflare challenge for the given URL and return the clearance
|
|
||||||
* cookies as a `-b "name=value; ..."` argument string suitable for curl.
|
|
||||||
* Returns null if FlareSolverr is unavailable or no cookies were set.
|
|
||||||
*/
|
|
||||||
export async function getFlareSolverrCookies(
|
|
||||||
url: string,
|
|
||||||
timeout = 30000,
|
|
||||||
): Promise<string | null> {
|
|
||||||
try {
|
|
||||||
const solution = await flareRequest(url, timeout);
|
|
||||||
if (!solution.cookies?.length) return null;
|
|
||||||
return solution.cookies.map((c) => `${c.name}=${c.value}`).join("; ");
|
|
||||||
} catch {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
import { promises as fs } from "node:fs";
|
import { promises as fs } from "node:fs";
|
||||||
import { getFlareSolverrCookies } from "@/lib/services/flare-solver";
|
import { getByparrCookies } from "@/lib/services/byparr";
|
||||||
import { parseNitroBundle } from "../../swf/nitro-builder";
|
import { parseNitroBundle } from "../../swf/nitro-builder";
|
||||||
import { browserHeaders } from "./browser-headers";
|
import { browserHeaders } from "./browser-headers";
|
||||||
import { curlDownload } from "./curl-fetch";
|
import { curlDownload } from "./curl-fetch";
|
||||||
@@ -62,7 +62,7 @@ export async function downloadFile(
|
|||||||
const curlFallback = async (): Promise<boolean> => {
|
const curlFallback = async (): Promise<boolean> => {
|
||||||
let curlRes = await curlDownload(url, destPath);
|
let curlRes = await curlDownload(url, destPath);
|
||||||
if (!curlRes.ok) {
|
if (!curlRes.ok) {
|
||||||
const cookieHeader = await getFlareSolverrCookies(url);
|
const cookieHeader = await getByparrCookies(url);
|
||||||
curlRes = await curlDownload(
|
curlRes = await curlDownload(
|
||||||
url,
|
url,
|
||||||
destPath,
|
destPath,
|
||||||
@@ -108,7 +108,7 @@ export async function downloadFile(
|
|||||||
res.status === 404 || res.status === 410 || res.status === 403;
|
res.status === 404 || res.status === 410 || res.status === 403;
|
||||||
if (deterministic || attempt === maxRetries) {
|
if (deterministic || attempt === maxRetries) {
|
||||||
// HTTP 403 is often a TLS-fingerprint / Cloudflare challenge
|
// HTTP 403 is often a TLS-fingerprint / Cloudflare challenge
|
||||||
// block (e.g. Leet.city) — retry via curl, with FlareSolverr
|
// block (e.g. Leet.city) — retry via curl, with Byparr
|
||||||
// clearance cookies if the plain curl is also challenged.
|
// clearance cookies if the plain curl is also challenged.
|
||||||
if (res.status === 403 && (await curlFallback())) {
|
if (res.status === 403 && (await curlFallback())) {
|
||||||
const stat = await fs.stat(destPath);
|
const stat = await fs.stat(destPath);
|
||||||
@@ -148,7 +148,7 @@ export async function downloadFile(
|
|||||||
return { ok: true, size: buffer.length };
|
return { ok: true, size: buffer.length };
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (attempt === maxRetries || stage === "write") {
|
if (attempt === maxRetries || stage === "write") {
|
||||||
// TLS fingerprint aborts land here too — try curl/FlareSolverr
|
// TLS fingerprint aborts land here too — try curl/Byparr
|
||||||
// before reporting the network failure.
|
// before reporting the network failure.
|
||||||
if (stage === "download" && (await curlFallback())) {
|
if (stage === "download" && (await curlFallback())) {
|
||||||
const stat = await fs.stat(destPath);
|
const stat = await fs.stat(destPath);
|
||||||
|
|||||||
+1
-1
@@ -1282,7 +1282,7 @@ with open(out, "w", encoding="utf-8") as f:
|
|||||||
|
|
||||||
# Sanity-check a .env file for the failure modes that actually break a deploy:
|
# Sanity-check a .env file for the failure modes that actually break a deploy:
|
||||||
# duplicate keys, and a value concatenated onto another key (missing newline),
|
# duplicate keys, and a value concatenated onto another key (missing newline),
|
||||||
# e.g. FLARESOLVERR_URL=http://localhost:8191AUTH_TRUST_HOST="..."
|
# e.g. BYPARR_URL=http://localhost:8191AUTH_TRUST_HOST="..."
|
||||||
validate_env_file() {
|
validate_env_file() {
|
||||||
local base_dir="${1:-$SCRIPT_DIR}"
|
local base_dir="${1:-$SCRIPT_DIR}"
|
||||||
local en="$base_dir/.env"
|
local en="$base_dir/.env"
|
||||||
|
|||||||
Reference in new issue
Block a user