feat: integrate FlareSolverr for Cloudflare bypass on clone sources
- Add FLARESOLVERR_URL env var to .env.example - Update fetchSourceFurnidata to fall back to FlareSolverr on CF challenges (403/HTML) - Add docker-compose.yml with FlareSolverr service - Add scripts/health-check.sh for FlareSolverr readiness check - Add health:check script to package.json - Document FlareSolverr setup in README
This commit is contained in:
1 parent
2e87e5bf09
commit
3edc987281
7 files changed
+125
-6
No files matched your search
@@ -0,0 +1,51 @@
|
||||
const FLARESOLVERR_URL =
|
||||
process.env.FLARESOLVERR_URL ?? "http://localhost:8191";
|
||||
|
||||
export async function fetchWithFlareSolver(
|
||||
url: string,
|
||||
timeout = 30000,
|
||||
): Promise<string> {
|
||||
const res = await fetch(FLARESOLVERR_URL, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
cmd: "request.get",
|
||||
url,
|
||||
maxTimeout: timeout,
|
||||
}),
|
||||
signal: AbortSignal.timeout(timeout + 5000),
|
||||
});
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(
|
||||
`FlareSolverr request failed: ${res.status} ${res.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const data = (await res.json()) as {
|
||||
solution?: { response: string };
|
||||
error?: string;
|
||||
};
|
||||
|
||||
if (data.error) {
|
||||
throw new Error(`FlareSolverr error: ${data.error}`);
|
||||
}
|
||||
|
||||
if (!data.solution) {
|
||||
throw new Error("FlareSolverr: no solution in response");
|
||||
}
|
||||
|
||||
return data.solution.response;
|
||||
}
|
||||
|
||||
export async function fetchJsonWithFlareSolver<T = unknown>(
|
||||
url: string,
|
||||
timeout = 30000,
|
||||
): Promise<T> {
|
||||
const body = await fetchWithFlareSolver(url, timeout);
|
||||
try {
|
||||
return JSON.parse(body) as T;
|
||||
} catch {
|
||||
throw new Error(`flare-solver: could not parse JSON from ${url}`);
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user