100 lines
2.3 KiB
TypeScript
100 lines
2.3 KiB
TypeScript
const UA =
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) " +
|
|
"Chrome/125.0.0.0 Safari/537.36";
|
|
|
|
let browser: any = null;
|
|
|
|
async function getBrowser() {
|
|
if (browser) return browser;
|
|
const puppeteer = await import("puppeteer-extra");
|
|
const StealthPlugin = await import("puppeteer-extra-plugin-stealth");
|
|
puppeteer.default.use(StealthPlugin.default());
|
|
browser = await puppeteer.default.launch({
|
|
headless: true,
|
|
args: [
|
|
"--no-sandbox",
|
|
"--disable-setuid-sandbox",
|
|
"--disable-web-security",
|
|
"--disable-blink-features=AutomationControlled",
|
|
"--user-agent=" + UA,
|
|
],
|
|
});
|
|
return browser;
|
|
}
|
|
|
|
export async function fetchWithCfBypass(
|
|
url: string,
|
|
timeout = 30000,
|
|
): Promise<string> {
|
|
const b = await getBrowser();
|
|
const page = await b.newPage();
|
|
await page.setUserAgent(UA);
|
|
await page.setDefaultTimeout(timeout);
|
|
|
|
try {
|
|
const response = await page.goto(url, {
|
|
waitUntil: "networkidle2",
|
|
timeout,
|
|
});
|
|
|
|
const status = response?.status() ?? 0;
|
|
const body = await page.content();
|
|
|
|
const isCfChallenge =
|
|
body.includes("Attention Required") ||
|
|
body.includes("Checking your browser") ||
|
|
body.includes("Just a moment") ||
|
|
body.includes("cf-challenge") ||
|
|
status === 403;
|
|
|
|
if (isCfChallenge) {
|
|
await page.waitForFunction(
|
|
() =>
|
|
!document.body.innerHTML.includes("Attention Required") &&
|
|
!document.body.innerHTML.includes("Checking your browser") &&
|
|
!document.body.innerHTML.includes("Just a moment"),
|
|
{ timeout: 10000 },
|
|
);
|
|
}
|
|
|
|
const finalBody = await page.content();
|
|
return finalBody;
|
|
} finally {
|
|
try {
|
|
await page.close();
|
|
} catch {
|
|
// ignore cleanup errors
|
|
}
|
|
}
|
|
}
|
|
|
|
export async function fetchJsonWithCfBypass<T = unknown>(
|
|
url: string,
|
|
timeout = 30000,
|
|
): Promise<T> {
|
|
const html = await fetchWithCfBypass(url, timeout);
|
|
const match = html.match(
|
|
/<pre[^>]*class="[^"]*json[^"]*"[^>]*>([\s\S]*?)<\/pre>/i,
|
|
);
|
|
if (match) {
|
|
return JSON.parse(match[1].trim()) as T;
|
|
}
|
|
try {
|
|
return JSON.parse(html) as T;
|
|
} catch {
|
|
throw new Error(`cf-fetch: could not parse JSON from ${url}`);
|
|
}
|
|
}
|
|
|
|
export async function closeCfBrowser(): Promise<void> {
|
|
if (browser) {
|
|
try {
|
|
await browser.close();
|
|
} catch {
|
|
// ignore cleanup errors
|
|
}
|
|
browser = null;
|
|
}
|
|
}
|