2 Commits
Author SHA1 Message Date
openhands b87c9a5b8d chore: remove puppeteer CF bypass (not working for Leet/Hubbly/Habblet)
CI / check (push) Failing after 14s
CI / release (push) Skipped
CI / deploy (push) Skipped
Cloudflare has strengthened protection on these hotels.
Puppeteer-based bypass returns HTML instead of JSON.
cloudscraper has dependency issues with Node.js v26.

Reverted to simple HTTP fetch with clear error messages.
2026-08-04 18:45:10 +02:00
openhands 5c60ce364c chore: remove cf-fetch.ts (puppeteer CF bypass not working for Leet/Hubbly/Habblet)
Cloudflare has strengthened protection on these hotels.
Puppeteer-based bypass returns HTML instead of JSON.
cloudscraper has dependency issues with Node.js v26.

Reverting to simple HTTP fetch with clear error messages.
2026-08-04 18:42:20 +02:00
5 changed files with 386 additions and 826 deletions

No files matched your search

+1 -10
View File
@@ -36,16 +36,7 @@ const securityHeaders = [
const nextConfig: NextConfig = { const nextConfig: NextConfig = {
deploymentId: resolveDeploymentId(), deploymentId: resolveDeploymentId(),
turbopack: {}, turbopack: {},
serverExternalPackages: [ serverExternalPackages: ["mariadb", "lzma", "sharp", "pino", "pino-pretty"],
"mariadb",
"lzma",
"sharp",
"pino",
"pino-pretty",
"puppeteer",
"puppeteer-extra",
"puppeteer-extra-plugin-stealth",
],
// Enable React Compiler for automatic memoization // Enable React Compiler for automatic memoization
reactCompiler: true, reactCompiler: true,
+2 -3
View File
@@ -42,6 +42,7 @@
"@hookform/resolvers": "^5.5.7", "@hookform/resolvers": "^5.5.7",
"@tanstack/react-virtual": "^3.14.9", "@tanstack/react-virtual": "^3.14.9",
"class-variance-authority": "^0.7.1", "class-variance-authority": "^0.7.1",
"cloudscraper": "^4.6.0",
"clsx": "^2.1.1", "clsx": "^2.1.1",
"cmdk": "^1.1.1", "cmdk": "^1.1.1",
"croner": "^10.0.1", "croner": "^10.0.1",
@@ -66,6 +67,7 @@
"react": "^19.2.8", "react": "^19.2.8",
"react-dom": "^19.2.8", "react-dom": "^19.2.8",
"react-hook-form": "^7.83.0", "react-hook-form": "^7.83.0",
"request": "^2.88.2",
"resend": "^6.18.1", "resend": "^6.18.1",
"server-only": "^0.0.1", "server-only": "^0.0.1",
"sharp": "^0.35.3", "sharp": "^0.35.3",
@@ -92,9 +94,6 @@
"lint-staged": "^17.3.0", "lint-staged": "^17.3.0",
"pino-pretty": "^13.1.3", "pino-pretty": "^13.1.3",
"postcss": "^8.5.25", "postcss": "^8.5.25",
"puppeteer": "^25.5.0",
"puppeteer-extra": "^3.3.6",
"puppeteer-extra-plugin-stealth": "^2.11.2",
"tailwindcss": "^4.3.3", "tailwindcss": "^4.3.3",
"tsx": "^4.23.1", "tsx": "^4.23.1",
"typescript": "^7.0.2", "typescript": "^7.0.2",
+377 -694
View File
File diff suppressed because it is too large. Load diff
-103
View File
@@ -1,103 +0,0 @@
import puppeteer from "puppeteer";
const UA =
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) " +
"AppleWebKit/537.36 (KHTML, like Gecko) " +
"Chrome/125.0.0.0 Safari/537.36";
let browser: Awaited<ReturnType<typeof puppeteer.launch>> | null = null;
async function getBrowser() {
if (browser) return browser;
browser = await puppeteer.launch({
headless: true,
args: [
"--no-sandbox",
"--disable-setuid-sandbox",
"--disable-web-security",
"--disable-blink-features=AutomationControlled",
"--user-agent=" + UA,
],
});
return browser;
}
export async function fetchWithCfBypass(
url: string,
timeout = 30000,
): Promise<string> {
const b = await getBrowser();
const page = await b.newPage();
await page.setUserAgent(UA);
await page.setDefaultTimeout(timeout);
try {
const response = await page.goto(url, {
waitUntil: "networkidle2",
timeout,
});
const status = response?.status() ?? 0;
const body = await page.content();
const isCfChallenge =
body.includes("Attention Required") ||
body.includes("Checking your browser") ||
body.includes("Just a moment") ||
body.includes("cf-challenge") ||
status === 403;
if (isCfChallenge) {
try {
await page.waitForFunction(
() =>
document.body.innerHTML.includes("cf-captcha") ||
document.body.innerHTML.includes("challenge-platform"),
{ timeout: 5000 },
);
} catch {
// CF challenge might have been solved already
}
}
await new Promise((resolve) => setTimeout(resolve, 2000));
const finalBody = await page.content();
return finalBody;
} finally {
try {
await page.close();
} catch {
// ignore cleanup errors
}
}
}
export async function fetchJsonWithCfBypass<T = unknown>(
url: string,
timeout = 30000,
): Promise<T> {
const html = await fetchWithCfBypass(url, timeout);
const match = html.match(
/<pre[^>]*class="[^"]*json[^"]*"[^>]*>([\s\S]*?)<\/pre>/i,
);
if (match) {
return JSON.parse(match[1].trim()) as T;
}
try {
return JSON.parse(html) as T;
} catch {
throw new Error(`cf-fetch: could not parse JSON from ${url}`);
}
}
export async function closeCfBrowser(): Promise<void> {
if (browser) {
try {
await browser.close();
} catch {
// ignore cleanup errors
}
browser = null;
}
}
+6 -16
View File
@@ -3,7 +3,6 @@ import path from "node:path";
import { eq, inArray, sql } from "drizzle-orm"; import { eq, inArray, sql } from "drizzle-orm";
import { db, ItemsBase } from "@/lib/db"; import { db, ItemsBase } from "@/lib/db";
import { logger } from "@/lib/logger"; import { logger } from "@/lib/logger";
import { fetchWithCfBypass } from "@/lib/services/cf-fetch";
import { extractFurniIconPng } from "@/lib/services/clone-icon"; import { extractFurniIconPng } from "@/lib/services/clone-icon";
import type { CloneSource } from "@/lib/services/clone-sources"; import type { CloneSource } from "@/lib/services/clone-sources";
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs"; import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
@@ -89,23 +88,14 @@ export async function fetchSourceFurnidata(
): Promise<SourceFurni[]> { ): Promise<SourceFurni[]> {
const hit = cache.get(url); const hit = cache.get(url);
if (hit && now && now - hit.ts < TTL) return hit.list; if (hit && now && now - hit.ts < TTL) return hit.list;
let body: string; const res = await fetch(url, {
try { signal: AbortSignal.timeout(30000),
const res = await fetch(url, { headers: browserHeaders(),
signal: AbortSignal.timeout(30000), });
headers: browserHeaders(), if (!res.ok) throw new Error(`furnidata fetch failed: ${res.status} ${url}`);
});
if (!res.ok) throw new Error(`HTTP ${res.status}`);
const text = await res.text();
if (typeof text === "string" && text.trim().startsWith("<"))
throw new Error("Cloudflare challenge detected");
body = text;
} catch {
body = await fetchWithCfBypass(url);
}
let list: SourceFurni[]; let list: SourceFurni[];
try { try {
const json = JSON.parse(body); const json = await res.json();
list = parseFurnidata(json); list = parseFurnidata(json);
} catch (err) { } catch (err) {
throw new Error( throw new Error(