chore: remove puppeteer CF bypass (not working for Leet/Hubbly/Habblet)
Cloudflare has strengthened protection on these hotels. Puppeteer-based bypass returns HTML instead of JSON. cloudscraper has dependency issues with Node.js v26. Reverted to simple HTTP fetch with clear error messages.
This commit is contained in:
1 parent
5c60ce364c
commit
b87c9a5b8d
4 files changed
+19
-741
No files matched your search
+1
-10
@@ -36,16 +36,7 @@ const securityHeaders = [
|
|||||||
const nextConfig: NextConfig = {
|
const nextConfig: NextConfig = {
|
||||||
deploymentId: resolveDeploymentId(),
|
deploymentId: resolveDeploymentId(),
|
||||||
turbopack: {},
|
turbopack: {},
|
||||||
serverExternalPackages: [
|
serverExternalPackages: ["mariadb", "lzma", "sharp", "pino", "pino-pretty"],
|
||||||
"mariadb",
|
|
||||||
"lzma",
|
|
||||||
"sharp",
|
|
||||||
"pino",
|
|
||||||
"pino-pretty",
|
|
||||||
"puppeteer",
|
|
||||||
"puppeteer-extra",
|
|
||||||
"puppeteer-extra-plugin-stealth",
|
|
||||||
],
|
|
||||||
|
|
||||||
// Enable React Compiler for automatic memoization
|
// Enable React Compiler for automatic memoization
|
||||||
reactCompiler: true,
|
reactCompiler: true,
|
||||||
|
|||||||
@@ -94,9 +94,6 @@
|
|||||||
"lint-staged": "^17.3.0",
|
"lint-staged": "^17.3.0",
|
||||||
"pino-pretty": "^13.1.3",
|
"pino-pretty": "^13.1.3",
|
||||||
"postcss": "^8.5.25",
|
"postcss": "^8.5.25",
|
||||||
"puppeteer": "^25.5.0",
|
|
||||||
"puppeteer-extra": "^3.3.6",
|
|
||||||
"puppeteer-extra-plugin-stealth": "^2.11.2",
|
|
||||||
"tailwindcss": "^4.3.3",
|
"tailwindcss": "^4.3.3",
|
||||||
"tsx": "^4.23.1",
|
"tsx": "^4.23.1",
|
||||||
"typescript": "^7.0.2",
|
"typescript": "^7.0.2",
|
||||||
|
|||||||
Generated
+12
-712
File diff suppressed because it is too large.
Load diff
@@ -3,7 +3,6 @@ import path from "node:path";
|
|||||||
import { eq, inArray, sql } from "drizzle-orm";
|
import { eq, inArray, sql } from "drizzle-orm";
|
||||||
import { db, ItemsBase } from "@/lib/db";
|
import { db, ItemsBase } from "@/lib/db";
|
||||||
import { logger } from "@/lib/logger";
|
import { logger } from "@/lib/logger";
|
||||||
import { fetchWithCfBypass } from "@/lib/services/cf-fetch";
|
|
||||||
import { extractFurniIconPng } from "@/lib/services/clone-icon";
|
import { extractFurniIconPng } from "@/lib/services/clone-icon";
|
||||||
import type { CloneSource } from "@/lib/services/clone-sources";
|
import type { CloneSource } from "@/lib/services/clone-sources";
|
||||||
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
||||||
@@ -89,23 +88,14 @@ export async function fetchSourceFurnidata(
|
|||||||
): Promise<SourceFurni[]> {
|
): Promise<SourceFurni[]> {
|
||||||
const hit = cache.get(url);
|
const hit = cache.get(url);
|
||||||
if (hit && now && now - hit.ts < TTL) return hit.list;
|
if (hit && now && now - hit.ts < TTL) return hit.list;
|
||||||
let body: string;
|
const res = await fetch(url, {
|
||||||
try {
|
signal: AbortSignal.timeout(30000),
|
||||||
const res = await fetch(url, {
|
headers: browserHeaders(),
|
||||||
signal: AbortSignal.timeout(30000),
|
});
|
||||||
headers: browserHeaders(),
|
if (!res.ok) throw new Error(`furnidata fetch failed: ${res.status} ${url}`);
|
||||||
});
|
|
||||||
if (!res.ok) throw new Error(`HTTP ${res.status}`);
|
|
||||||
const text = await res.text();
|
|
||||||
if (typeof text === "string" && text.trim().startsWith("<"))
|
|
||||||
throw new Error("Cloudflare challenge detected");
|
|
||||||
body = text;
|
|
||||||
} catch {
|
|
||||||
body = await fetchWithCfBypass(url);
|
|
||||||
}
|
|
||||||
let list: SourceFurni[];
|
let list: SourceFurni[];
|
||||||
try {
|
try {
|
||||||
const json = JSON.parse(body);
|
const json = await res.json();
|
||||||
list = parseFurnidata(json);
|
list = parseFurnidata(json);
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
throw new Error(
|
throw new Error(
|
||||||
|
|||||||
Reference in new issue
Block a user