fix(import): support HTML-wrapped clone furnidata and skip empty icon sources
Leet serves its furnidata as HTML with the JSON embedded in a <pre> block, and Cloudflare blocks Node's direct fetch. Extract the JSON payload from the HTML (direct or via the FlareSolverr fallback) before giving up on a Cloudflare challenge. Also skip the standalone icon download when a clone source has no iconBaseUrl configured (Habbo, Hubba, Fresh, Kyzegs, RidgeRP), which previously produced invalid relative URLs like /xxx_icon.png and a flood of download errors before falling back to extracting the icon from the .nitro bundle.
This commit is contained in:
1 parent
172941c542
commit
1851653afb
2 files changed
+125
-11
No files matched your search
@@ -85,6 +85,24 @@ export function parseFurnidata(data: RemoteFurnidata): SourceFurni[] {
|
||||
const cache = new Map<string, { list: SourceFurni[]; ts: number }>();
|
||||
const TTL = 5 * 60 * 1000;
|
||||
|
||||
/**
|
||||
* Some sources serve their furnidata wrapped in an HTML document (e.g. Leet's
|
||||
* JSON is embedded in a `<pre>` block inside the page, often surfaced by the
|
||||
* FlareSolverr fallback). Extract the JSON payload before giving up.
|
||||
*/
|
||||
function extractJsonFromHtml(body: string): string | null {
|
||||
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/);
|
||||
const candidate = pre?.[1] ?? body;
|
||||
const trimmed = candidate.trim();
|
||||
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return null;
|
||||
try {
|
||||
JSON.parse(trimmed);
|
||||
return trimmed;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchSourceFurnidata(
|
||||
url: string,
|
||||
now = Date.now(),
|
||||
@@ -115,9 +133,13 @@ export async function fetchSourceFurnidata(
|
||||
}
|
||||
}
|
||||
if (body.trimStart().startsWith("<")) {
|
||||
throw new Error(
|
||||
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`,
|
||||
);
|
||||
const extracted = extractJsonFromHtml(body);
|
||||
if (!extracted) {
|
||||
throw new Error(
|
||||
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`,
|
||||
);
|
||||
}
|
||||
body = extracted;
|
||||
}
|
||||
let list: SourceFurni[];
|
||||
try {
|
||||
@@ -253,14 +275,21 @@ export async function cloneSingleFurni(params: {
|
||||
logger.warn("[clone-import] invalid .nitro bundle for", { classname });
|
||||
return { ok: false, classname, warnings, error: "invalid .nitro bundle" };
|
||||
}
|
||||
const iconDl = await downloadFile(
|
||||
`${source.iconBaseUrl}/${classname}_icon.png`,
|
||||
iconPath,
|
||||
{
|
||||
maxRetries: 1,
|
||||
validate: "png",
|
||||
},
|
||||
);
|
||||
let iconDl: { ok: boolean; size: number };
|
||||
if (source.iconBaseUrl) {
|
||||
iconDl = await downloadFile(
|
||||
`${source.iconBaseUrl}/${classname}_icon.png`,
|
||||
iconPath,
|
||||
{
|
||||
maxRetries: 1,
|
||||
validate: "png",
|
||||
},
|
||||
);
|
||||
} else {
|
||||
// No standalone icons on this source (e.g. Habbo/Hubba) — fall through to
|
||||
// extracting the catalog icon from the .nitro bundle we just downloaded.
|
||||
iconDl = { ok: false, size: 0 };
|
||||
}
|
||||
if (!iconDl.ok) {
|
||||
// Source serves no standalone icon (e.g. icons embedded in the .nitro) —
|
||||
// extract the catalog icon from the bundle we just downloaded.
|
||||
|
||||
Reference in new issue
Block a user