fix(import): support HTML-wrapped clone furnidata and skip empty icon sources
Leet serves its furnidata as HTML with the JSON embedded in a <pre> block, and Cloudflare blocks Node's direct fetch. Extract the JSON payload from the HTML (direct or via the FlareSolverr fallback) before giving up on a Cloudflare challenge. Also skip the standalone icon download when a clone source has no iconBaseUrl configured (Habbo, Hubba, Fresh, Kyzegs, RidgeRP), which previously produced invalid relative URLs like /xxx_icon.png and a flood of download errors before falling back to extracting the icon from the .nitro bundle.
This commit is contained in:
1 parent
172941c542
commit
1851653afb
2 files changed
+125
-11
No files matched your search
File diff suppressed because it is too large.
Load diff
@@ -85,6 +85,24 @@ export function parseFurnidata(data: RemoteFurnidata): SourceFurni[] {
|
|||||||
const cache = new Map<string, { list: SourceFurni[]; ts: number }>();
|
const cache = new Map<string, { list: SourceFurni[]; ts: number }>();
|
||||||
const TTL = 5 * 60 * 1000;
|
const TTL = 5 * 60 * 1000;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Some sources serve their furnidata wrapped in an HTML document (e.g. Leet's
|
||||||
|
* JSON is embedded in a `<pre>` block inside the page, often surfaced by the
|
||||||
|
* FlareSolverr fallback). Extract the JSON payload before giving up.
|
||||||
|
*/
|
||||||
|
function extractJsonFromHtml(body: string): string | null {
|
||||||
|
const pre = body.match(/<pre[^>]*>([\s\S]*?)<\/pre>/);
|
||||||
|
const candidate = pre?.[1] ?? body;
|
||||||
|
const trimmed = candidate.trim();
|
||||||
|
if (!trimmed.startsWith("{") && !trimmed.startsWith("[")) return null;
|
||||||
|
try {
|
||||||
|
JSON.parse(trimmed);
|
||||||
|
return trimmed;
|
||||||
|
} catch {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
export async function fetchSourceFurnidata(
|
export async function fetchSourceFurnidata(
|
||||||
url: string,
|
url: string,
|
||||||
now = Date.now(),
|
now = Date.now(),
|
||||||
@@ -115,9 +133,13 @@ export async function fetchSourceFurnidata(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
if (body.trimStart().startsWith("<")) {
|
if (body.trimStart().startsWith("<")) {
|
||||||
throw new Error(
|
const extracted = extractJsonFromHtml(body);
|
||||||
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`,
|
if (!extracted) {
|
||||||
);
|
throw new Error(
|
||||||
|
`Source ${url} is behind a Cloudflare challenge that could not be bypassed. Start FlareSolverr and check its logs.`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
body = extracted;
|
||||||
}
|
}
|
||||||
let list: SourceFurni[];
|
let list: SourceFurni[];
|
||||||
try {
|
try {
|
||||||
@@ -253,14 +275,21 @@ export async function cloneSingleFurni(params: {
|
|||||||
logger.warn("[clone-import] invalid .nitro bundle for", { classname });
|
logger.warn("[clone-import] invalid .nitro bundle for", { classname });
|
||||||
return { ok: false, classname, warnings, error: "invalid .nitro bundle" };
|
return { ok: false, classname, warnings, error: "invalid .nitro bundle" };
|
||||||
}
|
}
|
||||||
const iconDl = await downloadFile(
|
let iconDl: { ok: boolean; size: number };
|
||||||
`${source.iconBaseUrl}/${classname}_icon.png`,
|
if (source.iconBaseUrl) {
|
||||||
iconPath,
|
iconDl = await downloadFile(
|
||||||
{
|
`${source.iconBaseUrl}/${classname}_icon.png`,
|
||||||
maxRetries: 1,
|
iconPath,
|
||||||
validate: "png",
|
{
|
||||||
},
|
maxRetries: 1,
|
||||||
);
|
validate: "png",
|
||||||
|
},
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// No standalone icons on this source (e.g. Habbo/Hubba) — fall through to
|
||||||
|
// extracting the catalog icon from the .nitro bundle we just downloaded.
|
||||||
|
iconDl = { ok: false, size: 0 };
|
||||||
|
}
|
||||||
if (!iconDl.ok) {
|
if (!iconDl.ok) {
|
||||||
// Source serves no standalone icon (e.g. icons embedded in the .nitro) —
|
// Source serves no standalone icon (e.g. icons embedded in the .nitro) —
|
||||||
// extract the catalog icon from the bundle we just downloaded.
|
// extract the catalog icon from the bundle we just downloaded.
|
||||||
|
|||||||
Reference in new issue
Block a user