feat(studio): recover matching Nitro sources and persist import phases
This commit is contained in:
1 parent
00e33623f7
commit
13bd3695cb
45 files changed
+903
-28
No files matched your search
@@ -1,4 +1,14 @@
|
||||
import type { SourceFurni } from "@/lib/services/clone-import";
|
||||
export const IMPORT_PHASES = [
|
||||
"validating",
|
||||
"downloading",
|
||||
"converting",
|
||||
"checking_sources",
|
||||
"writing_db",
|
||||
"writing_furnidata",
|
||||
"translating",
|
||||
] as const;
|
||||
export type ImportPhase = (typeof IMPORT_PHASES)[number];
|
||||
export interface ImportJobItem {
|
||||
id: number;
|
||||
classname: string;
|
||||
@@ -34,6 +44,14 @@ export interface ImportJob {
|
||||
| "failed"
|
||||
| "cancelled"
|
||||
| "interrupted";
|
||||
phase?: ImportPhase;
|
||||
phaseStartedAt?: string;
|
||||
sourceAttempt?: string;
|
||||
recoveredSource?: {
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
revision: number;
|
||||
};
|
||||
error?: string;
|
||||
warnings?: string[];
|
||||
itemId?: number;
|
||||
|
||||
@@ -247,7 +247,7 @@ export function __resetItemsBaseAllocatorForTests(): void {
|
||||
export async function cloneSingleFurni(params: {
|
||||
source: CloneSource;
|
||||
entry: SourceFurni;
|
||||
onProgress?: (status: string) => void;
|
||||
onProgress?: (status: string) => void | Promise<void>;
|
||||
/** Skip the per-item FurnitureData.json write and return the entry instead,
|
||||
* so batch callers can append everything in one lock + write cycle. */
|
||||
deferFurniData?: boolean;
|
||||
@@ -291,7 +291,7 @@ export async function cloneSingleFurni(params: {
|
||||
const { iconDir, nitroDir } = await getFurniAssetDirs();
|
||||
|
||||
// Download .nitro + icon directly from the source hotel.
|
||||
onProgress?.("downloading");
|
||||
await onProgress?.("downloading");
|
||||
const nitroPath = getRuntimePath(
|
||||
/*turbopackIgnore: true*/ nitroDir,
|
||||
`${classname}.nitro`,
|
||||
@@ -370,7 +370,7 @@ export async function cloneSingleFurni(params: {
|
||||
}
|
||||
}
|
||||
|
||||
onProgress?.("writing_db");
|
||||
await onProgress?.("writing_db");
|
||||
const stackHeight =
|
||||
entry.canlayon || entry.cansiton ? 1.0 : entry.canstandon ? 1.0 : 0.0;
|
||||
// allow_stack: derived from stackHeight (mirrors furni-import.ts `dims.z > 0`).
|
||||
@@ -426,7 +426,7 @@ export async function cloneSingleFurni(params: {
|
||||
// FurnitureData entry — reuse the source's furnitype object, but with our id
|
||||
// AND our offerid. The source's offerid points at the source hotel's sprite
|
||||
// ids, which would silently break the emulator's catalog offer lookup.
|
||||
onProgress?.("writing_furnidata");
|
||||
await onProgress?.("writing_furnidata");
|
||||
const furniDataEntry = {
|
||||
...entry,
|
||||
id: newId,
|
||||
|
||||
@@ -6,7 +6,9 @@ import { importRoot, validJobId } from "./furni-job-store";
|
||||
export function validateFurnitureAttachment(buffer: Buffer, classname: string) {
|
||||
if (buffer.length > 50 * 1024 * 1024)
|
||||
throw Error("Maximum file size is 50 MB");
|
||||
const parsed = parseNitroBundle(buffer);
|
||||
const parsed = parseNitroBundle(buffer, {
|
||||
maxOutputLength: 50 * 1024 * 1024,
|
||||
});
|
||||
const base = classname.split("*")[0];
|
||||
if (
|
||||
parsed.jsonFileName !== `${base}.json` ||
|
||||
|
||||
@@ -4,6 +4,8 @@ import { MySqlDialect } from "drizzle-orm/mysql-core";
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
nitroExists: true,
|
||||
write: vi.fn(),
|
||||
official: vi.fn(),
|
||||
download: vi.fn(),
|
||||
execute: vi.fn(),
|
||||
@@ -24,9 +26,10 @@ vi.mock("@/lib/db", () => ({
|
||||
CatalogPages: {},
|
||||
}));
|
||||
vi.mock("node:fs", () => ({
|
||||
existsSync: () => true,
|
||||
existsSync: (path: string) => !path.endsWith(".nitro") || mocks.nitroExists,
|
||||
promises: {
|
||||
stat: mocks.stat,
|
||||
writeFile: mocks.write,
|
||||
readFile: vi.fn().mockResolvedValue(Buffer.alloc(256)),
|
||||
},
|
||||
}));
|
||||
@@ -57,6 +60,9 @@ vi.mock("@/lib/services/import/core/download", () => ({
|
||||
vi.mock("@/lib/services/swf-to-nitro", () => ({
|
||||
parseNitroBundle: () => ({ json: {} }),
|
||||
extractIconFromSwfAsync: async () => null,
|
||||
convertSwfToNitroAsync: async () => {
|
||||
throw Error("Source conversion failed");
|
||||
},
|
||||
}));
|
||||
|
||||
import { importSingleFurni } from "./furni-import";
|
||||
@@ -78,6 +84,10 @@ function queries() {
|
||||
describe("completion of existing furniture", () => {
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.nitroExists = true;
|
||||
mocks.write.mockImplementation(async () => {
|
||||
mocks.nitroExists = true;
|
||||
});
|
||||
mocks.official.mockResolvedValue(null);
|
||||
mocks.download.mockResolvedValue(false);
|
||||
const rows = [
|
||||
@@ -92,6 +102,23 @@ describe("completion of existing furniture", () => {
|
||||
mocks.execute.mockResolvedValue([[{ id: 123, page_id: "15" }], []]);
|
||||
mocks.build.mockImplementation((value) => value);
|
||||
});
|
||||
it("recovers missing Nitro before completing the existing item without changing its IDs", async () => {
|
||||
mocks.nitroExists = false;
|
||||
const recover = vi.fn(async () => Buffer.alloc(256));
|
||||
const result = await importSingleFurni({
|
||||
...input,
|
||||
resolveMissingNitro: recover,
|
||||
});
|
||||
expect(recover).toHaveBeenCalledWith(1);
|
||||
expect(mocks.write).toHaveBeenCalledWith(
|
||||
expect.stringContaining("recycler_scrapbookA.nitro"),
|
||||
expect.any(Buffer),
|
||||
{ flag: "wx" },
|
||||
);
|
||||
expect(result.ok).toBe(true);
|
||||
expect(result.itemId).toBe(2000037257);
|
||||
});
|
||||
|
||||
it("reuses local item and sprite IDs without rewriting SQL settings or offers", async () => {
|
||||
const result = await importSingleFurni(input);
|
||||
expect(result.ok).toBe(true);
|
||||
|
||||
@@ -517,7 +517,8 @@ export async function importSingleFurni(params: {
|
||||
preserveAssets?: boolean;
|
||||
/** Validated matching bundle supplied by a queued import. */
|
||||
providedNitro?: Buffer;
|
||||
onProgress?: (status: string) => void;
|
||||
onProgress?: (status: string) => void | Promise<void>;
|
||||
resolveMissingNitro?: (revision: number) => Promise<Buffer | undefined>;
|
||||
/** Per-source SWF download base URL (e.g. "https://virtualc.nl/dcr"). */
|
||||
sourceSwfBaseUrl?: string;
|
||||
/** Per-source nitro bundle + icon base URL. */
|
||||
@@ -660,7 +661,7 @@ export async function importSingleFurni(params: {
|
||||
// For color variants (classname has `*`): try color-specific first,
|
||||
// fall back to base-classname icon (Nitro reuses base for all colors
|
||||
// when no color-specific icon exists).
|
||||
onProgress?.("downloading");
|
||||
await onProgress?.("downloading");
|
||||
|
||||
const iconColorAware = classname.replace(/\*/g, "_");
|
||||
|
||||
@@ -770,7 +771,7 @@ export async function importSingleFurni(params: {
|
||||
warnings.push(`SWF download failed: ${swfFailure || "unknown cause"}`);
|
||||
|
||||
// ── Convert SWF to Nitro ──────────────────────────────────────────
|
||||
onProgress?.("converting");
|
||||
await onProgress?.("converting");
|
||||
let conversionResult: ConversionResult | null = null;
|
||||
|
||||
if (existsSync(/*turbopackIgnore: true*/ nitroPath)) {
|
||||
@@ -803,6 +804,16 @@ export async function importSingleFurni(params: {
|
||||
}
|
||||
}
|
||||
|
||||
if (!existsSync(nitroPath) && params.resolveMissingNitro) {
|
||||
await onProgress?.("checking_sources");
|
||||
const recovered = await params.resolveMissingNitro(rev);
|
||||
if (recovered) await fs.writeFile(nitroPath, recovered, { flag: "wx" });
|
||||
else
|
||||
warnings.push(
|
||||
"No matching bundle was recovered from the checked sources. Review source availability or attach the original .nitro file.",
|
||||
);
|
||||
}
|
||||
|
||||
if (!iconOk && existsSync(nitroPath)) {
|
||||
try {
|
||||
const icon = extractFurniIconPng(await fs.readFile(nitroPath));
|
||||
@@ -847,7 +858,7 @@ export async function importSingleFurni(params: {
|
||||
}
|
||||
|
||||
// ── Insert into DB with enriched data ──────────────────────────────
|
||||
onProgress?.("writing_db");
|
||||
await onProgress?.("writing_db");
|
||||
try {
|
||||
if (preservingExisting) {
|
||||
warnings.push(
|
||||
@@ -1032,6 +1043,7 @@ export async function importSingleFurni(params: {
|
||||
// Batch mode: return the entry so the caller writes once at the end
|
||||
} else {
|
||||
try {
|
||||
await onProgress?.("writing_furnidata");
|
||||
await appendFurniEntry(furniEntry, itemType);
|
||||
} catch (err) {
|
||||
logger.warn("[import-furni] Failed to update FurnitureData.json", {
|
||||
|
||||
@@ -242,3 +242,36 @@ it("attaches recovery files only to safe failed items and preserves retry histor
|
||||
s.retry(original.id, 4, { chair: randomUUID() }),
|
||||
).rejects.toThrow("latest retry");
|
||||
});
|
||||
|
||||
it("clears old source provenance and phase when creating a retry", async () => {
|
||||
const s = await store();
|
||||
const original: ImportJob = {
|
||||
...job(),
|
||||
state: "completed",
|
||||
items: [
|
||||
{
|
||||
id: 1,
|
||||
classname: "chair",
|
||||
name: "Chair",
|
||||
description: "",
|
||||
type: "flooritem",
|
||||
revision: 1,
|
||||
category: "other",
|
||||
state: "failed",
|
||||
phase: "converting",
|
||||
phaseStartedAt: new Date().toISOString(),
|
||||
sourceAttempt: "Old source",
|
||||
recoveredSource: {
|
||||
sourceId: "old",
|
||||
sourceName: "Old source",
|
||||
revision: 1,
|
||||
},
|
||||
},
|
||||
],
|
||||
};
|
||||
await s.create(original);
|
||||
const child = await s.retry(original.id, 4);
|
||||
expect(child?.items[0].phase).toBeUndefined();
|
||||
expect(child?.items[0].recoveredSource).toBeUndefined();
|
||||
expect(child?.items[0].sourceAttempt).toBeUndefined();
|
||||
});
|
||||
@@ -111,7 +111,16 @@ export class ImportJobStore {
|
||||
state: "queued",
|
||||
cancelRequested: false,
|
||||
items: items.map(
|
||||
({ error: _error, warnings: _warnings, itemId: _itemId, ...item }) => ({
|
||||
({
|
||||
error: _error,
|
||||
warnings: _warnings,
|
||||
itemId: _itemId,
|
||||
phase: _phase,
|
||||
phaseStartedAt: _phaseStartedAt,
|
||||
sourceAttempt: _sourceAttempt,
|
||||
recoveredSource: _recoveredSource,
|
||||
...item
|
||||
}) => ({
|
||||
...item,
|
||||
...(Object.hasOwn(attachments, item.classname)
|
||||
? { attachmentId: attachments[item.classname] }
|
||||
|
||||
@@ -221,7 +221,7 @@ it("does not save a translated outcome after ownership is lost during translatio
|
||||
});
|
||||
await drainFurnitureImports();
|
||||
expect(mocks.translate).toHaveBeenCalledOnce();
|
||||
expect(snapshots).toHaveLength(2);
|
||||
expect(snapshots).toHaveLength(3);
|
||||
expect(snapshots.at(-1)?.items[0].state).toBe("running");
|
||||
expect(mocks.updateCatalog).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -249,3 +249,16 @@ it("keeps successful imports complete if their audit write fails", async () => {
|
||||
expect(job.items[0].state).toBe("done");
|
||||
expect(job.items[0].warnings).toContain("Audit record could not be saved");
|
||||
});
|
||||
|
||||
it("persists progress before continuing an import and retains the failed phase", async () => {
|
||||
mocks.import.mockImplementationOnce(async (params) => {
|
||||
await params.onProgress("downloading");
|
||||
expect(job.items[0].phase).toBe("downloading");
|
||||
expect(mocks.save).toHaveBeenCalled();
|
||||
await params.onProgress("converting");
|
||||
throw Error("Conversion failed");
|
||||
});
|
||||
await drainFurnitureImports();
|
||||
expect(job.items[0]).toMatchObject({ state: "failed", phase: "converting" });
|
||||
expect(job.items[0].phaseStartedAt).toBeTruthy();
|
||||
});
|
||||
@@ -1,7 +1,7 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
import { createStore, runWithStore } from "@/lib/foundation/request-context";
|
||||
import type { IpAddress, RequestId, UserId } from "@/lib/foundation/types";
|
||||
import type { ImportJob } from "@/lib/furni/import-job";
|
||||
import { IMPORT_PHASES, type ImportJob } from "@/lib/furni/import-job";
|
||||
import { redis } from "@/lib/redis";
|
||||
import { logServerError } from "@/lib/server-log";
|
||||
import { logAudit } from "./audit";
|
||||
@@ -14,6 +14,7 @@ import { ensureDirectories, importSingleFurni } from "./furni-import";
|
||||
import { ImportJobStore } from "./furni-job-store";
|
||||
import { runSyncJobItem } from "./furni-sync-item";
|
||||
import { repairFurniture } from "./furniture-repair";
|
||||
import { recoverFurnitureNitro } from "./furniture-source-recovery";
|
||||
import { rcon } from "./rcon";
|
||||
|
||||
const LOCK = "furniture-import-worker:v1";
|
||||
@@ -120,7 +121,15 @@ async function drain() {
|
||||
break;
|
||||
}
|
||||
item.state = "running";
|
||||
await saveOwned(job);
|
||||
const onProgress = async (status: string) => {
|
||||
const phase = IMPORT_PHASES.find((value) => value === status);
|
||||
if (!phase) return;
|
||||
if (item.phase !== phase)
|
||||
item.phaseStartedAt = new Date().toISOString();
|
||||
item.phase = phase;
|
||||
await saveOwned(job);
|
||||
};
|
||||
await onProgress("validating");
|
||||
try {
|
||||
if (job.sourceId && !source)
|
||||
throw Error("Import source no longer exists");
|
||||
@@ -133,12 +142,32 @@ async function drain() {
|
||||
: undefined;
|
||||
await requireLease();
|
||||
const result = job.syncKind
|
||||
? await runSyncJobItem(item)
|
||||
? await runSyncJobItem(item, onProgress)
|
||||
: await (job.mode === "repair"
|
||||
? repairFurniture
|
||||
: importSingleFurni)({
|
||||
...item,
|
||||
repairExisting: true,
|
||||
onProgress,
|
||||
resolveMissingNitro: async (revision) => {
|
||||
const recovered = await recoverFurnitureNitro(
|
||||
{ ...item, revision },
|
||||
{
|
||||
excludeSourceId: job.sourceId,
|
||||
onAttempt: async (name) => {
|
||||
item.sourceAttempt = name;
|
||||
await onProgress("checking_sources");
|
||||
},
|
||||
},
|
||||
);
|
||||
await requireLease();
|
||||
if (!recovered) return undefined;
|
||||
const { buffer, ...provenance } = recovered;
|
||||
item.recoveredSource = provenance;
|
||||
item.sourceAttempt = undefined;
|
||||
await saveOwned(job);
|
||||
return buffer;
|
||||
},
|
||||
providedNitro,
|
||||
sourceSwfBaseUrl: source?.sourceSwfBaseUrl,
|
||||
nitroBaseUrl: source?.nitroBaseUrl,
|
||||
@@ -148,7 +177,6 @@ async function drain() {
|
||||
item.warnings = result.warnings;
|
||||
item.itemId = result.itemId;
|
||||
if (!result.ok) throw Error(result.error || "Import failed");
|
||||
item.state = "done";
|
||||
await logAudit({
|
||||
userId: job.userId,
|
||||
action: "furni_import",
|
||||
@@ -171,12 +199,14 @@ async function drain() {
|
||||
});
|
||||
if (job.translate && job.mode !== "repair")
|
||||
try {
|
||||
await onProgress("translating");
|
||||
await patchLocalizedFurniDataEntries([item], true, job.langs);
|
||||
} catch {
|
||||
item.warnings.push(
|
||||
"Translation failed; furniture imported successfully",
|
||||
);
|
||||
}
|
||||
item.state = "done";
|
||||
} catch (error) {
|
||||
if (error instanceof ImportLeaseLostError) throw error;
|
||||
await requireLease();
|
||||
|
||||
@@ -6,7 +6,10 @@ import { getSource } from "./clone-sources";
|
||||
import { importSingleFurni } from "./furni-import";
|
||||
|
||||
/** Called only inside the shared import lease and catalog export lock. */
|
||||
export async function runSyncJobItem(item: ImportJobItem) {
|
||||
export async function runSyncJobItem(
|
||||
item: ImportJobItem,
|
||||
onProgress?: (status: string) => void | Promise<void>,
|
||||
) {
|
||||
const [existing] = await db
|
||||
.select({ id: ItemsBase.id })
|
||||
.from(ItemsBase)
|
||||
@@ -22,7 +25,11 @@ export async function runSyncJobItem(item: ImportJobItem) {
|
||||
const source = await getSource(item.cloneSourceId);
|
||||
if (!source || !item.cloneEntry)
|
||||
throw Error("Synchronization source no longer available");
|
||||
const result = await cloneSingleFurni({ source, entry: item.cloneEntry });
|
||||
const result = await cloneSingleFurni({
|
||||
source,
|
||||
entry: item.cloneEntry,
|
||||
onProgress,
|
||||
});
|
||||
const [created] = result.ok
|
||||
? await db
|
||||
.select({ id: ItemsBase.id })
|
||||
@@ -32,5 +39,5 @@ export async function runSyncJobItem(item: ImportJobItem) {
|
||||
: [];
|
||||
return { ...result, itemId: created?.id };
|
||||
}
|
||||
return importSingleFurni({ ...item, repairExisting: false });
|
||||
return importSingleFurni({ ...item, repairExisting: false, onProgress });
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
import { beforeEach, expect, it, vi } from "vitest";
|
||||
|
||||
const mocks = vi.hoisted(() => ({
|
||||
sources: vi.fn(),
|
||||
validate: vi.fn((buffer: Buffer) => buffer),
|
||||
}));
|
||||
vi.mock("./clone-sources", () => ({ listSources: mocks.sources }));
|
||||
vi.mock("./furni-attachment", () => ({
|
||||
validateFurnitureAttachment: mocks.validate,
|
||||
}));
|
||||
|
||||
import { recoverFurnitureNitro } from "./furniture-source-recovery";
|
||||
|
||||
const item = { classname: "chair", type: "flooritem", revision: 42 };
|
||||
const source = (id: string) => ({
|
||||
id,
|
||||
name: id,
|
||||
furnidataUrl: `https://${id}.test/data`,
|
||||
nitroBaseUrl: `https://${id}.test/nitro`,
|
||||
});
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
mocks.validate.mockImplementation((buffer) => buffer);
|
||||
});
|
||||
it("skips the selected and disabled sources and rejects a different type or revision", async () => {
|
||||
mocks.sources.mockResolvedValue([
|
||||
source("selected"),
|
||||
{ ...source("disabled"), enabled: false },
|
||||
source("wrong"),
|
||||
source("matching"),
|
||||
]);
|
||||
const fetcher = vi.fn(async (url: string) =>
|
||||
url.endsWith("/data")
|
||||
? Response.json({
|
||||
roomitemtypes: {
|
||||
furnitype: [
|
||||
{ classname: "chair", revision: url.includes("wrong") ? 41 : 42 },
|
||||
],
|
||||
},
|
||||
})
|
||||
: new Response("bundle"),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetcher);
|
||||
const result = await recoverFurnitureNitro(item, {
|
||||
excludeSourceId: "selected",
|
||||
});
|
||||
expect(result?.sourceId).toBe("matching");
|
||||
expect(fetcher.mock.calls.map(([url]) => url)).toEqual([
|
||||
"https://wrong.test/data",
|
||||
"https://matching.test/data",
|
||||
"https://matching.test/nitro/chair.nitro",
|
||||
]);
|
||||
expect(mocks.validate).toHaveBeenCalledWith(Buffer.from("bundle"), "chair");
|
||||
});
|
||||
it("rejects bundles with the wrong identity without writing them", async () => {
|
||||
mocks.sources.mockResolvedValue([source("badbundle")]);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (url: string) =>
|
||||
url.endsWith("/data")
|
||||
? Response.json({
|
||||
roomitemtypes: {
|
||||
furnitype: [{ classname: "chair", revision: 42 }],
|
||||
},
|
||||
})
|
||||
: new Response("other furniture"),
|
||||
),
|
||||
);
|
||||
mocks.validate.mockImplementation(() => {
|
||||
throw Error("Wrong identity");
|
||||
});
|
||||
expect(await recoverFurnitureNitro(item)).toBeNull();
|
||||
});
|
||||
it("does not download a floor bundle for a wall item or a similar classname", async () => {
|
||||
mocks.sources.mockResolvedValue([source("floor")]);
|
||||
const fetcher = vi.fn(async () =>
|
||||
Response.json({
|
||||
roomitemtypes: { furnitype: [{ classname: "chair", revision: 42 }] },
|
||||
}),
|
||||
);
|
||||
vi.stubGlobal("fetch", fetcher);
|
||||
expect(await recoverFurnitureNitro({ ...item, type: "wallitem" })).toBeNull();
|
||||
expect(fetcher).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
it("rejects excessive content-length before buffering the bundle", async () => {
|
||||
mocks.sources.mockResolvedValue([source("oversize")]);
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async (url: string) =>
|
||||
url.endsWith("/data")
|
||||
? Response.json({
|
||||
roomitemtypes: {
|
||||
furnitype: [{ classname: "chair", revision: 42 }],
|
||||
},
|
||||
})
|
||||
: new Response("huge", { headers: { "content-length": "1000000000" } }),
|
||||
),
|
||||
);
|
||||
expect(await recoverFurnitureNitro(item)).toBeNull();
|
||||
expect(mocks.validate).not.toHaveBeenCalled();
|
||||
});
|
||||
@@ -0,0 +1,162 @@
|
||||
import { listSources } from "./clone-sources";
|
||||
import { validateFurnitureAttachment } from "./furni-attachment";
|
||||
import { browserHeaders } from "./import/core/browser-headers";
|
||||
|
||||
type Item = { classname: string; type: string; revision: number };
|
||||
export interface RecoveredFurnitureSource {
|
||||
buffer: Buffer;
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
revision: number;
|
||||
}
|
||||
type CatalogEntry = Pick<Item, "classname" | "revision">;
|
||||
type Catalog = {
|
||||
roomitemtypes?: { furnitype?: CatalogEntry[] };
|
||||
wallitemtypes?: { furnitype?: CatalogEntry[] };
|
||||
furnitype?: CatalogEntry[];
|
||||
};
|
||||
function compactCatalog(raw: Catalog): Catalog {
|
||||
const entries = (value: unknown): CatalogEntry[] => {
|
||||
if (!Array.isArray(value)) return [];
|
||||
return value.slice(0, 100_000).flatMap((entry) => {
|
||||
if (
|
||||
!entry ||
|
||||
typeof entry.classname !== "string" ||
|
||||
entry.classname.length > 256 ||
|
||||
!Number.isSafeInteger(Number(entry.revision))
|
||||
)
|
||||
return [];
|
||||
return [{ classname: entry.classname, revision: Number(entry.revision) }];
|
||||
});
|
||||
};
|
||||
|
||||
return {
|
||||
roomitemtypes: {
|
||||
furnitype: entries(raw?.roomitemtypes?.furnitype ?? raw?.furnitype),
|
||||
},
|
||||
wallitemtypes: { furnitype: entries(raw?.wallitemtypes?.furnitype) },
|
||||
};
|
||||
}
|
||||
const cache = new Map<string, { expires: number; catalog: Catalog | null }>();
|
||||
|
||||
async function boundedDownload(
|
||||
url: string,
|
||||
maxBytes: number,
|
||||
signal: AbortSignal,
|
||||
) {
|
||||
const parsed = new URL(url);
|
||||
if (!["http:", "https:"].includes(parsed.protocol))
|
||||
throw Error("Unsupported source protocol");
|
||||
const response = await fetch(url, { signal, headers: browserHeaders() });
|
||||
if (
|
||||
!response.ok ||
|
||||
Number(response.headers.get("content-length")) > maxBytes ||
|
||||
!response.body
|
||||
) {
|
||||
await response.body?.cancel();
|
||||
throw Error("Source unavailable or too large");
|
||||
}
|
||||
const reader = response.body.getReader();
|
||||
const chunks: Uint8Array[] = [];
|
||||
let size = 0;
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
size += value.byteLength;
|
||||
if (size > maxBytes) throw Error("Source exceeds size limit");
|
||||
chunks.push(value);
|
||||
}
|
||||
} finally {
|
||||
await reader.cancel().catch(() => undefined);
|
||||
reader.releaseLock();
|
||||
}
|
||||
return Buffer.concat(chunks, size);
|
||||
}
|
||||
|
||||
/** Only exact classname/type/revision matches; never borrow another furniture's metadata. */
|
||||
export async function recoverFurnitureNitro(
|
||||
item: Item,
|
||||
options: {
|
||||
excludeSourceId?: string;
|
||||
onAttempt?: (sourceName: string) => Promise<void>;
|
||||
} = {},
|
||||
): Promise<RecoveredFurnitureSource | null> {
|
||||
if (!Number.isSafeInteger(item.revision) || item.revision <= 0) return null;
|
||||
const deadline = Date.now() + 20_000;
|
||||
const sources = (await listSources())
|
||||
.filter(
|
||||
(source) =>
|
||||
source.enabled !== false &&
|
||||
source.id !== options.excludeSourceId &&
|
||||
source.nitroBaseUrl &&
|
||||
source.furnidataUrl,
|
||||
)
|
||||
.slice(0, 8);
|
||||
for (const source of sources) {
|
||||
if (Date.now() >= deadline) break;
|
||||
// Await outside the catch: a lost worker lease must stop the import.
|
||||
await options.onAttempt?.(source.name);
|
||||
try {
|
||||
const signal = AbortSignal.timeout(
|
||||
Math.max(1, Math.min(4000, deadline - Date.now())),
|
||||
);
|
||||
let catalog = cache.get(source.furnidataUrl);
|
||||
if (!catalog || catalog.expires <= Date.now()) {
|
||||
let data: Catalog | null = null;
|
||||
try {
|
||||
data = compactCatalog(
|
||||
JSON.parse(
|
||||
(
|
||||
await boundedDownload(
|
||||
source.furnidataUrl,
|
||||
20 * 1024 * 1024,
|
||||
signal,
|
||||
)
|
||||
).toString("utf8"),
|
||||
),
|
||||
);
|
||||
} catch {
|
||||
/* Try the next configured source. */
|
||||
}
|
||||
if (cache.size >= 20) cache.clear();
|
||||
catalog = {
|
||||
expires: Date.now() + (data ? 300_000 : 60_000),
|
||||
catalog: data,
|
||||
};
|
||||
cache.set(source.furnidataUrl, catalog);
|
||||
}
|
||||
const entries =
|
||||
item.type === "flooritem"
|
||||
? (catalog.catalog?.roomitemtypes?.furnitype ??
|
||||
catalog.catalog?.furnitype)
|
||||
: catalog.catalog?.wallitemtypes?.furnitype;
|
||||
if (
|
||||
!Array.isArray(entries) ||
|
||||
!entries.some(
|
||||
(entry) =>
|
||||
entry.classname === item.classname &&
|
||||
Number(entry.revision) === item.revision,
|
||||
)
|
||||
)
|
||||
continue;
|
||||
if (Date.now() >= deadline) break;
|
||||
const base = encodeURIComponent(item.classname.split("*")[0]);
|
||||
const buffer = await boundedDownload(
|
||||
`${source.nitroBaseUrl.replace(/\/+$/, "")}/${base}.nitro`,
|
||||
50 * 1024 * 1024,
|
||||
AbortSignal.timeout(Math.max(1, Math.min(4000, deadline - Date.now()))),
|
||||
);
|
||||
validateFurnitureAttachment(buffer, item.classname);
|
||||
return {
|
||||
buffer,
|
||||
sourceId: source.id,
|
||||
sourceName: source.name,
|
||||
revision: item.revision,
|
||||
};
|
||||
} catch {
|
||||
/* Invalid, blocked or missing bundles never become local assets. */
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -238,7 +238,10 @@ export function createNitroBundle(
|
||||
|
||||
// ── .nitro Bundle Parser ──────────────────────────────────────────
|
||||
|
||||
export function parseNitroBundle(buffer: Buffer): {
|
||||
export function parseNitroBundle(
|
||||
buffer: Buffer,
|
||||
options?: { maxOutputLength?: number },
|
||||
): {
|
||||
json: Record<string, unknown>;
|
||||
jsonFileName: string;
|
||||
png: Buffer;
|
||||
@@ -267,7 +270,7 @@ export function parseNitroBundle(buffer: Buffer): {
|
||||
offset += 4;
|
||||
const jsonCompressed = buffer.subarray(offset, offset + jsonCompressedLen);
|
||||
offset += jsonCompressedLen;
|
||||
const jsonStr = inflateJsonOrGzip(jsonCompressed).toString("utf-8");
|
||||
const jsonStr = inflateJsonOrGzip(jsonCompressed, options).toString("utf-8");
|
||||
const json = JSON.parse(jsonStr);
|
||||
|
||||
// File 2: PNG
|
||||
@@ -279,7 +282,7 @@ export function parseNitroBundle(buffer: Buffer): {
|
||||
offset += 4;
|
||||
const pngCompressed = buffer.subarray(offset, offset + pngCompressedLen);
|
||||
offset += pngCompressedLen;
|
||||
const png = inflateJsonOrGzip(pngCompressed);
|
||||
const png = inflateJsonOrGzip(pngCompressed, options);
|
||||
|
||||
return { json, jsonFileName, png, pngFileName };
|
||||
}
|
||||
@@ -289,10 +292,13 @@ export function parseNitroBundle(buffer: Buffer): {
|
||||
* gzip their contents instead of raw-deflate. Try inflate first, then fall
|
||||
* back to gunzip so both formats parse.
|
||||
*/
|
||||
function inflateJsonOrGzip(input: Buffer): Buffer {
|
||||
function inflateJsonOrGzip(
|
||||
input: Buffer,
|
||||
options?: { maxOutputLength?: number },
|
||||
): Buffer {
|
||||
try {
|
||||
return inflateSync(input);
|
||||
return inflateSync(input, options);
|
||||
} catch {
|
||||
return gunzipSync(input);
|
||||
return gunzipSync(input, options);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { expect, it } from "vitest";
|
||||
import { createNitroBundle, parseNitroBundle } from "./nitro-builder";
|
||||
|
||||
it("honors an explicit decompression limit for imported bundles", () => {
|
||||
const bundle = createNitroBundle(
|
||||
{ name: "chair", value: "a".repeat(4096) },
|
||||
Buffer.from("png"),
|
||||
"chair",
|
||||
);
|
||||
expect(() => parseNitroBundle(bundle, { maxOutputLength: 1024 })).toThrow();
|
||||
expect(parseNitroBundle(bundle).json.name).toBe("chair");
|
||||
});
|
||||
Reference in new issue
Block a user