feat(nitro): convert imported SWF bundles to WebP Lossless
Gitea Actions Runner Test / test-job (push) Successful in 1s
CI / check (push) Successful in 31s
CI / tests-unit (push) Successful in 1m59s
CI / tests-integration (push) Successful in 2m19s
CI / tests-ui (push) Successful in 2m56s
CI / preflight (push) Skipped
CI / deploy (push) Successful in 2m18s

Newly converted .nitro bundles now store their spritesheet as WebP VP8L
instead of PNG, so imports land much smaller without changing a single
pixel. The texture member and spritesheet.meta.image are both labelled
from the actual bytes, never from a caller's assumption.

- encode through sharp with lossless and exact, so colour hidden under
  alpha 0 survives; this mirrors ImageSharp's TransparentColorMode.Preserve
- detect PNG/WebP by magic bytes and reject anything the client cannot
  render, on create, download and upload paths
- keep the source format when deriving size-32 sheets, scaling composites
  and editing metadata, so existing bundles are never silently rewritten
- report fidelity in the studio: the compression panel re-encodes with the
  same options the importer uses, so it cannot drift and invent false
  warnings, and shows PNG/WebP size estimates

convertSwfToNitro and buildSpritesheet are now async, so the worker, the
main-thread fallback and every import call site await them. PNG stays
supported for existing bundles and icon sidecars are untouched.
This commit is contained in:
openhands committed 2026-09-27 15:42:21 +02:00
1 parent 420210ffa0
commit 17de94d984
22 files changed
+1118 -125

No files matched your search

@@ -199,7 +199,10 @@ export const POST = withAdmin(
});
try {
const swfBuffer = await fs.readFile(swfPath);
const conversion = convertSwfToNitro(swfBuffer, baseClassname);
const conversion = await convertSwfToNitro(
swfBuffer,
baseClassname,
);
await fs.writeFile(nitroPath, conversion.bundle);
// Build FurnitureData entry if missing
@@ -104,13 +104,16 @@ export const PUT = withAdmin(
const saved = await withFurniDataLock(async () => {
const existingBuffer = await fs.readFile(nitroPath);
const { png, json: currentMetadata } = parseNitroBundle(existingBuffer);
const { texture, json: currentMetadata } =
parseNitroBundle(existingBuffer);
if (
JSON.stringify(currentMetadata.catalogStudio) !==
JSON.stringify(metadata.catalogStudio)
)
return false;
const newBundle = createNitroBundle(metadata, png, base);
// `texture` keeps its original encoding, so editing metadata never
// changes the bundle's texture format.
const newBundle = createNitroBundle(metadata, texture, base);
await fs.writeFile(nitroPath, newBundle);
return true;
});
+1 -1
View File
@@ -775,7 +775,7 @@ export const PATCH = withAdmin(
// Convert SWF → Nitro + update FurnitureData.json
try {
const swfBuffer = await fs.readFile(swfPath);
const conversion = convertSwfToNitro(swfBuffer, baseClassname);
const conversion = await convertSwfToNitro(swfBuffer, baseClassname);
await fs.writeFile(nitroPath, conversion.bundle);
// Add to FurnitureData.json if not already present
@@ -4,6 +4,7 @@ import sharp from "sharp";
import { apiError, apiOk } from "@/lib/api";
import { withAdmin } from "@/lib/api-handler";
import {
buildNitroCompressionReport,
inspectNitroScales,
object,
resolveNitroFrame,
@@ -21,10 +22,11 @@ export const GET = withAdmin(
const base = String(classname).split("*")[0],
{ nitroDir } = await getFurniAssetDirs();
try {
const { json, png } = parseNitroBundle(
const parsed = parseNitroBundle(
await fs.readFile(path.join(nitroDir, `${base}.nitro`)),
);
const image = await sharp(png).metadata();
const { json, texture } = parsed;
const image = await sharp(texture).metadata();
if (!image.width || !image.height)
return apiError("Invalid bundle texture", 422);
const asset = request.nextUrl.searchParams.get("asset");
@@ -33,7 +35,7 @@ export const GET = withAdmin(
return apiError("Sprite not found", 404);
const resolved = resolveNitroFrame(json, asset),
rect = object(resolved.frame.frame);
let pipeline = sharp(png).extract({
let pipeline = sharp(texture).extract({
left: Number(rect.x),
top: Number(rect.y),
width: Number(rect.w),
@@ -51,7 +53,10 @@ export const GET = withAdmin(
});
}
return apiOk({
report: inspectNitroScales(json, image.width, image.height),
report: {
...inspectNitroScales(json, image.width, image.height),
compression: await buildNitroCompressionReport(parsed),
},
});
} catch (error) {
if ((error as NodeJS.ErrnoException).code === "ENOENT")
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+29 -13
View File
@@ -45,7 +45,7 @@ export async function generateScale32(input: Buffer): Promise<{
count: number;
skipped: number;
}> {
const { json, png } = parseNitroBundle(input);
const { json, texture, textureFormat } = parseNitroBundle(input);
const assets = object(json.assets),
sheet = object(json.spritesheet),
frames = object(sheet.frames);
@@ -67,7 +67,7 @@ export async function generateScale32(input: Buffer): Promise<{
if (!names.length)
throw Error("No supported scale 64 furniture sprites were found");
if (names.length > 10000) throw Error("Too many sprites to generate safely");
const dimensions = await sharp(png).metadata();
const dimensions = await sharp(texture).metadata();
if (!dimensions.width || !dimensions.height) throw Error("Invalid texture");
const images: {
id: number;
@@ -103,7 +103,7 @@ export async function generateScale32(input: Buffer): Promise<{
}
const frame = resolved.frame,
rect = object(frame.frame);
let sprite = await sharp(png)
let sprite = await sharp(texture)
.extract({
left: Number(rect.x),
top: Number(rect.y),
@@ -149,25 +149,35 @@ export async function generateScale32(input: Buffer): Promise<{
});
}
if (!images.length) throw Error("No independent texture frames were found");
const atlas = buildSpritesheet(images);
const atlas = await buildSpritesheet(images, { format: textureFormat });
const width = Math.max(dimensions.width, atlas.width),
height = dimensions.height + atlas.height;
if (width * height > 64_000_000)
throw Error("Combined texture exceeds the texture budget");
const combined = await sharp({
// The merged sheet must keep the source bundle's texture format so an
// existing PNG bundle is not silently converted (or vice versa).
const composited = sharp({
create: {
width,
height,
channels: 4,
background: { r: 0, g: 0, b: 0, alpha: 0 },
},
})
.composite([
{ input: png, left: 0, top: 0 },
{ input: atlas.png, left: 0, top: dimensions.height },
])
.png()
.toBuffer();
}).composite([
{ input: texture, left: 0, top: 0 },
{ input: atlas.texture, left: 0, top: dimensions.height },
]);
const combined =
textureFormat === "webp"
? await composited
.webp({
lossless: true,
nearLossless: false,
smartSubsample: false,
effort: 6,
})
.toBuffer()
: await composited.png().toBuffer();
const generated = new Map<string, boolean>();
for (const name of names)
if (!skipped.has(name)) generated.set(scaleName(name), true);
@@ -198,7 +208,13 @@ export async function generateScale32(input: Buffer): Promise<{
json.spritesheet = {
...sheet,
frames,
meta: { ...object(sheet.meta), size: { w: width, h: height } },
meta: {
...object(sheet.meta),
// Keep the client-facing image pointer aligned with the member
// `createNitroBundle` is about to write.
image: `${String(json.name || "furniture")}.${textureFormat}`,
size: { w: width, h: height },
},
};
json.catalogStudio = {
...object(json.catalogStudio),
+1 -1
View File
@@ -113,7 +113,7 @@ export async function importSingleEffect(params: {
onProgress?.("converting");
try {
const swfBuffer = await fs.readFile(/*turbopackIgnore: true*/ swfPath);
const result = convertSwfToNitro(swfBuffer, lib);
const result = await convertSwfToNitro(swfBuffer, lib);
await fs.writeFile(/*turbopackIgnore: true*/ nitroPath, result.bundle);
if (result.warnings?.length) warnings.push(...result.warnings);
} catch (err) {
+1 -1
View File
@@ -112,7 +112,7 @@ export async function importSingleFigure(params: {
onProgress?.("converting");
try {
const swfBuffer = await fs.readFile(/*turbopackIgnore: true*/ swfPath);
const result = convertSwfToNitro(swfBuffer, lib);
const result = await convertSwfToNitro(swfBuffer, lib);
await fs.writeFile(
/*turbopackIgnore: true*/ await nitroPathFor(lib),
result.bundle,
+1 -1
View File
@@ -30,5 +30,5 @@ it("rejects missing texture data", () => {
createNitroBundle({ name: "chair" }, Buffer.from("not png"), "chair"),
"chair",
),
).toThrow("not a PNG");
).toThrow("PNG or WebP");
});
+9 -7
View File
@@ -1,7 +1,10 @@
import { randomUUID } from "node:crypto";
import { promises as fs } from "node:fs";
import path from "node:path";
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
import {
detectNitroTextureFormat,
parseNitroBundle,
} from "@/lib/services/swf/nitro-builder";
import { importRoot, validJobId } from "./furni-job-store";
export function validateFurnitureAttachment(buffer: Buffer, classname: string) {
if (buffer.length > 50 * 1024 * 1024)
@@ -15,12 +18,11 @@ export function validateFurnitureAttachment(buffer: Buffer, classname: string) {
(parsed.json.name && parsed.json.name !== base)
)
throw Error(`This bundle does not belong to ${classname}`);
if (
!parsed.png
.subarray(0, 8)
.equals(Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]))
)
throw Error("The bundle texture is not a PNG");
// The renderer reads WebP Lossless and PNG; parseNitroBundle already
// rejects anything else, so this only names the format in the error.
const format = detectNitroTextureFormat(parsed.texture);
if (!format)
throw Error("The bundle texture is neither a PNG nor a WebP image");
return buffer;
}
export async function stageFurnitureAttachment(
+7 -2
View File
@@ -1,6 +1,9 @@
import { promises as fs } from "node:fs";
import { getByparrCookies } from "@/lib/services/byparr";
import { parseNitroBundle } from "../../swf/nitro-builder";
import {
detectNitroTextureFormat,
parseNitroBundle,
} from "../../swf/nitro-builder";
import { browserHeaders } from "./browser-headers";
import { curlDownload } from "./curl-fetch";
@@ -27,7 +30,9 @@ export function validateNitroBytes(buffer: Buffer): boolean {
!!bundle.json &&
typeof bundle.json === "object" &&
!Array.isArray(bundle.json) &&
validatePngBytes(bundle.png)
// parseNitroBundle already rejects non-PNG/WebP textures, so this
// is a cheap re-assertion rather than the real gate.
detectNitroTextureFormat(bundle.texture) !== null
);
} catch {
return false;
+1 -1
View File
@@ -78,7 +78,7 @@ export async function importSinglePet(params: {
onProgress?.("converting");
try {
const swfBuffer = await fs.readFile(/*turbopackIgnore: true*/ swfPath);
const result = convertSwfToNitro(swfBuffer, lib);
const result = await convertSwfToNitro(swfBuffer, lib);
await fs.writeFile(
/*turbopackIgnore: true*/ nitroPathFor(lib),
result.bundle,
+18 -5
View File
@@ -34,7 +34,8 @@ export function nitroHasSize32(json: Record<string, unknown>): boolean {
export async function addSize32ToBundle(
buffer: Buffer,
): Promise<Buffer | null> {
const { json, png, jsonFileName } = parseNitroBundle(buffer);
const { json, texture, textureFormat, jsonFileName } =
parseNitroBundle(buffer);
if (nitroHasSize32(json)) return null;
const name = (json.name as string) || jsonFileName.replace(/\.json$/, "");
@@ -68,7 +69,7 @@ export async function addSize32ToBundle(
for (const [frameName, frame] of Object.entries(frames)) {
const f = frame.frame;
if (!f || f.w <= 0 || f.h <= 0) continue;
const { data, info } = await sharp(png)
const { data, info } = await sharp(texture)
.extract({ left: f.x, top: f.y, width: f.w, height: f.h })
.raw()
.ensureAlpha()
@@ -95,7 +96,7 @@ export async function addSize32ToBundle(
const f = frame.frame;
if (f.w <= 0 || f.h <= 0) continue;
const src = await sharp(png)
const src = await sharp(texture)
.extract({ left: f.x, top: f.y, width: f.w, height: f.h })
.raw()
.ensureAlpha()
@@ -131,7 +132,12 @@ export async function addSize32ToBundle(
if (Object.keys(newAssets).length === 0) return null;
const { png: newPng, frames: newFrames } = buildSpritesheet(images);
// Preserve the source bundle's texture format: regenerating a scale must
// not silently downgrade a WebP bundle back to PNG.
const { texture: newTexture, frames: newFrames } = await buildSpritesheet(
images,
{ format: textureFormat },
);
// Update existing frame coordinates (sheet was rebuilt).
for (const [frameName, frame] of Object.entries(frames)) {
@@ -171,5 +177,12 @@ export async function addSize32ToBundle(
vis.push({ ...sourceVis, size: 32 });
}
return createNitroBundle(json, newPng, name);
// Keep the client-facing image pointer aligned with the member
// `createNitroBundle` is about to write.
const sheet = (json.spritesheet ?? {}) as Record<string, unknown>;
const meta = (sheet.meta ?? {}) as Record<string, unknown>;
sheet.meta = { ...meta, image: `${name}.${textureFormat}` };
json.spritesheet = sheet;
return createNitroBundle(json, newTexture, name);
}
@@ -32,7 +32,19 @@ describe("conversion worker recovery", () => {
const data = Buffer.from("SWF");
const result = await convertSwfToNitroAsync(data, "chair");
expect(result.bundle.toString()).toBe("converted");
expect(mocks.convert).toHaveBeenCalledWith(data, "chair");
expect(mocks.convert).toHaveBeenCalledWith(data, "chair", {
textureFormat: undefined,
});
});
it("passes the requested texture format through to the main-thread fallback", async () => {
vi.stubEnv("NODE_ENV", "production");
vi.stubEnv("VITEST", undefined);
const { convertSwfToNitroAsync } = await import("./conversion-pool");
const data = Buffer.from("SWF");
await convertSwfToNitroAsync(data, "chair", { textureFormat: "webp" });
expect(mocks.convert).toHaveBeenCalledWith(data, "chair", {
textureFormat: "webp",
});
});
it("recovers icon extraction from a worker startup error", async () => {
vi.stubEnv("NODE_ENV", "production");
+37 -19
View File
@@ -6,6 +6,7 @@ import {
convertSwfToNitro,
extractIconFromSwf,
} from "./index";
import type { NitroTextureFormat } from "./nitro-builder";
// Offloads the synchronous, CPU-heavy SWF→Nitro conversion (and icon
// extraction) to a small pool of worker threads so the main Node event loop
@@ -89,28 +90,39 @@ function pump() {
const CONVERSION_TIMEOUT_MS = 60_000;
function run(req: Omit<WorkerRequest, "id">): Promise<WorkerResponse> {
// `Omit` does not distribute over a union, which would erase `textureFormat`
// from the convert variant.
type DistributiveOmit<T, K extends keyof never> = T extends unknown
? Omit<T, K>
: never;
function run(
req: DistributiveOmit<WorkerRequest, "id">,
): Promise<WorkerResponse> {
if (!ensurePool()) {
try {
if (req.type === "convert") {
// No pool available: run on the main thread. Icon extraction stays
// synchronous; the conversion is async because it encodes a spritesheet.
if (req.type === "icon") {
try {
return Promise.resolve({
id: 0,
ok: true,
result: convertSwfToNitro(req.buffer, req.classname),
icon: extractIconFromSwf(req.buffer, req.classname),
});
} catch (e) {
return Promise.resolve({
id: 0,
ok: false,
error: (e as Error).message,
});
}
return Promise.resolve({
id: 0,
ok: true,
icon: extractIconFromSwf(req.buffer, req.classname),
});
} catch (e) {
return Promise.resolve({
id: 0,
ok: false,
error: (e as Error).message,
});
}
return convertSwfToNitro(req.buffer, req.classname, {
textureFormat: req.textureFormat,
}).then(
(result) => ({ id: 0, ok: true, result }) as WorkerResponse,
(e: Error) => ({ id: 0, ok: false, error: e.message }) as WorkerResponse,
);
}
return new Promise<WorkerResponse>((resolve, reject) => {
const job: Job = {
@@ -153,13 +165,19 @@ function run(req: Omit<WorkerRequest, "id">): Promise<WorkerResponse> {
export async function convertSwfToNitroAsync(
buffer: Buffer,
classname: string,
options?: { textureFormat?: NitroTextureFormat },
): Promise<ConversionResult> {
const res = await run({ type: "convert", buffer, classname }).catch(
() => null,
);
const res = await run({
type: "convert",
buffer,
classname,
textureFormat: options?.textureFormat,
}).catch(() => null);
if (res?.ok && res.result) return res.result;
// Worker failed for this item — fall back to the main thread.
return convertSwfToNitro(buffer, classname);
return convertSwfToNitro(buffer, classname, {
textureFormat: options?.textureFormat,
});
}
export async function extractIconFromSwfAsync(
+31 -15
View File
@@ -1,11 +1,14 @@
import { parentPort } from "node:worker_threads";
import { convertSwfToNitro, extractIconFromSwf } from "./index";
import type { NitroTextureFormat } from "./nitro-builder";
export interface ConvertRequest {
id: number;
type: "convert";
buffer: Buffer;
classname: string;
/** Omitted means the default (WebP Lossless). */
textureFormat?: NitroTextureFormat;
}
export interface IconRequest {
@@ -28,28 +31,41 @@ export type WorkerResponse =
if (parentPort) {
parentPort.on("message", (msg: WorkerRequest) => {
try {
if (msg.type === "convert") {
const result = convertSwfToNitro(msg.buffer, msg.classname);
parentPort?.postMessage({
id: msg.id,
ok: true,
result,
} satisfies WorkerResponse);
} else {
if (msg.type === "icon") {
try {
const icon = extractIconFromSwf(msg.buffer, msg.classname);
parentPort?.postMessage({
id: msg.id,
ok: true,
icon: icon ?? null,
} satisfies WorkerResponse);
} catch (err) {
parentPort?.postMessage({
id: msg.id,
ok: false,
error: (err as Error).message,
} satisfies WorkerResponse);
}
} catch (err) {
parentPort?.postMessage({
id: msg.id,
ok: false,
error: (err as Error).message,
} satisfies WorkerResponse);
return;
}
// The conversion encodes a spritesheet, which is async (sharp), so it
// cannot share the synchronous try/catch above.
void convertSwfToNitro(msg.buffer, msg.classname, {
textureFormat: msg.textureFormat,
})
.then((result) => {
parentPort?.postMessage({
id: msg.id,
ok: true,
result,
} satisfies WorkerResponse);
})
.catch((err: Error) => {
parentPort?.postMessage({
id: msg.id,
ok: false,
error: err.message,
} satisfies WorkerResponse);
});
});
}
+23 -7
View File
@@ -17,6 +17,7 @@ import {
buildSpritesheet,
createNitroBundle,
encodePng,
type NitroTextureFormat,
} from "./nitro-builder";
import {
decompressSwf,
@@ -56,15 +57,25 @@ export {
extractIconFromSwfAsync,
} from "./conversion-pool";
// Re-export public API from submodules
export { createNitroBundle, parseNitroBundle } from "./nitro-builder";
export {
buildSpritesheet,
createNitroBundle,
detectNitroTextureFormat,
encodePng,
encodeWebpLossless,
NITRO_TEXTURE_LABEL,
type NitroTextureFormat,
parseNitroBundle,
} from "./nitro-builder";
export { computeDefaultDir, extractPartColors } from "./xml-processor";
// ── Main Conversion ────────────────────────────────────────────────
export function convertSwfToNitro(
export async function convertSwfToNitro(
swfBuffer: Buffer,
classname: string,
): ConversionResult {
options?: { textureFormat?: NitroTextureFormat },
): Promise<ConversionResult> {
const warnings: string[] = [];
// 1. Decompress SWF
@@ -241,11 +252,14 @@ export function convertSwfToNitro(
// 7. Build spritesheet using bin-packing
const {
png,
texture,
format: sheetFormat,
frames,
width: sheetWidth,
height: sheetHeight,
} = buildSpritesheet(namedImages);
} = await buildSpritesheet(namedImages, {
format: options?.textureFormat ?? "webp",
});
// 8. Process XML into Nitro JSON structures
const { palettes } = processAssetsXml(assetsXml);
@@ -392,7 +406,9 @@ export function convertSwfToNitro(
logicType,
spritesheet: {
meta: {
image: `${classname}.png`,
// Must match the texture member `createNitroBundle` writes, or
// the client resolves a file that is not in the bundle.
image: `${classname}.${sheetFormat}`,
format: "RGBA8888",
size: { w: sheetWidth, h: sheetHeight },
scale: 1,
@@ -410,7 +426,7 @@ export function convertSwfToNitro(
nitroJson.logic = logicObj;
// 10. Create .nitro bundle
const bundle = createNitroBundle(nitroJson, png, classname);
const bundle = createNitroBundle(nitroJson, texture, classname);
return {
bundle,
File diff suppressed because it is too large. Load diff
@@ -4,9 +4,15 @@ import { createNitroBundle, parseNitroBundle } from "./nitro-builder";
it("honors an explicit decompression limit for imported bundles", () => {
const bundle = createNitroBundle(
{ name: "chair", value: "a".repeat(4096) },
Buffer.from("png"),
Buffer.from([137, 80, 78, 71, 13, 10, 26, 10]),
"chair",
);
expect(() => parseNitroBundle(bundle, { maxOutputLength: 1024 })).toThrow();
expect(parseNitroBundle(bundle).json.name).toBe("chair");
});
it("refuses to write a bundle whose texture is neither PNG nor WebP", () => {
expect(() =>
createNitroBundle({ name: "chair" }, Buffer.from("not an image"), "chair"),
).toThrow(/PNG or WebP/);
});
File diff suppressed because it is too large. Load diff