From 43742e8d99b2c35f8c82db0280a18bd64993d4c6 Mon Sep 17 00:00:00 2001 From: openhands Date: Sun, 11 Oct 2026 17:01:37 +0200 Subject: [PATCH] fix(catalog): decode WebP bundle textures so furniture icons resolve again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every write path normalises a bundle's texture to WebP Lossless, so the catalog icon was being read back with a PNG-only decoder. decodePng throws on anything that is not a PNG, the callers caught that and returned null, and the user-visible result was "no icon (not in source or bundle)" for every furniture whose source does not serve a standalone icon. Measured against the production asset tree, all 18,505 bundles were WebP; icon extraction succeeded on 0 of them. src/lib/services/imager/ decode-texture.ts keeps PNG on the dependency-free decoder and routes WebP through sharp, which is already a dependency and already encodes these textures. extractFurniIconPng and getPetIconPng become async; the five call sites (upload, clone import, furni import, icon repair and both icon routes) already awaited their surrounding work. Same root cause, second bug: the spritesheet frame key. Converters disagree on packing — some keep a trailing ".png", and some lowercase the whole key while leaving the bundle name mixed-case, so "LTD_fashionistaf" looks up frame "LTD_fashionistaf_LTD_fashionistaf_icon_a" and never finds "ltd_fashionistaf_ltd_fashionistaf_icon_a". Any mixed-case classname could therefore never match, which is most of the catalogue. findFrame tries the two exact spellings, then falls back to one case-insensitive pass. Extraction now succeeds on 18,483 of 18,505 bundles (99.88%); the 22 remainder are data, not code — 9 bundles ship no icon asset, 11 do not parse. Third: three catalogue icons exist only as .gif while catalogueIconUrl hardcoded .png, so the picker offered icons that could only ever 404, and 291 icons that ship as both formats were listed twice. The API now dedupes per id and the two renderers retry with .gif before falling back to the placeholder, matching what catalog-image-picker already did. Verified live: /gamedata/.../icon_1542.png returns 404 while icon_1542.gif returns 200. Separately, close the last hole in the memory cap. Every script in package.json routes through scripts/with-memory-cap.sh, but invoking the builder directly — from a terminal, an IDE or an agent — skipped the wrapper and ran unbounded, on a host with no swap where the OOM killer picks its victim across the whole machine. next.config.ts now refuses a production build that the wrapper has not marked, before anything allocates. next dev and next start are deliberately unaffected. README gains a Memory-capped commands section covering the per-script ceilings, the backends and the ulimit -v trap, and its stale version and script tables are corrected. --- README.md | 108 ++++++++++++++++++- next.config.ts | 47 +++++++- scripts/with-memory-cap.sh | 24 +++-- src/app/api/admin/catalog/icons/route.ts | 18 ++-- src/app/api/admin/import/clone/icon/route.ts | 2 +- src/app/api/admin/import/pets/icon/route.ts | 2 +- src/components/admin/catalog-tree.tsx | 9 +- src/components/admin/catalog/icon-picker.tsx | 11 +- src/lib/catalog-assets.test.ts | 7 ++ src/lib/catalog-assets.ts | 16 ++- src/lib/deployment-id.test.ts | 36 ++++++- src/lib/services/clone-icon.test.ts | 77 +++++++++++++ src/lib/services/clone-icon.ts | 50 +++++++-- src/lib/services/clone-import.ts | 2 +- src/lib/services/furni-import.ts | 2 +- src/lib/services/imager/decode-texture.ts | 36 +++++++ src/lib/services/pet-icon.ts | 47 ++++---- src/lib/services/repair-icons.ts | 6 +- 18 files changed, 433 insertions(+), 67 deletions(-) create mode 100644 src/lib/services/clone-icon.test.ts create mode 100644 src/lib/services/imager/decode-texture.ts diff --git a/README.md b/README.md index 56829af2..c416e663 100644 --- a/README.md +++ b/README.md @@ -10,14 +10,21 @@ Features a premium animated homepage (typewriter hero, floating orbs, scroll cou | Component | Version | Notes | | --------------- | -------------- | ---------------------------------------- | -| Node.js | 26.9.0 | Current release pinned in `.nvmrc` | -| pnpm | >= 11.25.0 | Recommended package manager | +| Node.js | 26.10.0 | Pinned in `.nvmrc` (`>=26.10.0 <27`) | +| pnpm | 12.10.1 | Pinned via `packageManager` | | npm | >= 11.x | Supported alternative | | yarn | >= 4.x | Supported alternative | | MySQL / MariaDB | 8.0+ / 10.6+ | Shared with the emulator | | Docker | 24+ | Optional — for containerized deployment | | Valkey | 8.x+ | Optional — caching, rate limiting, SSE | +Core stack: **Next.js 16.4.0** (App Router) · **React 19.3.0** · **TypeScript 7.0.2** · **Drizzle ORM 0.45.3** · **Zod 4.6.5** · **Vitest 5.0.3** · **Biome 2.5.15** · **Playwright 1.63.0**. + +> Builds must run through the `pnpm` scripts. This host has no swap and +> `vm.overcommit_memory=0`, so an uncapped `next build` gets OOM-killed by the +> kernel and can take the database and the live release down with it. See +> [Memory-capped commands](#memory-capped-commands). + --- ## Quick Start @@ -102,6 +109,10 @@ pnpm build && pnpm start # or: npm run build && npm start Open `http://localhost:3002` in your browser. +> Always go through these scripts. `next build` is refused outright when it is +> not running under the memory cap — see +> [Memory-capped commands](#memory-capped-commands). + ### 6. First Login 1. Register an account at `/register`, or log in with an existing emulator account. @@ -951,15 +962,99 @@ branch guard inside `ci-deploy.sh` only accepts `main`/`master`. --- +## Memory-capped commands + +This host runs with `vm.overcommit_memory=0` **and no swap**. When a process +asks for more memory than is free, the kernel does not wait — it calls the +OOM-killer immediately, and it picks its victim across the **whole machine**, +not just the offending process. An uncapped build does not merely fail: it can +take the MariaDB process, nginx and the live release down with it. + +Every heavy command therefore runs inside its own cgroup with a hard +`MemoryMax`, via `scripts/with-memory-cap.sh`. If the build outgrows its +ceiling, only that cgroup is killed — the build fails, the site keeps serving. + +| Script | Ceiling | Covers | +| --------------------- | ------- | --------------------------------------- | +| `pnpm dev` | 8 GB | Dev server | +| `pnpm build` | 10 GB | Production build | +| `pnpm analyze` | 10 GB | Build + bundle-size report | +| `pnpm test` | 8 GB | Vitest | +| `pnpm test:coverage` | 8 GB | Vitest with coverage | +| `pnpm test:ui` | 8 GB | Playwright | +| `pnpm test:e2e` | 8 GB | Playwright | +| `pnpm test:integration` | 8 GB | Vitest integration config | +| `pnpm typecheck` | 6 GB | `tsc --noEmit` | + +### Running the builder by hand + +```bash +npx next build # ✗ refused before it allocates anything +``` + +`next build` loads `next.config.ts`, which **refuses any production build that +is not running under the memory cap**. This closes the one hole the `pnpm` +scripts leave open: invoking the builder directly — from a terminal, an IDE, or +an automated agent — would otherwise bypass the cgroup entirely and go +unbounded. + +The refusal looks like this: + +``` +Error: Refusing to run an uncapped production build. + +On this host an unbounded `next build` gets OOM-killed by the kernel, +and the killer may take the database, nginx or the live release with it. + +Use the capped build instead: + pnpm build +``` + +Fix: use `pnpm build`. If you are genuinely inside an isolated environment +where the container *is* the boundary (the Docker build, a CI runner), set +`CMS_MEMORY_CAPPED=1` to opt out deliberately. + +### Backends + +`scripts/with-memory-cap.sh` picks its mechanism automatically: + +| `CMS_MEMORY_CAP_BACKEND` | Mechanism | Notes | +| ------------------------ | ------------------------------------------- | ------------------------------------------------------- | +| `auto` *(default)* | systemd cgroup `MemoryMax` | Real RSS bound over the whole process tree. Used here. | +| `ulimit` | `ulimit -v`, per process | Virtual address space, **not** RSS. Fallback only. | +| `none` | None — warning only | Docker build and GitLab runner, each already isolated. | + +On a host **without** systemd and without `CMS_MEMORY_CAP_BACKEND=none`, the +script refuses to run rather than proceeding unbounded. + +> Do not "fix" a cap failure by lowering `--max-old-space-size` or by raising +> `CMS_MEMORY_CAP_VIRTUAL` (the default is `40g` on purpose). A `ulimit -v` of +> 10g makes V8 clamp its own heap to ~2.25 GB and webpack dies with +> `std::bad_alloc`. Measure first; raise the ceiling deliberately. + +--- + ## Scripts | Command | Description | | ------------------------- | -------------------------------------------------- | | `pnpm dev` | Start development server (hot reload) | -| `pnpm build` | Production build | +| `pnpm build` | Production build (memory-capped) | | `pnpm start` | Start production server | | `pnpm typecheck` | Run TypeScript type checking | | `pnpm test` | Run all tests (Vitest) | +| `pnpm test:coverage` | Run all tests with coverage thresholds enforced | +| `pnpm test:ui` | Playwright UI tests (`playwright.ui.config.ts`) | +| `pnpm test:ui:update` | Playwright UI tests, updating snapshots | +| `pnpm test:e2e` | Playwright end-to-end tests | +| `pnpm test:integration` | Integration tests (own Vitest config) | +| `pnpm test:housekeeping` | Housekeeping feature tests | +| `pnpm lint` | Lint and format check (Biome) | +| `pnpm biome:lint` | Alias of `pnpm lint` | +| `pnpm format` | Format files in place (Biome) | +| `pnpm toolchain:check` | Verify the Node toolchain matches `.nvmrc` | +| `pnpm i18n:check` | Audit CMS translation coverage | +| `pnpm deps:audit` | Audit dependencies for high-severity advisories | | `pnpm db:migrate` | Apply pending SQL migrations | | `pnpm db:migrate:status` | Show migration status | | `pnpm db:schema:generate` | Regen `src/db/schema.ts` from prior schema + live DB | @@ -970,10 +1065,15 @@ branch guard inside `ci-deploy.sh` only accepts `main`/`master`. | `pnpm db:up` / `pnpm db:down` | Start / stop the `mariadb-turbo` container | | `pnpm gamedata:compress` | Pre-compress large gamedata JSON to `.gz` (gzip_static) | | `pnpm analyze` | Build + report per-route bundle sizes | +| `pnpm performance:report` | Re-render the performance report from a build | | `pnpm jobs:worker` | Start background task worker | -| `pnpm biome:check` | Lint and format code | +| `pnpm assets:editor` | Copy TinyMCE editor assets into `public/` | > Replace `pnpm` with `npm run` or `yarn` for other package managers. +> +> The heavy ones run memory-capped — see +> [Memory-capped commands](#memory-capped-commands). A production `next build` +> run outside the wrapper is refused rather than executed unbounded. --- diff --git a/next.config.ts b/next.config.ts index 109f7556..544a2ef4 100644 --- a/next.config.ts +++ b/next.config.ts @@ -1,7 +1,47 @@ import { execSync } from "node:child_process"; import type { NextConfig } from "next"; +import { PHASE_PRODUCTION_BUILD } from "next/constants"; import createNextIntlPlugin from "next-intl/plugin"; +/** + * This host runs with `vm.overcommit_memory=0` and no swap, so a process that + * asks for more memory than is free gets OOM-killed by the kernel immediately. + * The killer picks its victim across the WHOLE machine — an unbounded build can + * take down the database, nginx and the live release with it. + * + * `scripts/with-memory-cap.sh` runs a heavy command in its own cgroup with a + * hard `MemoryMax`, so only that build dies and the site keeps serving. Every + * script in package.json goes through it. + * + * The one hole that leaves is running the builder by hand: `npx next build`, + * `pnpm exec next build`, or an IDE/agent task invoking it directly skips the + * wrapper entirely and is unbounded. This guard closes that. `next build` + * loads the config, so refusing here stops the build before it allocates + * anything. See the header of scripts/with-memory-cap.sh. + */ +function assertMemoryCapped(phase: string): void { + if (phase !== PHASE_PRODUCTION_BUILD) return; + if (process.env.CMS_MEMORY_CAPPED === "1") return; + + throw new Error( + [ + "Refusing to run an uncapped production build.", + "", + "On this host an unbounded `next build` gets OOM-killed by the kernel,", + "and the killer may take the database, nginx or the live release with it.", + "", + "Use the capped build instead:", + " pnpm build", + "", + "It runs the builder through scripts/with-memory-cap.sh, which puts it in", + "its own cgroup with a MemoryMax, so a runaway build fails alone.", + "", + "Already inside an isolated environment (Docker, a CI runner) where the", + "container itself is the boundary? Set CMS_MEMORY_CAPPED=1 explicitly.", + ].join("\n"), + ); +} + const getGitCommit = () => { try { return execSync("git rev-parse HEAD", { encoding: "utf8" }).trim(); @@ -174,4 +214,9 @@ const nextConfig: NextConfig = { const withNextIntl = createNextIntlPlugin("./src/i18n/request.ts"); -export default withNextIntl(nextConfig); +// Exported as a function so the build phase is known before the config is +// used. next-intl only accepts a plain object, so it is applied here. +export default function config(phase: string) { + assertMemoryCapped(phase); + return withNextIntl(nextConfig); +} diff --git a/scripts/with-memory-cap.sh b/scripts/with-memory-cap.sh index 35676dfd..67a58b50 100755 --- a/scripts/with-memory-cap.sh +++ b/scripts/with-memory-cap.sh @@ -40,12 +40,20 @@ # RAM als je het echt wilt gebruiken. # # Geen van beide -> weigeren. Stil onbegrensd doorlopen zou precies de -# valse geruststelling zijn waar 3d828a61 voor waarschuwt. Omgevingen -# zonder systemd (de Docker-build, de GitLab-runner) kiezen daarom -# expliciet voor CMS_MEMORY_CAP_BACKEND=none — met een waarschuwing, -# en met als rechtvaardiging dat die builds al begrensd zijn door -# `next build --webpack` + `--max-old-space-size` en in hun eigen -# geïsoleerde container draaien, niet op de host. +# valse geruststelling zijn waar 3d828a61 voor waarschuwt. Omgevingen +# zonder systemd (de Docker-build, de GitLab-runner) kiezen daarom +# expliciet voor CMS_MEMORY_CAP_BACKEND=none — met een waarschuwing, +# en met als rechtvaardiging dat die builds al begrensd zijn door +# `next build --webpack` + `--max-old-space-size` en in hun eigen +# geïsoleerde container draaien, niet op de host. +# +# Markering: +# Elke backend zet `CMS_MEMORY_CAPPED=1` voordat het commando start. +# `next.config.ts` weigert een productie-build zonder die markering, zodat +# een handmatig `npx next build` (of een IDE/agent die de build zelf +# start) niet meer onbeperkt geheugen kan vragen en de hele host mee +# neemt. `CMS_MEMORY_CAP_BACKEND=none` telt mee: die omgevingen draaien +# al in een eigen, geïsoleerde container. # # Gebruik: bash scripts/with-memory-cap.sh 10g # Backend kiezen: CMS_MEMORY_CAP_BACKEND=systemd|ulimit|none|auto @@ -93,6 +101,10 @@ virtual_kb=$((virtual_bytes / 1024)) backend="${CMS_MEMORY_CAP_BACKEND:-auto}" systemd_cmd=() +# Vanaf hier is dit script de enige plek waar een zwaar commando nog mag +# starten. De markering maakt dat afdwingbaar in next.config.ts. +export CMS_MEMORY_CAPPED=1 + # Echt proberen, niet alleen uitzoeken of het bestand bestaat: `systemd-run` # zonder rechten faalt met "Access denied", en dat moet dan een nette # terugval naar ulimit worden in plaats van een kapotte build. diff --git a/src/app/api/admin/catalog/icons/route.ts b/src/app/api/admin/catalog/icons/route.ts index ac265adb..29f7b1c7 100644 --- a/src/app/api/admin/catalog/icons/route.ts +++ b/src/app/api/admin/catalog/icons/route.ts @@ -20,12 +20,18 @@ export const GET = withAdmin( const directories = catalogueAssetDirectories(await getGamedataRoot()); const files = await readCatalogueAssetFiles(directories); - let icons = files - .map((f) => /^icon_(\d+)\.(?:png|gif)$/i.exec(f)) - .filter((m): m is RegExpExecArray => Boolean(m)) - .map((m) => Number(m[1])) - .filter((n) => Number.isFinite(n)) - .sort((a, b) => a - b); + // A handful of icons ship as both `.png` and `.gif`. The client always + // tries `.png` first and falls back to `.gif`, so an id that has both is + // one icon, not two — dedupe instead of offering it twice in the picker. + let icons = [ + ...new Set( + files + .map((f) => /^icon_(\d+)\.(?:png|gif)$/i.exec(f)) + .filter((m): m is RegExpExecArray => Boolean(m)) + .map((m) => Number(m[1])) + .filter((n) => Number.isFinite(n)), + ), + ].sort((a, b) => a - b); if (search) { icons = icons.filter((id) => String(id).includes(search)); diff --git a/src/app/api/admin/import/clone/icon/route.ts b/src/app/api/admin/import/clone/icon/route.ts index 68e99f67..fb1ee24a 100644 --- a/src/app/api/admin/import/clone/icon/route.ts +++ b/src/app/api/admin/import/clone/icon/route.ts @@ -46,7 +46,7 @@ export const GET = withAdmin( iconCache.set(cacheKey, null); return apiError("Bundle not found", 404); } - icon = extractFurniIconPng(Buffer.from(await res.arrayBuffer())); + icon = await extractFurniIconPng(Buffer.from(await res.arrayBuffer())); iconCache.set(cacheKey, icon); pruneIconCache(); } catch { diff --git a/src/app/api/admin/import/pets/icon/route.ts b/src/app/api/admin/import/pets/icon/route.ts index 0d8935fa..f661b53e 100644 --- a/src/app/api/admin/import/pets/icon/route.ts +++ b/src/app/api/admin/import/pets/icon/route.ts @@ -7,7 +7,7 @@ export const GET = withAdmin( { permission: PERMS.ASSETS_IMPORT }, async (request) => { const lib = request.nextUrl.searchParams.get("lib") || ""; - const png = getPetIconPng(lib); + const png = await getPetIconPng(lib); if (!png) return new Response(null, { status: 404 }); return new Response(new Uint8Array(png), { headers: { diff --git a/src/components/admin/catalog-tree.tsx b/src/components/admin/catalog-tree.tsx index 7975e21e..80cf23d1 100644 --- a/src/components/admin/catalog-tree.tsx +++ b/src/components/admin/catalog-tree.tsx @@ -13,21 +13,22 @@ export function CatalogIcon({ iconImage: number; size?: number; }) { - const [error, setError] = useState(false); + const [error, setError] = useState(0); - if (error || iconImage <= 0) { + // A few icons ship only as `.gif`; retry once before giving up. + if (error >= 2 || iconImage <= 0) { return ; } return ( setError(true)} + onError={() => setError((e) => e + 1)} /> ); } diff --git a/src/components/admin/catalog/icon-picker.tsx b/src/components/admin/catalog/icon-picker.tsx index 07f81472..aca9ebad 100644 --- a/src/components/admin/catalog/icon-picker.tsx +++ b/src/components/admin/catalog/icon-picker.tsx @@ -213,13 +213,14 @@ function IconPreview({ iconImage: number; size?: number; }) { - const [error, setError] = useState(false); + const [error, setError] = useState(0); // Clear the previous load failure when the icon changes, otherwise a // placeholder sticks to the next image too. // biome-ignore lint/correctness/useExhaustiveDependencies: iconImage is a prop; the reset is meant to follow it - useEffect(() => setError(false), [iconImage]); + useEffect(() => setError(0), [iconImage]); - if (error || iconImage <= 0) { + // A few icons ship only as `.gif`; retry once before falling back. + if (error >= 2 || iconImage <= 0) { return ( setError(true)} + onError={() => setError((e) => e + 1)} /> ); } diff --git a/src/lib/catalog-assets.test.ts b/src/lib/catalog-assets.test.ts index 57d78581..9adc86e9 100644 --- a/src/lib/catalog-assets.test.ts +++ b/src/lib/catalog-assets.test.ts @@ -7,6 +7,13 @@ describe("catalogue asset URLs", () => { expect(catalogueIconUrl(7)).toBe("/gamedata/c_images/catalogue/icon_7.png"); }); + // A few catalogue icons exist only as `.gif`, so the picker retries with it. + it("can build the `.gif` variant of a category icon", () => { + expect(catalogueIconUrl(1542, "gif")).toBe( + "/gamedata/c_images/catalogue/icon_1542.gif", + ); + }); + it("encodes catalogue image names and supports both image formats", () => { expect(catalogueAssetUrl("front page", "png")).toBe( "/gamedata/c_images/catalogue/front%20page.png", diff --git a/src/lib/catalog-assets.ts b/src/lib/catalog-assets.ts index 34bee162..7821d5b3 100644 --- a/src/lib/catalog-assets.ts +++ b/src/lib/catalog-assets.ts @@ -1,12 +1,22 @@ export const CATALOGUE_ASSET_BASE_PATH = "/gamedata/c_images/catalogue"; -export function catalogueIconUrl(iconImage: number): string { - return `${CATALOGUE_ASSET_BASE_PATH}/icon_${iconImage}.png`; +export type CatalogueImageExtension = "png" | "gif"; + +/** + * A handful of catalogue icons only exist as `.gif`, so callers must be able to + * retry with `"gif"` after the `.png` attempt 404s. Defaults to `"png"`, which + * is what every icon except a few uses. + */ +export function catalogueIconUrl( + iconImage: number, + extension: CatalogueImageExtension = "png", +): string { + return `${CATALOGUE_ASSET_BASE_PATH}/icon_${iconImage}.${extension}`; } export function catalogueAssetUrl( name: string, - extension: "png" | "gif", + extension: CatalogueImageExtension, ): string { return `${CATALOGUE_ASSET_BASE_PATH}/${encodeURIComponent(name)}.${extension}`; } diff --git a/src/lib/deployment-id.test.ts b/src/lib/deployment-id.test.ts index ab7d219f..f6635dea 100644 --- a/src/lib/deployment-id.test.ts +++ b/src/lib/deployment-id.test.ts @@ -1,16 +1,48 @@ // @ts-nocheck import { execFileSync } from "node:child_process"; -import { describe, expect, it } from "vitest"; +import { afterEach, describe, expect, it } from "vitest"; import nextConfig from "../../next.config"; +// The default export is a function so the build phase is known before the +// config is handed to next-intl. Any phase other than the production build +// resolves it; deploymentId itself is phase-independent. +const resolveConfig = () => nextConfig("phase-production-server"); + +afterEach(() => { + delete process.env.CMS_MEMORY_CAPPED; +}); + describe("deployment version skew protection", () => { it("uses an explicit deployment ID or the current Git commit", () => { const gitCommit = execFileSync("git", ["rev-parse", "HEAD"], { encoding: "utf8", }).trim(); - expect(nextConfig.deploymentId).toBe( + expect(resolveConfig().deploymentId).toBe( process.env.NEXT_DEPLOYMENT_ID?.trim() || gitCommit, ); }); }); + +describe("production build memory cap", () => { + // An uncapped `next build` gets OOM-killed by the kernel on this host, and + // the killer may take the database and the live release with it. Refusing + // here stops the build before it allocates anything. + it("refuses a production build that is not running under the memory cap", () => { + expect(() => nextConfig("phase-production-build")).toThrow( + /Refusing to run an uncapped production build/, + ); + }); + + it("allows a production build when the wrapper marked it as capped", () => { + process.env.CMS_MEMORY_CAPPED = "1"; + expect(() => nextConfig("phase-production-build")).not.toThrow(); + }); + + // Only production builds are heavy enough to kill the machine; dev and + // start must keep working uncapped. + it("does not restrict other phases", () => { + expect(() => nextConfig("phase-development-server")).not.toThrow(); + expect(() => nextConfig("phase-production-server")).not.toThrow(); + }); +}); diff --git a/src/lib/services/clone-icon.test.ts b/src/lib/services/clone-icon.test.ts new file mode 100644 index 00000000..071cd27b --- /dev/null +++ b/src/lib/services/clone-icon.test.ts @@ -0,0 +1,77 @@ +import { describe, expect, it } from "vitest"; +import { extractFurniIconPng } from "@/lib/services/clone-icon"; +import { decodePng } from "@/lib/services/imager/png-decode"; +import { + createNitroBundle, + encodePng, + isLosslessWebp, + parseNitroBundle, + toWebpLosslessBundle, +} from "@/lib/services/swf/nitro-builder"; + +const NAME = "chair_test"; +const W = 16; +const H = 16; + +function makeRgba(): Buffer { + const rgba = Buffer.alloc(W * H * 4); + for (let i = 0; i < W * H; i++) { + rgba[i * 4 + 0] = 200; + rgba[i * 4 + 1] = 40; + rgba[i * 4 + 2] = 90; + rgba[i * 4 + 3] = 255; + } + return rgba; +} + +function buildBundle(texture: Buffer): Buffer { + return createNitroBundle( + { + name: NAME, + assets: { [`${NAME}_icon_a`]: { x: 0, y: 0 } }, + spritesheet: { + frames: { + [`${NAME}_${NAME}_icon_a`]: { frame: { x: 0, y: 0, w: W, h: H } }, + }, + }, + }, + texture, + NAME, + ); +} + +describe("extractFurniIconPng", () => { + // Every write path normalises bundles to WebP Lossless, so a PNG-only + // decoder made this silently return null for nearly every real bundle. + it("extracts the icon from a PNG-textured bundle", async () => { + const icon = await extractFurniIconPng( + buildBundle(encodePng(W, H, makeRgba())), + ); + expect(icon).not.toBeNull(); + expect(decodePng(icon as Buffer)).toMatchObject({ width: W, height: H }); + }); + + it("extracts the icon from a WebP-lossless-textured bundle", async () => { + const webpBundle = await toWebpLosslessBundle( + buildBundle(encodePng(W, H, makeRgba())), + ); + expect(isLosslessWebp(parseNitroBundle(webpBundle).texture)).toBe(true); + + const icon = await extractFurniIconPng(webpBundle); + expect(icon).not.toBeNull(); + expect(decodePng(icon as Buffer)).toMatchObject({ width: W, height: H }); + }); + + it("returns null for a bundle with no icon asset", async () => { + const bundle = createNitroBundle( + { name: NAME, assets: { other: { x: 0, y: 0 } } }, + encodePng(W, H, makeRgba()), + NAME, + ); + expect(await extractFurniIconPng(bundle)).toBeNull(); + }); + + it("returns null for bytes that are not a bundle at all", async () => { + expect(await extractFurniIconPng(Buffer.from([1, 2, 3, 4]))).toBeNull(); + }); +}); diff --git a/src/lib/services/clone-icon.ts b/src/lib/services/clone-icon.ts index b4c23493..637f0fe0 100644 --- a/src/lib/services/clone-icon.ts +++ b/src/lib/services/clone-icon.ts @@ -1,4 +1,5 @@ -import { cropRgba, decodePng } from "@/lib/services/imager/png-decode"; +import { decodeTextureRgba } from "@/lib/services/imager/decode-texture"; +import { cropRgba } from "@/lib/services/imager/png-decode"; import { encodePng, parseNitroBundle } from "@/lib/services/swf/nitro-builder"; interface FurniAsset { @@ -16,19 +17,48 @@ interface FurniNitroJson { spritesheet?: { frames?: Record }; } +/** + * Locate the spritesheet frame for an asset. Try the two exact spellings + * first (with and without a trailing `.png`), then one case-insensitive pass + * so a converter that lowercased the frame keys still resolves. + */ +function findFrame( + frames: Record, + name: string, + pixelAsset: string, +): FurniFrame | undefined { + const direct = + frames[`${name}_${pixelAsset}`] ?? frames[`${name}_${pixelAsset}.png`]; + if (direct) return direct; + + const wanted = new Set([ + `${name}_${pixelAsset}`.toLowerCase(), + `${name}_${pixelAsset}.png`.toLowerCase(), + ]); + for (const [key, frame] of Object.entries(frames)) { + if (wanted.has(key.toLowerCase())) return frame; + } + return undefined; +} + /** * Extract the furni catalog icon — the `{name}_icon_a` sprite — from a Nitro * bundle and return it as a standalone PNG, or null if the bundle has no icon * asset / usable frame. Used for sources that embed furni icons inside the * `.nitro` instead of serving a separate `{classname}_icon.png`. + * + * Async because the texture is usually WebP Lossless, which needs a real codec + * to read back out. */ -export function extractFurniIconPng(nitro: Buffer): Buffer | null { +export async function extractFurniIconPng( + nitro: Buffer, +): Promise { let json: FurniNitroJson; - let png: Buffer; + let texture: Buffer; try { const parsed = parseNitroBundle(nitro); json = parsed.json as FurniNitroJson; - png = parsed.png; + texture = parsed.texture; } catch { return null; } @@ -44,18 +74,20 @@ export function extractFurniIconPng(nitro: Buffer): Buffer | null { const asset = json.assets[iconKey]; // An asset may alias another's pixels via `source`; the frame key is the - // bundle name prefixed onto the asset name. Different converters pack the - // key with or without a trailing `.png`, so try both. + // bundle name prefixed onto the asset name. Converters disagree on the + // packing: some keep the trailing `.png`, and some lowercase the whole key + // while leaving `name` mixed-case (`LTD_fashionistaf` -> key + // `ltd_fashionistaf_ltd_fashionistaf_icon_a`). Match exactly first, then + // fall back to a case-insensitive lookup so mixed-case classnames work. const pixelAsset = asset.source ?? iconKey; const frames = json.spritesheet?.frames ?? {}; - const frame = - frames[`${name}_${pixelAsset}`] ?? frames[`${name}_${pixelAsset}.png`]; + const frame = findFrame(frames, name, pixelAsset); if (!frame || frame.rotated) return null; const { x, y, w, h } = frame.frame; if (w <= 0 || h <= 0) return null; try { - const sheet = decodePng(png); + const sheet = await decodeTextureRgba(texture); const px = cropRgba(sheet.rgba, sheet.width, x, y, w, h); return encodePng(w, h, px); } catch { diff --git a/src/lib/services/clone-import.ts b/src/lib/services/clone-import.ts index 26595430..63d6f3a0 100644 --- a/src/lib/services/clone-import.ts +++ b/src/lib/services/clone-import.ts @@ -378,7 +378,7 @@ export async function cloneSingleFurni(params: { // Source serves no standalone icon (e.g. icons embedded in the .nitro) — // extract the catalog icon from the bundle we just downloaded. try { - const icon = extractFurniIconPng( + const icon = await extractFurniIconPng( await fs.readFile(/*turbopackIgnore: true*/ nitroPath), ); if (icon) { diff --git a/src/lib/services/furni-import.ts b/src/lib/services/furni-import.ts index 358736b7..b01314bb 100644 --- a/src/lib/services/furni-import.ts +++ b/src/lib/services/furni-import.ts @@ -926,7 +926,7 @@ export async function importSingleFurni(params: { if (!iconOk && nitroExists()) { try { - const icon = extractFurniIconPng( + const icon = await extractFurniIconPng( await fs.readFile(resolveNitro() ?? nitroPath), ); if (icon) { diff --git a/src/lib/services/imager/decode-texture.ts b/src/lib/services/imager/decode-texture.ts new file mode 100644 index 00000000..72be019b --- /dev/null +++ b/src/lib/services/imager/decode-texture.ts @@ -0,0 +1,36 @@ +import sharp from "sharp"; +import type { DecodedPng } from "@/lib/services/imager/png-decode"; +import { decodePng } from "@/lib/services/imager/png-decode"; +import { detectNitroTextureFormat } from "@/lib/services/swf/nitro-builder"; + +/** + * Decode a Nitro spritesheet texture to raw RGBA pixels. + * + * Every write path normalises bundles to WebP Lossless, so a PNG-only decoder + * silently returns null for almost every bundle on disk. PNG keeps the + * dependency-free decoder (it handles the sprite format natively and avoids a + * native round-trip); WebP — the common case — goes through sharp. + * + * Throws when the bytes are neither PNG nor WebP, or when sharp cannot decode. + */ +export async function decodeTextureRgba(bytes: Buffer): Promise { + const format = detectNitroTextureFormat(bytes); + if (format === "png") return decodePng(bytes); + if (format !== "webp") throw new Error("not a PNG or WebP texture"); + + const { data, info } = await sharp(bytes) + .ensureAlpha() + .raw() + .toBuffer({ resolveWithObject: true }); + + // sharp can report a colour space the spritesheet maths does not expect; + // normalise to the 4-bytes-per-pixel RGBA that cropRgba assumes. + if (info.channels !== 4) { + throw new Error(`unsupported WebP channel count ${info.channels}`); + } + if (info.width * info.height * 4 !== data.length) { + throw new Error("WebP texture has an inconsistent pixel buffer size"); + } + + return { width: info.width, height: info.height, rgba: data }; +} diff --git a/src/lib/services/pet-icon.ts b/src/lib/services/pet-icon.ts index 499c37fc..5ac387de 100644 --- a/src/lib/services/pet-icon.ts +++ b/src/lib/services/pet-icon.ts @@ -1,6 +1,7 @@ import { readFileSync } from "node:fs"; import { resolveBundleInDir } from "@/lib/furni/bundle-file"; -import { cropRgba, decodePng } from "@/lib/services/imager/png-decode"; +import { decodeTextureRgba } from "@/lib/services/imager/decode-texture"; +import { cropRgba } from "@/lib/services/imager/png-decode"; import { encodePng, parseNitroBundle } from "@/lib/services/swf/nitro-builder"; import { getRuntimePath } from "@/lib/utils/runtime-path"; @@ -50,8 +51,10 @@ interface LayerSpec { // Directions present on pets are a subset of 0..7; 2 reads best as a portrait. const DIR_PREFERENCE = [2, 0, 4, 3, 1, 7, 6, 5]; -// Generated PNG icons are tiny; cache them in-process keyed by lib. -const iconCache = new Map(); +// Generated PNG icons are tiny; cache them in-process keyed by lib. The cache +// holds the in-flight promise so concurrent requests for the same pet decode +// once, and it never holds a rejection. +const iconCache = new Map>(); function findNitro(lib: string): string | null { for (const dir of PET_DIRS) { @@ -357,27 +360,29 @@ function composite( /** * Build a composited PNG icon for a pet from its `.nitro` spritesheet, or null * if the pet has no bundle / no usable frames. Cached in-process. + * + * Async because the texture is usually WebP Lossless, which needs a real codec + * to read back out. */ -export function getPetIconPng(lib: string): Buffer | null { +export async function getPetIconPng(lib: string): Promise { if (!/^[A-Za-z0-9_]+$/.test(lib)) return null; const cached = iconCache.get(lib); if (cached !== undefined) return cached; - const nitroPath = findNitro(lib); - if (!nitroPath) { - iconCache.set(lib, null); - return null; - } - try { - const { json, png } = parseNitroBundle( - readFileSync(/*turbopackIgnore: true*/ nitroPath), - ); - const sheet = decodePng(png); - const icon = composite(json as NitroJson, sheet); - iconCache.set(lib, icon); - return icon; - } catch { - iconCache.set(lib, null); - return null; - } + const pending = (async (): Promise => { + const nitroPath = findNitro(lib); + if (!nitroPath) return null; + try { + const { json, texture } = parseNitroBundle( + readFileSync(/*turbopackIgnore: true*/ nitroPath), + ); + const sheet = await decodeTextureRgba(texture); + return composite(json as NitroJson, sheet); + } catch { + return null; + } + })(); + + iconCache.set(lib, pending); + return pending; } diff --git a/src/lib/services/repair-icons.ts b/src/lib/services/repair-icons.ts index fc8e47f9..5fd037ff 100644 --- a/src/lib/services/repair-icons.ts +++ b/src/lib/services/repair-icons.ts @@ -274,7 +274,7 @@ export async function repairMissingIcons( foundNitro = true; try { const nitroBuf = await fs.readFile(nitroPath); - const icon = extractFurniIconPng(nitroBuf); + const icon = await extractFurniIconPng(nitroBuf); if (icon) { const writeErrors = await writeIconToDirs( icon, @@ -375,7 +375,9 @@ export async function repairMissingIcons( ); wroteTmp = dl.ok; if (dl.ok) { - const icon = extractFurniIconPng(await fs.readFile(nitroTmp)); + const icon = await extractFurniIconPng( + await fs.readFile(nitroTmp), + ); if (icon) { const writeErrors = await writeIconToDirs( icon,