import { existsSync, promises as fs } from "node:fs"; import os from "node:os"; import { sql } from "drizzle-orm"; import { queryRows } from "@/lib/db"; import { getBaseClassname, getSafeClassnameStem } from "@/lib/furni/classname"; import { DEFAULT_NITRO_SOURCES, listSources, } from "@/lib/services/clone-sources"; import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs"; import { downloadFile } from "@/lib/services/import/core/download"; import { parseNitroBundle } from "@/lib/services/swf/nitro-builder"; import { getRuntimePath } from "@/lib/utils/runtime-path"; /** * Catalog Studio nitro cleanup. * * Scans every asset directory (public + mirrored gamedata) for furniture * leftovers and splits the found bundles into removable groups: * * - "fake" .nitro files whose stem does not match any item in `items_base` * (leftover files that belong to no furniture — including items that were * deleted or imported under a different name); * - "broken" .nitro files that do belong to an item but cannot be parsed as a * valid Nitro bundle (truncated, corrupted or renamed files — e.g. a * download that landed as an HTML error page instead of a bundle); * * The scan validates bundles with a cheap header-only read (a few KB per file, * no decompression) so it stays fast even on 200k+-file nitro directories. A * full parse is only attempted for files whose header looks suspicious, so * valid bundles are never flagged by a header false-positive. * - orphaned `.swf` / icon files whose stem matches no item in `items_base`. * * Deletion only touches the asset files on disk — never database rows. Broken * bundles can be re-downloaded from the configured nitro sources instead of * being deleted, and fake bundles older than a configurable age can be removed * automatically for self-cleaning nitro directories. */ export type CleanupAssetKind = "nitro" | "swf" | "icon"; export interface AssetCleanupEntry { fileName: string; base: string; size: number; dirs: string[]; lastModified: number; } export interface NitroCleanupEntry extends AssetCleanupEntry { error?: string; } export interface NitroCleanupScan { fake: NitroCleanupEntry[]; broken: NitroCleanupEntry[]; orphanedSwf: AssetCleanupEntry[]; orphanedIcon: AssetCleanupEntry[]; total: number; } export interface NitroCleanupDeleteResult { /** Files that were fully removed from every matching directory. */ deleted: number; /** Individual files removed across all directories (mirrors). */ copiesRemoved: number; errors: string[]; files: Array<{ fileName: string; deleted: boolean; remaining: string[] }>; } export interface NitroRepairResult { repaired: number; failed: number; errors: string[]; files: Array<{ fileName: string; ok: boolean; error?: string }>; } export interface NitroAutoCleanResult extends NitroCleanupDeleteResult { /** Fake bundles skipped because they are younger than the age threshold. */ skippedRecent: number; /** Broken bundles skipped — only fake leftovers are auto-removed. */ skippedBroken: number; } const NITRO_FILE_RE = /^[a-z0-9_*\-.]+\.nitro$/i; const SWF_FILE_RE = /^[a-z0-9_*\-.]+\.swf$/i; const ICON_FILE_RE = /^[a-z0-9_*\-.]+\.(?:gif|png)$/i; /** * Only this many leading bytes of a `.nitro` bundle are read during the fake / * broken scan. The header (file count + two embedded file names + zlib payload * lengths) is tiny, and a zlib/gzip-signed JSON stream is enough to confirm a * bundle is structurally valid without decompressing the large PNG texture. */ const NITRO_HEADER_SAMPLE_BYTES = 2048; /** * Minimum + maximum plausible header fields for a Nitro bundle. A bundle always * contains at least two files (a `.json` config stream and a `.png` texture * stream). File-name length fields and compressed stream lengths are bounded so * a bundle whose header claims absurd sizes (e.g. a download that landed as an * HTML error page, or a truncated/zero-filled file) can be rejected without * decompressing anything. */ const NITRO_MIN_FILES = 2; const NITRO_NAME_LEN_MAX = 1024; const NITRO_COMPRESSED_LEN_MAX = 512 * 1024 * 1024; const KIND_FILE_RE: Record = { nitro: NITRO_FILE_RE, swf: SWF_FILE_RE, icon: ICON_FILE_RE, }; const DAY_MS = 24 * 60 * 60 * 1000; function uniqueDirs(dirs: string[]): string[] { const seen = new Set(); const out: string[] = []; for (const dir of dirs) { const key = process.platform === "win32" ? dir.toLowerCase() : dir; if (seen.has(key)) continue; seen.add(key); out.push(dir); } return out; } /** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */ async function runPool( items: T[], concurrency: number, worker: (item: T) => Promise, ): Promise { const limit = Math.min(Math.max(concurrency, 1), items.length); if (limit === 0) return; let next = 0; await Promise.allSettled( Array.from({ length: limit }, async () => { while (true) { const i = next++; if (i >= items.length) break; await worker(items[i]); } }), ); } /** * Cheap header-only validation that mirrors `parseNitroBundle`'s byte layout * (used by `swf/nitro-builder`): a bundle starts with a big-endian file count * followed by `(nameLen:int16, name:utf8, compressedLen:int32)` pairs. We read * only `NITRO_HEADER_SAMPLE_BYTES` of leading bytes (no decompression) and * reject bundles whose header claims absurd sizes — e.g. a download that landed * as an HTML error page, or a truncated/zero-filled file. * * Returns `false` when the header is clearly implausible. When the file cannot * be opened/read (including mocked test environments where the path does not * exist) it also returns `false`, which makes the caller fall back to the * expensive full `parseNitroBundle` — so tests that supply buffers via * `readFileFn` keep working unchanged. */ async function hasValidNitroHeader(filePath: string): Promise { let handle: import("node:fs/promises").FileHandle | undefined; try { handle = await fs.open(filePath, "r"); const sample = Buffer.alloc(NITRO_HEADER_SAMPLE_BYTES); const { bytesRead } = await handle.read(sample, 0, sample.length, 0); if (bytesRead < 8) return false; let offset = 0; const fileCount = sample.readInt16BE(offset); offset += 2; if (fileCount < NITRO_MIN_FILES) return false; if (fileCount > 16) return false; for (let i = 0; i < fileCount && offset < bytesRead - 2; i++) { const nameLen = sample.readInt16BE(offset); offset += 2; if (nameLen < 1 || nameLen > NITRO_NAME_LEN_MAX) return false; // The name field must be printable utf-8; an HTML error page or // zero-filled buffer almost always trips this. if (offset + nameLen > bytesRead) return false; const name = sample.toString("utf-8", offset, offset + nameLen); offset += nameLen; if (!/^[\x20-\x7e]+$/.test(name)) return false; if (offset + 4 > bytesRead) return true; const compressedLen = sample.readInt32BE(offset); offset += 4; if (compressedLen <= 0 || compressedLen > NITRO_COMPRESSED_LEN_MAX) { return false; } } return true; } catch { return false; } finally { await handle?.close().catch(() => {}); } } /** Register every .nitro file stem a classname could legitimately use. */ function addStems(set: Set, classname: string): void { const normalized = classname.trim().toLowerCase(); if (!normalized) return; const base = getBaseClassname(normalized).toLowerCase(); if (base) set.add(base); if (normalized.includes("*")) { const safe = getSafeClassnameStem(normalized).toLowerCase(); if (safe) set.add(safe); } } interface DirectoryRead { name: string; base: string; size: number; lastModified: number; } async function readAssetDir( dir: string, regex: RegExp, ): Promise { const names = await fs.readdir(dir).catch(() => [] as string[]); const entries: DirectoryRead[] = []; for (const name of names) { if (!regex.test(name)) continue; let size = 0; let mtimeMs = 0; try { const stat = await fs.stat(getRuntimePath(dir, name)); size = stat.size; mtimeMs = stat.mtimeMs; } catch { /* ignore unreadable entry */ } entries.push({ name, base: name.replace(/\.[a-z0-9]+$/i, ""), size, lastModified: mtimeMs, }); } return entries; } async function collectAssets( dirs: string[], regex: RegExp, ): Promise> { const byName = new Map(); for (const dir of dirs) { for (const read of await readAssetDir(dir, regex)) { const entry = byName.get(read.name) ?? { fileName: read.name, base: read.base, size: 0, dirs: [], lastModified: 0, }; if (!entry.dirs.includes(dir)) entry.dirs.push(dir); entry.size = Math.max(entry.size, read.size); entry.lastModified = Math.max(entry.lastModified, read.lastModified); byName.set(read.name, entry); } } return byName; } async function loadValidStems(): Promise> { // Authoritative stem set from items_base (item_name + public_name // fallback), mirroring repair-nitros.ts classname resolution. const validStems = new Set(); try { const result = await queryRows<{ item_name: string; public_name: string; }>(sql` SELECT item_name, public_name FROM items_base `); for (const row of result) { addStems(validStems, row.item_name); if ( row.public_name && row.public_name !== row.item_name && !row.public_name.includes(" ") && row.public_name !== "0" ) { addStems(validStems, row.public_name); } } } catch { /* empty set → every asset is treated as orphaned */ } return validStems; } export async function scanFakeBrokenNitros(): Promise { const targets = await getFurniAssetWriteTargets(); const nitroDirs = uniqueDirs([ targets.nitroDir, ...targets.mirrorDirs.map((d) => d.nitroDir), ]); const swfDirs = uniqueDirs([ targets.swfDir, ...targets.mirrorDirs.map((d) => d.swfDir), ]); const iconDirs = uniqueDirs([ targets.iconDir, ...targets.mirrorDirs.map((d) => d.iconDir), ]); const [byNitro, bySwf, byIcon, validStems] = await Promise.all([ collectAssets(nitroDirs, NITRO_FILE_RE), collectAssets(swfDirs, SWF_FILE_RE), collectAssets(iconDirs, ICON_FILE_RE), loadValidStems(), ]); const entries: NitroCleanupEntry[] = [...byNitro.values()]; const fake: NitroCleanupEntry[] = []; const broken: NitroCleanupEntry[] = []; await runPool(entries, 8, async (entry) => { // A .nitro that no DB item maps to is a leftover / fake bundle. if (!validStems.has(entry.base.toLowerCase())) { fake.push(entry); return; } // Otherwise validate the bundle content. Most bundles pass a cheap // header-only check (a few KB, no decompression); only files whose // header looks suspicious get the expensive full parse. This keeps the // scan fast even on directories with hundreds of thousands of bundles. let parsed = false; let firstError = ""; for (const dir of entry.dirs) { if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) { parsed = true; break; } try { parseNitroBundle( await fs.readFile(getRuntimePath(dir, entry.fileName)), ); parsed = true; break; } catch (err) { firstError = (err as Error).message; } } if (!parsed) { broken.push({ ...entry, error: firstError }); } }); const orphanedSwf = [...bySwf.values()].filter( (entry) => !validStems.has(entry.base.toLowerCase()), ); const orphanedIcon = [...byIcon.values()].filter( (entry) => !validStems.has(entry.base.toLowerCase()), ); const byName = (entries: T[]): T[] => entries.sort((a, b) => a.fileName.localeCompare(b.fileName)); return { fake: byName(fake), broken: byName(broken), orphanedSwf: byName(orphanedSwf), orphanedIcon: byName(orphanedIcon), total: byNitro.size + bySwf.size + byIcon.size, }; } function dirsForKind( targets: Awaited>, kind: CleanupAssetKind, ): string[] { const primary = { nitro: targets.nitroDir, swf: targets.swfDir, icon: targets.iconDir, }[kind]; const mirrors = targets.mirrorDirs.map((d) => d[`${kind}Dir` as const]); return uniqueDirs([primary, ...mirrors]); } /** Delete asset files from every matching directory (primary + mirrors). */ export async function deleteNitroCleanupFiles( fileNames: string[], kind: CleanupAssetKind = "nitro", ): Promise { const targets = await getFurniAssetWriteTargets(); const dirs = dirsForKind(targets, kind); const fileRegex = KIND_FILE_RE[kind]; let deleted = 0; let copiesRemoved = 0; const errors: string[] = []; const files: NitroCleanupDeleteResult["files"] = []; for (const rawName of [...new Set(fileNames)]) { const fileName = String(rawName ?? ""); if (!fileRegex.test(fileName)) { errors.push(`Invalid ${kind} file name: ${fileName}`); files.push({ fileName, deleted: false, remaining: [] }); continue; } const remaining: string[] = []; let existed = false; for (const dir of dirs) { const filePath = getRuntimePath(dir, fileName); if (!existsSync(filePath)) continue; existed = true; try { await fs.unlink(filePath); copiesRemoved++; } catch (err) { remaining.push(dir); errors.push(`${fileName} in ${dir}: ${(err as Error).message}`); } } const fullyDeleted = existed && remaining.length === 0; if (fullyDeleted) deleted++; files.push({ fileName, deleted: fullyDeleted, remaining }); } return { deleted, copiesRemoved, errors, files }; } function ageDays(lastModified: number): number { if (!lastModified) return 0; return Math.max(0, Math.floor((Date.now() - lastModified) / DAY_MS)); } /** * Automatically remove fake .nitro leftovers that are older than * `maxAgeDays`. Broken bundles and young fake bundles are never touched — * only former leftovers whose items are long gone from the database. */ export async function autoCleanFakeNitros( maxAgeDays = 30, ): Promise { const scan = await scanFakeBrokenNitros(); const threshold = Math.max(0, Math.floor(maxAgeDays)); const removable = scan.fake.filter( (entry) => ageDays(entry.lastModified) >= threshold, ); const result = await deleteNitroCleanupFiles( removable.map((entry) => entry.fileName), "nitro", ); return { ...result, skippedRecent: scan.fake.length - removable.length, skippedBroken: scan.broken.length, }; } async function copyFileToDirs( buffer: Buffer, fileName: string, dirs: string[], ): Promise { const errors: string[] = []; for (const dir of dirs) { try { await fs.mkdir(dir, { recursive: true }); await fs.writeFile(getRuntimePath(dir, fileName), buffer); } catch (err) { errors.push(`write .nitro to ${dir} failed: ${(err as Error).message}`); } } return errors; } /** * Re-download broken .nitro bundles from the configured nitro sources. The * file stem matched an item in `items_base`, so the same classname candidates * used by repair-nitros.ts are tried; every downloaded bundle is validated * before it replaces the broken copies across all nitro directories. */ export async function repairBrokenNitros( fileNames: string[], ): Promise { const targets = await getFurniAssetWriteTargets(); const nitroDirs = dirsForKind(targets, "nitro"); let sources = (await listSources().catch(() => [] as never)).filter( (source: { nitroBaseUrl?: string }) => source.nitroBaseUrl, ); if (sources.length === 0) sources = DEFAULT_NITRO_SOURCES as never; let repaired = 0; let failed = 0; const errors: string[] = []; const files: NitroRepairResult["files"] = []; await runPool([...new Set(fileNames)], 6, async (rawName) => { const fileName = String(rawName ?? ""); if (!NITRO_FILE_RE.test(fileName) || !fileName.endsWith(".nitro")) { failed++; const message = `Invalid nitro file name: ${fileName}`; errors.push(message); files.push({ fileName, ok: false, error: message }); return; } const name = fileName.slice(0, -".nitro".length); const candidates = [ name, ...(name.includes("*") ? [name.split("*")[0], name.replace(/\*/g, "_")] : []), ...(name.includes(".") ? [name.replace(/\./g, "_")] : []), ]; for (const source of sources as { name: string; nitroBaseUrl?: string; }[]) { for (const candidate of candidates) { const base = source.nitroBaseUrl; if (!base) continue; const tmpPath = getRuntimePath( os.tmpdir(), `.repair_nitro_${name}_${Math.random().toString(36).slice(2, 8)}.nitro`, ); try { const dl = await downloadFile( `${base.replace(/\/+$/, "")}/${encodeURIComponent(candidate)}.nitro`, tmpPath, { maxRetries: 1, validate: "nitro" }, ); if (dl.ok) { const buffer = await fs.readFile(tmpPath); parseNitroBundle(buffer); const writeErrors = await copyFileToDirs( buffer, fileName, nitroDirs, ); if (writeErrors.length === 0) { repaired++; files.push({ fileName, ok: true }); } else { failed++; errors.push(...writeErrors); files.push({ fileName, ok: false, error: writeErrors[0], }); } return; } } catch (err) { errors.push( `repair ${fileName} from ${source.name} failed: ${(err as Error).message}`, ); } finally { await fs.unlink(tmpPath).catch(() => {}); } } } failed++; errors.push(`could not re-obtain ${fileName} from any nitro source`); files.push({ fileName, ok: false, error: "could not re-obtain bundle from any nitro source", }); }); return { repaired, failed, errors, files }; }