perf(studio): cache nitro scan, stream progress, virtualize cleanup list

src/lib/services/furni-import-cache.ts

@@ -16,7 +16,11 @@ import { existsSync } from "node:fs";
 import path from "node:path";
 import { sql } from "drizzle-orm";
 import { db, ItemsBase, queryRows } from "@/lib/db";
-import { normalizeClassname } from "@/lib/furni/classname";
+import {
+	getBaseClassname,
+	getSafeClassnameStem,
+	normalizeClassname,
+} from "@/lib/furni/classname";
 import { indexLocalFurniture } from "@/lib/furni/local-presence";
 import { readFurniData } from "./furni-data";
 
@@ -107,6 +111,58 @@ export async function getCatalogIdSet(): Promise<Set<number>> {
 
 const nitroPresenceCache = new Map<string, boolean>();
 
+/** Every classname stem a `.nitro`/`.swf`/icon file may legitimately use. */
+function addClassnameStem(set: Set<string>, classname: string): void {
+	const normalized = classname.trim().toLowerCase();
+	if (!normalized) return;
+	const base = getBaseClassname(normalized).toLowerCase();
+	if (base) set.add(base);
+	if (normalized.includes("*")) {
+		const safe = getSafeClassnameStem(normalized).toLowerCase();
+		if (safe) set.add(safe);
+	}
+}
+
+let cleanupStemsSnapshot: { set: Set<string>; timestamp: number } | null = null;
+
+/**
+ * Authoritative stems for the nitro-cleanup scan: every `item_name` plus the
+ * `public_name` fallback (mirroring repair-nitros.ts resolution). Cached the
+ * same way as the Studio listings so repeated scans (auto-clean, panel) don't
+ * re-query `items_base` on every call.
+ */
+export async function getCleanupValidStems(): Promise<Set<string>> {
+	const now = Date.now();
+	if (
+		cleanupStemsSnapshot &&
+		now - cleanupStemsSnapshot.timestamp < CACHE_TTL
+	) {
+		return cleanupStemsSnapshot.set;
+	}
+	const set = new Set<string>();
+	try {
+		const rows = await queryRows<{
+			item_name: string;
+			public_name: string;
+		}>(sql`SELECT item_name, public_name FROM items_base`);
+		for (const row of rows) {
+			addClassnameStem(set, row.item_name);
+			if (
+				row.public_name &&
+				row.public_name !== row.item_name &&
+				!row.public_name.includes(" ") &&
+				row.public_name !== "0"
+			) {
+				addClassnameStem(set, row.public_name);
+			}
+		}
+	} catch {
+		/* empty set → every asset is treated as orphaned */
+	}
+	cleanupStemsSnapshot = { set, timestamp: now };
+	return set;
+}
+
 /** `existsSync` memoized per (dir + base) — nitro files change rarely. */
 export function nitroExistsCached(dir: string, base: string): boolean {
 	const key = `${dir}/${base}`;
@@ -210,5 +266,6 @@ export function clearFurniImportCache(): void {
 	furniDataSetSnapshot = null;
 	catalogIdsSnapshot = null;
 	nitroPresenceCache.clear();
+	cleanupStemsSnapshot = null;
 	statsSnapshot = null;
 }

src/lib/services/nitro-cleanup.test.ts

@@ -1,4 +1,5 @@
 // @vitest-environment node
+import { createHash } from "node:crypto";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 
 type AnyFn = (...args: any[]) => any;
@@ -130,6 +131,13 @@ function mockReaddir(byDir: Record<string, string[]>): void {
 	readdirFn.mockImplementation(async (dir: string) => byDir[dir] ?? []);
 }
 
+/** Replicates `dirNameSignature` in the service so cache fixtures match. */
+function dirSignature(names: string[]): string {
+	const hash = createHash("sha1");
+	for (const name of names) hash.update(name).update("\n");
+	return hash.digest("hex");
+}
+
 describe("scanFakeBrokenNitros", () => {
 	it("classifies fake (no matching item) and broken (unparseable) bundles", async () => {
 		mockReaddir({
@@ -233,6 +241,73 @@ describe("scanFakeBrokenNitros", () => {
 		expect(result.fake[0].size).toBe(900);
 		expect(result.fake[0].lastModified).toBe(2000);
 	});
+
+	it("serves results from the on-disk cache when directories are unchanged", async () => {
+		mockReaddir({
+			"/assets/nitro": ["chair.nitro", "ghost.nitro"],
+		});
+		const dirs = {
+			"nitro\u0000/assets/nitro": dirSignature(["chair.nitro", "ghost.nitro"]),
+			"swf\u0000/assets/swf": dirSignature([]),
+			"icon\u0000/assets/icons": dirSignature([]),
+		};
+		const cachedResult = {
+			fake: [
+				{
+					fileName: "ghost.nitro",
+					base: "ghost",
+					size: 500,
+					dirs: ["/assets/nitro"],
+					lastModified: 0,
+					error: undefined,
+				},
+			],
+			broken: [],
+			orphanedSwf: [],
+			orphanedIcon: [],
+			total: 4,
+		};
+		existsFn.mockImplementation((filePath: string) =>
+			filePath.includes("scan-cache.json"),
+		);
+		readFileFn.mockImplementation(async (filePath: string) => {
+			if (filePath.includes("scan-cache.json")) {
+				return JSON.stringify({ dirs, result: cachedResult });
+			}
+			return Buffer.from("nitro-data");
+		});
+
+		const result = await scanFakeBrokenNitros();
+
+		// `total: 4` could never come from a fresh scan of 2 files — the cache
+		// result came back verbatim, so classification was skipped entirely.
+		expect(result).toEqual(expect.objectContaining(cachedResult));
+		expect(result.cached).toBe(true);
+		expect(result.total).toBe(4);
+		expect(executeFn).not.toHaveBeenCalled();
+		expect(statFn).not.toHaveBeenCalled();
+		expect(readFileFn).toHaveBeenCalledTimes(1);
+		expect(writeFileFn).not.toHaveBeenCalled();
+	});
+
+	it("force bypasses the cache and re-validates the directories", async () => {
+		mockReaddir({
+			"/assets/nitro": ["chair.nitro", "ghost.nitro"],
+		});
+		// A matching-on-disk cache exists, but `force: true` ignores it.
+		existsFn.mockImplementation(() => true);
+		readFileFn.mockResolvedValue(Buffer.from("nitro-data"));
+		parseNitroBundle.mockReturnValue({});
+		statFn.mockImplementation(async () => ({ size: 512, mtimeMs: 0 }));
+
+		const result = await scanFakeBrokenNitros({ force: true });
+
+		expect(result.cached).toBeUndefined();
+		expect(result.fake.map((e) => e.fileName)).toEqual(["ghost.nitro"]);
+		expect(result.broken).toEqual([]);
+		expect(result.total).toBe(2);
+		expect(statFn).toHaveBeenCalled();
+	});
 });
 
 describe("deleteNitroCleanupFiles", () => {

src/lib/services/nitro-cleanup.ts

@@ -1,13 +1,13 @@
+import { createHash } from "node:crypto";
 import { existsSync, promises as fs } from "node:fs";
 import os from "node:os";
-import { sql } from "drizzle-orm";
-import { queryRows } from "@/lib/db";
-import { getBaseClassname, getSafeClassnameStem } from "@/lib/furni/classname";
+import path from "node:path";
 import {
 	DEFAULT_NITRO_SOURCES,
 	listSources,
 } from "@/lib/services/clone-sources";
 import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs";
+import { getCleanupValidStems } from "@/lib/services/furni-import-cache";
 import { downloadFile } from "@/lib/services/import/core/download";
 import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
 import { getRuntimePath } from "@/lib/utils/runtime-path";
@@ -57,6 +57,23 @@ export interface NitroCleanupScan {
 	orphanedSwf: AssetCleanupEntry[];
 	orphanedIcon: AssetCleanupEntry[];
 	total: number;
+	/** True when the result was served from the on-disk scan cache. */
+	cached?: boolean;
+}
+
+export interface CleanupProgress {
+	phase: "stems" | "readdir" | "validate" | "done";
+	scanned: number;
+}
+
+export interface CleanupScanOptions {
+	/** Bypass the scan cache and re-validate every bundle. */
+	force?: boolean;
+	/**
+	 * Coarse progress callback used to stream scan status to the UI instead of
+	 * leaving the tab looking frozen while a large directory is processed.
+	 */
+	onProgress?: (progress: CleanupProgress) => void;
 }
 
 export interface NitroCleanupDeleteResult {
@@ -198,18 +215,6 @@ async function hasValidNitroHeader(filePath: string): Promise<boolean> {
 	}
 }
 
-/** Register every .nitro file stem a classname could legitimately use. */
-function addStems(set: Set<string>, classname: string): void {
-	const normalized = classname.trim().toLowerCase();
-	if (!normalized) return;
-	const base = getBaseClassname(normalized).toLowerCase();
-	if (base) set.add(base);
-	if (normalized.includes("*")) {
-		const safe = getSafeClassnameStem(normalized).toLowerCase();
-		if (safe) set.add(safe);
-	}
-}
-
 interface DirectoryRead {
 	name: string;
 	base: string;
@@ -217,6 +222,23 @@ interface DirectoryRead {
 	lastModified: number;
 }
 
+/** Discriminated cache key per asset directory (kind + path). */
+function kindKey(kind: CleanupAssetKind, dir: string): string {
+	return `${kind}\0${dir}`;
+}
+
+async function readDirNames(dir: string, regex: RegExp): Promise<string[]> {
+	const names = await fs.readdir(dir).catch(() => [] as string[]);
+	return names.filter((name) => regex.test(name)).sort();
+}
+
+/** Cheap fingerprint of a directory listing — a rename/add/remove changes it. */
+function dirNameSignature(names: string[]): string {
+	const hash = createHash("sha1");
+	for (const name of names) hash.update(name).update("\n");
+	return hash.digest("hex");
+}
+
 /**
  * How many `stat` calls run in parallel while reading an asset directory.
  * Directories can hold hundreds of thousands of files; serial `stat`ing them
@@ -224,14 +246,12 @@ interface DirectoryRead {
  */
 const DIR_STAT_CONCURRENCY = 32;
 
-async function readAssetDir(
+async function buildDirEntries(
 	dir: string,
-	regex: RegExp,
+	names: string[],
 ): Promise<DirectoryRead[]> {
-	const names = await fs.readdir(dir).catch(() => [] as string[]);
 	const entries: DirectoryRead[] = [];
 	await runPool(names, DIR_STAT_CONCURRENCY, async (name) => {
-		if (!regex.test(name)) return;
 		let size = 0;
 		let mtimeMs = 0;
 		try {
@@ -252,12 +272,14 @@ async function readAssetDir(
 }
 
 async function collectAssets(
+	kind: CleanupAssetKind,
 	dirs: string[],
-	regex: RegExp,
+	namesByKey: Map<string, string[]>,
 ): Promise<Map<string, AssetCleanupEntry>> {
 	const byName = new Map<string, AssetCleanupEntry>();
 	for (const dir of dirs) {
-		for (const read of await readAssetDir(dir, regex)) {
+		const names = namesByKey.get(kindKey(kind, dir)) ?? [];
+		for (const read of await buildDirEntries(dir, names)) {
 			const entry = byName.get(read.name) ?? {
 				fileName: read.name,
 				base: read.base,
@@ -274,35 +296,66 @@ async function collectAssets(
 	return byName;
 }
 
-async function loadValidStems(): Promise<Set<string>> {
-	// Authoritative stem set from items_base (item_name + public_name
-	// fallback), mirroring repair-nitros.ts classname resolution.
-	const validStems = new Set<string>();
-	try {
-		const result = await queryRows<{
-			item_name: string;
-			public_name: string;
-		}>(sql`
-			SELECT item_name, public_name FROM items_base
-		`);
-		for (const row of result) {
-			addStems(validStems, row.item_name);
-			if (
-				row.public_name &&
-				row.public_name !== row.item_name &&
-				!row.public_name.includes(" ") &&
-				row.public_name !== "0"
-			) {
-				addStems(validStems, row.public_name);
-			}
-		}
-	} catch {
-		/* empty set → every asset is treated as orphaned */
-	}
-	return validStems;
+// ── On-disk scan cache ──────────────────────────────────────────────────────
+//
+// The heavy parts of a scan are the per-bundle header validation (one open +
+// small read per .nitro) and the parallel stat pass. Both can be skipped when
+// no asset directory changed since the last scan: a signature per directory is
+// derived from the sortable file *list* (readdir + hash, no stat). Any add /
+// remove / rename invalidates the cache and triggers a full re-validation.
+// Bundles rewritten in place without a name change are not detected by the
+// signature; run a forced scan (`force: true`) to pick those up.
+
+const SCAN_CACHE_DIR = path.join(process.cwd(), "storage", "nitro-cleanup");
+const scanCachePath = () => path.join(SCAN_CACHE_DIR, "scan-cache.json");
+
+interface ScanCacheFile {
+	dirs: Record<string, string>;
+	result: Omit<NitroCleanupScan, "cached">;
 }
 
-export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
+async function readScanCache(): Promise<ScanCacheFile | null> {
+	try {
+		if (!existsSync(scanCachePath())) return null;
+		const raw = await fs.readFile(scanCachePath(), "utf8");
+		const parsed = JSON.parse(raw) as ScanCacheFile;
+		if (!parsed || typeof parsed.dirs !== "object" || !parsed.result) {
+			return null;
+		}
+		return parsed;
+	} catch {
+		return null;
+	}
+}
+
+async function writeScanCache(
+	dirs: Record<string, string>,
+	result: Omit<NitroCleanupScan, "cached">,
+): Promise<void> {
+	try {
+		await fs.mkdir(SCAN_CACHE_DIR, { recursive: true });
+		await fs.writeFile(scanCachePath(), JSON.stringify({ dirs, result }));
+	} catch {
+		/* cache is best effort — a broken cache simply re-scans next time */
+	}
+}
+
+function cacheDirsMatch(
+	cached: Record<string, string>,
+	current: Record<string, string>,
+): boolean {
+	const cachedKeys = Object.keys(cached);
+	if (cachedKeys.length !== Object.keys(current).length) return false;
+	for (const key of cachedKeys) {
+		if (cached[key] !== current[key]) return false;
+	}
+	return true;
+}
+
+export async function scanFakeBrokenNitros(
+	options: CleanupScanOptions = {},
+): Promise<NitroCleanupScan> {
+	const { force = false, onProgress } = options;
 	const targets = await getFurniAssetWriteTargets();
 	const nitroDirs = uniqueDirs([
 		targets.nitroDir,
@@ -316,47 +369,94 @@ export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
 		targets.iconDir,
 		...targets.mirrorDirs.map((d) => d.iconDir),
 	]);
+	const directoryPlan: Array<{
+		kind: CleanupAssetKind;
+		dirs: string[];
+	}> = [
+		{ kind: "nitro", dirs: nitroDirs },
+		{ kind: "swf", dirs: swfDirs },
+		{ kind: "icon", dirs: iconDirs },
+	];
 
-	const [byNitro, bySwf, byIcon, validStems] = await Promise.all([
-		collectAssets(nitroDirs, NITRO_FILE_RE),
-		collectAssets(swfDirs, SWF_FILE_RE),
-		collectAssets(iconDirs, ICON_FILE_RE),
-		loadValidStems(),
+	onProgress?.({ phase: "readdir", scanned: 0 });
+
+	// One readdir pass shared by both the cache signature and the full scan —
+	// never stat a file just to learn the directory changed.
+	const namesByKey = new Map<string, string[]>();
+	await Promise.all(
+		directoryPlan.flatMap(({ kind, dirs }) =>
+			dirs.map(async (dir) => {
+				namesByKey.set(
+					kindKey(kind, dir),
+					await readDirNames(dir, KIND_FILE_RE[kind]),
+				);
+			}),
+		),
+	);
+	const currentDigests: Record<string, string> = {};
+	for (const key of namesByKey.keys()) {
+		currentDigests[key] = dirNameSignature(namesByKey.get(key) ?? []);
+	}
+
+	onProgress?.({ phase: "readdir", scanned: 0 });
+	if (!force) {
+		const cache = await readScanCache();
+		if (cache && cacheDirsMatch(cache.dirs, currentDigests)) {
+			onProgress?.({ phase: "done", scanned: cache.result.total });
+			return { ...cache.result, cached: true };
+		}
+	}
+
+	// Needed only for classification — skip the DB query on a cache hit.
+	onProgress?.({ phase: "stems", scanned: 0 });
+	const validStems = await getCleanupValidStems();
+
+	const [byNitro, bySwf, byIcon] = await Promise.all([
+		collectAssets("nitro", nitroDirs, namesByKey),
+		collectAssets("swf", swfDirs, namesByKey),
+		collectAssets("icon", iconDirs, namesByKey),
 	]);
 
 	const entries: NitroCleanupEntry[] = [...byNitro.values()];
 	const fake: NitroCleanupEntry[] = [];
 	const broken: NitroCleanupEntry[] = [];
+	let validated = 0;
+	const totalToValidate = entries.length;
 
 	await runPool(entries, 16, async (entry) => {
 		// A .nitro that no DB item maps to is a leftover / fake bundle.
 		if (!validStems.has(entry.base.toLowerCase())) {
 			fake.push(entry);
-			return;
-		}
-		// Otherwise validate the bundle content. Most bundles pass a cheap
-		// header-only check (a few KB, no decompression); only files whose
-		// header looks suspicious get the expensive full parse. This keeps the
-		// scan fast even on directories with hundreds of thousands of bundles.
-		let parsed = false;
-		let firstError = "";
-		for (const dir of entry.dirs) {
-			if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) {
-				parsed = true;
-				break;
+		} else {
+			// Otherwise validate the bundle content. Most bundles pass a cheap
+			// header-only check (a few KB, no decompression); only files whose
+			// header looks suspicious get the expensive full parse. This keeps
+			// the scan fast even on directories with hundreds of thousands of
+			// bundles.
+			let parsed = false;
+			let firstError = "";
+			for (const dir of entry.dirs) {
+				if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) {
+					parsed = true;
+					break;
+				}
+				try {
+					parseNitroBundle(
+						await fs.readFile(getRuntimePath(dir, entry.fileName)),
+					);
+					parsed = true;
+					break;
+				} catch (err) {
+					firstError = (err as Error).message;
+				}
 			}
-			try {
-				parseNitroBundle(
-					await fs.readFile(getRuntimePath(dir, entry.fileName)),
-				);
-				parsed = true;
-				break;
-			} catch (err) {
-				firstError = (err as Error).message;
+			if (!parsed) {
+				broken.push({ ...entry, error: firstError });
 			}
 		}
-		if (!parsed) {
-			broken.push({ ...entry, error: firstError });
+		validated++;
+		if (validated % 2500 === 0 || validated === totalToValidate) {
+			onProgress?.({ phase: "validate", scanned: validated });
 		}
 	});
 
@@ -370,13 +470,16 @@ export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
 	const byName = <T extends { fileName: string }>(entries: T[]): T[] =>
 		entries.sort((a, b) => a.fileName.localeCompare(b.fileName));
 
-	return {
+	const result: NitroCleanupScan = {
 		fake: byName(fake),
 		broken: byName(broken),
 		orphanedSwf: byName(orphanedSwf),
 		orphanedIcon: byName(orphanedIcon),
 		total: byNitro.size + bySwf.size + byIcon.size,
 	};
+	await writeScanCache(currentDigests, result);
+	onProgress?.({ phase: "done", scanned: result.total });
+	return result;
 }
 
 function dirsForKind(
@@ -435,7 +538,7 @@ export async function deleteNitroCleanupFiles(
 	return { deleted, copiesRemoved, errors, files };
 }
 
-function ageDays(lastModified: number): number {
+export function ageDays(lastModified: number): number {
 	if (!lastModified) return 0;
 	return Math.max(0, Math.floor((Date.now() - lastModified) / DAY_MS));
 }