perf(studio): cache nitro scan, stream progress, virtualize cleanup list

2026-09-19 13:01:59 +02:00
1 parent c916e42572
commit ba81d16f00
@@ -1,13 +1,13 @@
+import { createHash } from "node:crypto";
 import { existsSync, promises as fs } from "node:fs";
 import os from "node:os";
-import { sql } from "drizzle-orm";
-import { queryRows } from "@/lib/db";
-import { getBaseClassname, getSafeClassnameStem } from "@/lib/furni/classname";
+import path from "node:path";
 import {
 	DEFAULT_NITRO_SOURCES,
 	listSources,
 } from "@/lib/services/clone-sources";
 import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs";
+import { getCleanupValidStems } from "@/lib/services/furni-import-cache";
 import { downloadFile } from "@/lib/services/import/core/download";
 import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
 import { getRuntimePath } from "@/lib/utils/runtime-path";
@@ -57,6 +57,23 @@ export interface NitroCleanupScan {
 	orphanedSwf: AssetCleanupEntry[];
 	orphanedIcon: AssetCleanupEntry[];
 	total: number;
+	/** True when the result was served from the on-disk scan cache. */
+	cached?: boolean;
+}
+
+export interface CleanupProgress {
+	phase: "stems" | "readdir" | "validate" | "done";
+	scanned: number;
+}
+
+export interface CleanupScanOptions {
+	/** Bypass the scan cache and re-validate every bundle. */
+	force?: boolean;
+	/**
+	 * Coarse progress callback used to stream scan status to the UI instead of
+	 * leaving the tab looking frozen while a large directory is processed.
+	 */
+	onProgress?: (progress: CleanupProgress) => void;
 }

 export interface NitroCleanupDeleteResult {
@@ -198,18 +215,6 @@ async function hasValidNitroHeader(filePath: string): Promise<boolean> {
 	}
 }

-/** Register every .nitro file stem a classname could legitimately use. */
-function addStems(set: Set<string>, classname: string): void {
-	const normalized = classname.trim().toLowerCase();
-	if (!normalized) return;
-	const base = getBaseClassname(normalized).toLowerCase();
-	if (base) set.add(base);
-	if (normalized.includes("*")) {
-		const safe = getSafeClassnameStem(normalized).toLowerCase();
-		if (safe) set.add(safe);
-	}
-}
-
 interface DirectoryRead {
 	name: string;
 	base: string;
@@ -217,6 +222,23 @@ interface DirectoryRead {
 	lastModified: number;
 }

+/** Discriminated cache key per asset directory (kind + path). */
+function kindKey(kind: CleanupAssetKind, dir: string): string {
+	return `${kind}\0${dir}`;
+}
+
+async function readDirNames(dir: string, regex: RegExp): Promise<string[]> {
+	const names = await fs.readdir(dir).catch(() => [] as string[]);
+	return names.filter((name) => regex.test(name)).sort();
+}
+
+/** Cheap fingerprint of a directory listing — a rename/add/remove changes it. */
+function dirNameSignature(names: string[]): string {
+	const hash = createHash("sha1");
+	for (const name of names) hash.update(name).update("\n");
+	return hash.digest("hex");
+}
+
 /**
 * How many `stat` calls run in parallel while reading an asset directory.
 * Directories can hold hundreds of thousands of files; serial `stat`ing them
@@ -224,14 +246,12 @@ interface DirectoryRead {
 */
 const DIR_STAT_CONCURRENCY = 32;

-async function readAssetDir(
+async function buildDirEntries(
 	dir: string,
-	regex: RegExp,
+	names: string[],
 ): Promise<DirectoryRead[]> {
-	const names = await fs.readdir(dir).catch(() => [] as string[]);
 	const entries: DirectoryRead[] = [];
 	await runPool(names, DIR_STAT_CONCURRENCY, async (name) => {
-		if (!regex.test(name)) return;
 		let size = 0;
 		let mtimeMs = 0;
 		try {
@@ -252,12 +272,14 @@ async function readAssetDir(
 }

 async function collectAssets(
+	kind: CleanupAssetKind,
 	dirs: string[],
-	regex: RegExp,
+	namesByKey: Map<string, string[]>,
 ): Promise<Map<string, AssetCleanupEntry>> {
 	const byName = new Map<string, AssetCleanupEntry>();
 	for (const dir of dirs) {
-		for (const read of await readAssetDir(dir, regex)) {
+		const names = namesByKey.get(kindKey(kind, dir)) ?? [];
+		for (const read of await buildDirEntries(dir, names)) {
 			const entry = byName.get(read.name) ?? {
 				fileName: read.name,
 				base: read.base,
@@ -274,35 +296,66 @@ async function collectAssets(
 	return byName;
 }

-async function loadValidStems(): Promise<Set<string>> {
-	// Authoritative stem set from items_base (item_name + public_name
-	// fallback), mirroring repair-nitros.ts classname resolution.
-	const validStems = new Set<string>();
-	try {
-		const result = await queryRows<{
-			item_name: string;
-			public_name: string;
-		}>(sql`
-			SELECT item_name, public_name FROM items_base
-		`);
-		for (const row of result) {
-			addStems(validStems, row.item_name);
-			if (
-				row.public_name &&
-				row.public_name !== row.item_name &&
-				!row.public_name.includes(" ") &&
-				row.public_name !== "0"
-			) {
-				addStems(validStems, row.public_name);
-			}
-		}
-	} catch {
-		/* empty set → every asset is treated as orphaned */
-	}
-	return validStems;
+// ── On-disk scan cache ──────────────────────────────────────────────────────
+//
+// The heavy parts of a scan are the per-bundle header validation (one open +
+// small read per .nitro) and the parallel stat pass. Both can be skipped when
+// no asset directory changed since the last scan: a signature per directory is
+// derived from the sortable file *list* (readdir + hash, no stat). Any add /
+// remove / rename invalidates the cache and triggers a full re-validation.
+// Bundles rewritten in place without a name change are not detected by the
+// signature; run a forced scan (`force: true`) to pick those up.
+
+const SCAN_CACHE_DIR = path.join(process.cwd(), "storage", "nitro-cleanup");
+const scanCachePath = () => path.join(SCAN_CACHE_DIR, "scan-cache.json");
+
+interface ScanCacheFile {
+	dirs: Record<string, string>;
+	result: Omit<NitroCleanupScan, "cached">;
 }

-export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
+async function readScanCache(): Promise<ScanCacheFile | null> {
+	try {
+		if (!existsSync(scanCachePath())) return null;
+		const raw = await fs.readFile(scanCachePath(), "utf8");
+		const parsed = JSON.parse(raw) as ScanCacheFile;
+		if (!parsed || typeof parsed.dirs !== "object" || !parsed.result) {
+			return null;
+		}
+		return parsed;
+	} catch {
+		return null;
+	}
+}
+
+async function writeScanCache(
+	dirs: Record<string, string>,
+	result: Omit<NitroCleanupScan, "cached">,
+): Promise<void> {
+	try {
+		await fs.mkdir(SCAN_CACHE_DIR, { recursive: true });
+		await fs.writeFile(scanCachePath(), JSON.stringify({ dirs, result }));
+	} catch {
+		/* cache is best effort — a broken cache simply re-scans next time */
+	}
+}
+
+function cacheDirsMatch(
+	cached: Record<string, string>,
+	current: Record<string, string>,
+): boolean {
+	const cachedKeys = Object.keys(cached);
+	if (cachedKeys.length !== Object.keys(current).length) return false;
+	for (const key of cachedKeys) {
+		if (cached[key] !== current[key]) return false;
+	}
+	return true;
+}
+
+export async function scanFakeBrokenNitros(
+	options: CleanupScanOptions = {},
+): Promise<NitroCleanupScan> {
+	const { force = false, onProgress } = options;
 	const targets = await getFurniAssetWriteTargets();
 	const nitroDirs = uniqueDirs([
 		targets.nitroDir,
@@ -316,47 +369,94 @@ export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
 		targets.iconDir,
 		...targets.mirrorDirs.map((d) => d.iconDir),
 	]);
+	const directoryPlan: Array<{
+		kind: CleanupAssetKind;
+		dirs: string[];
+	}> = [
+		{ kind: "nitro", dirs: nitroDirs },
+		{ kind: "swf", dirs: swfDirs },
+		{ kind: "icon", dirs: iconDirs },
+	];

-	const [byNitro, bySwf, byIcon, validStems] = await Promise.all([
-		collectAssets(nitroDirs, NITRO_FILE_RE),
-		collectAssets(swfDirs, SWF_FILE_RE),
-		collectAssets(iconDirs, ICON_FILE_RE),
-		loadValidStems(),
+	onProgress?.({ phase: "readdir", scanned: 0 });
+
+	// One readdir pass shared by both the cache signature and the full scan —
+	// never stat a file just to learn the directory changed.
+	const namesByKey = new Map<string, string[]>();
+	await Promise.all(
+		directoryPlan.flatMap(({ kind, dirs }) =>
+			dirs.map(async (dir) => {
+				namesByKey.set(
+					kindKey(kind, dir),
+					await readDirNames(dir, KIND_FILE_RE[kind]),
+				);
+			}),
+		),
+	);
+	const currentDigests: Record<string, string> = {};
+	for (const key of namesByKey.keys()) {
+		currentDigests[key] = dirNameSignature(namesByKey.get(key) ?? []);
+	}
+
+	onProgress?.({ phase: "readdir", scanned: 0 });
+	if (!force) {
+		const cache = await readScanCache();
+		if (cache && cacheDirsMatch(cache.dirs, currentDigests)) {
+			onProgress?.({ phase: "done", scanned: cache.result.total });
+			return { ...cache.result, cached: true };
+		}
+	}
+
+	// Needed only for classification — skip the DB query on a cache hit.
+	onProgress?.({ phase: "stems", scanned: 0 });
+	const validStems = await getCleanupValidStems();
+
+	const [byNitro, bySwf, byIcon] = await Promise.all([
+		collectAssets("nitro", nitroDirs, namesByKey),
+		collectAssets("swf", swfDirs, namesByKey),
+		collectAssets("icon", iconDirs, namesByKey),
 	]);

 	const entries: NitroCleanupEntry[] = [...byNitro.values()];
 	const fake: NitroCleanupEntry[] = [];
 	const broken: NitroCleanupEntry[] = [];
+	let validated = 0;
+	const totalToValidate = entries.length;

 	await runPool(entries, 16, async (entry) => {
 		// A .nitro that no DB item maps to is a leftover / fake bundle.
 		if (!validStems.has(entry.base.toLowerCase())) {
 			fake.push(entry);
-			return;
-		}
-		// Otherwise validate the bundle content. Most bundles pass a cheap
-		// header-only check (a few KB, no decompression); only files whose
-		// header looks suspicious get the expensive full parse. This keeps the
-		// scan fast even on directories with hundreds of thousands of bundles.
-		let parsed = false;
-		let firstError = "";
-		for (const dir of entry.dirs) {
-			if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) {
-				parsed = true;
-				break;
+		} else {
+			// Otherwise validate the bundle content. Most bundles pass a cheap
+			// header-only check (a few KB, no decompression); only files whose
+			// header looks suspicious get the expensive full parse. This keeps
+			// the scan fast even on directories with hundreds of thousands of
+			// bundles.
+			let parsed = false;
+			let firstError = "";
+			for (const dir of entry.dirs) {
+				if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) {
+					parsed = true;
+					break;
+				}
+				try {
+					parseNitroBundle(
+						await fs.readFile(getRuntimePath(dir, entry.fileName)),
+					);
+					parsed = true;
+					break;
+				} catch (err) {
+					firstError = (err as Error).message;
+				}
 			}
-			try {
-				parseNitroBundle(
-					await fs.readFile(getRuntimePath(dir, entry.fileName)),
-				);
-				parsed = true;
-				break;
-			} catch (err) {
-				firstError = (err as Error).message;
+			if (!parsed) {
+				broken.push({ ...entry, error: firstError });
 			}
 		}
-		if (!parsed) {
-			broken.push({ ...entry, error: firstError });
+		validated++;
+		if (validated % 2500 === 0 || validated === totalToValidate) {
+			onProgress?.({ phase: "validate", scanned: validated });
 		}
 	});

@@ -370,13 +470,16 @@ export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
 	const byName = <T extends { fileName: string }>(entries: T[]): T[] =>
 		entries.sort((a, b) => a.fileName.localeCompare(b.fileName));

-	return {
+	const result: NitroCleanupScan = {
 		fake: byName(fake),
 		broken: byName(broken),
 		orphanedSwf: byName(orphanedSwf),
 		orphanedIcon: byName(orphanedIcon),
 		total: byNitro.size + bySwf.size + byIcon.size,
 	};
+	await writeScanCache(currentDigests, result);
+	onProgress?.({ phase: "done", scanned: result.total });
+	return result;
 }

 function dirsForKind(
@@ -435,7 +538,7 @@ export async function deleteNitroCleanupFiles(
 	return { deleted, copiesRemoved, errors, files };
 }

-function ageDays(lastModified: number): number {
+export function ageDays(lastModified: number): number {
 	if (!lastModified) return 0;
 	return Math.max(0, Math.floor((Date.now() - lastModified) / DAY_MS));
 }