diff --git a/src/app/api/admin/studio/nitro-cleanup/route.ts b/src/app/api/admin/studio/nitro-cleanup/route.ts index 4c69c0c5..8d5c5436 100644 --- a/src/app/api/admin/studio/nitro-cleanup/route.ts +++ b/src/app/api/admin/studio/nitro-cleanup/route.ts @@ -4,31 +4,115 @@ import { PERMS } from "@/lib/permissions"; import { logAudit } from "@/lib/services/audit"; import { clearFurniImportCache } from "@/lib/services/furni-import-cache"; import { + ageDays, type CleanupAssetKind, + type CleanupProgress, deleteNitroCleanupFiles, scanFakeBrokenNitros, } from "@/lib/services/nitro-cleanup"; const VALID_KINDS: readonly CleanupAssetKind[] = ["nitro", "swf", "icon"]; +function intParam(value: string | null): number | null { + if (value === null) return null; + const n = Number(value); + return Number.isFinite(n) && n >= 0 ? Math.floor(n) : null; +} + /** * Scan all asset directories for fake (orphaned) nitro / swf / icon files and * broken (unparseable) .nitro bundles. + * + * Query params: + * - `force=1` skips the on-disk scan cache (fresh re-validation); + * - `minAgeDays=N` only returns entries older than N days; + * - `limit=N` caps each returned list (use `full=1` to override); + * - `full=1` never applies the limit (used by the manifest export). + * + * When the client asks for `text/event-stream` the scan is streamed as + * `progress` / `result` / `error` events, keeping the connection alive while a + * large nitro directory is walked so reverse proxies don't time the request out. */ -export const GET = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => { - try { - const scan = await scanFakeBrokenNitros(); - return apiOk({ - fake: scan.fake, - broken: scan.broken, - orphanedSwf: scan.orphanedSwf, - orphanedIcon: scan.orphanedIcon, - total: scan.total, - }); - } catch (err) { - return apiError(`Asset scan failed: ${(err as Error).message}`, 500); - } -}); +export const GET = withAdmin( + { permission: PERMS.ASSETS_IMPORT }, + async (request) => { + const wantsSse = request.headers + .get("accept") + ?.includes("text/event-stream"); + const force = request.nextUrl.searchParams.get("force") === "1"; + const full = request.nextUrl.searchParams.get("full") === "1"; + const minAgeDays = intParam(request.nextUrl.searchParams.get("minAgeDays")); + const limit = intParam(request.nextUrl.searchParams.get("limit")); + + const run = async (onProgress?: (p: CleanupProgress) => void) => { + const scan = await scanFakeBrokenNitros({ force, onProgress }); + const ageFilter = ( + entries: T[], + ): T[] => + minAgeDays !== null && minAgeDays > 0 + ? entries.filter((entry) => ageDays(entry.lastModified) >= minAgeDays) + : entries; + const cap = (entries: T[]): T[] => + full || limit === null || limit <= 0 + ? entries + : entries.slice(0, limit); + return { + fake: cap(ageFilter(scan.fake)), + broken: cap(ageFilter(scan.broken)), + orphanedSwf: cap(ageFilter(scan.orphanedSwf)), + orphanedIcon: cap(ageFilter(scan.orphanedIcon)), + total: scan.total, + cached: scan.cached ?? false, + }; + }; + + try { + if (!wantsSse) { + return apiOk(await run()); + } + const encoder = new TextEncoder(); + const stream = new ReadableStream({ + async start(controller) { + const send = (data: unknown) => { + try { + controller.enqueue( + encoder.encode(`data: ${JSON.stringify(data)}\n\n`), + ); + } catch { + /* stream closed by client */ + } + }; + try { + const result = await run((progress) => + send({ type: "progress", ...progress }), + ); + send({ type: "result", ...result }); + } catch (err) { + send({ + type: "error", + message: `Asset scan failed: ${(err as Error).message}`, + }); + } + try { + controller.close(); + } catch { + /* ignore */ + } + }, + }); + + return new Response(stream, { + headers: { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }, + }); + } catch (err) { + return apiError(`Asset scan failed: ${(err as Error).message}`, 500); + } + }, +); /** * Delete the given asset files (default .nitro) from every matching directory diff --git a/src/components/admin/studio/nitro-cleanup-panel.tsx b/src/components/admin/studio/nitro-cleanup-panel.tsx index 0a4cea2d..1c8fff73 100644 --- a/src/components/admin/studio/nitro-cleanup-panel.tsx +++ b/src/components/admin/studio/nitro-cleanup-panel.tsx @@ -1,5 +1,6 @@ "use client"; +import { useVirtualizer } from "@tanstack/react-virtual"; import { AlertTriangle, CheckCircle2, @@ -11,8 +12,9 @@ import { Sparkles, Trash2, Wrench, + X, } from "lucide-react"; -import { useCallback, useMemo, useState } from "react"; +import { useCallback, useRef, useState } from "react"; import { toast } from "sonner"; import { useConfirmDialog } from "@/components/admin/confirm-dialog"; import { Button } from "@/components/ui/button"; @@ -20,6 +22,7 @@ import { Checkbox } from "@/components/ui/checkbox"; import { Input } from "@/components/ui/input"; import { adminFetch } from "@/lib/admin-fetch"; import { formatBytes } from "@/lib/services/disk-usage"; +import { readSseStream } from "@/lib/sse-client"; interface AssetEntry { fileName: string; @@ -36,6 +39,7 @@ interface NitroCleanupScan { orphanedSwf: AssetEntry[]; orphanedIcon: AssetEntry[]; total: number; + cached?: boolean; } interface DeleteResponse { @@ -62,13 +66,7 @@ type GroupTone = "error" | "warning" | "neutral"; type AssetKind = "nitro" | "swf" | "icon"; const DAY_MS = 24 * 60 * 60 * 1000; - -/** - * How many rows a group renders at once. Directories can contain tens of - * thousands of leftover files; mounting every row as DOM nodes freezes the - * tab, so rows are added in chunks via a "load more" button. - */ -const RENDER_CHUNK = 500; +const SCAN_IDLE_TIMEOUT_MS = 120_000; function ageLabel(lastModified: number): string { if (!lastModified) return "age unknown"; @@ -91,6 +89,87 @@ function downloadJson(filename: string, data: unknown): void { URL.revokeObjectURL(url); } +/** + * Virtualized list for one cleanup group. Directories can hold tens of + * thousands of leftover files; only the visible rows are mounted as DOM nodes, + * so the tab never freezes no matter how long the list is. + */ +function VirtualAssetList({ + entries, + selected, + disabled = false, + onToggle, +}: { + entries: AssetEntry[]; + selected: Set; + disabled?: boolean; + onToggle: (fileName: string) => void; +}) { + const parentRef = useRef(null); + const virtualizer = useVirtualizer({ + count: entries.length, + getScrollElement: () => parentRef.current, + estimateSize: () => 30, + overscan: 10, + }); + + return ( +
+
+ {virtualizer.getVirtualItems().map((virtualItem) => { + const entry = entries[virtualItem.index]; + return ( +
+ onToggle(entry.fileName)} + aria-label={`Select ${entry.fileName}`} + /> + + {entry.fileName} + + + {formatBytes(entry.size)} + + + {ageLabel(entry.lastModified)} + + {entry.error && ( + + {entry.error} + + )} + {entry.dirs.length > 1 && ( + + {entry.dirs.length} copies + + )} +
+ ); + })} +
+
+ ); +} + function CleanupGroup({ label, description, @@ -110,11 +189,8 @@ function CleanupGroup({ onToggle: (fileName: string) => void; onToggleAll: (names: string[]) => void; }) { - const [shown, setShown] = useState(RENDER_CHUNK); - const checked = entries.filter((e) => selected.has(e.fileName)); const allChecked = entries.length > 0 && checked.length === entries.length; - const visibleEntries = entries.slice(0, shown); const toneClass = tone === "error" ? "border-[var(--admin-error-border)] bg-[var(--admin-error-subtle)]" @@ -158,63 +234,12 @@ function CleanupGroup({ No {label.toLowerCase()} found

) : ( - <> -
    - {visibleEntries.map((entry) => ( -
  • - onToggle(entry.fileName)} - aria-label={`Select ${entry.fileName}`} - /> - - {entry.fileName} - - - {formatBytes(entry.size)} - - - {ageLabel(entry.lastModified)} - - {entry.error && ( - - {entry.error} - - )} - {entry.dirs.length > 1 && ( - - {entry.dirs.length} copies - - )} -
    • - ))} -
- {entries.length > shown && ( -
- - Showing {shown} of {entries.length} - - -
- )} - + )} ); @@ -225,63 +250,113 @@ export function NitroCleanupPanel() { const [deleting, setDeleting] = useState(false); const [repairing, setRepairing] = useState(false); const [autoCleaning, setAutoCleaning] = useState(false); + const [exporting, setExporting] = useState(false); const [autoAge, setAutoAge] = useState(30); const [minAgeDays, setMinAgeDays] = useState(0); const [onlyOld, setOnlyOld] = useState(false); + const [progress, setProgress] = useState<{ + phase: string; + scanned: number; + } | null>(null); const [scan, setScan] = useState(null); const [selected, setSelected] = useState>(new Set()); const [selectedSwf, setSelectedSwf] = useState>(new Set()); const [selectedIcon, setSelectedIcon] = useState>(new Set()); + const abortRef = useRef(null); const { confirm, dialog } = useConfirmDialog(); - const scanNow = useCallback(async () => { - setScanning(true); - setSelected(new Set()); - setSelectedSwf(new Set()); - setSelectedIcon(new Set()); - try { - const res = await adminFetch("/api/admin/studio/nitro-cleanup"); - const type = res.headers.get("content-type") ?? ""; - if (!res.ok) { - const raw = await res.text(); - throw new Error( - raw && !type.includes("application/json") - ? `Scan failed (HTTP ${res.status}) — the server or reverse proxy returned a non-JSON response. This usually means a proxy/worker timeout; try again or raise the upstream timeout.` - : `Scan failed (HTTP ${res.status})`, + const scanNow = useCallback( + async (opts: { force?: boolean; minAgeDays?: number } = {}) => { + const force = opts.force ?? true; + const ageFilter = opts.minAgeDays ?? (onlyOld ? minAgeDays : 0); + abortRef.current?.abort(); + setScanning(true); + setProgress(null); + setSelected(new Set()); + setSelectedSwf(new Set()); + setSelectedIcon(new Set()); + + const abort = new AbortController(); + abortRef.current = abort; + const params = new URLSearchParams(); + if (force) params.set("force", "1"); + if (ageFilter > 0) params.set("minAgeDays", String(ageFilter)); + + const holder: { result: NitroCleanupScan | null } = { result: null }; + let lastError: string | null = null; + try { + const res = await adminFetch( + `/api/admin/studio/nitro-cleanup${ + params.size > 0 ? `?${params.toString()}` : "" + }`, + { headers: { Accept: "text/event-stream" }, signal: abort.signal }, ); - } - if (!type.includes("application/json")) { - throw new Error( - "Scan failed — the server returned a non-JSON response (likely an upstream timeout or proxy error page).", + if (!res.ok) { + const data = (await res.json().catch(() => null)) as { + error?: string; + } | null; + throw new Error( + typeof data?.error === "string" + ? data.error + : `Scan failed (HTTP ${res.status})`, + ); + } + if (!res.body) throw new Error("Scan returned no response body"); + + await readSseStream( + res.body, + (evt) => { + if (evt.type === "progress") { + setProgress({ + phase: String(evt.phase ?? ""), + scanned: Number(evt.scanned ?? 0), + }); + } else if (evt.type === "result") { + holder.result = { + fake: (evt.fake as AssetEntry[]) ?? [], + broken: (evt.broken as AssetEntry[]) ?? [], + orphanedSwf: (evt.orphanedSwf as AssetEntry[]) ?? [], + orphanedIcon: (evt.orphanedIcon as AssetEntry[]) ?? [], + total: Number(evt.total ?? 0), + cached: evt.cached === true, + }; + setScan(holder.result); + } else if (evt.type === "error") { + lastError = String(evt.message ?? "Scan failed"); + } + }, + abort.signal, + SCAN_IDLE_TIMEOUT_MS, ); + + if (lastError) throw new Error(lastError); + const lastResult = holder.result; + if (lastResult) { + const found = + lastResult.fake.length + + lastResult.broken.length + + lastResult.orphanedSwf.length + + lastResult.orphanedIcon.length; + if (found === 0) { + toast.success("No orphaned or broken furniture assets found"); + } else { + toast.info(`Found ${found} removable asset file(s)`); + } + } + } catch (err) { + if ((err as Error).name === "AbortError") { + toast.info("Scan cancelled"); + } else { + toast.error(err instanceof Error ? err.message : "Scan failed"); + } + } finally { + setScanning(false); + setProgress(null); + if (abortRef.current === abort) abortRef.current = null; } - const data = (await res.json()) as NitroCleanupScan & { - error?: string; - }; - if (data.error) throw new Error(data.error); - setScan({ - fake: data.fake ?? [], - broken: data.broken ?? [], - orphanedSwf: data.orphanedSwf ?? [], - orphanedIcon: data.orphanedIcon ?? [], - total: data.total ?? 0, - }); - const found = - data.fake.length + - data.broken.length + - data.orphanedSwf.length + - data.orphanedIcon.length; - if (found === 0) { - toast.success("No orphaned or broken furniture assets found"); - } else { - toast.info(`Found ${found} removable asset file(s)`); - } - } catch (err) { - toast.error(err instanceof Error ? err.message : "Scan failed"); - } finally { - setScanning(false); - } - }, []); + }, + [onlyOld, minAgeDays], + ); const toggle = (kind: AssetKind) => (fileName: string) => { const setter = @@ -318,6 +393,25 @@ export function NitroCleanupPanel() { const totalSelected = selected.size + selectedSwf.size + selectedIcon.size; + /** Drop deleted/attempted files from the loaded scan without re-scanning. */ + const dropFromScan = (kind: AssetKind, names: string[]) => { + const remove = new Set(names); + if (remove.size === 0) return; + setScan((prev) => { + if (!prev) return prev; + const drop = (list: T[]): T[] => + list.filter((entry) => !remove.has(entry.fileName)); + return { + ...prev, + fake: kind === "nitro" ? drop(prev.fake) : prev.fake, + broken: kind === "nitro" ? drop(prev.broken) : prev.broken, + orphanedSwf: kind === "swf" ? drop(prev.orphanedSwf) : prev.orphanedSwf, + orphanedIcon: + kind === "icon" ? drop(prev.orphanedIcon) : prev.orphanedIcon, + }; + }); + }; + const removeSelected = async () => { const groups: Array<{ kind: AssetKind; names: string[] }> = []; for (const [kind, set] of [ @@ -358,6 +452,7 @@ export function NitroCleanupPanel() { deleted += data.deleted ?? 0; copiesRemoved += data.copiesRemoved ?? 0; if (data.errors?.length) hadErrors = true; + dropFromScan(group.kind, group.names); } if (hadErrors) { toast.warning( @@ -371,7 +466,6 @@ export function NitroCleanupPanel() { setSelected(new Set()); setSelectedSwf(new Set()); setSelectedIcon(new Set()); - await scanNow(); } catch (err) { toast.error(err instanceof Error ? err.message : "Delete failed"); } finally { @@ -408,8 +502,12 @@ export function NitroCleanupPanel() { } else { toast.warning(`Could not re-obtain any bundle (${data.failed} failed)`); } + // Only drop the attempted bundles when every one succeeded; partial + // failures stay visible so they can be retried. + if (data.repaired >= fileNames.length) { + dropFromScan("nitro", fileNames); + } setSelected(new Set()); - await scanNow(); } catch (err) { toast.error(err instanceof Error ? err.message : "Repair failed"); } finally { @@ -445,12 +543,18 @@ export function NitroCleanupPanel() { toast.success( `Auto-cleaned ${data.deleted} old fake .nitro file(s) (${data.copiesRemoved} copies). Skipped ${data.skippedRecent} recent + ${data.skippedBroken} broken.`, ); + const removable = (scan.fake ?? []) + .filter( + (entry) => + Math.floor((Date.now() - entry.lastModified) / DAY_MS) >= autoAge, + ) + .map((entry) => entry.fileName); + dropFromScan("nitro", removable); } else { toast.info( `Nothing old enough to remove (skipped ${data.skippedRecent} recent, ${data.skippedBroken} broken)`, ); } - await scanNow(); } catch (err) { toast.error(err instanceof Error ? err.message : "Auto-clean failed"); } finally { @@ -458,57 +562,72 @@ export function NitroCleanupPanel() { } }; - const exportManifest = () => { + const exportManifest = async () => { if (scan === null) return; - const entries = { - scannedAt: new Date().toISOString(), - total: scan.total, - fake: scan.fake, - broken: scan.broken, - orphanedSwf: scan.orphanedSwf, - orphanedIcon: scan.orphanedIcon, - selected: { - nitro: [...selected].sort(), - swf: [...selectedSwf].sort(), - icon: [...selectedIcon].sort(), - }, - }; - downloadJson(`nitro-cleanup-scan-${Date.now()}.json`, entries); - toast.success("Scan manifest downloaded"); + setExporting(true); + try { + const res = await adminFetch("/api/admin/studio/nitro-cleanup?full=1", { + headers: { Accept: "application/json" }, + }); + const data = (await res.json()) as NitroCleanupScan & { + error?: string; + }; + if (!res.ok || data.error) { + throw new Error(data.error || `Export failed (HTTP ${res.status})`); + } + const entries = { + scannedAt: new Date().toISOString(), + total: data.total ?? 0, + fake: data.fake ?? [], + broken: data.broken ?? [], + orphanedSwf: data.orphanedSwf ?? [], + orphanedIcon: data.orphanedIcon ?? [], + selected: { + nitro: [...selected].sort(), + swf: [...selectedSwf].sort(), + icon: [...selectedIcon].sort(), + }, + }; + downloadJson(`nitro-cleanup-scan-${Date.now()}.json`, entries); + toast.success("Scan manifest downloaded"); + } catch (err) { + toast.error(err instanceof Error ? err.message : "Export failed"); + } finally { + setExporting(false); + } }; - const visibleFake = useMemo(() => { - if (scan === null) return []; - if (!onlyOld || minAgeDays <= 0) return scan.fake; - const threshold = minAgeDays * DAY_MS; - return scan.fake.filter((e) => e.lastModified <= Date.now() - threshold); - }, [scan, onlyOld, minAgeDays]); + const toggleOnlyOld = (value: boolean) => { + setOnlyOld(value); + const threshold = value ? Math.max(0, minAgeDays) : undefined; + void scanNow({ force: false, minAgeDays: threshold ?? 0 }); + }; - const visibleSwf = useMemo(() => { - if (scan === null) return []; - if (!onlyOld || minAgeDays <= 0) return scan.orphanedSwf; - const threshold = minAgeDays * DAY_MS; - return scan.orphanedSwf.filter( - (e) => e.lastModified <= Date.now() - threshold, - ); - }, [scan, onlyOld, minAgeDays]); + const changeMinAgeDays = (value: number) => { + const days = Number.isFinite(value) && value >= 0 ? Math.floor(value) : 0; + setMinAgeDays(days); + if (onlyOld) void scanNow({ force: false, minAgeDays: days }); + }; - const visibleIcon = useMemo(() => { - if (scan === null) return []; - if (!onlyOld || minAgeDays <= 0) return scan.orphanedIcon; - const threshold = minAgeDays * DAY_MS; - return scan.orphanedIcon.filter( - (e) => e.lastModified <= Date.now() - threshold, - ); - }, [scan, onlyOld, minAgeDays]); - - const busy = scanning || deleting || repairing || autoCleaning; + const busy = scanning || deleting || repairing || autoCleaning || exporting; + const phaseLabel = (phase: string): string => + phase === "stems" + ? "Loading furniture stems…" + : phase === "readdir" + ? "Reading asset directories…" + : phase === "validate" + ? "Validating .nitro bundles…" + : "Scanning…"; return (
{dialog}
- + {scanning && ( + + )} {scan && ( )}
+ {scanning && progress && ( +
+ + {phaseLabel(progress.phase)} + {progress.scanned > 0 && ( + + {progress.scanned.toLocaleString()} files + + )} +
+ )} + {scan && (
@@ -602,7 +747,7 @@ export function NitroCleanupPanel() { className="w-20" value={minAgeDays} disabled={busy} - onChange={(e) => setMinAgeDays(Number(e.target.value))} + onChange={(e) => changeMinAgeDays(Number(e.target.value))} aria-label="Minimum age in days to display" /> days @@ -613,7 +758,7 @@ export function NitroCleanupPanel() { setOnlyOld(value === true)} + onCheckedChange={(value) => toggleOnlyOld(value === true)} /> Hide recent files @@ -624,6 +769,12 @@ export function NitroCleanupPanel() {
{scan.total}
asset files scanned + {scan.cached && ( + + {" "} + (cached — directories unchanged) + + )}
@@ -648,7 +799,7 @@ export function NitroCleanupPanel() { label="Fake nitros" description="These files match no item in items_base — leftovers from deleted or renamed furniture." tone="error" - entries={visibleFake} + entries={scan.fake} selected={selected} onToggle={toggle("nitro")} onToggleAll={toggleAll("nitro")} @@ -657,7 +808,7 @@ export function NitroCleanupPanel() { label="Broken nitros" description="These files belong to an item but cannot be parsed as a valid Nitro bundle." tone="warning" - entries={onlyOld ? [] : scan.broken} + entries={scan.broken} selected={selected} onToggle={toggle("nitro")} onToggleAll={toggleAll("nitro")} @@ -667,7 +818,7 @@ export function NitroCleanupPanel() { label="Orphaned SWFs" description=".swf files that match no item in items_base." tone="warning" - entries={visibleSwf} + entries={scan.orphanedSwf} selected={selectedSwf} onToggle={toggle("swf")} onToggleAll={toggleAll("swf")} @@ -676,7 +827,7 @@ export function NitroCleanupPanel() { label="Orphaned icons" description="Icon files that match no item in items_base." tone="neutral" - entries={visibleIcon} + entries={scan.orphanedIcon} selected={selectedIcon} onToggle={toggle("icon")} onToggleAll={toggleAll("icon")} diff --git a/src/lib/services/furni-import-cache.ts b/src/lib/services/furni-import-cache.ts index 1ce59549..eb38b278 100644 --- a/src/lib/services/furni-import-cache.ts +++ b/src/lib/services/furni-import-cache.ts @@ -16,7 +16,11 @@ import { existsSync } from "node:fs"; import path from "node:path"; import { sql } from "drizzle-orm"; import { db, ItemsBase, queryRows } from "@/lib/db"; -import { normalizeClassname } from "@/lib/furni/classname"; +import { + getBaseClassname, + getSafeClassnameStem, + normalizeClassname, +} from "@/lib/furni/classname"; import { indexLocalFurniture } from "@/lib/furni/local-presence"; import { readFurniData } from "./furni-data"; @@ -107,6 +111,58 @@ export async function getCatalogIdSet(): Promise> { const nitroPresenceCache = new Map(); +/** Every classname stem a `.nitro`/`.swf`/icon file may legitimately use. */ +function addClassnameStem(set: Set, classname: string): void { + const normalized = classname.trim().toLowerCase(); + if (!normalized) return; + const base = getBaseClassname(normalized).toLowerCase(); + if (base) set.add(base); + if (normalized.includes("*")) { + const safe = getSafeClassnameStem(normalized).toLowerCase(); + if (safe) set.add(safe); + } +} + +let cleanupStemsSnapshot: { set: Set; timestamp: number } | null = null; + +/** + * Authoritative stems for the nitro-cleanup scan: every `item_name` plus the + * `public_name` fallback (mirroring repair-nitros.ts resolution). Cached the + * same way as the Studio listings so repeated scans (auto-clean, panel) don't + * re-query `items_base` on every call. + */ +export async function getCleanupValidStems(): Promise> { + const now = Date.now(); + if ( + cleanupStemsSnapshot && + now - cleanupStemsSnapshot.timestamp < CACHE_TTL + ) { + return cleanupStemsSnapshot.set; + } + const set = new Set(); + try { + const rows = await queryRows<{ + item_name: string; + public_name: string; + }>(sql`SELECT item_name, public_name FROM items_base`); + for (const row of rows) { + addClassnameStem(set, row.item_name); + if ( + row.public_name && + row.public_name !== row.item_name && + !row.public_name.includes(" ") && + row.public_name !== "0" + ) { + addClassnameStem(set, row.public_name); + } + } + } catch { + /* empty set → every asset is treated as orphaned */ + } + cleanupStemsSnapshot = { set, timestamp: now }; + return set; +} + /** `existsSync` memoized per (dir + base) — nitro files change rarely. */ export function nitroExistsCached(dir: string, base: string): boolean { const key = `${dir}/${base}`; @@ -210,5 +266,6 @@ export function clearFurniImportCache(): void { furniDataSetSnapshot = null; catalogIdsSnapshot = null; nitroPresenceCache.clear(); + cleanupStemsSnapshot = null; statsSnapshot = null; } diff --git a/src/lib/services/nitro-cleanup.test.ts b/src/lib/services/nitro-cleanup.test.ts index dba657b5..319884db 100644 --- a/src/lib/services/nitro-cleanup.test.ts +++ b/src/lib/services/nitro-cleanup.test.ts @@ -1,4 +1,5 @@ // @vitest-environment node +import { createHash } from "node:crypto"; import { beforeEach, describe, expect, it, vi } from "vitest"; type AnyFn = (...args: any[]) => any; @@ -130,6 +131,13 @@ function mockReaddir(byDir: Record): void { readdirFn.mockImplementation(async (dir: string) => byDir[dir] ?? []); } +/** Replicates `dirNameSignature` in the service so cache fixtures match. */ +function dirSignature(names: string[]): string { + const hash = createHash("sha1"); + for (const name of names) hash.update(name).update("\n"); + return hash.digest("hex"); +} + describe("scanFakeBrokenNitros", () => { it("classifies fake (no matching item) and broken (unparseable) bundles", async () => { mockReaddir({ @@ -233,6 +241,73 @@ describe("scanFakeBrokenNitros", () => { expect(result.fake[0].size).toBe(900); expect(result.fake[0].lastModified).toBe(2000); }); + + it("serves results from the on-disk cache when directories are unchanged", async () => { + mockReaddir({ + "/assets/nitro": ["chair.nitro", "ghost.nitro"], + }); + const dirs = { + "nitro\u0000/assets/nitro": dirSignature(["chair.nitro", "ghost.nitro"]), + "swf\u0000/assets/swf": dirSignature([]), + "icon\u0000/assets/icons": dirSignature([]), + }; + const cachedResult = { + fake: [ + { + fileName: "ghost.nitro", + base: "ghost", + size: 500, + dirs: ["/assets/nitro"], + lastModified: 0, + error: undefined, + }, + ], + broken: [], + orphanedSwf: [], + orphanedIcon: [], + total: 4, + }; + existsFn.mockImplementation((filePath: string) => + filePath.includes("scan-cache.json"), + ); + readFileFn.mockImplementation(async (filePath: string) => { + if (filePath.includes("scan-cache.json")) { + return JSON.stringify({ dirs, result: cachedResult }); + } + return Buffer.from("nitro-data"); + }); + + const result = await scanFakeBrokenNitros(); + + // `total: 4` could never come from a fresh scan of 2 files — the cache + // result came back verbatim, so classification was skipped entirely. + expect(result).toEqual(expect.objectContaining(cachedResult)); + expect(result.cached).toBe(true); + expect(result.total).toBe(4); + expect(executeFn).not.toHaveBeenCalled(); + expect(statFn).not.toHaveBeenCalled(); + expect(readFileFn).toHaveBeenCalledTimes(1); + expect(writeFileFn).not.toHaveBeenCalled(); + }); + + it("force bypasses the cache and re-validates the directories", async () => { + mockReaddir({ + "/assets/nitro": ["chair.nitro", "ghost.nitro"], + }); + // A matching-on-disk cache exists, but `force: true` ignores it. + existsFn.mockImplementation(() => true); + readFileFn.mockResolvedValue(Buffer.from("nitro-data")); + parseNitroBundle.mockReturnValue({}); + statFn.mockImplementation(async () => ({ size: 512, mtimeMs: 0 })); + + const result = await scanFakeBrokenNitros({ force: true }); + + expect(result.cached).toBeUndefined(); + expect(result.fake.map((e) => e.fileName)).toEqual(["ghost.nitro"]); + expect(result.broken).toEqual([]); + expect(result.total).toBe(2); + expect(statFn).toHaveBeenCalled(); + }); }); describe("deleteNitroCleanupFiles", () => { diff --git a/src/lib/services/nitro-cleanup.ts b/src/lib/services/nitro-cleanup.ts index 2efe4321..0c9aa42d 100644 --- a/src/lib/services/nitro-cleanup.ts +++ b/src/lib/services/nitro-cleanup.ts @@ -1,13 +1,13 @@ +import { createHash } from "node:crypto"; import { existsSync, promises as fs } from "node:fs"; import os from "node:os"; -import { sql } from "drizzle-orm"; -import { queryRows } from "@/lib/db"; -import { getBaseClassname, getSafeClassnameStem } from "@/lib/furni/classname"; +import path from "node:path"; import { DEFAULT_NITRO_SOURCES, listSources, } from "@/lib/services/clone-sources"; import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs"; +import { getCleanupValidStems } from "@/lib/services/furni-import-cache"; import { downloadFile } from "@/lib/services/import/core/download"; import { parseNitroBundle } from "@/lib/services/swf/nitro-builder"; import { getRuntimePath } from "@/lib/utils/runtime-path"; @@ -57,6 +57,23 @@ export interface NitroCleanupScan { orphanedSwf: AssetCleanupEntry[]; orphanedIcon: AssetCleanupEntry[]; total: number; + /** True when the result was served from the on-disk scan cache. */ + cached?: boolean; +} + +export interface CleanupProgress { + phase: "stems" | "readdir" | "validate" | "done"; + scanned: number; +} + +export interface CleanupScanOptions { + /** Bypass the scan cache and re-validate every bundle. */ + force?: boolean; + /** + * Coarse progress callback used to stream scan status to the UI instead of + * leaving the tab looking frozen while a large directory is processed. + */ + onProgress?: (progress: CleanupProgress) => void; } export interface NitroCleanupDeleteResult { @@ -198,18 +215,6 @@ async function hasValidNitroHeader(filePath: string): Promise { } } -/** Register every .nitro file stem a classname could legitimately use. */ -function addStems(set: Set, classname: string): void { - const normalized = classname.trim().toLowerCase(); - if (!normalized) return; - const base = getBaseClassname(normalized).toLowerCase(); - if (base) set.add(base); - if (normalized.includes("*")) { - const safe = getSafeClassnameStem(normalized).toLowerCase(); - if (safe) set.add(safe); - } -} - interface DirectoryRead { name: string; base: string; @@ -217,6 +222,23 @@ interface DirectoryRead { lastModified: number; } +/** Discriminated cache key per asset directory (kind + path). */ +function kindKey(kind: CleanupAssetKind, dir: string): string { + return `${kind}\0${dir}`; +} + +async function readDirNames(dir: string, regex: RegExp): Promise { + const names = await fs.readdir(dir).catch(() => [] as string[]); + return names.filter((name) => regex.test(name)).sort(); +} + +/** Cheap fingerprint of a directory listing — a rename/add/remove changes it. */ +function dirNameSignature(names: string[]): string { + const hash = createHash("sha1"); + for (const name of names) hash.update(name).update("\n"); + return hash.digest("hex"); +} + /** * How many `stat` calls run in parallel while reading an asset directory. * Directories can hold hundreds of thousands of files; serial `stat`ing them @@ -224,14 +246,12 @@ interface DirectoryRead { */ const DIR_STAT_CONCURRENCY = 32; -async function readAssetDir( +async function buildDirEntries( dir: string, - regex: RegExp, + names: string[], ): Promise { - const names = await fs.readdir(dir).catch(() => [] as string[]); const entries: DirectoryRead[] = []; await runPool(names, DIR_STAT_CONCURRENCY, async (name) => { - if (!regex.test(name)) return; let size = 0; let mtimeMs = 0; try { @@ -252,12 +272,14 @@ async function readAssetDir( } async function collectAssets( + kind: CleanupAssetKind, dirs: string[], - regex: RegExp, + namesByKey: Map, ): Promise> { const byName = new Map(); for (const dir of dirs) { - for (const read of await readAssetDir(dir, regex)) { + const names = namesByKey.get(kindKey(kind, dir)) ?? []; + for (const read of await buildDirEntries(dir, names)) { const entry = byName.get(read.name) ?? { fileName: read.name, base: read.base, @@ -274,35 +296,66 @@ async function collectAssets( return byName; } -async function loadValidStems(): Promise> { - // Authoritative stem set from items_base (item_name + public_name - // fallback), mirroring repair-nitros.ts classname resolution. - const validStems = new Set(); - try { - const result = await queryRows<{ - item_name: string; - public_name: string; - }>(sql` - SELECT item_name, public_name FROM items_base - `); - for (const row of result) { - addStems(validStems, row.item_name); - if ( - row.public_name && - row.public_name !== row.item_name && - !row.public_name.includes(" ") && - row.public_name !== "0" - ) { - addStems(validStems, row.public_name); - } - } - } catch { - /* empty set → every asset is treated as orphaned */ - } - return validStems; +// ── On-disk scan cache ────────────────────────────────────────────────────── +// +// The heavy parts of a scan are the per-bundle header validation (one open + +// small read per .nitro) and the parallel stat pass. Both can be skipped when +// no asset directory changed since the last scan: a signature per directory is +// derived from the sortable file *list* (readdir + hash, no stat). Any add / +// remove / rename invalidates the cache and triggers a full re-validation. +// Bundles rewritten in place without a name change are not detected by the +// signature; run a forced scan (`force: true`) to pick those up. + +const SCAN_CACHE_DIR = path.join(process.cwd(), "storage", "nitro-cleanup"); +const scanCachePath = () => path.join(SCAN_CACHE_DIR, "scan-cache.json"); + +interface ScanCacheFile { + dirs: Record; + result: Omit; } -export async function scanFakeBrokenNitros(): Promise { +async function readScanCache(): Promise { + try { + if (!existsSync(scanCachePath())) return null; + const raw = await fs.readFile(scanCachePath(), "utf8"); + const parsed = JSON.parse(raw) as ScanCacheFile; + if (!parsed || typeof parsed.dirs !== "object" || !parsed.result) { + return null; + } + return parsed; + } catch { + return null; + } +} + +async function writeScanCache( + dirs: Record, + result: Omit, +): Promise { + try { + await fs.mkdir(SCAN_CACHE_DIR, { recursive: true }); + await fs.writeFile(scanCachePath(), JSON.stringify({ dirs, result })); + } catch { + /* cache is best effort — a broken cache simply re-scans next time */ + } +} + +function cacheDirsMatch( + cached: Record, + current: Record, +): boolean { + const cachedKeys = Object.keys(cached); + if (cachedKeys.length !== Object.keys(current).length) return false; + for (const key of cachedKeys) { + if (cached[key] !== current[key]) return false; + } + return true; +} + +export async function scanFakeBrokenNitros( + options: CleanupScanOptions = {}, +): Promise { + const { force = false, onProgress } = options; const targets = await getFurniAssetWriteTargets(); const nitroDirs = uniqueDirs([ targets.nitroDir, @@ -316,47 +369,94 @@ export async function scanFakeBrokenNitros(): Promise { targets.iconDir, ...targets.mirrorDirs.map((d) => d.iconDir), ]); + const directoryPlan: Array<{ + kind: CleanupAssetKind; + dirs: string[]; + }> = [ + { kind: "nitro", dirs: nitroDirs }, + { kind: "swf", dirs: swfDirs }, + { kind: "icon", dirs: iconDirs }, + ]; - const [byNitro, bySwf, byIcon, validStems] = await Promise.all([ - collectAssets(nitroDirs, NITRO_FILE_RE), - collectAssets(swfDirs, SWF_FILE_RE), - collectAssets(iconDirs, ICON_FILE_RE), - loadValidStems(), + onProgress?.({ phase: "readdir", scanned: 0 }); + + // One readdir pass shared by both the cache signature and the full scan — + // never stat a file just to learn the directory changed. + const namesByKey = new Map(); + await Promise.all( + directoryPlan.flatMap(({ kind, dirs }) => + dirs.map(async (dir) => { + namesByKey.set( + kindKey(kind, dir), + await readDirNames(dir, KIND_FILE_RE[kind]), + ); + }), + ), + ); + const currentDigests: Record = {}; + for (const key of namesByKey.keys()) { + currentDigests[key] = dirNameSignature(namesByKey.get(key) ?? []); + } + + onProgress?.({ phase: "readdir", scanned: 0 }); + if (!force) { + const cache = await readScanCache(); + if (cache && cacheDirsMatch(cache.dirs, currentDigests)) { + onProgress?.({ phase: "done", scanned: cache.result.total }); + return { ...cache.result, cached: true }; + } + } + + // Needed only for classification — skip the DB query on a cache hit. + onProgress?.({ phase: "stems", scanned: 0 }); + const validStems = await getCleanupValidStems(); + + const [byNitro, bySwf, byIcon] = await Promise.all([ + collectAssets("nitro", nitroDirs, namesByKey), + collectAssets("swf", swfDirs, namesByKey), + collectAssets("icon", iconDirs, namesByKey), ]); const entries: NitroCleanupEntry[] = [...byNitro.values()]; const fake: NitroCleanupEntry[] = []; const broken: NitroCleanupEntry[] = []; + let validated = 0; + const totalToValidate = entries.length; await runPool(entries, 16, async (entry) => { // A .nitro that no DB item maps to is a leftover / fake bundle. if (!validStems.has(entry.base.toLowerCase())) { fake.push(entry); - return; - } - // Otherwise validate the bundle content. Most bundles pass a cheap - // header-only check (a few KB, no decompression); only files whose - // header looks suspicious get the expensive full parse. This keeps the - // scan fast even on directories with hundreds of thousands of bundles. - let parsed = false; - let firstError = ""; - for (const dir of entry.dirs) { - if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) { - parsed = true; - break; + } else { + // Otherwise validate the bundle content. Most bundles pass a cheap + // header-only check (a few KB, no decompression); only files whose + // header looks suspicious get the expensive full parse. This keeps + // the scan fast even on directories with hundreds of thousands of + // bundles. + let parsed = false; + let firstError = ""; + for (const dir of entry.dirs) { + if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) { + parsed = true; + break; + } + try { + parseNitroBundle( + await fs.readFile(getRuntimePath(dir, entry.fileName)), + ); + parsed = true; + break; + } catch (err) { + firstError = (err as Error).message; + } } - try { - parseNitroBundle( - await fs.readFile(getRuntimePath(dir, entry.fileName)), - ); - parsed = true; - break; - } catch (err) { - firstError = (err as Error).message; + if (!parsed) { + broken.push({ ...entry, error: firstError }); } } - if (!parsed) { - broken.push({ ...entry, error: firstError }); + validated++; + if (validated % 2500 === 0 || validated === totalToValidate) { + onProgress?.({ phase: "validate", scanned: validated }); } }); @@ -370,13 +470,16 @@ export async function scanFakeBrokenNitros(): Promise { const byName = (entries: T[]): T[] => entries.sort((a, b) => a.fileName.localeCompare(b.fileName)); - return { + const result: NitroCleanupScan = { fake: byName(fake), broken: byName(broken), orphanedSwf: byName(orphanedSwf), orphanedIcon: byName(orphanedIcon), total: byNitro.size + bySwf.size + byIcon.size, }; + await writeScanCache(currentDigests, result); + onProgress?.({ phase: "done", scanned: result.total }); + return result; } function dirsForKind( @@ -435,7 +538,7 @@ export async function deleteNitroCleanupFiles( return { deleted, copiesRemoved, errors, files }; } -function ageDays(lastModified: number): number { +export function ageDays(lastModified: number): number { if (!lastModified) return 0; return Math.max(0, Math.floor((Date.now() - lastModified) / DAY_MS)); }