perf(studio): cache nitro scan, stream progress, virtualize cleanup list
CI / check (push) Successful in 4m14s
CI / preflight (push) Skipped
CI / deploy (push) Successful in 2m10s

This commit is contained in:
openhands committed 2026-09-19 13:01:59 +02:00
1 parent c916e42572
commit ba81d16f00
5 files changed
+727 -257

No files matched your search

+98 -14
View File
@@ -4,31 +4,115 @@ import { PERMS } from "@/lib/permissions";
import { logAudit } from "@/lib/services/audit"; import { logAudit } from "@/lib/services/audit";
import { clearFurniImportCache } from "@/lib/services/furni-import-cache"; import { clearFurniImportCache } from "@/lib/services/furni-import-cache";
import { import {
ageDays,
type CleanupAssetKind, type CleanupAssetKind,
type CleanupProgress,
deleteNitroCleanupFiles, deleteNitroCleanupFiles,
scanFakeBrokenNitros, scanFakeBrokenNitros,
} from "@/lib/services/nitro-cleanup"; } from "@/lib/services/nitro-cleanup";
const VALID_KINDS: readonly CleanupAssetKind[] = ["nitro", "swf", "icon"]; const VALID_KINDS: readonly CleanupAssetKind[] = ["nitro", "swf", "icon"];
function intParam(value: string | null): number | null {
if (value === null) return null;
const n = Number(value);
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : null;
}
/** /**
* Scan all asset directories for fake (orphaned) nitro / swf / icon files and * Scan all asset directories for fake (orphaned) nitro / swf / icon files and
* broken (unparseable) .nitro bundles. * broken (unparseable) .nitro bundles.
*
* Query params:
* - `force=1` skips the on-disk scan cache (fresh re-validation);
* - `minAgeDays=N` only returns entries older than N days;
* - `limit=N` caps each returned list (use `full=1` to override);
* - `full=1` never applies the limit (used by the manifest export).
*
* When the client asks for `text/event-stream` the scan is streamed as
* `progress` / `result` / `error` events, keeping the connection alive while a
* large nitro directory is walked so reverse proxies don't time the request out.
*/ */
export const GET = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => { export const GET = withAdmin(
try { { permission: PERMS.ASSETS_IMPORT },
const scan = await scanFakeBrokenNitros(); async (request) => {
return apiOk({ const wantsSse = request.headers
fake: scan.fake, .get("accept")
broken: scan.broken, ?.includes("text/event-stream");
orphanedSwf: scan.orphanedSwf, const force = request.nextUrl.searchParams.get("force") === "1";
orphanedIcon: scan.orphanedIcon, const full = request.nextUrl.searchParams.get("full") === "1";
total: scan.total, const minAgeDays = intParam(request.nextUrl.searchParams.get("minAgeDays"));
}); const limit = intParam(request.nextUrl.searchParams.get("limit"));
} catch (err) {
return apiError(`Asset scan failed: ${(err as Error).message}`, 500); const run = async (onProgress?: (p: CleanupProgress) => void) => {
} const scan = await scanFakeBrokenNitros({ force, onProgress });
}); const ageFilter = <T extends { lastModified: number }>(
entries: T[],
): T[] =>
minAgeDays !== null && minAgeDays > 0
? entries.filter((entry) => ageDays(entry.lastModified) >= minAgeDays)
: entries;
const cap = <T>(entries: T[]): T[] =>
full || limit === null || limit <= 0
? entries
: entries.slice(0, limit);
return {
fake: cap(ageFilter(scan.fake)),
broken: cap(ageFilter(scan.broken)),
orphanedSwf: cap(ageFilter(scan.orphanedSwf)),
orphanedIcon: cap(ageFilter(scan.orphanedIcon)),
total: scan.total,
cached: scan.cached ?? false,
};
};
try {
if (!wantsSse) {
return apiOk(await run());
}
const encoder = new TextEncoder();
const stream = new ReadableStream({
async start(controller) {
const send = (data: unknown) => {
try {
controller.enqueue(
encoder.encode(`data: ${JSON.stringify(data)}\n\n`),
);
} catch {
/* stream closed by client */
}
};
try {
const result = await run((progress) =>
send({ type: "progress", ...progress }),
);
send({ type: "result", ...result });
} catch (err) {
send({
type: "error",
message: `Asset scan failed: ${(err as Error).message}`,
});
}
try {
controller.close();
} catch {
/* ignore */
}
},
});
return new Response(stream, {
headers: {
"Content-Type": "text/event-stream",
"Cache-Control": "no-cache",
Connection: "keep-alive",
},
});
} catch (err) {
return apiError(`Asset scan failed: ${(err as Error).message}`, 500);
}
},
);
/** /**
* Delete the given asset files (default .nitro) from every matching directory * Delete the given asset files (default .nitro) from every matching directory
@@ -1,5 +1,6 @@
"use client"; "use client";
import { useVirtualizer } from "@tanstack/react-virtual";
import { import {
AlertTriangle, AlertTriangle,
CheckCircle2, CheckCircle2,
@@ -11,8 +12,9 @@ import {
Sparkles, Sparkles,
Trash2, Trash2,
Wrench, Wrench,
X,
} from "lucide-react"; } from "lucide-react";
import { useCallback, useMemo, useState } from "react"; import { useCallback, useRef, useState } from "react";
import { toast } from "sonner"; import { toast } from "sonner";
import { useConfirmDialog } from "@/components/admin/confirm-dialog"; import { useConfirmDialog } from "@/components/admin/confirm-dialog";
import { Button } from "@/components/ui/button"; import { Button } from "@/components/ui/button";
@@ -20,6 +22,7 @@ import { Checkbox } from "@/components/ui/checkbox";
import { Input } from "@/components/ui/input"; import { Input } from "@/components/ui/input";
import { adminFetch } from "@/lib/admin-fetch"; import { adminFetch } from "@/lib/admin-fetch";
import { formatBytes } from "@/lib/services/disk-usage"; import { formatBytes } from "@/lib/services/disk-usage";
import { readSseStream } from "@/lib/sse-client";
interface AssetEntry { interface AssetEntry {
fileName: string; fileName: string;
@@ -36,6 +39,7 @@ interface NitroCleanupScan {
orphanedSwf: AssetEntry[]; orphanedSwf: AssetEntry[];
orphanedIcon: AssetEntry[]; orphanedIcon: AssetEntry[];
total: number; total: number;
cached?: boolean;
} }
interface DeleteResponse { interface DeleteResponse {
@@ -62,13 +66,7 @@ type GroupTone = "error" | "warning" | "neutral";
type AssetKind = "nitro" | "swf" | "icon"; type AssetKind = "nitro" | "swf" | "icon";
const DAY_MS = 24 * 60 * 60 * 1000; const DAY_MS = 24 * 60 * 60 * 1000;
const SCAN_IDLE_TIMEOUT_MS = 120_000;
/**
* How many rows a group renders at once. Directories can contain tens of
* thousands of leftover files; mounting every row as DOM nodes freezes the
* tab, so rows are added in chunks via a "load more" button.
*/
const RENDER_CHUNK = 500;
function ageLabel(lastModified: number): string { function ageLabel(lastModified: number): string {
if (!lastModified) return "age unknown"; if (!lastModified) return "age unknown";
@@ -91,6 +89,87 @@ function downloadJson(filename: string, data: unknown): void {
URL.revokeObjectURL(url); URL.revokeObjectURL(url);
} }
/**
* Virtualized list for one cleanup group. Directories can hold tens of
* thousands of leftover files; only the visible rows are mounted as DOM nodes,
* so the tab never freezes no matter how long the list is.
*/
function VirtualAssetList({
entries,
selected,
disabled = false,
onToggle,
}: {
entries: AssetEntry[];
selected: Set<string>;
disabled?: boolean;
onToggle: (fileName: string) => void;
}) {
const parentRef = useRef<HTMLDivElement>(null);
const virtualizer = useVirtualizer({
count: entries.length,
getScrollElement: () => parentRef.current,
estimateSize: () => 30,
overscan: 10,
});
return (
<div
ref={parentRef}
className="max-h-80 overflow-auto rounded-lg border border-[var(--admin-border)] bg-[var(--admin-surface)] p-2"
>
<div style={{ height: virtualizer.getTotalSize(), position: "relative" }}>
{virtualizer.getVirtualItems().map((virtualItem) => {
const entry = entries[virtualItem.index];
return (
<div
key={entry.fileName}
data-index={virtualItem.index}
ref={virtualizer.measureElement}
className="absolute left-0 top-0 flex w-full items-center gap-2 rounded-md px-2 text-sm hover:bg-[var(--admin-accent)]/5"
style={{
height: virtualItem.size,
transform: `translateY(${virtualItem.start}px)`,
}}
>
<Checkbox
style={{ display: "inline-flex", width: 16, height: 16 }}
className="border-[var(--admin-border)] bg-[var(--admin-surface)] data-checked:border-[var(--admin-accent)] data-checked:bg-[var(--admin-accent)] data-checked:text-[var(--admin-accent-foreground)]"
checked={selected.has(entry.fileName)}
disabled={disabled}
onCheckedChange={() => onToggle(entry.fileName)}
aria-label={`Select ${entry.fileName}`}
/>
<code className="min-w-0 flex-1 truncate font-mono text-xs">
{entry.fileName}
</code>
<span className="shrink-0 text-xs text-[var(--admin-text-muted)]">
{formatBytes(entry.size)}
</span>
<span className="shrink-0 text-xs text-[var(--admin-text-muted)]">
{ageLabel(entry.lastModified)}
</span>
{entry.error && (
<span
title={entry.error}
className="hidden max-w-[320px] truncate text-xs text-[var(--admin-error)] lg:block"
>
{entry.error}
</span>
)}
{entry.dirs.length > 1 && (
<span className="shrink-0 rounded bg-[var(--admin-accent)]/10 px-1.5 py-0.5 text-[10px] font-medium text-[var(--admin-text-muted)]">
{entry.dirs.length} copies
</span>
)}
</div>
);
})}
</div>
</div>
);
}
function CleanupGroup({ function CleanupGroup({
label, label,
description, description,
@@ -110,11 +189,8 @@ function CleanupGroup({
onToggle: (fileName: string) => void; onToggle: (fileName: string) => void;
onToggleAll: (names: string[]) => void; onToggleAll: (names: string[]) => void;
}) { }) {
const [shown, setShown] = useState(RENDER_CHUNK);
const checked = entries.filter((e) => selected.has(e.fileName)); const checked = entries.filter((e) => selected.has(e.fileName));
const allChecked = entries.length > 0 && checked.length === entries.length; const allChecked = entries.length > 0 && checked.length === entries.length;
const visibleEntries = entries.slice(0, shown);
const toneClass = const toneClass =
tone === "error" tone === "error"
? "border-[var(--admin-error-border)] bg-[var(--admin-error-subtle)]" ? "border-[var(--admin-error-border)] bg-[var(--admin-error-subtle)]"
@@ -158,63 +234,12 @@ function CleanupGroup({
No {label.toLowerCase()} found No {label.toLowerCase()} found
</p> </p>
) : ( ) : (
<> <VirtualAssetList
<ul className="max-h-80 space-y-1 overflow-auto rounded-lg border border-[var(--admin-border)] bg-[var(--admin-surface)] p-2"> entries={entries}
{visibleEntries.map((entry) => ( selected={selected}
<li disabled={disabled}
key={entry.fileName} onToggle={onToggle}
className="flex items-center gap-2 rounded-md px-2 py-1.5 text-sm hover:bg-[var(--admin-accent)]/5" />
>
<Checkbox
style={{ display: "inline-flex", width: 16, height: 16 }}
className="border-[var(--admin-border)] bg-[var(--admin-surface)] data-checked:border-[var(--admin-accent)] data-checked:bg-[var(--admin-accent)] data-checked:text-[var(--admin-accent-foreground)]"
checked={selected.has(entry.fileName)}
disabled={disabled}
onCheckedChange={() => onToggle(entry.fileName)}
aria-label={`Select ${entry.fileName}`}
/>
<code className="min-w-0 flex-1 truncate font-mono text-xs">
{entry.fileName}
</code>
<span className="shrink-0 text-xs text-[var(--admin-text-muted)]">
{formatBytes(entry.size)}
</span>
<span className="shrink-0 text-xs text-[var(--admin-text-muted)]">
{ageLabel(entry.lastModified)}
</span>
{entry.error && (
<span
title={entry.error}
className="hidden max-w-[320px] truncate text-xs text-[var(--admin-error)] lg:block"
>
{entry.error}
</span>
)}
{entry.dirs.length > 1 && (
<span className="shrink-0 rounded bg-[var(--admin-accent)]/10 px-1.5 py-0.5 text-[10px] font-medium text-[var(--admin-text-muted)]">
{entry.dirs.length} copies
</span>
)}
</li>
))}
</ul>
{entries.length > shown && (
<div className="flex items-center justify-between px-2 text-xs text-[var(--admin-text-muted)]">
<span>
Showing {shown} of {entries.length}
</span>
<Button
type="button"
variant="outline"
size="sm"
disabled={disabled}
onClick={() => setShown((s) => s + RENDER_CHUNK)}
>
Load {Math.min(RENDER_CHUNK, entries.length - shown)} more
</Button>
</div>
)}
</>
)} )}
</section> </section>
); );
@@ -225,63 +250,113 @@ export function NitroCleanupPanel() {
const [deleting, setDeleting] = useState(false); const [deleting, setDeleting] = useState(false);
const [repairing, setRepairing] = useState(false); const [repairing, setRepairing] = useState(false);
const [autoCleaning, setAutoCleaning] = useState(false); const [autoCleaning, setAutoCleaning] = useState(false);
const [exporting, setExporting] = useState(false);
const [autoAge, setAutoAge] = useState(30); const [autoAge, setAutoAge] = useState(30);
const [minAgeDays, setMinAgeDays] = useState(0); const [minAgeDays, setMinAgeDays] = useState(0);
const [onlyOld, setOnlyOld] = useState(false); const [onlyOld, setOnlyOld] = useState(false);
const [progress, setProgress] = useState<{
phase: string;
scanned: number;
} | null>(null);
const [scan, setScan] = useState<NitroCleanupScan | null>(null); const [scan, setScan] = useState<NitroCleanupScan | null>(null);
const [selected, setSelected] = useState<Set<string>>(new Set()); const [selected, setSelected] = useState<Set<string>>(new Set());
const [selectedSwf, setSelectedSwf] = useState<Set<string>>(new Set()); const [selectedSwf, setSelectedSwf] = useState<Set<string>>(new Set());
const [selectedIcon, setSelectedIcon] = useState<Set<string>>(new Set()); const [selectedIcon, setSelectedIcon] = useState<Set<string>>(new Set());
const abortRef = useRef<AbortController | null>(null);
const { confirm, dialog } = useConfirmDialog(); const { confirm, dialog } = useConfirmDialog();
const scanNow = useCallback(async () => { const scanNow = useCallback(
setScanning(true); async (opts: { force?: boolean; minAgeDays?: number } = {}) => {
setSelected(new Set()); const force = opts.force ?? true;
setSelectedSwf(new Set()); const ageFilter = opts.minAgeDays ?? (onlyOld ? minAgeDays : 0);
setSelectedIcon(new Set()); abortRef.current?.abort();
try { setScanning(true);
const res = await adminFetch("/api/admin/studio/nitro-cleanup"); setProgress(null);
const type = res.headers.get("content-type") ?? ""; setSelected(new Set());
if (!res.ok) { setSelectedSwf(new Set());
const raw = await res.text(); setSelectedIcon(new Set());
throw new Error(
raw && !type.includes("application/json") const abort = new AbortController();
? `Scan failed (HTTP ${res.status}) — the server or reverse proxy returned a non-JSON response. This usually means a proxy/worker timeout; try again or raise the upstream timeout.` abortRef.current = abort;
: `Scan failed (HTTP ${res.status})`, const params = new URLSearchParams();
if (force) params.set("force", "1");
if (ageFilter > 0) params.set("minAgeDays", String(ageFilter));
const holder: { result: NitroCleanupScan | null } = { result: null };
let lastError: string | null = null;
try {
const res = await adminFetch(
`/api/admin/studio/nitro-cleanup${
params.size > 0 ? `?${params.toString()}` : ""
}`,
{ headers: { Accept: "text/event-stream" }, signal: abort.signal },
); );
} if (!res.ok) {
if (!type.includes("application/json")) { const data = (await res.json().catch(() => null)) as {
throw new Error( error?: string;
"Scan failed — the server returned a non-JSON response (likely an upstream timeout or proxy error page).", } | null;
throw new Error(
typeof data?.error === "string"
? data.error
: `Scan failed (HTTP ${res.status})`,
);
}
if (!res.body) throw new Error("Scan returned no response body");
await readSseStream(
res.body,
(evt) => {
if (evt.type === "progress") {
setProgress({
phase: String(evt.phase ?? ""),
scanned: Number(evt.scanned ?? 0),
});
} else if (evt.type === "result") {
holder.result = {
fake: (evt.fake as AssetEntry[]) ?? [],
broken: (evt.broken as AssetEntry[]) ?? [],
orphanedSwf: (evt.orphanedSwf as AssetEntry[]) ?? [],
orphanedIcon: (evt.orphanedIcon as AssetEntry[]) ?? [],
total: Number(evt.total ?? 0),
cached: evt.cached === true,
};
setScan(holder.result);
} else if (evt.type === "error") {
lastError = String(evt.message ?? "Scan failed");
}
},
abort.signal,
SCAN_IDLE_TIMEOUT_MS,
); );
if (lastError) throw new Error(lastError);
const lastResult = holder.result;
if (lastResult) {
const found =
lastResult.fake.length +
lastResult.broken.length +
lastResult.orphanedSwf.length +
lastResult.orphanedIcon.length;
if (found === 0) {
toast.success("No orphaned or broken furniture assets found");
} else {
toast.info(`Found ${found} removable asset file(s)`);
}
}
} catch (err) {
if ((err as Error).name === "AbortError") {
toast.info("Scan cancelled");
} else {
toast.error(err instanceof Error ? err.message : "Scan failed");
}
} finally {
setScanning(false);
setProgress(null);
if (abortRef.current === abort) abortRef.current = null;
} }
const data = (await res.json()) as NitroCleanupScan & { },
error?: string; [onlyOld, minAgeDays],
}; );
if (data.error) throw new Error(data.error);
setScan({
fake: data.fake ?? [],
broken: data.broken ?? [],
orphanedSwf: data.orphanedSwf ?? [],
orphanedIcon: data.orphanedIcon ?? [],
total: data.total ?? 0,
});
const found =
data.fake.length +
data.broken.length +
data.orphanedSwf.length +
data.orphanedIcon.length;
if (found === 0) {
toast.success("No orphaned or broken furniture assets found");
} else {
toast.info(`Found ${found} removable asset file(s)`);
}
} catch (err) {
toast.error(err instanceof Error ? err.message : "Scan failed");
} finally {
setScanning(false);
}
}, []);
const toggle = (kind: AssetKind) => (fileName: string) => { const toggle = (kind: AssetKind) => (fileName: string) => {
const setter = const setter =
@@ -318,6 +393,25 @@ export function NitroCleanupPanel() {
const totalSelected = selected.size + selectedSwf.size + selectedIcon.size; const totalSelected = selected.size + selectedSwf.size + selectedIcon.size;
/** Drop deleted/attempted files from the loaded scan without re-scanning. */
const dropFromScan = (kind: AssetKind, names: string[]) => {
const remove = new Set(names);
if (remove.size === 0) return;
setScan((prev) => {
if (!prev) return prev;
const drop = <T extends { fileName: string }>(list: T[]): T[] =>
list.filter((entry) => !remove.has(entry.fileName));
return {
...prev,
fake: kind === "nitro" ? drop(prev.fake) : prev.fake,
broken: kind === "nitro" ? drop(prev.broken) : prev.broken,
orphanedSwf: kind === "swf" ? drop(prev.orphanedSwf) : prev.orphanedSwf,
orphanedIcon:
kind === "icon" ? drop(prev.orphanedIcon) : prev.orphanedIcon,
};
});
};
const removeSelected = async () => { const removeSelected = async () => {
const groups: Array<{ kind: AssetKind; names: string[] }> = []; const groups: Array<{ kind: AssetKind; names: string[] }> = [];
for (const [kind, set] of [ for (const [kind, set] of [
@@ -358,6 +452,7 @@ export function NitroCleanupPanel() {
deleted += data.deleted ?? 0; deleted += data.deleted ?? 0;
copiesRemoved += data.copiesRemoved ?? 0; copiesRemoved += data.copiesRemoved ?? 0;
if (data.errors?.length) hadErrors = true; if (data.errors?.length) hadErrors = true;
dropFromScan(group.kind, group.names);
} }
if (hadErrors) { if (hadErrors) {
toast.warning( toast.warning(
@@ -371,7 +466,6 @@ export function NitroCleanupPanel() {
setSelected(new Set()); setSelected(new Set());
setSelectedSwf(new Set()); setSelectedSwf(new Set());
setSelectedIcon(new Set()); setSelectedIcon(new Set());
await scanNow();
} catch (err) { } catch (err) {
toast.error(err instanceof Error ? err.message : "Delete failed"); toast.error(err instanceof Error ? err.message : "Delete failed");
} finally { } finally {
@@ -408,8 +502,12 @@ export function NitroCleanupPanel() {
} else { } else {
toast.warning(`Could not re-obtain any bundle (${data.failed} failed)`); toast.warning(`Could not re-obtain any bundle (${data.failed} failed)`);
} }
// Only drop the attempted bundles when every one succeeded; partial
// failures stay visible so they can be retried.
if (data.repaired >= fileNames.length) {
dropFromScan("nitro", fileNames);
}
setSelected(new Set()); setSelected(new Set());
await scanNow();
} catch (err) { } catch (err) {
toast.error(err instanceof Error ? err.message : "Repair failed"); toast.error(err instanceof Error ? err.message : "Repair failed");
} finally { } finally {
@@ -445,12 +543,18 @@ export function NitroCleanupPanel() {
toast.success( toast.success(
`Auto-cleaned ${data.deleted} old fake .nitro file(s) (${data.copiesRemoved} copies). Skipped ${data.skippedRecent} recent + ${data.skippedBroken} broken.`, `Auto-cleaned ${data.deleted} old fake .nitro file(s) (${data.copiesRemoved} copies). Skipped ${data.skippedRecent} recent + ${data.skippedBroken} broken.`,
); );
const removable = (scan.fake ?? [])
.filter(
(entry) =>
Math.floor((Date.now() - entry.lastModified) / DAY_MS) >= autoAge,
)
.map((entry) => entry.fileName);
dropFromScan("nitro", removable);
} else { } else {
toast.info( toast.info(
`Nothing old enough to remove (skipped ${data.skippedRecent} recent, ${data.skippedBroken} broken)`, `Nothing old enough to remove (skipped ${data.skippedRecent} recent, ${data.skippedBroken} broken)`,
); );
} }
await scanNow();
} catch (err) { } catch (err) {
toast.error(err instanceof Error ? err.message : "Auto-clean failed"); toast.error(err instanceof Error ? err.message : "Auto-clean failed");
} finally { } finally {
@@ -458,57 +562,72 @@ export function NitroCleanupPanel() {
} }
}; };
const exportManifest = () => { const exportManifest = async () => {
if (scan === null) return; if (scan === null) return;
const entries = { setExporting(true);
scannedAt: new Date().toISOString(), try {
total: scan.total, const res = await adminFetch("/api/admin/studio/nitro-cleanup?full=1", {
fake: scan.fake, headers: { Accept: "application/json" },
broken: scan.broken, });
orphanedSwf: scan.orphanedSwf, const data = (await res.json()) as NitroCleanupScan & {
orphanedIcon: scan.orphanedIcon, error?: string;
selected: { };
nitro: [...selected].sort(), if (!res.ok || data.error) {
swf: [...selectedSwf].sort(), throw new Error(data.error || `Export failed (HTTP ${res.status})`);
icon: [...selectedIcon].sort(), }
}, const entries = {
}; scannedAt: new Date().toISOString(),
downloadJson(`nitro-cleanup-scan-${Date.now()}.json`, entries); total: data.total ?? 0,
toast.success("Scan manifest downloaded"); fake: data.fake ?? [],
broken: data.broken ?? [],
orphanedSwf: data.orphanedSwf ?? [],
orphanedIcon: data.orphanedIcon ?? [],
selected: {
nitro: [...selected].sort(),
swf: [...selectedSwf].sort(),
icon: [...selectedIcon].sort(),
},
};
downloadJson(`nitro-cleanup-scan-${Date.now()}.json`, entries);
toast.success("Scan manifest downloaded");
} catch (err) {
toast.error(err instanceof Error ? err.message : "Export failed");
} finally {
setExporting(false);
}
}; };
const visibleFake = useMemo(() => { const toggleOnlyOld = (value: boolean) => {
if (scan === null) return []; setOnlyOld(value);
if (!onlyOld || minAgeDays <= 0) return scan.fake; const threshold = value ? Math.max(0, minAgeDays) : undefined;
const threshold = minAgeDays * DAY_MS; void scanNow({ force: false, minAgeDays: threshold ?? 0 });
return scan.fake.filter((e) => e.lastModified <= Date.now() - threshold); };
}, [scan, onlyOld, minAgeDays]);
const visibleSwf = useMemo(() => { const changeMinAgeDays = (value: number) => {
if (scan === null) return []; const days = Number.isFinite(value) && value >= 0 ? Math.floor(value) : 0;
if (!onlyOld || minAgeDays <= 0) return scan.orphanedSwf; setMinAgeDays(days);
const threshold = minAgeDays * DAY_MS; if (onlyOld) void scanNow({ force: false, minAgeDays: days });
return scan.orphanedSwf.filter( };
(e) => e.lastModified <= Date.now() - threshold,
);
}, [scan, onlyOld, minAgeDays]);
const visibleIcon = useMemo(() => { const busy = scanning || deleting || repairing || autoCleaning || exporting;
if (scan === null) return []; const phaseLabel = (phase: string): string =>
if (!onlyOld || minAgeDays <= 0) return scan.orphanedIcon; phase === "stems"
const threshold = minAgeDays * DAY_MS; ? "Loading furniture stems…"
return scan.orphanedIcon.filter( : phase === "readdir"
(e) => e.lastModified <= Date.now() - threshold, ? "Reading asset directories…"
); : phase === "validate"
}, [scan, onlyOld, minAgeDays]); ? "Validating .nitro bundles…"
: "Scanning…";
const busy = scanning || deleting || repairing || autoCleaning;
return ( return (
<div className="space-y-4"> <div className="space-y-4">
{dialog} {dialog}
<div className="flex flex-wrap items-center gap-2"> <div className="flex flex-wrap items-center gap-2">
<Button onClick={scanNow} disabled={busy} size="lg"> <Button
onClick={() => scanNow({ force: true })}
disabled={busy}
size="lg"
>
{scanning ? ( {scanning ? (
<Loader2 className="w-4 h-4 animate-spin mr-2" /> <Loader2 className="w-4 h-4 animate-spin mr-2" />
) : ( ) : (
@@ -516,6 +635,16 @@ export function NitroCleanupPanel() {
)} )}
{scanning ? "Scanning assets…" : "Scan furniture assets"} {scanning ? "Scanning assets…" : "Scan furniture assets"}
</Button> </Button>
{scanning && (
<Button
variant="outline"
size="lg"
onClick={() => abortRef.current?.abort()}
>
<X className="w-4 h-4 mr-2" />
Cancel
</Button>
)}
{scan && ( {scan && (
<Button <Button
variant="destructive" variant="destructive"
@@ -583,12 +712,28 @@ export function NitroCleanupPanel() {
disabled={busy} disabled={busy}
onClick={exportManifest} onClick={exportManifest}
> >
<Download className="w-4 h-4 mr-2" /> {exporting ? (
Export manifest <Loader2 className="w-4 h-4 animate-spin mr-2" />
) : (
<Download className="w-4 h-4 mr-2" />
)}
{exporting ? "Exporting…" : "Export manifest"}
</Button> </Button>
)} )}
</div> </div>
{scanning && progress && (
<div className="flex items-center gap-2 rounded-lg border border-[var(--admin-border)] bg-[var(--admin-surface)] p-3 text-sm text-[var(--admin-text-muted)]">
<Loader2 className="w-4 h-4 animate-spin shrink-0" />
{phaseLabel(progress.phase)}
{progress.scanned > 0 && (
<span className="font-mono text-xs">
{progress.scanned.toLocaleString()} files
</span>
)}
</div>
)}
{scan && ( {scan && (
<div className="space-y-4"> <div className="space-y-4">
<div className="flex flex-wrap items-center gap-3 rounded-lg border border-[var(--admin-border)] bg-[var(--admin-surface)] p-3"> <div className="flex flex-wrap items-center gap-3 rounded-lg border border-[var(--admin-border)] bg-[var(--admin-surface)] p-3">
@@ -602,7 +747,7 @@ export function NitroCleanupPanel() {
className="w-20" className="w-20"
value={minAgeDays} value={minAgeDays}
disabled={busy} disabled={busy}
onChange={(e) => setMinAgeDays(Number(e.target.value))} onChange={(e) => changeMinAgeDays(Number(e.target.value))}
aria-label="Minimum age in days to display" aria-label="Minimum age in days to display"
/> />
<span className="text-xs text-[var(--admin-text-muted)]">days</span> <span className="text-xs text-[var(--admin-text-muted)]">days</span>
@@ -613,7 +758,7 @@ export function NitroCleanupPanel() {
<Checkbox <Checkbox
id="nitro-hide-recent" id="nitro-hide-recent"
checked={onlyOld} checked={onlyOld}
onCheckedChange={(value) => setOnlyOld(value === true)} onCheckedChange={(value) => toggleOnlyOld(value === true)}
/> />
<span>Hide recent files</span> <span>Hide recent files</span>
</label> </label>
@@ -624,6 +769,12 @@ export function NitroCleanupPanel() {
<div className="text-xl font-bold">{scan.total}</div> <div className="text-xl font-bold">{scan.total}</div>
<div className="text-xs text-muted-foreground"> <div className="text-xs text-muted-foreground">
asset files scanned asset files scanned
{scan.cached && (
<span className="text-[var(--admin-accent)]">
{" "}
(cached — directories unchanged)
</span>
)}
</div> </div>
</div> </div>
<div className="rounded-xl border bg-card p-3 space-y-1"> <div className="rounded-xl border bg-card p-3 space-y-1">
@@ -648,7 +799,7 @@ export function NitroCleanupPanel() {
label="Fake nitros" label="Fake nitros"
description="These files match no item in items_base — leftovers from deleted or renamed furniture." description="These files match no item in items_base — leftovers from deleted or renamed furniture."
tone="error" tone="error"
entries={visibleFake} entries={scan.fake}
selected={selected} selected={selected}
onToggle={toggle("nitro")} onToggle={toggle("nitro")}
onToggleAll={toggleAll("nitro")} onToggleAll={toggleAll("nitro")}
@@ -657,7 +808,7 @@ export function NitroCleanupPanel() {
label="Broken nitros" label="Broken nitros"
description="These files belong to an item but cannot be parsed as a valid Nitro bundle." description="These files belong to an item but cannot be parsed as a valid Nitro bundle."
tone="warning" tone="warning"
entries={onlyOld ? [] : scan.broken} entries={scan.broken}
selected={selected} selected={selected}
onToggle={toggle("nitro")} onToggle={toggle("nitro")}
onToggleAll={toggleAll("nitro")} onToggleAll={toggleAll("nitro")}
@@ -667,7 +818,7 @@ export function NitroCleanupPanel() {
label="Orphaned SWFs" label="Orphaned SWFs"
description=".swf files that match no item in items_base." description=".swf files that match no item in items_base."
tone="warning" tone="warning"
entries={visibleSwf} entries={scan.orphanedSwf}
selected={selectedSwf} selected={selectedSwf}
onToggle={toggle("swf")} onToggle={toggle("swf")}
onToggleAll={toggleAll("swf")} onToggleAll={toggleAll("swf")}
@@ -676,7 +827,7 @@ export function NitroCleanupPanel() {
label="Orphaned icons" label="Orphaned icons"
description="Icon files that match no item in items_base." description="Icon files that match no item in items_base."
tone="neutral" tone="neutral"
entries={visibleIcon} entries={scan.orphanedIcon}
selected={selectedIcon} selected={selectedIcon}
onToggle={toggle("icon")} onToggle={toggle("icon")}
onToggleAll={toggleAll("icon")} onToggleAll={toggleAll("icon")}
+58 -1
View File
@@ -16,7 +16,11 @@ import { existsSync } from "node:fs";
import path from "node:path"; import path from "node:path";
import { sql } from "drizzle-orm"; import { sql } from "drizzle-orm";
import { db, ItemsBase, queryRows } from "@/lib/db"; import { db, ItemsBase, queryRows } from "@/lib/db";
import { normalizeClassname } from "@/lib/furni/classname"; import {
getBaseClassname,
getSafeClassnameStem,
normalizeClassname,
} from "@/lib/furni/classname";
import { indexLocalFurniture } from "@/lib/furni/local-presence"; import { indexLocalFurniture } from "@/lib/furni/local-presence";
import { readFurniData } from "./furni-data"; import { readFurniData } from "./furni-data";
@@ -107,6 +111,58 @@ export async function getCatalogIdSet(): Promise<Set<number>> {
const nitroPresenceCache = new Map<string, boolean>(); const nitroPresenceCache = new Map<string, boolean>();
/** Every classname stem a `.nitro`/`.swf`/icon file may legitimately use. */
function addClassnameStem(set: Set<string>, classname: string): void {
const normalized = classname.trim().toLowerCase();
if (!normalized) return;
const base = getBaseClassname(normalized).toLowerCase();
if (base) set.add(base);
if (normalized.includes("*")) {
const safe = getSafeClassnameStem(normalized).toLowerCase();
if (safe) set.add(safe);
}
}
let cleanupStemsSnapshot: { set: Set<string>; timestamp: number } | null = null;
/**
* Authoritative stems for the nitro-cleanup scan: every `item_name` plus the
* `public_name` fallback (mirroring repair-nitros.ts resolution). Cached the
* same way as the Studio listings so repeated scans (auto-clean, panel) don't
* re-query `items_base` on every call.
*/
export async function getCleanupValidStems(): Promise<Set<string>> {
const now = Date.now();
if (
cleanupStemsSnapshot &&
now - cleanupStemsSnapshot.timestamp < CACHE_TTL
) {
return cleanupStemsSnapshot.set;
}
const set = new Set<string>();
try {
const rows = await queryRows<{
item_name: string;
public_name: string;
}>(sql`SELECT item_name, public_name FROM items_base`);
for (const row of rows) {
addClassnameStem(set, row.item_name);
if (
row.public_name &&
row.public_name !== row.item_name &&
!row.public_name.includes(" ") &&
row.public_name !== "0"
) {
addClassnameStem(set, row.public_name);
}
}
} catch {
/* empty set → every asset is treated as orphaned */
}
cleanupStemsSnapshot = { set, timestamp: now };
return set;
}
/** `existsSync` memoized per (dir + base) — nitro files change rarely. */ /** `existsSync` memoized per (dir + base) — nitro files change rarely. */
export function nitroExistsCached(dir: string, base: string): boolean { export function nitroExistsCached(dir: string, base: string): boolean {
const key = `${dir}/${base}`; const key = `${dir}/${base}`;
@@ -210,5 +266,6 @@ export function clearFurniImportCache(): void {
furniDataSetSnapshot = null; furniDataSetSnapshot = null;
catalogIdsSnapshot = null; catalogIdsSnapshot = null;
nitroPresenceCache.clear(); nitroPresenceCache.clear();
cleanupStemsSnapshot = null;
statsSnapshot = null; statsSnapshot = null;
} }
+75
View File
@@ -1,4 +1,5 @@
// @vitest-environment node // @vitest-environment node
import { createHash } from "node:crypto";
import { beforeEach, describe, expect, it, vi } from "vitest"; import { beforeEach, describe, expect, it, vi } from "vitest";
type AnyFn = (...args: any[]) => any; type AnyFn = (...args: any[]) => any;
@@ -130,6 +131,13 @@ function mockReaddir(byDir: Record<string, string[]>): void {
readdirFn.mockImplementation(async (dir: string) => byDir[dir] ?? []); readdirFn.mockImplementation(async (dir: string) => byDir[dir] ?? []);
} }
/** Replicates `dirNameSignature` in the service so cache fixtures match. */
function dirSignature(names: string[]): string {
const hash = createHash("sha1");
for (const name of names) hash.update(name).update("\n");
return hash.digest("hex");
}
describe("scanFakeBrokenNitros", () => { describe("scanFakeBrokenNitros", () => {
it("classifies fake (no matching item) and broken (unparseable) bundles", async () => { it("classifies fake (no matching item) and broken (unparseable) bundles", async () => {
mockReaddir({ mockReaddir({
@@ -233,6 +241,73 @@ describe("scanFakeBrokenNitros", () => {
expect(result.fake[0].size).toBe(900); expect(result.fake[0].size).toBe(900);
expect(result.fake[0].lastModified).toBe(2000); expect(result.fake[0].lastModified).toBe(2000);
}); });
it("serves results from the on-disk cache when directories are unchanged", async () => {
mockReaddir({
"/assets/nitro": ["chair.nitro", "ghost.nitro"],
});
const dirs = {
"nitro\u0000/assets/nitro": dirSignature(["chair.nitro", "ghost.nitro"]),
"swf\u0000/assets/swf": dirSignature([]),
"icon\u0000/assets/icons": dirSignature([]),
};
const cachedResult = {
fake: [
{
fileName: "ghost.nitro",
base: "ghost",
size: 500,
dirs: ["/assets/nitro"],
lastModified: 0,
error: undefined,
},
],
broken: [],
orphanedSwf: [],
orphanedIcon: [],
total: 4,
};
existsFn.mockImplementation((filePath: string) =>
filePath.includes("scan-cache.json"),
);
readFileFn.mockImplementation(async (filePath: string) => {
if (filePath.includes("scan-cache.json")) {
return JSON.stringify({ dirs, result: cachedResult });
}
return Buffer.from("nitro-data");
});
const result = await scanFakeBrokenNitros();
// `total: 4` could never come from a fresh scan of 2 files — the cache
// result came back verbatim, so classification was skipped entirely.
expect(result).toEqual(expect.objectContaining(cachedResult));
expect(result.cached).toBe(true);
expect(result.total).toBe(4);
expect(executeFn).not.toHaveBeenCalled();
expect(statFn).not.toHaveBeenCalled();
expect(readFileFn).toHaveBeenCalledTimes(1);
expect(writeFileFn).not.toHaveBeenCalled();
});
it("force bypasses the cache and re-validates the directories", async () => {
mockReaddir({
"/assets/nitro": ["chair.nitro", "ghost.nitro"],
});
// A matching-on-disk cache exists, but `force: true` ignores it.
existsFn.mockImplementation(() => true);
readFileFn.mockResolvedValue(Buffer.from("nitro-data"));
parseNitroBundle.mockReturnValue({});
statFn.mockImplementation(async () => ({ size: 512, mtimeMs: 0 }));
const result = await scanFakeBrokenNitros({ force: true });
expect(result.cached).toBeUndefined();
expect(result.fake.map((e) => e.fileName)).toEqual(["ghost.nitro"]);
expect(result.broken).toEqual([]);
expect(result.total).toBe(2);
expect(statFn).toHaveBeenCalled();
});
}); });
describe("deleteNitroCleanupFiles", () => { describe("deleteNitroCleanupFiles", () => {
+180 -77
View File
@@ -1,13 +1,13 @@
import { createHash } from "node:crypto";
import { existsSync, promises as fs } from "node:fs"; import { existsSync, promises as fs } from "node:fs";
import os from "node:os"; import os from "node:os";
import { sql } from "drizzle-orm"; import path from "node:path";
import { queryRows } from "@/lib/db";
import { getBaseClassname, getSafeClassnameStem } from "@/lib/furni/classname";
import { import {
DEFAULT_NITRO_SOURCES, DEFAULT_NITRO_SOURCES,
listSources, listSources,
} from "@/lib/services/clone-sources"; } from "@/lib/services/clone-sources";
import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs"; import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs";
import { getCleanupValidStems } from "@/lib/services/furni-import-cache";
import { downloadFile } from "@/lib/services/import/core/download"; import { downloadFile } from "@/lib/services/import/core/download";
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder"; import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
import { getRuntimePath } from "@/lib/utils/runtime-path"; import { getRuntimePath } from "@/lib/utils/runtime-path";
@@ -57,6 +57,23 @@ export interface NitroCleanupScan {
orphanedSwf: AssetCleanupEntry[]; orphanedSwf: AssetCleanupEntry[];
orphanedIcon: AssetCleanupEntry[]; orphanedIcon: AssetCleanupEntry[];
total: number; total: number;
/** True when the result was served from the on-disk scan cache. */
cached?: boolean;
}
export interface CleanupProgress {
phase: "stems" | "readdir" | "validate" | "done";
scanned: number;
}
export interface CleanupScanOptions {
/** Bypass the scan cache and re-validate every bundle. */
force?: boolean;
/**
* Coarse progress callback used to stream scan status to the UI instead of
* leaving the tab looking frozen while a large directory is processed.
*/
onProgress?: (progress: CleanupProgress) => void;
} }
export interface NitroCleanupDeleteResult { export interface NitroCleanupDeleteResult {
@@ -198,18 +215,6 @@ async function hasValidNitroHeader(filePath: string): Promise<boolean> {
} }
} }
/** Register every .nitro file stem a classname could legitimately use. */
function addStems(set: Set<string>, classname: string): void {
const normalized = classname.trim().toLowerCase();
if (!normalized) return;
const base = getBaseClassname(normalized).toLowerCase();
if (base) set.add(base);
if (normalized.includes("*")) {
const safe = getSafeClassnameStem(normalized).toLowerCase();
if (safe) set.add(safe);
}
}
interface DirectoryRead { interface DirectoryRead {
name: string; name: string;
base: string; base: string;
@@ -217,6 +222,23 @@ interface DirectoryRead {
lastModified: number; lastModified: number;
} }
/** Discriminated cache key per asset directory (kind + path). */
function kindKey(kind: CleanupAssetKind, dir: string): string {
return `${kind}\0${dir}`;
}
async function readDirNames(dir: string, regex: RegExp): Promise<string[]> {
const names = await fs.readdir(dir).catch(() => [] as string[]);
return names.filter((name) => regex.test(name)).sort();
}
/** Cheap fingerprint of a directory listing — a rename/add/remove changes it. */
function dirNameSignature(names: string[]): string {
const hash = createHash("sha1");
for (const name of names) hash.update(name).update("\n");
return hash.digest("hex");
}
/** /**
* How many `stat` calls run in parallel while reading an asset directory. * How many `stat` calls run in parallel while reading an asset directory.
* Directories can hold hundreds of thousands of files; serial `stat`ing them * Directories can hold hundreds of thousands of files; serial `stat`ing them
@@ -224,14 +246,12 @@ interface DirectoryRead {
*/ */
const DIR_STAT_CONCURRENCY = 32; const DIR_STAT_CONCURRENCY = 32;
async function readAssetDir( async function buildDirEntries(
dir: string, dir: string,
regex: RegExp, names: string[],
): Promise<DirectoryRead[]> { ): Promise<DirectoryRead[]> {
const names = await fs.readdir(dir).catch(() => [] as string[]);
const entries: DirectoryRead[] = []; const entries: DirectoryRead[] = [];
await runPool(names, DIR_STAT_CONCURRENCY, async (name) => { await runPool(names, DIR_STAT_CONCURRENCY, async (name) => {
if (!regex.test(name)) return;
let size = 0; let size = 0;
let mtimeMs = 0; let mtimeMs = 0;
try { try {
@@ -252,12 +272,14 @@ async function readAssetDir(
} }
async function collectAssets( async function collectAssets(
kind: CleanupAssetKind,
dirs: string[], dirs: string[],
regex: RegExp, namesByKey: Map<string, string[]>,
): Promise<Map<string, AssetCleanupEntry>> { ): Promise<Map<string, AssetCleanupEntry>> {
const byName = new Map<string, AssetCleanupEntry>(); const byName = new Map<string, AssetCleanupEntry>();
for (const dir of dirs) { for (const dir of dirs) {
for (const read of await readAssetDir(dir, regex)) { const names = namesByKey.get(kindKey(kind, dir)) ?? [];
for (const read of await buildDirEntries(dir, names)) {
const entry = byName.get(read.name) ?? { const entry = byName.get(read.name) ?? {
fileName: read.name, fileName: read.name,
base: read.base, base: read.base,
@@ -274,35 +296,66 @@ async function collectAssets(
return byName; return byName;
} }
async function loadValidStems(): Promise<Set<string>> { // ── On-disk scan cache ──────────────────────────────────────────────────────
// Authoritative stem set from items_base (item_name + public_name //
// fallback), mirroring repair-nitros.ts classname resolution. // The heavy parts of a scan are the per-bundle header validation (one open +
const validStems = new Set<string>(); // small read per .nitro) and the parallel stat pass. Both can be skipped when
try { // no asset directory changed since the last scan: a signature per directory is
const result = await queryRows<{ // derived from the sortable file *list* (readdir + hash, no stat). Any add /
item_name: string; // remove / rename invalidates the cache and triggers a full re-validation.
public_name: string; // Bundles rewritten in place without a name change are not detected by the
}>(sql` // signature; run a forced scan (`force: true`) to pick those up.
SELECT item_name, public_name FROM items_base
`); const SCAN_CACHE_DIR = path.join(process.cwd(), "storage", "nitro-cleanup");
for (const row of result) { const scanCachePath = () => path.join(SCAN_CACHE_DIR, "scan-cache.json");
addStems(validStems, row.item_name);
if ( interface ScanCacheFile {
row.public_name && dirs: Record<string, string>;
row.public_name !== row.item_name && result: Omit<NitroCleanupScan, "cached">;
!row.public_name.includes(" ") &&
row.public_name !== "0"
) {
addStems(validStems, row.public_name);
}
}
} catch {
/* empty set → every asset is treated as orphaned */
}
return validStems;
} }
export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> { async function readScanCache(): Promise<ScanCacheFile | null> {
try {
if (!existsSync(scanCachePath())) return null;
const raw = await fs.readFile(scanCachePath(), "utf8");
const parsed = JSON.parse(raw) as ScanCacheFile;
if (!parsed || typeof parsed.dirs !== "object" || !parsed.result) {
return null;
}
return parsed;
} catch {
return null;
}
}
async function writeScanCache(
dirs: Record<string, string>,
result: Omit<NitroCleanupScan, "cached">,
): Promise<void> {
try {
await fs.mkdir(SCAN_CACHE_DIR, { recursive: true });
await fs.writeFile(scanCachePath(), JSON.stringify({ dirs, result }));
} catch {
/* cache is best effort — a broken cache simply re-scans next time */
}
}
function cacheDirsMatch(
cached: Record<string, string>,
current: Record<string, string>,
): boolean {
const cachedKeys = Object.keys(cached);
if (cachedKeys.length !== Object.keys(current).length) return false;
for (const key of cachedKeys) {
if (cached[key] !== current[key]) return false;
}
return true;
}
export async function scanFakeBrokenNitros(
options: CleanupScanOptions = {},
): Promise<NitroCleanupScan> {
const { force = false, onProgress } = options;
const targets = await getFurniAssetWriteTargets(); const targets = await getFurniAssetWriteTargets();
const nitroDirs = uniqueDirs([ const nitroDirs = uniqueDirs([
targets.nitroDir, targets.nitroDir,
@@ -316,47 +369,94 @@ export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
targets.iconDir, targets.iconDir,
...targets.mirrorDirs.map((d) => d.iconDir), ...targets.mirrorDirs.map((d) => d.iconDir),
]); ]);
const directoryPlan: Array<{
kind: CleanupAssetKind;
dirs: string[];
}> = [
{ kind: "nitro", dirs: nitroDirs },
{ kind: "swf", dirs: swfDirs },
{ kind: "icon", dirs: iconDirs },
];
const [byNitro, bySwf, byIcon, validStems] = await Promise.all([ onProgress?.({ phase: "readdir", scanned: 0 });
collectAssets(nitroDirs, NITRO_FILE_RE),
collectAssets(swfDirs, SWF_FILE_RE), // One readdir pass shared by both the cache signature and the full scan —
collectAssets(iconDirs, ICON_FILE_RE), // never stat a file just to learn the directory changed.
loadValidStems(), const namesByKey = new Map<string, string[]>();
await Promise.all(
directoryPlan.flatMap(({ kind, dirs }) =>
dirs.map(async (dir) => {
namesByKey.set(
kindKey(kind, dir),
await readDirNames(dir, KIND_FILE_RE[kind]),
);
}),
),
);
const currentDigests: Record<string, string> = {};
for (const key of namesByKey.keys()) {
currentDigests[key] = dirNameSignature(namesByKey.get(key) ?? []);
}
onProgress?.({ phase: "readdir", scanned: 0 });
if (!force) {
const cache = await readScanCache();
if (cache && cacheDirsMatch(cache.dirs, currentDigests)) {
onProgress?.({ phase: "done", scanned: cache.result.total });
return { ...cache.result, cached: true };
}
}
// Needed only for classification — skip the DB query on a cache hit.
onProgress?.({ phase: "stems", scanned: 0 });
const validStems = await getCleanupValidStems();
const [byNitro, bySwf, byIcon] = await Promise.all([
collectAssets("nitro", nitroDirs, namesByKey),
collectAssets("swf", swfDirs, namesByKey),
collectAssets("icon", iconDirs, namesByKey),
]); ]);
const entries: NitroCleanupEntry[] = [...byNitro.values()]; const entries: NitroCleanupEntry[] = [...byNitro.values()];
const fake: NitroCleanupEntry[] = []; const fake: NitroCleanupEntry[] = [];
const broken: NitroCleanupEntry[] = []; const broken: NitroCleanupEntry[] = [];
let validated = 0;
const totalToValidate = entries.length;
await runPool(entries, 16, async (entry) => { await runPool(entries, 16, async (entry) => {
// A .nitro that no DB item maps to is a leftover / fake bundle. // A .nitro that no DB item maps to is a leftover / fake bundle.
if (!validStems.has(entry.base.toLowerCase())) { if (!validStems.has(entry.base.toLowerCase())) {
fake.push(entry); fake.push(entry);
return; } else {
} // Otherwise validate the bundle content. Most bundles pass a cheap
// Otherwise validate the bundle content. Most bundles pass a cheap // header-only check (a few KB, no decompression); only files whose
// header-only check (a few KB, no decompression); only files whose // header looks suspicious get the expensive full parse. This keeps
// header looks suspicious get the expensive full parse. This keeps the // the scan fast even on directories with hundreds of thousands of
// scan fast even on directories with hundreds of thousands of bundles. // bundles.
let parsed = false; let parsed = false;
let firstError = ""; let firstError = "";
for (const dir of entry.dirs) { for (const dir of entry.dirs) {
if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) { if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) {
parsed = true; parsed = true;
break; break;
}
try {
parseNitroBundle(
await fs.readFile(getRuntimePath(dir, entry.fileName)),
);
parsed = true;
break;
} catch (err) {
firstError = (err as Error).message;
}
} }
try { if (!parsed) {
parseNitroBundle( broken.push({ ...entry, error: firstError });
await fs.readFile(getRuntimePath(dir, entry.fileName)),
);
parsed = true;
break;
} catch (err) {
firstError = (err as Error).message;
} }
} }
if (!parsed) { validated++;
broken.push({ ...entry, error: firstError }); if (validated % 2500 === 0 || validated === totalToValidate) {
onProgress?.({ phase: "validate", scanned: validated });
} }
}); });
@@ -370,13 +470,16 @@ export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
const byName = <T extends { fileName: string }>(entries: T[]): T[] => const byName = <T extends { fileName: string }>(entries: T[]): T[] =>
entries.sort((a, b) => a.fileName.localeCompare(b.fileName)); entries.sort((a, b) => a.fileName.localeCompare(b.fileName));
return { const result: NitroCleanupScan = {
fake: byName(fake), fake: byName(fake),
broken: byName(broken), broken: byName(broken),
orphanedSwf: byName(orphanedSwf), orphanedSwf: byName(orphanedSwf),
orphanedIcon: byName(orphanedIcon), orphanedIcon: byName(orphanedIcon),
total: byNitro.size + bySwf.size + byIcon.size, total: byNitro.size + bySwf.size + byIcon.size,
}; };
await writeScanCache(currentDigests, result);
onProgress?.({ phase: "done", scanned: result.total });
return result;
} }
function dirsForKind( function dirsForKind(
@@ -435,7 +538,7 @@ export async function deleteNitroCleanupFiles(
return { deleted, copiesRemoved, errors, files }; return { deleted, copiesRemoved, errors, files };
} }
function ageDays(lastModified: number): number { export function ageDays(lastModified: number): number {
if (!lastModified) return 0; if (!lastModified) return 0;
return Math.max(0, Math.floor((Date.now() - lastModified) / DAY_MS)); return Math.max(0, Math.floor((Date.now() - lastModified) / DAY_MS));
} }