feat(studio): expand nitro cleanup with repair, auto-clean, and orphaned assets
Scan distinguishes fake, broken, and orphaned SWF/icon assets with age metadata, deletes per asset kind, re-downloads broken nitro bundles from configured sources, auto-cleans old fake leftovers, and exports a JSON manifest. Adds rebuild and auto-clean API endpoints with audit coverage and a housekeeping preview route under the hotel domain. Verified: full vitest suite (2213 tests), typecheck, and biome all pass.
This commit is contained in:
openhands committed
1 parent
ea4fd375b4
commit
438277a17a
13 files changed
+1118
-155
No files matched your search
@@ -1,50 +1,91 @@
|
||||
import { existsSync, promises as fs } from "node:fs";
|
||||
import os from "node:os";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { db } from "@/lib/db";
|
||||
import { getBaseClassname, getSafeClassnameStem } from "@/lib/furni/classname";
|
||||
import {
|
||||
DEFAULT_NITRO_SOURCES,
|
||||
listSources,
|
||||
} from "@/lib/services/clone-sources";
|
||||
import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs";
|
||||
import { downloadFile } from "@/lib/services/import/core/download";
|
||||
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
|
||||
import { getRuntimePath } from "@/lib/utils/runtime-path";
|
||||
|
||||
/**
|
||||
* Catalog Studio nitro cleanup.
|
||||
*
|
||||
* Scans every .nitro directory (public + mirrored gamedata) and splits the
|
||||
* found bundles into two removable groups:
|
||||
* Scans every asset directory (public + mirrored gamedata) for furniture
|
||||
* leftovers and splits the found bundles into removable groups:
|
||||
*
|
||||
* - "fake" .nitro files whose stem does not match any item in `items_base`
|
||||
* (leftover files that belong to no furniture — including items that were
|
||||
* deleted or imported under a different name);
|
||||
* - "broken" .nitro files that do belong to an item but cannot be parsed as a
|
||||
* valid Nitro bundle (truncated, corrupted or renamed files).
|
||||
* valid Nitro bundle (truncated, corrupted or renamed files);
|
||||
* - orphaned `.swf` / icon files whose stem matches no item in `items_base`.
|
||||
*
|
||||
* Deletion only touches the .nitro files on disk — never database rows.
|
||||
* Deletion only touches the asset files on disk — never database rows. Broken
|
||||
* bundles can be re-downloaded from the configured nitro sources instead of
|
||||
* being deleted, and fake bundles older than a configurable age can be removed
|
||||
* automatically for self-cleaning nitro directories.
|
||||
*/
|
||||
|
||||
export interface NitroCleanupEntry {
|
||||
export type CleanupAssetKind = "nitro" | "swf" | "icon";
|
||||
|
||||
export interface AssetCleanupEntry {
|
||||
fileName: string;
|
||||
base: string;
|
||||
size: number;
|
||||
dirs: string[];
|
||||
lastModified: number;
|
||||
}
|
||||
|
||||
export interface NitroCleanupEntry extends AssetCleanupEntry {
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface NitroCleanupScan {
|
||||
fake: NitroCleanupEntry[];
|
||||
broken: NitroCleanupEntry[];
|
||||
orphanedSwf: AssetCleanupEntry[];
|
||||
orphanedIcon: AssetCleanupEntry[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface NitroCleanupDeleteResult {
|
||||
/** Files that were fully removed from every nitro directory. */
|
||||
/** Files that were fully removed from every matching directory. */
|
||||
deleted: number;
|
||||
/** Individual .nitro files removed across all directories (mirrors). */
|
||||
/** Individual files removed across all directories (mirrors). */
|
||||
copiesRemoved: number;
|
||||
errors: string[];
|
||||
files: Array<{ fileName: string; deleted: boolean; remaining: string[] }>;
|
||||
}
|
||||
|
||||
export interface NitroRepairResult {
|
||||
repaired: number;
|
||||
failed: number;
|
||||
errors: string[];
|
||||
files: Array<{ fileName: string; ok: boolean; error?: string }>;
|
||||
}
|
||||
|
||||
export interface NitroAutoCleanResult extends NitroCleanupDeleteResult {
|
||||
/** Fake bundles skipped because they are younger than the age threshold. */
|
||||
skippedRecent: number;
|
||||
/** Broken bundles skipped — only fake leftovers are auto-removed. */
|
||||
skippedBroken: number;
|
||||
}
|
||||
|
||||
const NITRO_FILE_RE = /^[a-z0-9_*\-.]+\.nitro$/i;
|
||||
const SWF_FILE_RE = /^[a-z0-9_*\-.]+\.swf$/i;
|
||||
const ICON_FILE_RE = /^[a-z0-9_*\-.]+\.(?:gif|png)$/i;
|
||||
const KIND_FILE_RE: Record<CleanupAssetKind, RegExp> = {
|
||||
nitro: NITRO_FILE_RE,
|
||||
swf: SWF_FILE_RE,
|
||||
icon: ICON_FILE_RE,
|
||||
};
|
||||
|
||||
const DAY_MS = 24 * 60 * 60 * 1000;
|
||||
|
||||
function uniqueDirs(dirs: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
@@ -90,34 +131,67 @@ function addStems(set: Set<string>, classname: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
|
||||
const targets = await getFurniAssetWriteTargets();
|
||||
const nitroDirs = uniqueDirs([
|
||||
targets.nitroDir,
|
||||
...targets.mirrorDirs.map((d) => d.nitroDir),
|
||||
]);
|
||||
interface DirectoryRead {
|
||||
name: string;
|
||||
base: string;
|
||||
size: number;
|
||||
lastModified: number;
|
||||
}
|
||||
|
||||
// name → dirs + largest size seen anywhere.
|
||||
const byName = new Map<string, { dirs: string[]; size: number }>();
|
||||
for (const dir of nitroDirs) {
|
||||
const names = await fs.readdir(dir).catch(() => [] as string[]);
|
||||
for (const name of names) {
|
||||
if (!NITRO_FILE_RE.test(name)) continue;
|
||||
const info = byName.get(name) ?? { dirs: [], size: 0 };
|
||||
info.dirs.push(dir);
|
||||
try {
|
||||
const stat = await fs.stat(getRuntimePath(dir, name));
|
||||
info.size = Math.max(info.size, stat.size);
|
||||
} catch {
|
||||
/* ignore unreadable entry */
|
||||
}
|
||||
byName.set(name, info);
|
||||
async function readAssetDir(
|
||||
dir: string,
|
||||
regex: RegExp,
|
||||
): Promise<DirectoryRead[]> {
|
||||
const names = await fs.readdir(dir).catch(() => [] as string[]);
|
||||
const entries: DirectoryRead[] = [];
|
||||
for (const name of names) {
|
||||
if (!regex.test(name)) continue;
|
||||
let size = 0;
|
||||
let mtimeMs = 0;
|
||||
try {
|
||||
const stat = await fs.stat(getRuntimePath(dir, name));
|
||||
size = stat.size;
|
||||
mtimeMs = stat.mtimeMs;
|
||||
} catch {
|
||||
/* ignore unreadable entry */
|
||||
}
|
||||
entries.push({
|
||||
name,
|
||||
base: name.replace(/\.[a-z0-9]+$/i, ""),
|
||||
size,
|
||||
lastModified: mtimeMs,
|
||||
});
|
||||
}
|
||||
return entries;
|
||||
}
|
||||
|
||||
async function collectAssets(
|
||||
dirs: string[],
|
||||
regex: RegExp,
|
||||
): Promise<Map<string, AssetCleanupEntry>> {
|
||||
const byName = new Map<string, AssetCleanupEntry>();
|
||||
for (const dir of dirs) {
|
||||
for (const read of await readAssetDir(dir, regex)) {
|
||||
const entry = byName.get(read.name) ?? {
|
||||
fileName: read.name,
|
||||
base: read.base,
|
||||
size: 0,
|
||||
dirs: [],
|
||||
lastModified: 0,
|
||||
};
|
||||
if (!entry.dirs.includes(dir)) entry.dirs.push(dir);
|
||||
entry.size = Math.max(entry.size, read.size);
|
||||
entry.lastModified = Math.max(entry.lastModified, read.lastModified);
|
||||
byName.set(read.name, entry);
|
||||
}
|
||||
}
|
||||
return byName;
|
||||
}
|
||||
|
||||
async function loadValidStems(): Promise<Set<string>> {
|
||||
// Authoritative stem set from items_base (item_name + public_name
|
||||
// fallback), mirroring repair-nitros.ts classname resolution.
|
||||
let rows: Array<{ item_name: string; public_name: string }> = [];
|
||||
const validStems = new Set<string>();
|
||||
try {
|
||||
const [result] = (await db.execute(sql`
|
||||
SELECT item_name, public_name FROM items_base
|
||||
@@ -125,30 +199,46 @@ export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
|
||||
Array<{ item_name: string; public_name: string }>,
|
||||
unknown,
|
||||
];
|
||||
rows = result;
|
||||
} catch {
|
||||
/* empty set → every .nitro is treated as fake */
|
||||
}
|
||||
const validStems = new Set<string>();
|
||||
for (const row of rows) {
|
||||
addStems(validStems, row.item_name);
|
||||
if (
|
||||
row.public_name &&
|
||||
row.public_name !== row.item_name &&
|
||||
!row.public_name.includes(" ") &&
|
||||
row.public_name !== "0"
|
||||
) {
|
||||
addStems(validStems, row.public_name);
|
||||
for (const row of result) {
|
||||
addStems(validStems, row.item_name);
|
||||
if (
|
||||
row.public_name &&
|
||||
row.public_name !== row.item_name &&
|
||||
!row.public_name.includes(" ") &&
|
||||
row.public_name !== "0"
|
||||
) {
|
||||
addStems(validStems, row.public_name);
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
/* empty set → every asset is treated as orphaned */
|
||||
}
|
||||
return validStems;
|
||||
}
|
||||
|
||||
const entries: NitroCleanupEntry[] = [...byName].map(([fileName, info]) => ({
|
||||
fileName,
|
||||
base: fileName.slice(0, -".nitro".length),
|
||||
size: info.size,
|
||||
dirs: info.dirs,
|
||||
}));
|
||||
export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
|
||||
const targets = await getFurniAssetWriteTargets();
|
||||
const nitroDirs = uniqueDirs([
|
||||
targets.nitroDir,
|
||||
...targets.mirrorDirs.map((d) => d.nitroDir),
|
||||
]);
|
||||
const swfDirs = uniqueDirs([
|
||||
targets.swfDir,
|
||||
...targets.mirrorDirs.map((d) => d.swfDir),
|
||||
]);
|
||||
const iconDirs = uniqueDirs([
|
||||
targets.iconDir,
|
||||
...targets.mirrorDirs.map((d) => d.iconDir),
|
||||
]);
|
||||
|
||||
const [byNitro, bySwf, byIcon, validStems] = await Promise.all([
|
||||
collectAssets(nitroDirs, NITRO_FILE_RE),
|
||||
collectAssets(swfDirs, SWF_FILE_RE),
|
||||
collectAssets(iconDirs, ICON_FILE_RE),
|
||||
loadValidStems(),
|
||||
]);
|
||||
|
||||
const entries: NitroCleanupEntry[] = [...byNitro.values()];
|
||||
const fake: NitroCleanupEntry[] = [];
|
||||
const broken: NitroCleanupEntry[] = [];
|
||||
|
||||
@@ -177,21 +267,46 @@ export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
|
||||
}
|
||||
});
|
||||
|
||||
fake.sort((a, b) => a.fileName.localeCompare(b.fileName));
|
||||
broken.sort((a, b) => a.fileName.localeCompare(b.fileName));
|
||||
const orphanedSwf = [...bySwf.values()].filter(
|
||||
(entry) => !validStems.has(entry.base.toLowerCase()),
|
||||
);
|
||||
const orphanedIcon = [...byIcon.values()].filter(
|
||||
(entry) => !validStems.has(entry.base.toLowerCase()),
|
||||
);
|
||||
|
||||
return { fake, broken, total: byName.size };
|
||||
const byName = <T extends { fileName: string }>(entries: T[]): T[] =>
|
||||
entries.sort((a, b) => a.fileName.localeCompare(b.fileName));
|
||||
|
||||
return {
|
||||
fake: byName(fake),
|
||||
broken: byName(broken),
|
||||
orphanedSwf: byName(orphanedSwf),
|
||||
orphanedIcon: byName(orphanedIcon),
|
||||
total: byNitro.size + bySwf.size + byIcon.size,
|
||||
};
|
||||
}
|
||||
|
||||
/** Delete .nitro files from every nitro directory (primary + mirrors). */
|
||||
function dirsForKind(
|
||||
targets: Awaited<ReturnType<typeof getFurniAssetWriteTargets>>,
|
||||
kind: CleanupAssetKind,
|
||||
): string[] {
|
||||
const primary = {
|
||||
nitro: targets.nitroDir,
|
||||
swf: targets.swfDir,
|
||||
icon: targets.iconDir,
|
||||
}[kind];
|
||||
const mirrors = targets.mirrorDirs.map((d) => d[`${kind}Dir` as const]);
|
||||
return uniqueDirs([primary, ...mirrors]);
|
||||
}
|
||||
|
||||
/** Delete asset files from every matching directory (primary + mirrors). */
|
||||
export async function deleteNitroCleanupFiles(
|
||||
fileNames: string[],
|
||||
kind: CleanupAssetKind = "nitro",
|
||||
): Promise<NitroCleanupDeleteResult> {
|
||||
const targets = await getFurniAssetWriteTargets();
|
||||
const nitroDirs = uniqueDirs([
|
||||
targets.nitroDir,
|
||||
...targets.mirrorDirs.map((d) => d.nitroDir),
|
||||
]);
|
||||
const dirs = dirsForKind(targets, kind);
|
||||
const fileRegex = KIND_FILE_RE[kind];
|
||||
|
||||
let deleted = 0;
|
||||
let copiesRemoved = 0;
|
||||
@@ -200,14 +315,14 @@ export async function deleteNitroCleanupFiles(
|
||||
|
||||
for (const rawName of [...new Set(fileNames)]) {
|
||||
const fileName = String(rawName ?? "");
|
||||
if (!NITRO_FILE_RE.test(fileName)) {
|
||||
errors.push(`Invalid nitro file name: ${fileName}`);
|
||||
if (!fileRegex.test(fileName)) {
|
||||
errors.push(`Invalid ${kind} file name: ${fileName}`);
|
||||
files.push({ fileName, deleted: false, remaining: [] });
|
||||
continue;
|
||||
}
|
||||
const remaining: string[] = [];
|
||||
let existed = false;
|
||||
for (const dir of nitroDirs) {
|
||||
for (const dir of dirs) {
|
||||
const filePath = getRuntimePath(dir, fileName);
|
||||
if (!existsSync(filePath)) continue;
|
||||
existed = true;
|
||||
@@ -226,3 +341,147 @@ export async function deleteNitroCleanupFiles(
|
||||
|
||||
return { deleted, copiesRemoved, errors, files };
|
||||
}
|
||||
|
||||
function ageDays(lastModified: number): number {
|
||||
if (!lastModified) return 0;
|
||||
return Math.max(0, Math.floor((Date.now() - lastModified) / DAY_MS));
|
||||
}
|
||||
|
||||
/**
|
||||
* Automatically remove fake .nitro leftovers that are older than
|
||||
* `maxAgeDays`. Broken bundles and young fake bundles are never touched —
|
||||
* only former leftovers whose items are long gone from the database.
|
||||
*/
|
||||
export async function autoCleanFakeNitros(
|
||||
maxAgeDays = 30,
|
||||
): Promise<NitroAutoCleanResult> {
|
||||
const scan = await scanFakeBrokenNitros();
|
||||
const threshold = Math.max(0, Math.floor(maxAgeDays));
|
||||
const removable = scan.fake.filter(
|
||||
(entry) => ageDays(entry.lastModified) >= threshold,
|
||||
);
|
||||
const result = await deleteNitroCleanupFiles(
|
||||
removable.map((entry) => entry.fileName),
|
||||
"nitro",
|
||||
);
|
||||
return {
|
||||
...result,
|
||||
skippedRecent: scan.fake.length - removable.length,
|
||||
skippedBroken: scan.broken.length,
|
||||
};
|
||||
}
|
||||
|
||||
async function copyFileToDirs(
|
||||
buffer: Buffer,
|
||||
fileName: string,
|
||||
dirs: string[],
|
||||
): Promise<string[]> {
|
||||
const errors: string[] = [];
|
||||
for (const dir of dirs) {
|
||||
try {
|
||||
await fs.mkdir(dir, { recursive: true });
|
||||
await fs.writeFile(getRuntimePath(dir, fileName), buffer);
|
||||
} catch (err) {
|
||||
errors.push(`write .nitro to ${dir} failed: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
return errors;
|
||||
}
|
||||
|
||||
/**
|
||||
* Re-download broken .nitro bundles from the configured nitro sources. The
|
||||
* file stem matched an item in `items_base`, so the same classname candidates
|
||||
* used by repair-nitros.ts are tried; every downloaded bundle is validated
|
||||
* before it replaces the broken copies across all nitro directories.
|
||||
*/
|
||||
export async function repairBrokenNitros(
|
||||
fileNames: string[],
|
||||
): Promise<NitroRepairResult> {
|
||||
const targets = await getFurniAssetWriteTargets();
|
||||
const nitroDirs = dirsForKind(targets, "nitro");
|
||||
let sources = (await listSources().catch(() => [] as never)).filter(
|
||||
(source: { nitroBaseUrl?: string }) => source.nitroBaseUrl,
|
||||
);
|
||||
if (sources.length === 0) sources = DEFAULT_NITRO_SOURCES as never;
|
||||
|
||||
let repaired = 0;
|
||||
let failed = 0;
|
||||
const errors: string[] = [];
|
||||
const files: NitroRepairResult["files"] = [];
|
||||
|
||||
await runPool([...new Set(fileNames)], 6, async (rawName) => {
|
||||
const fileName = String(rawName ?? "");
|
||||
if (!NITRO_FILE_RE.test(fileName) || !fileName.endsWith(".nitro")) {
|
||||
failed++;
|
||||
const message = `Invalid nitro file name: ${fileName}`;
|
||||
errors.push(message);
|
||||
files.push({ fileName, ok: false, error: message });
|
||||
return;
|
||||
}
|
||||
const name = fileName.slice(0, -".nitro".length);
|
||||
const candidates = [
|
||||
name,
|
||||
...(name.includes("*")
|
||||
? [name.split("*")[0], name.replace(/\*/g, "_")]
|
||||
: []),
|
||||
...(name.includes(".") ? [name.replace(/\./g, "_")] : []),
|
||||
];
|
||||
for (const source of sources as {
|
||||
name: string;
|
||||
nitroBaseUrl?: string;
|
||||
}[]) {
|
||||
for (const candidate of candidates) {
|
||||
const base = source.nitroBaseUrl;
|
||||
if (!base) continue;
|
||||
const tmpPath = getRuntimePath(
|
||||
os.tmpdir(),
|
||||
`.repair_nitro_${name}_${Math.random().toString(36).slice(2, 8)}.nitro`,
|
||||
);
|
||||
try {
|
||||
const dl = await downloadFile(
|
||||
`${base.replace(/\/+$/, "")}/${encodeURIComponent(candidate)}.nitro`,
|
||||
tmpPath,
|
||||
{ maxRetries: 1, validate: "nitro" },
|
||||
);
|
||||
if (dl.ok) {
|
||||
const buffer = await fs.readFile(tmpPath);
|
||||
parseNitroBundle(buffer);
|
||||
const writeErrors = await copyFileToDirs(
|
||||
buffer,
|
||||
fileName,
|
||||
nitroDirs,
|
||||
);
|
||||
if (writeErrors.length === 0) {
|
||||
repaired++;
|
||||
files.push({ fileName, ok: true });
|
||||
} else {
|
||||
failed++;
|
||||
errors.push(...writeErrors);
|
||||
files.push({
|
||||
fileName,
|
||||
ok: false,
|
||||
error: writeErrors[0],
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
} catch (err) {
|
||||
errors.push(
|
||||
`repair ${fileName} from ${source.name} failed: ${(err as Error).message}`,
|
||||
);
|
||||
} finally {
|
||||
await fs.unlink(tmpPath).catch(() => {});
|
||||
}
|
||||
}
|
||||
}
|
||||
failed++;
|
||||
errors.push(`could not re-obtain ${fileName} from any nitro source`);
|
||||
files.push({
|
||||
fileName,
|
||||
ok: false,
|
||||
error: "could not re-obtain bundle from any nitro source",
|
||||
});
|
||||
});
|
||||
|
||||
return { repaired, failed, errors, files };
|
||||
}
|
||||
Reference in new issue
Block a user