feat(studio): add nitro cleanup tool for fake and broken bundles
This commit is contained in:
openhands committed
1 parent
ce93b92a81
commit
694f87d98c
6 files changed
+869
No files matched your search
@@ -0,0 +1,195 @@
|
||||
// @vitest-environment node
|
||||
import { beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
type AnyFn = (...args: any[]) => any;
|
||||
|
||||
const {
|
||||
executeFn,
|
||||
readdirFn,
|
||||
readFileFn,
|
||||
statFn,
|
||||
unlinkFn,
|
||||
existsFn,
|
||||
parseNitroBundle,
|
||||
getTargetsFn,
|
||||
} = vi.hoisted(() => ({
|
||||
executeFn: vi.fn<AnyFn>(async () => [
|
||||
[
|
||||
{ item_name: "chair", public_name: "Chair" },
|
||||
{ item_name: "broken_thing", public_name: "Broken Thing" },
|
||||
{ item_name: "xmas_*1", public_name: "Xmas" },
|
||||
],
|
||||
[],
|
||||
]),
|
||||
readdirFn: vi.fn<AnyFn>(),
|
||||
readFileFn: vi.fn<AnyFn>(),
|
||||
statFn: vi.fn<AnyFn>(),
|
||||
unlinkFn: vi.fn<AnyFn>(async () => {}),
|
||||
existsFn: vi.fn<AnyFn>(() => false),
|
||||
parseNitroBundle: vi.fn<AnyFn>(),
|
||||
getTargetsFn: vi.fn<AnyFn>(async () => ({
|
||||
nitroDir: "/assets/nitro",
|
||||
swfDir: "/assets/swf",
|
||||
iconDir: "/assets/icons",
|
||||
mirrorDirs: [],
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("@/lib/db", () => ({ db: { execute: executeFn } }));
|
||||
vi.mock("@/lib/services/furni-asset-dirs", () => ({
|
||||
getFurniAssetWriteTargets: getTargetsFn,
|
||||
}));
|
||||
vi.mock("@/lib/services/swf/nitro-builder", () => ({ parseNitroBundle }));
|
||||
vi.mock("node:fs", async (orig) => {
|
||||
const real = (await orig()) as typeof import("node:fs");
|
||||
return {
|
||||
...real,
|
||||
promises: {
|
||||
...real.promises,
|
||||
readdir: readdirFn as AnyFn,
|
||||
readFile: readFileFn as AnyFn,
|
||||
stat: statFn as AnyFn,
|
||||
unlink: unlinkFn as AnyFn,
|
||||
},
|
||||
existsSync: existsFn,
|
||||
};
|
||||
});
|
||||
|
||||
import {
|
||||
deleteNitroCleanupFiles,
|
||||
scanFakeBrokenNitros,
|
||||
} from "@/lib/services/nitro-cleanup";
|
||||
|
||||
beforeEach(() => {
|
||||
vi.clearAllMocks();
|
||||
readdirFn.mockReset();
|
||||
readFileFn.mockReset();
|
||||
parseNitroBundle.mockReset();
|
||||
existsFn.mockReset();
|
||||
getTargetsFn.mockReset();
|
||||
getTargetsFn.mockResolvedValue({
|
||||
nitroDir: "/assets/nitro",
|
||||
swfDir: "/assets/swf",
|
||||
iconDir: "/assets/icons",
|
||||
mirrorDirs: [],
|
||||
});
|
||||
});
|
||||
|
||||
describe("scanFakeBrokenNitros", () => {
|
||||
it("classifies fake (no matching item) and broken (unparseable) bundles", async () => {
|
||||
readdirFn.mockResolvedValue([
|
||||
"chair.nitro",
|
||||
"fake_stuff.nitro",
|
||||
"broken_thing.nitro",
|
||||
"xmas_.nitro",
|
||||
]);
|
||||
readFileFn.mockImplementation(async (filePath: string) =>
|
||||
Buffer.from(filePath.includes("broken") ? "broken-data" : "nitro-data"),
|
||||
);
|
||||
statFn.mockImplementation(async () => ({ size: 512 }));
|
||||
parseNitroBundle.mockImplementation((buffer: Buffer) => {
|
||||
if (buffer.toString("utf-8").startsWith("broken")) {
|
||||
throw new Error("Not a valid .nitro bundle");
|
||||
}
|
||||
return {
|
||||
json: {},
|
||||
jsonFileName: "visualization.json",
|
||||
png: Buffer.alloc(0),
|
||||
pngFileName: "texture.png",
|
||||
};
|
||||
});
|
||||
|
||||
const result = await scanFakeBrokenNitros();
|
||||
|
||||
expect(result.total).toBe(4);
|
||||
expect(result.fake.map((e) => e.fileName)).toEqual(["fake_stuff.nitro"]);
|
||||
expect(result.broken.map((e) => e.fileName)).toEqual([
|
||||
"broken_thing.nitro",
|
||||
]);
|
||||
expect(result.broken[0].error).toBe("Not a valid .nitro bundle");
|
||||
});
|
||||
|
||||
it("resolves star classnames to their base nitro stem", async () => {
|
||||
readdirFn.mockResolvedValue(["xmas_.nitro"]);
|
||||
readFileFn.mockResolvedValue(Buffer.from("nitro-data"));
|
||||
statFn.mockImplementation(async () => ({ size: 16 }));
|
||||
parseNitroBundle.mockReturnValue({
|
||||
json: {},
|
||||
jsonFileName: "visualization.json",
|
||||
png: Buffer.alloc(0),
|
||||
pngFileName: "texture.png",
|
||||
});
|
||||
|
||||
const result = await scanFakeBrokenNitros();
|
||||
|
||||
expect(result.fake).toEqual([]);
|
||||
expect(result.broken).toEqual([]);
|
||||
expect(result.total).toBe(1);
|
||||
});
|
||||
|
||||
it("marks an unparseable bundle whose item is also missing as fake", async () => {
|
||||
readdirFn.mockResolvedValue(["ghost.nitro"]);
|
||||
statFn.mockImplementation(async () => ({ size: 8 }));
|
||||
|
||||
const result = await scanFakeBrokenNitros();
|
||||
|
||||
expect(result.fake[0].fileName).toBe("ghost.nitro");
|
||||
expect(result.broken).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("deleteNitroCleanupFiles", () => {
|
||||
beforeEach(() => {
|
||||
getTargetsFn.mockResolvedValue({
|
||||
nitroDir: "/a/nitro",
|
||||
swfDir: "/a/swf",
|
||||
iconDir: "/a/icons",
|
||||
mirrorDirs: [
|
||||
{ nitroDir: "/b/nitro", swfDir: "/b/swf", iconDir: "/b/icons" },
|
||||
],
|
||||
});
|
||||
existsFn.mockImplementation((filePath: string) =>
|
||||
filePath.includes("chair"),
|
||||
);
|
||||
});
|
||||
|
||||
it("removes each file from every directory where it exists", async () => {
|
||||
const result = await deleteNitroCleanupFiles([
|
||||
"chair.nitro",
|
||||
"missing.nitro",
|
||||
]);
|
||||
|
||||
expect(result.deleted).toBe(1);
|
||||
expect(result.copiesRemoved).toBe(2);
|
||||
expect(unlinkFn).toHaveBeenCalledTimes(2);
|
||||
expect(unlinkFn).toHaveBeenCalledWith("/a/nitro/chair.nitro");
|
||||
expect(unlinkFn).toHaveBeenCalledWith("/b/nitro/chair.nitro");
|
||||
expect(result.errors).toEqual([]);
|
||||
const chair = result.files.find((f) => f.fileName === "chair.nitro");
|
||||
expect(chair?.deleted).toBe(true);
|
||||
expect(chair?.remaining).toEqual([]);
|
||||
});
|
||||
|
||||
it("keeps a file in remaining dirs when an unlink fails", async () => {
|
||||
unlinkFn.mockImplementation(async (filePath: string) => {
|
||||
if (filePath.startsWith("/b/")) throw new Error("locked");
|
||||
});
|
||||
|
||||
const result = await deleteNitroCleanupFiles(["chair.nitro"]);
|
||||
|
||||
expect(result.deleted).toBe(0);
|
||||
expect(result.copiesRemoved).toBe(1);
|
||||
expect(result.errors.some((e) => e.includes("locked"))).toBe(true);
|
||||
const chair = result.files.find((f) => f.fileName === "chair.nitro");
|
||||
expect(chair?.deleted).toBe(false);
|
||||
expect(chair?.remaining).toEqual(["/b/nitro"]);
|
||||
});
|
||||
|
||||
it("rejects unsafe file names", async () => {
|
||||
const result = await deleteNitroCleanupFiles(["../evil.nitro"]);
|
||||
|
||||
expect(result.deleted).toBe(0);
|
||||
expect(result.errors[0]).toContain("Invalid nitro file name");
|
||||
expect(unlinkFn).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,228 @@
|
||||
import { existsSync, promises as fs } from "node:fs";
|
||||
import { sql } from "drizzle-orm";
|
||||
import { db } from "@/lib/db";
|
||||
import { getBaseClassname, getSafeClassnameStem } from "@/lib/furni/classname";
|
||||
import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs";
|
||||
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
|
||||
import { getRuntimePath } from "@/lib/utils/runtime-path";
|
||||
|
||||
/**
|
||||
* Catalog Studio nitro cleanup.
|
||||
*
|
||||
* Scans every .nitro directory (public + mirrored gamedata) and splits the
|
||||
* found bundles into two removable groups:
|
||||
*
|
||||
* - "fake" .nitro files whose stem does not match any item in `items_base`
|
||||
* (leftover files that belong to no furniture — including items that were
|
||||
* deleted or imported under a different name);
|
||||
* - "broken" .nitro files that do belong to an item but cannot be parsed as a
|
||||
* valid Nitro bundle (truncated, corrupted or renamed files).
|
||||
*
|
||||
* Deletion only touches the .nitro files on disk — never database rows.
|
||||
*/
|
||||
|
||||
export interface NitroCleanupEntry {
|
||||
fileName: string;
|
||||
base: string;
|
||||
size: number;
|
||||
dirs: string[];
|
||||
error?: string;
|
||||
}
|
||||
|
||||
export interface NitroCleanupScan {
|
||||
fake: NitroCleanupEntry[];
|
||||
broken: NitroCleanupEntry[];
|
||||
total: number;
|
||||
}
|
||||
|
||||
export interface NitroCleanupDeleteResult {
|
||||
/** Files that were fully removed from every nitro directory. */
|
||||
deleted: number;
|
||||
/** Individual .nitro files removed across all directories (mirrors). */
|
||||
copiesRemoved: number;
|
||||
errors: string[];
|
||||
files: Array<{ fileName: string; deleted: boolean; remaining: string[] }>;
|
||||
}
|
||||
|
||||
const NITRO_FILE_RE = /^[a-z0-9_*\-.]+\.nitro$/i;
|
||||
|
||||
function uniqueDirs(dirs: string[]): string[] {
|
||||
const seen = new Set<string>();
|
||||
const out: string[] = [];
|
||||
for (const dir of dirs) {
|
||||
const key = process.platform === "win32" ? dir.toLowerCase() : dir;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push(dir);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */
|
||||
async function runPool<T>(
|
||||
items: T[],
|
||||
concurrency: number,
|
||||
worker: (item: T) => Promise<void>,
|
||||
): Promise<void> {
|
||||
const limit = Math.min(Math.max(concurrency, 1), items.length);
|
||||
if (limit === 0) return;
|
||||
let next = 0;
|
||||
await Promise.allSettled(
|
||||
Array.from({ length: limit }, async () => {
|
||||
while (true) {
|
||||
const i = next++;
|
||||
if (i >= items.length) break;
|
||||
await worker(items[i]);
|
||||
}
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
/** Register every .nitro file stem a classname could legitimately use. */
|
||||
function addStems(set: Set<string>, classname: string): void {
|
||||
const normalized = classname.trim().toLowerCase();
|
||||
if (!normalized) return;
|
||||
const base = getBaseClassname(normalized).toLowerCase();
|
||||
if (base) set.add(base);
|
||||
if (normalized.includes("*")) {
|
||||
const safe = getSafeClassnameStem(normalized).toLowerCase();
|
||||
if (safe) set.add(safe);
|
||||
}
|
||||
}
|
||||
|
||||
export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
|
||||
const targets = await getFurniAssetWriteTargets();
|
||||
const nitroDirs = uniqueDirs([
|
||||
targets.nitroDir,
|
||||
...targets.mirrorDirs.map((d) => d.nitroDir),
|
||||
]);
|
||||
|
||||
// name → dirs + largest size seen anywhere.
|
||||
const byName = new Map<string, { dirs: string[]; size: number }>();
|
||||
for (const dir of nitroDirs) {
|
||||
const names = await fs.readdir(dir).catch(() => [] as string[]);
|
||||
for (const name of names) {
|
||||
if (!NITRO_FILE_RE.test(name)) continue;
|
||||
const info = byName.get(name) ?? { dirs: [], size: 0 };
|
||||
info.dirs.push(dir);
|
||||
try {
|
||||
const stat = await fs.stat(getRuntimePath(dir, name));
|
||||
info.size = Math.max(info.size, stat.size);
|
||||
} catch {
|
||||
/* ignore unreadable entry */
|
||||
}
|
||||
byName.set(name, info);
|
||||
}
|
||||
}
|
||||
|
||||
// Authoritative stem set from items_base (item_name + public_name
|
||||
// fallback), mirroring repair-nitros.ts classname resolution.
|
||||
let rows: Array<{ item_name: string; public_name: string }> = [];
|
||||
try {
|
||||
const [result] = (await db.execute(sql`
|
||||
SELECT item_name, public_name FROM items_base
|
||||
`)) as unknown as [
|
||||
Array<{ item_name: string; public_name: string }>,
|
||||
unknown,
|
||||
];
|
||||
rows = result;
|
||||
} catch {
|
||||
/* empty set → every .nitro is treated as fake */
|
||||
}
|
||||
const validStems = new Set<string>();
|
||||
for (const row of rows) {
|
||||
addStems(validStems, row.item_name);
|
||||
if (
|
||||
row.public_name &&
|
||||
row.public_name !== row.item_name &&
|
||||
!row.public_name.includes(" ") &&
|
||||
row.public_name !== "0"
|
||||
) {
|
||||
addStems(validStems, row.public_name);
|
||||
}
|
||||
}
|
||||
|
||||
const entries: NitroCleanupEntry[] = [...byName].map(([fileName, info]) => ({
|
||||
fileName,
|
||||
base: fileName.slice(0, -".nitro".length),
|
||||
size: info.size,
|
||||
dirs: info.dirs,
|
||||
}));
|
||||
|
||||
const fake: NitroCleanupEntry[] = [];
|
||||
const broken: NitroCleanupEntry[] = [];
|
||||
|
||||
await runPool(entries, 8, async (entry) => {
|
||||
// A .nitro that no DB item maps to is a leftover / fake bundle.
|
||||
if (!validStems.has(entry.base.toLowerCase())) {
|
||||
fake.push(entry);
|
||||
return;
|
||||
}
|
||||
// Otherwise validate the bundle content.
|
||||
let parsed = false;
|
||||
let firstError = "";
|
||||
for (const dir of entry.dirs) {
|
||||
try {
|
||||
parseNitroBundle(
|
||||
await fs.readFile(getRuntimePath(dir, entry.fileName)),
|
||||
);
|
||||
parsed = true;
|
||||
break;
|
||||
} catch (err) {
|
||||
firstError = (err as Error).message;
|
||||
}
|
||||
}
|
||||
if (!parsed) {
|
||||
broken.push({ ...entry, error: firstError });
|
||||
}
|
||||
});
|
||||
|
||||
fake.sort((a, b) => a.fileName.localeCompare(b.fileName));
|
||||
broken.sort((a, b) => a.fileName.localeCompare(b.fileName));
|
||||
|
||||
return { fake, broken, total: byName.size };
|
||||
}
|
||||
|
||||
/** Delete .nitro files from every nitro directory (primary + mirrors). */
|
||||
export async function deleteNitroCleanupFiles(
|
||||
fileNames: string[],
|
||||
): Promise<NitroCleanupDeleteResult> {
|
||||
const targets = await getFurniAssetWriteTargets();
|
||||
const nitroDirs = uniqueDirs([
|
||||
targets.nitroDir,
|
||||
...targets.mirrorDirs.map((d) => d.nitroDir),
|
||||
]);
|
||||
|
||||
let deleted = 0;
|
||||
let copiesRemoved = 0;
|
||||
const errors: string[] = [];
|
||||
const files: NitroCleanupDeleteResult["files"] = [];
|
||||
|
||||
for (const rawName of [...new Set(fileNames)]) {
|
||||
const fileName = String(rawName ?? "");
|
||||
if (!NITRO_FILE_RE.test(fileName)) {
|
||||
errors.push(`Invalid nitro file name: ${fileName}`);
|
||||
files.push({ fileName, deleted: false, remaining: [] });
|
||||
continue;
|
||||
}
|
||||
const remaining: string[] = [];
|
||||
let existed = false;
|
||||
for (const dir of nitroDirs) {
|
||||
const filePath = getRuntimePath(dir, fileName);
|
||||
if (!existsSync(filePath)) continue;
|
||||
existed = true;
|
||||
try {
|
||||
await fs.unlink(filePath);
|
||||
copiesRemoved++;
|
||||
} catch (err) {
|
||||
remaining.push(dir);
|
||||
errors.push(`${fileName} in ${dir}: ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
const fullyDeleted = existed && remaining.length === 0;
|
||||
if (fullyDeleted) deleted++;
|
||||
files.push({ fileName, deleted: fullyDeleted, remaining });
|
||||
}
|
||||
|
||||
return { deleted, copiesRemoved, errors, files };
|
||||
}
|
||||
Reference in new issue
Block a user