openhands
8638e81444
Replace raw db.execute tuple casts with queryRows/rowsFrom/execResult/ affectedRows helpers from lib/db, drop redundant mysql2 casts on typed query builders, and centralize per-test fakeForm into test/fake-form. Update db mocks in tests so helpers resolve against mocked execute.
574 lines
17 KiB
TypeScript
574 lines
17 KiB
TypeScript
import { existsSync, promises as fs } from "node:fs";
|
|
import os from "node:os";
|
|
import { sql } from "drizzle-orm";
|
|
import { queryRows } from "@/lib/db";
|
|
import { getBaseClassname, getSafeClassnameStem } from "@/lib/furni/classname";
|
|
import {
|
|
DEFAULT_NITRO_SOURCES,
|
|
listSources,
|
|
} from "@/lib/services/clone-sources";
|
|
import { getFurniAssetWriteTargets } from "@/lib/services/furni-asset-dirs";
|
|
import { downloadFile } from "@/lib/services/import/core/download";
|
|
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
|
|
import { getRuntimePath } from "@/lib/utils/runtime-path";
|
|
|
|
/**
|
|
* Catalog Studio nitro cleanup.
|
|
*
|
|
* Scans every asset directory (public + mirrored gamedata) for furniture
|
|
* leftovers and splits the found bundles into removable groups:
|
|
*
|
|
* - "fake" .nitro files whose stem does not match any item in `items_base`
|
|
* (leftover files that belong to no furniture — including items that were
|
|
* deleted or imported under a different name);
|
|
* - "broken" .nitro files that do belong to an item but cannot be parsed as a
|
|
* valid Nitro bundle (truncated, corrupted or renamed files — e.g. a
|
|
* download that landed as an HTML error page instead of a bundle);
|
|
*
|
|
* The scan validates bundles with a cheap header-only read (a few KB per file,
|
|
* no decompression) so it stays fast even on 200k+-file nitro directories. A
|
|
* full parse is only attempted for files whose header looks suspicious, so
|
|
* valid bundles are never flagged by a header false-positive.
|
|
* - orphaned `.swf` / icon files whose stem matches no item in `items_base`.
|
|
*
|
|
* Deletion only touches the asset files on disk — never database rows. Broken
|
|
* bundles can be re-downloaded from the configured nitro sources instead of
|
|
* being deleted, and fake bundles older than a configurable age can be removed
|
|
* automatically for self-cleaning nitro directories.
|
|
*/
|
|
|
|
export type CleanupAssetKind = "nitro" | "swf" | "icon";
|
|
|
|
export interface AssetCleanupEntry {
|
|
fileName: string;
|
|
base: string;
|
|
size: number;
|
|
dirs: string[];
|
|
lastModified: number;
|
|
}
|
|
|
|
export interface NitroCleanupEntry extends AssetCleanupEntry {
|
|
error?: string;
|
|
}
|
|
|
|
export interface NitroCleanupScan {
|
|
fake: NitroCleanupEntry[];
|
|
broken: NitroCleanupEntry[];
|
|
orphanedSwf: AssetCleanupEntry[];
|
|
orphanedIcon: AssetCleanupEntry[];
|
|
total: number;
|
|
}
|
|
|
|
export interface NitroCleanupDeleteResult {
|
|
/** Files that were fully removed from every matching directory. */
|
|
deleted: number;
|
|
/** Individual files removed across all directories (mirrors). */
|
|
copiesRemoved: number;
|
|
errors: string[];
|
|
files: Array<{ fileName: string; deleted: boolean; remaining: string[] }>;
|
|
}
|
|
|
|
export interface NitroRepairResult {
|
|
repaired: number;
|
|
failed: number;
|
|
errors: string[];
|
|
files: Array<{ fileName: string; ok: boolean; error?: string }>;
|
|
}
|
|
|
|
export interface NitroAutoCleanResult extends NitroCleanupDeleteResult {
|
|
/** Fake bundles skipped because they are younger than the age threshold. */
|
|
skippedRecent: number;
|
|
/** Broken bundles skipped — only fake leftovers are auto-removed. */
|
|
skippedBroken: number;
|
|
}
|
|
|
|
const NITRO_FILE_RE = /^[a-z0-9_*\-.]+\.nitro$/i;
|
|
const SWF_FILE_RE = /^[a-z0-9_*\-.]+\.swf$/i;
|
|
const ICON_FILE_RE = /^[a-z0-9_*\-.]+\.(?:gif|png)$/i;
|
|
|
|
/**
|
|
* Only this many leading bytes of a `.nitro` bundle are read during the fake /
|
|
* broken scan. The header (file count + two embedded file names + zlib payload
|
|
* lengths) is tiny, and a zlib/gzip-signed JSON stream is enough to confirm a
|
|
* bundle is structurally valid without decompressing the large PNG texture.
|
|
*/
|
|
const NITRO_HEADER_SAMPLE_BYTES = 2048;
|
|
|
|
/**
|
|
* Minimum + maximum plausible header fields for a Nitro bundle. A bundle always
|
|
* contains at least two files (a `.json` config stream and a `.png` texture
|
|
* stream). File-name length fields and compressed stream lengths are bounded so
|
|
* a bundle whose header claims absurd sizes (e.g. a download that landed as an
|
|
* HTML error page, or a truncated/zero-filled file) can be rejected without
|
|
* decompressing anything.
|
|
*/
|
|
const NITRO_MIN_FILES = 2;
|
|
const NITRO_NAME_LEN_MAX = 1024;
|
|
const NITRO_COMPRESSED_LEN_MAX = 512 * 1024 * 1024;
|
|
const KIND_FILE_RE: Record<CleanupAssetKind, RegExp> = {
|
|
nitro: NITRO_FILE_RE,
|
|
swf: SWF_FILE_RE,
|
|
icon: ICON_FILE_RE,
|
|
};
|
|
|
|
const DAY_MS = 24 * 60 * 60 * 1000;
|
|
|
|
function uniqueDirs(dirs: string[]): string[] {
|
|
const seen = new Set<string>();
|
|
const out: string[] = [];
|
|
for (const dir of dirs) {
|
|
const key = process.platform === "win32" ? dir.toLowerCase() : dir;
|
|
if (seen.has(key)) continue;
|
|
seen.add(key);
|
|
out.push(dir);
|
|
}
|
|
return out;
|
|
}
|
|
|
|
/** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */
|
|
async function runPool<T>(
|
|
items: T[],
|
|
concurrency: number,
|
|
worker: (item: T) => Promise<void>,
|
|
): Promise<void> {
|
|
const limit = Math.min(Math.max(concurrency, 1), items.length);
|
|
if (limit === 0) return;
|
|
let next = 0;
|
|
await Promise.allSettled(
|
|
Array.from({ length: limit }, async () => {
|
|
while (true) {
|
|
const i = next++;
|
|
if (i >= items.length) break;
|
|
await worker(items[i]);
|
|
}
|
|
}),
|
|
);
|
|
}
|
|
|
|
/**
|
|
* Cheap header-only validation that mirrors `parseNitroBundle`'s byte layout
|
|
* (used by `swf/nitro-builder`): a bundle starts with a big-endian file count
|
|
* followed by `(nameLen:int16, name:utf8, compressedLen:int32)` pairs. We read
|
|
* only `NITRO_HEADER_SAMPLE_BYTES` of leading bytes (no decompression) and
|
|
* reject bundles whose header claims absurd sizes — e.g. a download that landed
|
|
* as an HTML error page, or a truncated/zero-filled file.
|
|
*
|
|
* Returns `false` when the header is clearly implausible. When the file cannot
|
|
* be opened/read (including mocked test environments where the path does not
|
|
* exist) it also returns `false`, which makes the caller fall back to the
|
|
* expensive full `parseNitroBundle` — so tests that supply buffers via
|
|
* `readFileFn` keep working unchanged.
|
|
*/
|
|
async function hasValidNitroHeader(filePath: string): Promise<boolean> {
|
|
let handle: import("node:fs/promises").FileHandle | undefined;
|
|
try {
|
|
handle = await fs.open(filePath, "r");
|
|
const sample = Buffer.alloc(NITRO_HEADER_SAMPLE_BYTES);
|
|
const { bytesRead } = await handle.read(sample, 0, sample.length, 0);
|
|
if (bytesRead < 8) return false;
|
|
|
|
let offset = 0;
|
|
const fileCount = sample.readInt16BE(offset);
|
|
offset += 2;
|
|
if (fileCount < NITRO_MIN_FILES) return false;
|
|
if (fileCount > 16) return false;
|
|
|
|
for (let i = 0; i < fileCount && offset < bytesRead - 2; i++) {
|
|
const nameLen = sample.readInt16BE(offset);
|
|
offset += 2;
|
|
if (nameLen < 1 || nameLen > NITRO_NAME_LEN_MAX) return false;
|
|
// The name field must be printable utf-8; an HTML error page or
|
|
// zero-filled buffer almost always trips this.
|
|
if (offset + nameLen > bytesRead) return false;
|
|
const name = sample.toString("utf-8", offset, offset + nameLen);
|
|
offset += nameLen;
|
|
if (!/^[\x20-\x7e]+$/.test(name)) return false;
|
|
if (offset + 4 > bytesRead) return true;
|
|
const compressedLen = sample.readInt32BE(offset);
|
|
offset += 4;
|
|
if (compressedLen <= 0 || compressedLen > NITRO_COMPRESSED_LEN_MAX) {
|
|
return false;
|
|
}
|
|
}
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
} finally {
|
|
await handle?.close().catch(() => {});
|
|
}
|
|
}
|
|
|
|
/** Register every .nitro file stem a classname could legitimately use. */
|
|
function addStems(set: Set<string>, classname: string): void {
|
|
const normalized = classname.trim().toLowerCase();
|
|
if (!normalized) return;
|
|
const base = getBaseClassname(normalized).toLowerCase();
|
|
if (base) set.add(base);
|
|
if (normalized.includes("*")) {
|
|
const safe = getSafeClassnameStem(normalized).toLowerCase();
|
|
if (safe) set.add(safe);
|
|
}
|
|
}
|
|
|
|
interface DirectoryRead {
|
|
name: string;
|
|
base: string;
|
|
size: number;
|
|
lastModified: number;
|
|
}
|
|
|
|
async function readAssetDir(
|
|
dir: string,
|
|
regex: RegExp,
|
|
): Promise<DirectoryRead[]> {
|
|
const names = await fs.readdir(dir).catch(() => [] as string[]);
|
|
const entries: DirectoryRead[] = [];
|
|
for (const name of names) {
|
|
if (!regex.test(name)) continue;
|
|
let size = 0;
|
|
let mtimeMs = 0;
|
|
try {
|
|
const stat = await fs.stat(getRuntimePath(dir, name));
|
|
size = stat.size;
|
|
mtimeMs = stat.mtimeMs;
|
|
} catch {
|
|
/* ignore unreadable entry */
|
|
}
|
|
entries.push({
|
|
name,
|
|
base: name.replace(/\.[a-z0-9]+$/i, ""),
|
|
size,
|
|
lastModified: mtimeMs,
|
|
});
|
|
}
|
|
return entries;
|
|
}
|
|
|
|
async function collectAssets(
|
|
dirs: string[],
|
|
regex: RegExp,
|
|
): Promise<Map<string, AssetCleanupEntry>> {
|
|
const byName = new Map<string, AssetCleanupEntry>();
|
|
for (const dir of dirs) {
|
|
for (const read of await readAssetDir(dir, regex)) {
|
|
const entry = byName.get(read.name) ?? {
|
|
fileName: read.name,
|
|
base: read.base,
|
|
size: 0,
|
|
dirs: [],
|
|
lastModified: 0,
|
|
};
|
|
if (!entry.dirs.includes(dir)) entry.dirs.push(dir);
|
|
entry.size = Math.max(entry.size, read.size);
|
|
entry.lastModified = Math.max(entry.lastModified, read.lastModified);
|
|
byName.set(read.name, entry);
|
|
}
|
|
}
|
|
return byName;
|
|
}
|
|
|
|
async function loadValidStems(): Promise<Set<string>> {
|
|
// Authoritative stem set from items_base (item_name + public_name
|
|
// fallback), mirroring repair-nitros.ts classname resolution.
|
|
const validStems = new Set<string>();
|
|
try {
|
|
const result = await queryRows<{
|
|
item_name: string;
|
|
public_name: string;
|
|
}>(sql`
|
|
SELECT item_name, public_name FROM items_base
|
|
`);
|
|
for (const row of result) {
|
|
addStems(validStems, row.item_name);
|
|
if (
|
|
row.public_name &&
|
|
row.public_name !== row.item_name &&
|
|
!row.public_name.includes(" ") &&
|
|
row.public_name !== "0"
|
|
) {
|
|
addStems(validStems, row.public_name);
|
|
}
|
|
}
|
|
} catch {
|
|
/* empty set → every asset is treated as orphaned */
|
|
}
|
|
return validStems;
|
|
}
|
|
|
|
export async function scanFakeBrokenNitros(): Promise<NitroCleanupScan> {
|
|
const targets = await getFurniAssetWriteTargets();
|
|
const nitroDirs = uniqueDirs([
|
|
targets.nitroDir,
|
|
...targets.mirrorDirs.map((d) => d.nitroDir),
|
|
]);
|
|
const swfDirs = uniqueDirs([
|
|
targets.swfDir,
|
|
...targets.mirrorDirs.map((d) => d.swfDir),
|
|
]);
|
|
const iconDirs = uniqueDirs([
|
|
targets.iconDir,
|
|
...targets.mirrorDirs.map((d) => d.iconDir),
|
|
]);
|
|
|
|
const [byNitro, bySwf, byIcon, validStems] = await Promise.all([
|
|
collectAssets(nitroDirs, NITRO_FILE_RE),
|
|
collectAssets(swfDirs, SWF_FILE_RE),
|
|
collectAssets(iconDirs, ICON_FILE_RE),
|
|
loadValidStems(),
|
|
]);
|
|
|
|
const entries: NitroCleanupEntry[] = [...byNitro.values()];
|
|
const fake: NitroCleanupEntry[] = [];
|
|
const broken: NitroCleanupEntry[] = [];
|
|
|
|
await runPool(entries, 8, async (entry) => {
|
|
// A .nitro that no DB item maps to is a leftover / fake bundle.
|
|
if (!validStems.has(entry.base.toLowerCase())) {
|
|
fake.push(entry);
|
|
return;
|
|
}
|
|
// Otherwise validate the bundle content. Most bundles pass a cheap
|
|
// header-only check (a few KB, no decompression); only files whose
|
|
// header looks suspicious get the expensive full parse. This keeps the
|
|
// scan fast even on directories with hundreds of thousands of bundles.
|
|
let parsed = false;
|
|
let firstError = "";
|
|
for (const dir of entry.dirs) {
|
|
if (await hasValidNitroHeader(getRuntimePath(dir, entry.fileName))) {
|
|
parsed = true;
|
|
break;
|
|
}
|
|
try {
|
|
parseNitroBundle(
|
|
await fs.readFile(getRuntimePath(dir, entry.fileName)),
|
|
);
|
|
parsed = true;
|
|
break;
|
|
} catch (err) {
|
|
firstError = (err as Error).message;
|
|
}
|
|
}
|
|
if (!parsed) {
|
|
broken.push({ ...entry, error: firstError });
|
|
}
|
|
});
|
|
|
|
const orphanedSwf = [...bySwf.values()].filter(
|
|
(entry) => !validStems.has(entry.base.toLowerCase()),
|
|
);
|
|
const orphanedIcon = [...byIcon.values()].filter(
|
|
(entry) => !validStems.has(entry.base.toLowerCase()),
|
|
);
|
|
|
|
const byName = <T extends { fileName: string }>(entries: T[]): T[] =>
|
|
entries.sort((a, b) => a.fileName.localeCompare(b.fileName));
|
|
|
|
return {
|
|
fake: byName(fake),
|
|
broken: byName(broken),
|
|
orphanedSwf: byName(orphanedSwf),
|
|
orphanedIcon: byName(orphanedIcon),
|
|
total: byNitro.size + bySwf.size + byIcon.size,
|
|
};
|
|
}
|
|
|
|
function dirsForKind(
|
|
targets: Awaited<ReturnType<typeof getFurniAssetWriteTargets>>,
|
|
kind: CleanupAssetKind,
|
|
): string[] {
|
|
const primary = {
|
|
nitro: targets.nitroDir,
|
|
swf: targets.swfDir,
|
|
icon: targets.iconDir,
|
|
}[kind];
|
|
const mirrors = targets.mirrorDirs.map((d) => d[`${kind}Dir` as const]);
|
|
return uniqueDirs([primary, ...mirrors]);
|
|
}
|
|
|
|
/** Delete asset files from every matching directory (primary + mirrors). */
|
|
export async function deleteNitroCleanupFiles(
|
|
fileNames: string[],
|
|
kind: CleanupAssetKind = "nitro",
|
|
): Promise<NitroCleanupDeleteResult> {
|
|
const targets = await getFurniAssetWriteTargets();
|
|
const dirs = dirsForKind(targets, kind);
|
|
const fileRegex = KIND_FILE_RE[kind];
|
|
|
|
let deleted = 0;
|
|
let copiesRemoved = 0;
|
|
const errors: string[] = [];
|
|
const files: NitroCleanupDeleteResult["files"] = [];
|
|
|
|
for (const rawName of [...new Set(fileNames)]) {
|
|
const fileName = String(rawName ?? "");
|
|
if (!fileRegex.test(fileName)) {
|
|
errors.push(`Invalid ${kind} file name: ${fileName}`);
|
|
files.push({ fileName, deleted: false, remaining: [] });
|
|
continue;
|
|
}
|
|
const remaining: string[] = [];
|
|
let existed = false;
|
|
for (const dir of dirs) {
|
|
const filePath = getRuntimePath(dir, fileName);
|
|
if (!existsSync(filePath)) continue;
|
|
existed = true;
|
|
try {
|
|
await fs.unlink(filePath);
|
|
copiesRemoved++;
|
|
} catch (err) {
|
|
remaining.push(dir);
|
|
errors.push(`${fileName} in ${dir}: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
const fullyDeleted = existed && remaining.length === 0;
|
|
if (fullyDeleted) deleted++;
|
|
files.push({ fileName, deleted: fullyDeleted, remaining });
|
|
}
|
|
|
|
return { deleted, copiesRemoved, errors, files };
|
|
}
|
|
|
|
function ageDays(lastModified: number): number {
|
|
if (!lastModified) return 0;
|
|
return Math.max(0, Math.floor((Date.now() - lastModified) / DAY_MS));
|
|
}
|
|
|
|
/**
|
|
* Automatically remove fake .nitro leftovers that are older than
|
|
* `maxAgeDays`. Broken bundles and young fake bundles are never touched —
|
|
* only former leftovers whose items are long gone from the database.
|
|
*/
|
|
export async function autoCleanFakeNitros(
|
|
maxAgeDays = 30,
|
|
): Promise<NitroAutoCleanResult> {
|
|
const scan = await scanFakeBrokenNitros();
|
|
const threshold = Math.max(0, Math.floor(maxAgeDays));
|
|
const removable = scan.fake.filter(
|
|
(entry) => ageDays(entry.lastModified) >= threshold,
|
|
);
|
|
const result = await deleteNitroCleanupFiles(
|
|
removable.map((entry) => entry.fileName),
|
|
"nitro",
|
|
);
|
|
return {
|
|
...result,
|
|
skippedRecent: scan.fake.length - removable.length,
|
|
skippedBroken: scan.broken.length,
|
|
};
|
|
}
|
|
|
|
async function copyFileToDirs(
|
|
buffer: Buffer,
|
|
fileName: string,
|
|
dirs: string[],
|
|
): Promise<string[]> {
|
|
const errors: string[] = [];
|
|
for (const dir of dirs) {
|
|
try {
|
|
await fs.mkdir(dir, { recursive: true });
|
|
await fs.writeFile(getRuntimePath(dir, fileName), buffer);
|
|
} catch (err) {
|
|
errors.push(`write .nitro to ${dir} failed: ${(err as Error).message}`);
|
|
}
|
|
}
|
|
return errors;
|
|
}
|
|
|
|
/**
|
|
* Re-download broken .nitro bundles from the configured nitro sources. The
|
|
* file stem matched an item in `items_base`, so the same classname candidates
|
|
* used by repair-nitros.ts are tried; every downloaded bundle is validated
|
|
* before it replaces the broken copies across all nitro directories.
|
|
*/
|
|
export async function repairBrokenNitros(
|
|
fileNames: string[],
|
|
): Promise<NitroRepairResult> {
|
|
const targets = await getFurniAssetWriteTargets();
|
|
const nitroDirs = dirsForKind(targets, "nitro");
|
|
let sources = (await listSources().catch(() => [] as never)).filter(
|
|
(source: { nitroBaseUrl?: string }) => source.nitroBaseUrl,
|
|
);
|
|
if (sources.length === 0) sources = DEFAULT_NITRO_SOURCES as never;
|
|
|
|
let repaired = 0;
|
|
let failed = 0;
|
|
const errors: string[] = [];
|
|
const files: NitroRepairResult["files"] = [];
|
|
|
|
await runPool([...new Set(fileNames)], 6, async (rawName) => {
|
|
const fileName = String(rawName ?? "");
|
|
if (!NITRO_FILE_RE.test(fileName) || !fileName.endsWith(".nitro")) {
|
|
failed++;
|
|
const message = `Invalid nitro file name: ${fileName}`;
|
|
errors.push(message);
|
|
files.push({ fileName, ok: false, error: message });
|
|
return;
|
|
}
|
|
const name = fileName.slice(0, -".nitro".length);
|
|
const candidates = [
|
|
name,
|
|
...(name.includes("*")
|
|
? [name.split("*")[0], name.replace(/\*/g, "_")]
|
|
: []),
|
|
...(name.includes(".") ? [name.replace(/\./g, "_")] : []),
|
|
];
|
|
for (const source of sources as {
|
|
name: string;
|
|
nitroBaseUrl?: string;
|
|
}[]) {
|
|
for (const candidate of candidates) {
|
|
const base = source.nitroBaseUrl;
|
|
if (!base) continue;
|
|
const tmpPath = getRuntimePath(
|
|
os.tmpdir(),
|
|
`.repair_nitro_${name}_${Math.random().toString(36).slice(2, 8)}.nitro`,
|
|
);
|
|
try {
|
|
const dl = await downloadFile(
|
|
`${base.replace(/\/+$/, "")}/${encodeURIComponent(candidate)}.nitro`,
|
|
tmpPath,
|
|
{ maxRetries: 1, validate: "nitro" },
|
|
);
|
|
if (dl.ok) {
|
|
const buffer = await fs.readFile(tmpPath);
|
|
parseNitroBundle(buffer);
|
|
const writeErrors = await copyFileToDirs(
|
|
buffer,
|
|
fileName,
|
|
nitroDirs,
|
|
);
|
|
if (writeErrors.length === 0) {
|
|
repaired++;
|
|
files.push({ fileName, ok: true });
|
|
} else {
|
|
failed++;
|
|
errors.push(...writeErrors);
|
|
files.push({
|
|
fileName,
|
|
ok: false,
|
|
error: writeErrors[0],
|
|
});
|
|
}
|
|
return;
|
|
}
|
|
} catch (err) {
|
|
errors.push(
|
|
`repair ${fileName} from ${source.name} failed: ${(err as Error).message}`,
|
|
);
|
|
} finally {
|
|
await fs.unlink(tmpPath).catch(() => {});
|
|
}
|
|
}
|
|
}
|
|
failed++;
|
|
errors.push(`could not re-obtain ${fileName} from any nitro source`);
|
|
files.push({
|
|
fileName,
|
|
ok: false,
|
|
error: "could not re-obtain bundle from any nitro source",
|
|
});
|
|
});
|
|
|
|
return { repaired, failed, errors, files };
|
|
}
|