feat(catalog): prevent and repair Arcturus emulator data errors

Block invalid catalog_items writes at the API level (points currency
allowlist, non-negative prices, positive amount, limited stack >= sold
count, unique sibling order numbers) and auto-assign unique order numbers
on bulk create. Add a catalog-maintenance scan + transactional repair that
fixes pre-existing rows: resets unsupported points_type, clamps negative
costs, sets amount to 1, raises limited_stack, renumbers duplicate orders
and deletes offers with missing page/item references. Surface the issue
count and a fix button in the admin maintenance panel.

Also: add enabled/retired flag to clone sources, classify poster and
currency furniture in item-kind, and remove the obsolete update-Nitrov3.sh.
This commit is contained in:
openhands committed 2026-09-10 11:29:28 +02:00
1 parent ad65d80d41
commit 3e69b72513
26 files changed
+17651 -3844

No files matched your search

@@ -0,0 +1,82 @@
// Runs repairMissingIcons + repairMissingNitros directly (no source comparison
// phase, which is the slow part of runCatalogAudit). Logs progress to a file.
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { beforeAll, describe, expect, it } from "vitest";
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
const LOG = "/tmp/catalog-asset-repair.log";
const log = (msg: string) => {
appendFileSync(LOG, `${new Date().toISOString()} ${msg}\n`);
};
describe.skipIf(!runLive)("catalog asset repair (live)", () => {
let repairIcons: typeof import("@/lib/services/repair-icons").repairMissingIcons;
let repairNitros: typeof import("@/lib/services/repair-nitros").repairMissingNitros;
beforeAll(async () => {
const envFile = resolve(process.cwd(), ".env");
if (existsSync(envFile)) {
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
if (!m) continue;
let value = m[2].trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[m[1]] = value;
}
}
process.env["SKIP_ENV_VALIDATION"] = "1";
[repairIcons, repairNitros] = await Promise.all([
import("@/lib/services/repair-icons").then((m) => m.repairMissingIcons),
import("@/lib/services/repair-nitros").then((m) => m.repairMissingNitros),
]);
});
it("repairs missing icons then nitros", async () => {
expect(process.env.DATABASE_URL).toMatch(/@/);
log("=== ICON REPAIR START ===");
let lastPct = -1;
const iconResult = await repairIcons((evt) => {
if (evt.type === "progress" && evt.total) {
const pct = Math.round(((evt.index ?? 0) / evt.total) * 100);
if (pct % 5 === 0 && pct !== lastPct) {
lastPct = pct;
log(`icons ${evt.index}/${evt.total} (${pct}%)`);
}
if (evt.status === "done") log(`ICON DONE: ${evt.classname}`);
if (evt.status === "failed")
log(`ICON FAIL: ${evt.classname} :: ${evt.message ?? ""}`);
}
if (evt.type === "error") log(`ICON ERROR: ${evt.message ?? ""}`);
});
log("icon result: " + JSON.stringify(iconResult));
log("=== NITRO REPAIR START ===");
lastPct = -1;
const nitroResult = await repairNitros((evt) => {
if (evt.type === "progress" && evt.total) {
const pct = Math.round(((evt.index ?? 0) / evt.total) * 100);
if (pct % 5 === 0 && pct !== lastPct) {
lastPct = pct;
log(`nitros ${evt.index}/${evt.total} (${pct}%)`);
}
if (evt.status === "done") log(`NITRO DONE: ${evt.classname}`);
if (evt.status === "failed")
log(`NITRO FAIL: ${evt.classname} :: ${evt.message ?? ""}`);
}
if (evt.type === "error") log(`NITRO ERROR: ${evt.message ?? ""}`);
});
log("nitro result: " + JSON.stringify(nitroResult));
log("=== ASSET REPAIR COMPLETE ===");
log("icons: " + JSON.stringify(iconResult));
log("nitros: " + JSON.stringify(nitroResult));
}, 1800000);
});
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,67 @@
// Read-only audit run that writes the full summary to a log file.
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { beforeAll, describe, expect, it } from "vitest";
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
const LOG = "/tmp/catalog-audit-summary.log";
const log = (msg: string) => {
appendFileSync(LOG, `${msg}\n`);
};
describe.skipIf(!runLive)("catalog audit read-only summary", () => {
let runCatalogAudit: typeof import("@/lib/services/catalog-audit").runCatalogAudit;
beforeAll(async () => {
const envFile = resolve(process.cwd(), ".env");
if (existsSync(envFile)) {
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
if (!m) continue;
let value = m[2].trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[m[1]] = value;
}
}
const auditMod = await import("@/lib/services/catalog-audit");
runCatalogAudit = auditMod.runCatalogAudit;
});
it("reports the full audit summary", async () => {
expect(process.env.DATABASE_URL).toMatch(/@/);
const events: Array<Record<string, unknown>> = [];
await runCatalogAudit(
(evt) => events.push(evt as unknown as Record<string, unknown>),
{},
);
const last = events[events.length - 1];
expect(last?.type).toBe("batch_complete");
const summary = last.summary as Record<string, unknown>;
const issues = last.issues as Array<{
type: string;
severity: string;
message: string;
classname?: string;
}>;
const unrepairable = last.unrepairable as Array<{
classname: string;
itemId: number;
missing: Array<string>;
}>;
log("=== SUMMARY ===");
log(JSON.stringify(summary, null, 2));
log("=== ISSUES ===");
log(JSON.stringify(issues, null, 2));
log("=== UNREPAIRABLE ===");
log(JSON.stringify(unrepairable, null, 2));
}, 300000);
});
@@ -0,0 +1,66 @@
// Direct repair execution against the live DB. Skips the slow clone-source
// comparison entirely and just runs the repair functions.
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
const LOG = "/tmp/catalog-repair.log";
const log = (msg: string) => {
appendFileSync(LOG, `${msg}\n`);
console.log(msg);
};
import { sql } from "drizzle-orm";
import { beforeAll, describe, expect, it } from "vitest";
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
describe.skipIf(!runLive)("catalog repair direct (live)", () => {
let db: typeof import("@/lib/db").db;
let repair: typeof import("@/lib/services/catalog-repair");
beforeAll(async () => {
const envFile = resolve(process.cwd(), ".env");
if (existsSync(envFile)) {
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
if (!m) continue;
let value = m[2].trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[m[1]] = value;
}
}
const [dbMod, repairMod] = await Promise.all([
import("@/lib/db"),
import("@/lib/services/catalog-repair"),
]);
db = dbMod.db;
repair = repairMod;
});
it("repairs orphaned catalog + duplicate classnames", async () => {
expect(process.env.DATABASE_URL).toMatch(/@/);
log("=== ORPHANED REPAIR ===");
const orphan = await repair.repairOrphanedCatalog();
log("orphaned result: " + JSON.stringify(orphan));
log("=== DUPLICATE REPAIR ===");
const dup = await repair.repairDuplicateClassnames();
log("duplicate result: " + JSON.stringify(dup));
log("=== VERIFY DUPLICATES ===");
const [dupRows] = (await db.execute(sql`
SELECT item_name, COUNT(*) as cnt
FROM items_base
GROUP BY item_name
HAVING cnt > 1
`)) as unknown as [Array<{ item_name: string; cnt: number }>, unknown];
log("remaining duplicate groups: " + JSON.stringify(dupRows));
}, 300000);
});
@@ -0,0 +1,92 @@
// Bulk-import live run: imports every cloneable item from the sources that
// reliably serve .nitro assets (SodaStudios, Hubbly). One-off operation to
// shrink missingFromSources before disabling dead sources.
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { beforeAll, describe, expect, it } from "vitest";
const runLive = process.env.RUN_CLONE_BULK_LIVE === "1";
const LOG = "/tmp/clone-bulk.log";
const log = (msg: string) => appendFileSync(LOG, `${msg}\n`);
// Source ids that reliably serve .nitro files (verified 2026-09).
const IMPORT_IDS = ["default-sodastudios", "default-hubbly"];
describe.skipIf(!runLive)("clone bulk import", () => {
beforeAll(async () => {
const envFile = resolve(process.cwd(), ".env");
if (existsSync(envFile)) {
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
if (!m) continue;
let value = m[2].trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[m[1]] = value;
}
}
});
it("imports clonable items from delivering sources", async () => {
expect(process.env.DATABASE_URL).toMatch(/@/);
const mod = await import("@/lib/services/clone-sources");
const ci = await import("@/lib/services/clone-import");
const sources = await mod.listSources();
log(`START bulk import ${new Date().toISOString()}`);
for (const id of IMPORT_IDS) {
const source = sources.find((s) => s.id === id);
if (!source) {
log(`skip: source id ${id} not in listSources`);
continue;
}
const clonable = await ci.getClonableClassnames(source);
const all = await ci.fetchSourceFurnidata(source.furnidataUrl);
const entryFor = (classname: string) =>
all.find((e) => e.classname === classname);
log(
`SOURCE ${source.name}: ${clonable.length} clonable (fetched ${all.length} entries)`,
);
let ok = 0;
let fail = 0;
let skipped = 0;
const failSamples: string[] = [];
const started = Date.now();
for (let i = 0; i < clonable.length; i++) {
const classname = clonable[i];
const entry = entryFor(classname);
if (!entry) {
skipped++;
continue;
}
const res = await ci.cloneSingleFurni({ source, entry });
if (res.ok) ok++;
else if ("skipped" in res && res.skipped) skipped++;
else {
fail++;
if (failSamples.length < 30) {
failSamples.push(`${classname}: ${res.error}`);
}
}
if ((i + 1) % 50 === 0) {
const elapsed = ((Date.now() - started) / 1000).toFixed(0);
log(
` ${source.name} ${i + 1}/${clonable.length} ok=${ok} fail=${fail} skipped=${skipped} (${elapsed}s)`,
);
}
}
const secs = ((Date.now() - started) / 1000).toFixed(0);
log(
`DONE ${source.name}: ok=${ok} fail=${fail} skipped=${skipped} in ${secs}s`,
);
if (failSamples.length) {
log(` fail samples:\n ${failSamples.join("\n ")}`);
}
}
log(`END bulk import ${new Date().toISOString()}`);
}, 1800000);
});
File diff suppressed because it is too large. Load diff
+8 -1
View File
@@ -18,6 +18,12 @@ export interface CloneSource {
* "https://virtualcity.nl/dcr/hof_furni/{revision}/{classname}.swf"
*/
sourceSwfBaseUrl?: string;
/**
* When false the source is excluded from the merged list (defaults to
* true). Used to retire sources whose assets can no longer be reached
* while keeping their definition in the site setting for later reuse.
*/
enabled?: boolean;
}
const KEY = "clone_sources";
@@ -215,7 +221,7 @@ export async function listSources(): Promise<CloneSource[]> {
for (const src of stored) {
if (src.id && src.furnidataUrl) merged.set(src.id, src);
}
return Array.from(merged.values());
return Array.from(merged.values()).filter((s) => s.enabled !== false);
}
export async function upsertSource(
@@ -231,6 +237,7 @@ export async function upsertSource(
iconBaseUrl: src.iconBaseUrl.trim().replace(/\/+$/, ""),
hotel: (src.hotel ?? "").trim(),
sourceSwfBaseUrl: swfBase || undefined,
enabled: src.enabled ?? true,
};
const idx = sources.findIndex((s) => s.id === entry.id);
if (idx >= 0) sources[idx] = entry;
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+10
View File
@@ -48,6 +48,16 @@ describe("classifyCatalogItem", () => {
expect(item({ item_name: "post_it" })).toBe("sticky");
expect(item({ item_name: "postit" })).toBe("sticky");
});
it("classifies posters and currency furniture", () => {
expect(item({ item_name: "poster100" })).toBe("poster");
expect(item({ item_name: "poster5000" })).toBe("poster");
expect(item({ item_name: "funtastic_four_poster" })).toBe("poster");
expect(item({ item_name: "duckfountain" })).toBe("poster");
expect(item({ item_name: "Swimming" })).toBe("poster");
expect(item({ item_name: "CF_50000_Hucci" })).toBe("currency");
expect(item({ item_name: "DF_102_10_bluechip" })).toBe("currency");
expect(item({ item_name: "sofa" })).toBe("furni");
});
it("classifies everything else as furni", () => {
expect(item({})).toBe("furni");
});
+30
View File
@@ -10,6 +10,12 @@
* assets live in the pet asset pipeline (`public/nitro-assets/bundled/pet`),
* not the furniture bundle dirs.
* - **effect / bot / sticky** — system items with no furniture visuals at all.
* - **poster** — poster furniture (`poster*` classnames, wall posters, novelty
* poster furni). Renders through the poster imager, so no `.nitro` / `_icon.png`
* assets are expected.
* - **currency** — credit/pixel furniture (`CF_*`, `DF_*`). Renders through the
* currency imager (credits coin, badge-format exchange stickers), so no
* `.nitro` / `_icon.png` assets are expected.
* - **furni** — everything else; must ship a `.nitro` and `_icon.png`.
*/
@@ -19,6 +25,8 @@ export type CatalogItemKind =
| "effect"
| "bot"
| "sticky"
| "poster"
| "currency"
| "furni";
export interface CatalogItemLike {
@@ -42,6 +50,24 @@ export function isPetInteraction(interactionType?: string | null): boolean {
return /^pet\d+$/i.test((interactionType ?? "").trim());
}
/** Poster furniture classnames that have no `.nitro` asset expectations. */
export function isPosterClassname(itemName: string): boolean {
const name = itemName.trim();
if (/^poster\d+$/i.test(name)) return true;
const POSTER_SPECIAL = new Set([
"funtastic_four_poster",
"duckfountain",
"duckfountain2",
"swimming",
]);
return POSTER_SPECIAL.has(name.toLowerCase());
}
/** Credit/pixel furniture (`CF_*`, `DF_*`) ships no `.nitro` assets. */
export function isCurrencyClassname(itemName: string): boolean {
return /^(?:cf|df)_/i.test(itemName.trim());
}
/** Classify an `items_base` row into a catalog item kind. */
export function classifyCatalogItem(item: CatalogItemLike): CatalogItemKind {
const name = item.item_name.trim();
@@ -52,6 +78,8 @@ export function classifyCatalogItem(item: CatalogItemLike): CatalogItemKind {
if (/^avatar_effect/i.test(name)) return "effect";
if (/^(bot_|rentable_bot_)/i.test(name)) return "bot";
if (/^post[._]?it/i.test(name)) return "sticky";
if (isPosterClassname(name)) return "poster";
if (isCurrencyClassname(name)) return "currency";
return "furni";
}
@@ -62,6 +90,8 @@ export const NO_ASSET_KINDS: ReadonlySet<CatalogItemKind> = new Set([
"effect",
"bot",
"sticky",
"poster",
"currency",
]);
/** Kinds that are not purchasable furniture and should not get catalog entries. */