feat(catalog): prevent and repair Arcturus emulator data errors
Block invalid catalog_items writes at the API level (points currency allowlist, non-negative prices, positive amount, limited stack >= sold count, unique sibling order numbers) and auto-assign unique order numbers on bulk create. Add a catalog-maintenance scan + transactional repair that fixes pre-existing rows: resets unsupported points_type, clamps negative costs, sets amount to 1, raises limited_stack, renumbers duplicate orders and deletes offers with missing page/item references. Surface the issue count and a fix button in the admin maintenance panel. Also: add enabled/retired flag to clone sources, classify poster and currency furniture in item-kind, and remove the obsolete update-Nitrov3.sh.
This commit is contained in:
1 parent
ad65d80d41
commit
3e69b72513
26 files changed
+17651
-3844
No files matched your search
@@ -0,0 +1,82 @@
|
||||
// Runs repairMissingIcons + repairMissingNitros directly (no source comparison
|
||||
// phase, which is the slow part of runCatalogAudit). Logs progress to a file.
|
||||
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
|
||||
const LOG = "/tmp/catalog-asset-repair.log";
|
||||
const log = (msg: string) => {
|
||||
appendFileSync(LOG, `${new Date().toISOString()} ${msg}\n`);
|
||||
};
|
||||
|
||||
describe.skipIf(!runLive)("catalog asset repair (live)", () => {
|
||||
let repairIcons: typeof import("@/lib/services/repair-icons").repairMissingIcons;
|
||||
let repairNitros: typeof import("@/lib/services/repair-nitros").repairMissingNitros;
|
||||
|
||||
beforeAll(async () => {
|
||||
const envFile = resolve(process.cwd(), ".env");
|
||||
if (existsSync(envFile)) {
|
||||
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
|
||||
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
|
||||
if (!m) continue;
|
||||
let value = m[2].trim();
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[m[1]] = value;
|
||||
}
|
||||
}
|
||||
process.env["SKIP_ENV_VALIDATION"] = "1";
|
||||
|
||||
[repairIcons, repairNitros] = await Promise.all([
|
||||
import("@/lib/services/repair-icons").then((m) => m.repairMissingIcons),
|
||||
import("@/lib/services/repair-nitros").then((m) => m.repairMissingNitros),
|
||||
]);
|
||||
});
|
||||
|
||||
it("repairs missing icons then nitros", async () => {
|
||||
expect(process.env.DATABASE_URL).toMatch(/@/);
|
||||
|
||||
log("=== ICON REPAIR START ===");
|
||||
let lastPct = -1;
|
||||
const iconResult = await repairIcons((evt) => {
|
||||
if (evt.type === "progress" && evt.total) {
|
||||
const pct = Math.round(((evt.index ?? 0) / evt.total) * 100);
|
||||
if (pct % 5 === 0 && pct !== lastPct) {
|
||||
lastPct = pct;
|
||||
log(`icons ${evt.index}/${evt.total} (${pct}%)`);
|
||||
}
|
||||
if (evt.status === "done") log(`ICON DONE: ${evt.classname}`);
|
||||
if (evt.status === "failed")
|
||||
log(`ICON FAIL: ${evt.classname} :: ${evt.message ?? ""}`);
|
||||
}
|
||||
if (evt.type === "error") log(`ICON ERROR: ${evt.message ?? ""}`);
|
||||
});
|
||||
log("icon result: " + JSON.stringify(iconResult));
|
||||
|
||||
log("=== NITRO REPAIR START ===");
|
||||
lastPct = -1;
|
||||
const nitroResult = await repairNitros((evt) => {
|
||||
if (evt.type === "progress" && evt.total) {
|
||||
const pct = Math.round(((evt.index ?? 0) / evt.total) * 100);
|
||||
if (pct % 5 === 0 && pct !== lastPct) {
|
||||
lastPct = pct;
|
||||
log(`nitros ${evt.index}/${evt.total} (${pct}%)`);
|
||||
}
|
||||
if (evt.status === "done") log(`NITRO DONE: ${evt.classname}`);
|
||||
if (evt.status === "failed")
|
||||
log(`NITRO FAIL: ${evt.classname} :: ${evt.message ?? ""}`);
|
||||
}
|
||||
if (evt.type === "error") log(`NITRO ERROR: ${evt.message ?? ""}`);
|
||||
});
|
||||
log("nitro result: " + JSON.stringify(nitroResult));
|
||||
|
||||
log("=== ASSET REPAIR COMPLETE ===");
|
||||
log("icons: " + JSON.stringify(iconResult));
|
||||
log("nitros: " + JSON.stringify(nitroResult));
|
||||
}, 1800000);
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,67 @@
|
||||
// Read-only audit run that writes the full summary to a log file.
|
||||
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
|
||||
const LOG = "/tmp/catalog-audit-summary.log";
|
||||
const log = (msg: string) => {
|
||||
appendFileSync(LOG, `${msg}\n`);
|
||||
};
|
||||
|
||||
describe.skipIf(!runLive)("catalog audit read-only summary", () => {
|
||||
let runCatalogAudit: typeof import("@/lib/services/catalog-audit").runCatalogAudit;
|
||||
|
||||
beforeAll(async () => {
|
||||
const envFile = resolve(process.cwd(), ".env");
|
||||
if (existsSync(envFile)) {
|
||||
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
|
||||
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
|
||||
if (!m) continue;
|
||||
let value = m[2].trim();
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[m[1]] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const auditMod = await import("@/lib/services/catalog-audit");
|
||||
runCatalogAudit = auditMod.runCatalogAudit;
|
||||
});
|
||||
|
||||
it("reports the full audit summary", async () => {
|
||||
expect(process.env.DATABASE_URL).toMatch(/@/);
|
||||
|
||||
const events: Array<Record<string, unknown>> = [];
|
||||
await runCatalogAudit(
|
||||
(evt) => events.push(evt as unknown as Record<string, unknown>),
|
||||
{},
|
||||
);
|
||||
|
||||
const last = events[events.length - 1];
|
||||
expect(last?.type).toBe("batch_complete");
|
||||
const summary = last.summary as Record<string, unknown>;
|
||||
const issues = last.issues as Array<{
|
||||
type: string;
|
||||
severity: string;
|
||||
message: string;
|
||||
classname?: string;
|
||||
}>;
|
||||
const unrepairable = last.unrepairable as Array<{
|
||||
classname: string;
|
||||
itemId: number;
|
||||
missing: Array<string>;
|
||||
}>;
|
||||
|
||||
log("=== SUMMARY ===");
|
||||
log(JSON.stringify(summary, null, 2));
|
||||
log("=== ISSUES ===");
|
||||
log(JSON.stringify(issues, null, 2));
|
||||
log("=== UNREPAIRABLE ===");
|
||||
log(JSON.stringify(unrepairable, null, 2));
|
||||
}, 300000);
|
||||
});
|
||||
@@ -0,0 +1,66 @@
|
||||
// Direct repair execution against the live DB. Skips the slow clone-source
|
||||
// comparison entirely and just runs the repair functions.
|
||||
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
|
||||
const LOG = "/tmp/catalog-repair.log";
|
||||
const log = (msg: string) => {
|
||||
appendFileSync(LOG, `${msg}\n`);
|
||||
console.log(msg);
|
||||
};
|
||||
|
||||
import { sql } from "drizzle-orm";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
|
||||
|
||||
describe.skipIf(!runLive)("catalog repair direct (live)", () => {
|
||||
let db: typeof import("@/lib/db").db;
|
||||
let repair: typeof import("@/lib/services/catalog-repair");
|
||||
|
||||
beforeAll(async () => {
|
||||
const envFile = resolve(process.cwd(), ".env");
|
||||
if (existsSync(envFile)) {
|
||||
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
|
||||
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
|
||||
if (!m) continue;
|
||||
let value = m[2].trim();
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[m[1]] = value;
|
||||
}
|
||||
}
|
||||
|
||||
const [dbMod, repairMod] = await Promise.all([
|
||||
import("@/lib/db"),
|
||||
import("@/lib/services/catalog-repair"),
|
||||
]);
|
||||
db = dbMod.db;
|
||||
repair = repairMod;
|
||||
});
|
||||
|
||||
it("repairs orphaned catalog + duplicate classnames", async () => {
|
||||
expect(process.env.DATABASE_URL).toMatch(/@/);
|
||||
|
||||
log("=== ORPHANED REPAIR ===");
|
||||
const orphan = await repair.repairOrphanedCatalog();
|
||||
log("orphaned result: " + JSON.stringify(orphan));
|
||||
|
||||
log("=== DUPLICATE REPAIR ===");
|
||||
const dup = await repair.repairDuplicateClassnames();
|
||||
log("duplicate result: " + JSON.stringify(dup));
|
||||
|
||||
log("=== VERIFY DUPLICATES ===");
|
||||
const [dupRows] = (await db.execute(sql`
|
||||
SELECT item_name, COUNT(*) as cnt
|
||||
FROM items_base
|
||||
GROUP BY item_name
|
||||
HAVING cnt > 1
|
||||
`)) as unknown as [Array<{ item_name: string; cnt: number }>, unknown];
|
||||
log("remaining duplicate groups: " + JSON.stringify(dupRows));
|
||||
}, 300000);
|
||||
});
|
||||
@@ -0,0 +1,92 @@
|
||||
// Bulk-import live run: imports every cloneable item from the sources that
|
||||
// reliably serve .nitro assets (SodaStudios, Hubbly). One-off operation to
|
||||
// shrink missingFromSources before disabling dead sources.
|
||||
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
const runLive = process.env.RUN_CLONE_BULK_LIVE === "1";
|
||||
const LOG = "/tmp/clone-bulk.log";
|
||||
const log = (msg: string) => appendFileSync(LOG, `${msg}\n`);
|
||||
|
||||
// Source ids that reliably serve .nitro files (verified 2026-09).
|
||||
const IMPORT_IDS = ["default-sodastudios", "default-hubbly"];
|
||||
|
||||
describe.skipIf(!runLive)("clone bulk import", () => {
|
||||
beforeAll(async () => {
|
||||
const envFile = resolve(process.cwd(), ".env");
|
||||
if (existsSync(envFile)) {
|
||||
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
|
||||
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
|
||||
if (!m) continue;
|
||||
let value = m[2].trim();
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[m[1]] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("imports clonable items from delivering sources", async () => {
|
||||
expect(process.env.DATABASE_URL).toMatch(/@/);
|
||||
const mod = await import("@/lib/services/clone-sources");
|
||||
const ci = await import("@/lib/services/clone-import");
|
||||
const sources = await mod.listSources();
|
||||
log(`START bulk import ${new Date().toISOString()}`);
|
||||
|
||||
for (const id of IMPORT_IDS) {
|
||||
const source = sources.find((s) => s.id === id);
|
||||
if (!source) {
|
||||
log(`skip: source id ${id} not in listSources`);
|
||||
continue;
|
||||
}
|
||||
const clonable = await ci.getClonableClassnames(source);
|
||||
const all = await ci.fetchSourceFurnidata(source.furnidataUrl);
|
||||
const entryFor = (classname: string) =>
|
||||
all.find((e) => e.classname === classname);
|
||||
log(
|
||||
`SOURCE ${source.name}: ${clonable.length} clonable (fetched ${all.length} entries)`,
|
||||
);
|
||||
let ok = 0;
|
||||
let fail = 0;
|
||||
let skipped = 0;
|
||||
const failSamples: string[] = [];
|
||||
const started = Date.now();
|
||||
for (let i = 0; i < clonable.length; i++) {
|
||||
const classname = clonable[i];
|
||||
const entry = entryFor(classname);
|
||||
if (!entry) {
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
const res = await ci.cloneSingleFurni({ source, entry });
|
||||
if (res.ok) ok++;
|
||||
else if ("skipped" in res && res.skipped) skipped++;
|
||||
else {
|
||||
fail++;
|
||||
if (failSamples.length < 30) {
|
||||
failSamples.push(`${classname}: ${res.error}`);
|
||||
}
|
||||
}
|
||||
if ((i + 1) % 50 === 0) {
|
||||
const elapsed = ((Date.now() - started) / 1000).toFixed(0);
|
||||
log(
|
||||
` ${source.name} ${i + 1}/${clonable.length} ok=${ok} fail=${fail} skipped=${skipped} (${elapsed}s)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const secs = ((Date.now() - started) / 1000).toFixed(0);
|
||||
log(
|
||||
`DONE ${source.name}: ok=${ok} fail=${fail} skipped=${skipped} in ${secs}s`,
|
||||
);
|
||||
if (failSamples.length) {
|
||||
log(` fail samples:\n ${failSamples.join("\n ")}`);
|
||||
}
|
||||
}
|
||||
log(`END bulk import ${new Date().toISOString()}`);
|
||||
}, 1800000);
|
||||
});
|
||||
File diff suppressed because it is too large.
Load diff
@@ -18,6 +18,12 @@ export interface CloneSource {
|
||||
* "https://virtualcity.nl/dcr/hof_furni/{revision}/{classname}.swf"
|
||||
*/
|
||||
sourceSwfBaseUrl?: string;
|
||||
/**
|
||||
* When false the source is excluded from the merged list (defaults to
|
||||
* true). Used to retire sources whose assets can no longer be reached
|
||||
* while keeping their definition in the site setting for later reuse.
|
||||
*/
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
const KEY = "clone_sources";
|
||||
@@ -215,7 +221,7 @@ export async function listSources(): Promise<CloneSource[]> {
|
||||
for (const src of stored) {
|
||||
if (src.id && src.furnidataUrl) merged.set(src.id, src);
|
||||
}
|
||||
return Array.from(merged.values());
|
||||
return Array.from(merged.values()).filter((s) => s.enabled !== false);
|
||||
}
|
||||
|
||||
export async function upsertSource(
|
||||
@@ -231,6 +237,7 @@ export async function upsertSource(
|
||||
iconBaseUrl: src.iconBaseUrl.trim().replace(/\/+$/, ""),
|
||||
hotel: (src.hotel ?? "").trim(),
|
||||
sourceSwfBaseUrl: swfBase || undefined,
|
||||
enabled: src.enabled ?? true,
|
||||
};
|
||||
const idx = sources.findIndex((s) => s.id === entry.id);
|
||||
if (idx >= 0) sources[idx] = entry;
|
||||
|
||||
File diff suppressed because it is too large.
Load diff
File diff suppressed because it is too large.
Load diff
@@ -48,6 +48,16 @@ describe("classifyCatalogItem", () => {
|
||||
expect(item({ item_name: "post_it" })).toBe("sticky");
|
||||
expect(item({ item_name: "postit" })).toBe("sticky");
|
||||
});
|
||||
it("classifies posters and currency furniture", () => {
|
||||
expect(item({ item_name: "poster100" })).toBe("poster");
|
||||
expect(item({ item_name: "poster5000" })).toBe("poster");
|
||||
expect(item({ item_name: "funtastic_four_poster" })).toBe("poster");
|
||||
expect(item({ item_name: "duckfountain" })).toBe("poster");
|
||||
expect(item({ item_name: "Swimming" })).toBe("poster");
|
||||
expect(item({ item_name: "CF_50000_Hucci" })).toBe("currency");
|
||||
expect(item({ item_name: "DF_102_10_bluechip" })).toBe("currency");
|
||||
expect(item({ item_name: "sofa" })).toBe("furni");
|
||||
});
|
||||
it("classifies everything else as furni", () => {
|
||||
expect(item({})).toBe("furni");
|
||||
});
|
||||
|
||||
@@ -10,6 +10,12 @@
|
||||
* assets live in the pet asset pipeline (`public/nitro-assets/bundled/pet`),
|
||||
* not the furniture bundle dirs.
|
||||
* - **effect / bot / sticky** — system items with no furniture visuals at all.
|
||||
* - **poster** — poster furniture (`poster*` classnames, wall posters, novelty
|
||||
* poster furni). Renders through the poster imager, so no `.nitro` / `_icon.png`
|
||||
* assets are expected.
|
||||
* - **currency** — credit/pixel furniture (`CF_*`, `DF_*`). Renders through the
|
||||
* currency imager (credits coin, badge-format exchange stickers), so no
|
||||
* `.nitro` / `_icon.png` assets are expected.
|
||||
* - **furni** — everything else; must ship a `.nitro` and `_icon.png`.
|
||||
*/
|
||||
|
||||
@@ -19,6 +25,8 @@ export type CatalogItemKind =
|
||||
| "effect"
|
||||
| "bot"
|
||||
| "sticky"
|
||||
| "poster"
|
||||
| "currency"
|
||||
| "furni";
|
||||
|
||||
export interface CatalogItemLike {
|
||||
@@ -42,6 +50,24 @@ export function isPetInteraction(interactionType?: string | null): boolean {
|
||||
return /^pet\d+$/i.test((interactionType ?? "").trim());
|
||||
}
|
||||
|
||||
/** Poster furniture classnames that have no `.nitro` asset expectations. */
|
||||
export function isPosterClassname(itemName: string): boolean {
|
||||
const name = itemName.trim();
|
||||
if (/^poster\d+$/i.test(name)) return true;
|
||||
const POSTER_SPECIAL = new Set([
|
||||
"funtastic_four_poster",
|
||||
"duckfountain",
|
||||
"duckfountain2",
|
||||
"swimming",
|
||||
]);
|
||||
return POSTER_SPECIAL.has(name.toLowerCase());
|
||||
}
|
||||
|
||||
/** Credit/pixel furniture (`CF_*`, `DF_*`) ships no `.nitro` assets. */
|
||||
export function isCurrencyClassname(itemName: string): boolean {
|
||||
return /^(?:cf|df)_/i.test(itemName.trim());
|
||||
}
|
||||
|
||||
/** Classify an `items_base` row into a catalog item kind. */
|
||||
export function classifyCatalogItem(item: CatalogItemLike): CatalogItemKind {
|
||||
const name = item.item_name.trim();
|
||||
@@ -52,6 +78,8 @@ export function classifyCatalogItem(item: CatalogItemLike): CatalogItemKind {
|
||||
if (/^avatar_effect/i.test(name)) return "effect";
|
||||
if (/^(bot_|rentable_bot_)/i.test(name)) return "bot";
|
||||
if (/^post[._]?it/i.test(name)) return "sticky";
|
||||
if (isPosterClassname(name)) return "poster";
|
||||
if (isCurrencyClassname(name)) return "currency";
|
||||
return "furni";
|
||||
}
|
||||
|
||||
@@ -62,6 +90,8 @@ export const NO_ASSET_KINDS: ReadonlySet<CatalogItemKind> = new Set([
|
||||
"effect",
|
||||
"bot",
|
||||
"sticky",
|
||||
"poster",
|
||||
"currency",
|
||||
]);
|
||||
|
||||
/** Kinds that are not purchasable furniture and should not get catalog entries. */
|
||||
|
||||
Reference in new issue
Block a user