feat(catalog): prevent and repair Arcturus emulator data errors
CI / check (push) Failing after 26s
CI / deploy (push) Skipped
CI / publish-container (push) Skipped

Block invalid catalog_items writes at the API level (points currency
allowlist, non-negative prices, positive amount, limited stack >= sold
count, unique sibling order numbers) and auto-assign unique order numbers
on bulk create. Add a catalog-maintenance scan + transactional repair that
fixes pre-existing rows: resets unsupported points_type, clamps negative
costs, sets amount to 1, raises limited_stack, renumbers duplicate orders
and deletes offers with missing page/item references. Surface the issue
count and a fix button in the admin maintenance panel.

Also: add enabled/retired flag to clone sources, classify poster and
currency furniture in item-kind, and remove the obsolete update-Nitrov3.sh.
This commit is contained in:
openhands committed 2026-09-10 11:29:28 +02:00
1 parent ad65d80d41
commit 3e69b72513
26 files changed
+17641 -3834

No files matched your search

File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+13
View File
@@ -0,0 +1,13 @@
{
"version": "7",
"dialect": "mysql",
"entries": [
{
"idx": 0,
"version": "5",
"when": 1788791201804,
"tag": "0000_premium_spirit",
"breakpoints": true
}
]
}
+7 -1
View File
@@ -144,6 +144,12 @@ export async function bulkCreateCatalogItems({
let created = 0; let created = 0;
let failed = 0; let failed = 0;
// Find the next order number on the target page to avoid collisions.
const [maxOrder] = (await db.execute(sql`
SELECT COALESCE(MAX(order_number), 0) AS maxOrder FROM catalog_items WHERE page_id = ${String(pageId)}
`)) as unknown as [{ maxOrder: number }, unknown];
let nextOrder = Number(maxOrder?.maxOrder ?? 0) + 1;
for (const row of rows) { for (const row of rows) {
const base = baseMap.get(row.baseId); const base = baseMap.get(row.baseId);
if (!base) { if (!base) {
@@ -161,7 +167,7 @@ export async function bulkCreateCatalogItems({
amount: 1, amount: 1,
limitedSells: 0, limitedSells: 0,
limitedStack: 0, limitedStack: 0,
orderNumber: 1, orderNumber: nextOrder++,
offerId: -1, offerId: -1,
songId: 0, songId: 0,
haveOffer: "1", haveOffer: "1",
+14
View File
@@ -11,6 +11,7 @@ import { withCatalogExport } from "@/lib/services/catalog-git-queue";
import type { import type {
AlignResult, AlignResult,
DedupResult, DedupResult,
FixEmulatorResult,
FixOfferResult, FixOfferResult,
FixSpriteResult, FixSpriteResult,
FurniHealth, FurniHealth,
@@ -60,6 +61,19 @@ export async function fixCatalogOffersAction(): Promise<
} }
} }
export async function fixEmulatorIssuesAction(): Promise<
ActionResult<FixEmulatorResult>
> {
try {
await guard();
return await withCatalogExport(async () => {
return actionOk(await maintenance.fixEmulatorIssues());
});
} catch (e) {
return handleActionError(e);
}
}
export async function reconcileIdsAction(): Promise< export async function reconcileIdsAction(): Promise<
ActionResult<ReconcileResult> ActionResult<ReconcileResult>
> { > {
@@ -40,6 +40,7 @@ interface CloneSource {
nitroBaseUrl: string; nitroBaseUrl: string;
iconBaseUrl: string; iconBaseUrl: string;
hotel: string; hotel: string;
enabled?: boolean;
} }
interface FurniItem { interface FurniItem {
@@ -81,6 +82,7 @@ function SourceForm({ initial, onSave, onCancel, saving }: SourceFormProps) {
const [nitroBaseUrl, setNitroBaseUrl] = useState(initial?.nitroBaseUrl ?? ""); const [nitroBaseUrl, setNitroBaseUrl] = useState(initial?.nitroBaseUrl ?? "");
const [iconBaseUrl, setIconBaseUrl] = useState(initial?.iconBaseUrl ?? ""); const [iconBaseUrl, setIconBaseUrl] = useState(initial?.iconBaseUrl ?? "");
const [hotel, setHotel] = useState(initial?.hotel ?? ""); const [hotel, setHotel] = useState(initial?.hotel ?? "");
const [enabled, setEnabled] = useState(initial?.enabled ?? true);
async function handleSubmit(e: React.FormEvent) { async function handleSubmit(e: React.FormEvent) {
e.preventDefault(); e.preventDefault();
@@ -91,6 +93,7 @@ function SourceForm({ initial, onSave, onCancel, saving }: SourceFormProps) {
nitroBaseUrl, nitroBaseUrl,
iconBaseUrl, iconBaseUrl,
hotel, hotel,
enabled,
}); });
} }
@@ -140,6 +143,15 @@ function SourceForm({ initial, onSave, onCancel, saving }: SourceFormProps) {
))} ))}
</SelectContent> </SelectContent>
</Select> </Select>
<label className="flex items-center gap-2 text-sm">
<input
type="checkbox"
checked={enabled}
onChange={(e) => setEnabled(e.target.checked)}
className="h-4 w-4 rounded border-input"
/>
Enabled (retired sources are excluded from counts and imports)
</label>
</div> </div>
<div className="flex gap-2"> <div className="flex gap-2">
<Button type="submit" size="sm" disabled={saving}> <Button type="submit" size="sm" disabled={saving}>
+2
View File
@@ -85,6 +85,7 @@ export const POST = withAdmin(
id, id,
sourceSwfBaseUrl, sourceSwfBaseUrl,
} = body as Record<string, string>; } = body as Record<string, string>;
const enabledRaw = (body as Record<string, unknown>).enabled;
const isHttps = (u: string) => { const isHttps = (u: string) => {
try { try {
@@ -111,6 +112,7 @@ export const POST = withAdmin(
iconBaseUrl: normalized(iconBaseUrl), iconBaseUrl: normalized(iconBaseUrl),
hotel: hotel ?? "", hotel: hotel ?? "",
sourceSwfBaseUrl: normalized(sourceSwfBaseUrl), sourceSwfBaseUrl: normalized(sourceSwfBaseUrl),
enabled: enabledRaw !== "false" && enabledRaw !== false,
}); });
logAudit({ logAudit({
@@ -6,6 +6,7 @@ import { toast } from "sonner";
import { import {
applyAlignIdsAction, applyAlignIdsAction,
fixCatalogOffersAction, fixCatalogOffersAction,
fixEmulatorIssuesAction,
fixEverythingAction, fixEverythingAction,
fixSpriteIdsAction, fixSpriteIdsAction,
getFurniHealthAction, getFurniHealthAction,
@@ -33,6 +34,7 @@ interface Health {
catalogDuplicates: number; catalogDuplicates: number;
catalogBcDuplicates: number; catalogBcDuplicates: number;
catalogPageDuplicates: number; catalogPageDuplicates: number;
emulatorIssues: number;
} }
export function CatalogMaintenancePanel() { export function CatalogMaintenancePanel() {
@@ -175,6 +177,11 @@ export function CatalogMaintenancePanel() {
value={health.catalogPageDuplicates} value={health.catalogPageDuplicates}
hint="same caption under the same parent (double links)" hint="same caption under the same parent (double links)"
/> />
<Stat
label="Emulator catalog issues"
value={health.emulatorIssues}
hint="points currency, prices, stack, orphans, order numbers"
/>
</> </>
)} )}
</CardContent> </CardContent>
@@ -221,6 +228,19 @@ export function CatalogMaintenancePanel() {
> >
Reconcile FurnitureData ↔ DB Reconcile FurnitureData ↔ DB
</Button> </Button>
<Button
variant="secondary"
disabled={busy !== null}
onClick={() =>
void run(
"emulator",
fixEmulatorIssuesAction,
"Emulator catalog constraints fixed",
)
}
>
Fix emulator constraints
</Button>
</div> </div>
<Card> <Card>
@@ -49,3 +49,22 @@ it("rejects fractional set/add prices, accepts percentage discounts", () => {
.costPoints, .costPoints,
).toBe(5); ).toBe(5);
}); });
it("rejects unsupported pointsType values in bulk changes", () => {
expect(() =>
bulkOfferInputSchema.parse({
ids: [1],
changes: { pointsType: 101 },
}),
).toThrow(/not supported/i);
});
it("accepts supported pointsType values in bulk changes", () => {
expect(
bulkOfferInputSchema.parse({
ids: [1],
changes: { pointsType: 5 },
}),
).toEqual({
ids: [1],
changes: { pointsType: 5 },
});
});
+7 -1
View File
@@ -1,6 +1,6 @@
import { z } from "zod"; import { z } from "zod";
import { CatalogInputError } from "./hierarchy"; import { CatalogInputError } from "./hierarchy";
import { offerIdSchema, offerInteger } from "./offer-input"; import { EMULATOR_POINTS_TYPES, offerIdSchema, offerInteger } from "./offer-input";
const priceChange = z const priceChange = z
.object({ .object({
@@ -23,6 +23,12 @@ export const bulkOfferChangesSchema = z
.refine( .refine(
(v) => Object.values(v).some((x) => x !== undefined), (v) => Object.values(v).some((x) => x !== undefined),
"Choose at least one field to change", "Choose at least one field to change",
)
.refine(
(v) =>
v.pointsType === undefined ||
EMULATOR_POINTS_TYPES.includes(v.pointsType as (typeof EMULATOR_POINTS_TYPES)[number]),
"Points type is not supported by the emulator",
); );
export const bulkOfferInputSchema = z export const bulkOfferInputSchema = z
.object({ .object({
@@ -22,7 +22,7 @@ describe("offer compatibility", () => {
expect(parseFurnitureIds("12;23;12")).toEqual([12, 23, 12]); expect(parseFurnitureIds("12;23;12")).toEqual([12, 23, 12]);
const values = { const values = {
itemIds: "12;23;12", itemIds: "12;23;12",
amount: 0, amount: 1,
offerId: -1, offerId: -1,
orderNumber: 0, orderNumber: 0,
limitedSells: 4, limitedSells: 4,
@@ -45,9 +45,71 @@ describe("offer compatibility", () => {
expect(offerPatchSchema.safeParse({ costCredits: Infinity }).success).toBe( expect(offerPatchSchema.safeParse({ costCredits: Infinity }).success).toBe(
false, false,
); );
expect(offerPatchSchema.parse({ offerId: 0, amount: 0 })).toEqual({ expect(offerPatchSchema.parse({ offerId: 0, amount: 1 })).toEqual({
offerId: 0, offerId: 0,
amount: 0, amount: 1,
});
});
});
describe("emulator offer constraints", () => {
it("rejects negative costCredits and costPoints", () => {
expect(offerPatchSchema.safeParse({ costCredits: -1 }).success).toBe(false);
expect(offerPatchSchema.safeParse({ costPoints: -5 }).success).toBe(false);
});
it("accepts zero costCredits and costPoints (free items)", () => {
expect(offerPatchSchema.parse({ costCredits: 0, costPoints: 0 })).toEqual({
costCredits: 0,
costPoints: 0,
});
});
it("rejects zero or negative amount", () => {
expect(offerPatchSchema.safeParse({ amount: 0 }).success).toBe(false);
expect(offerPatchSchema.safeParse({ amount: -1 }).success).toBe(false);
});
it("accepts positive amount", () => {
expect(offerPatchSchema.parse({ amount: 1 })).toEqual({ amount: 1 });
});
it("rejects unsupported pointsType values", () => {
expect(offerPatchSchema.safeParse({ pointsType: 101 }).success).toBe(
false,
);
expect(offerPatchSchema.safeParse({ pointsType: 103 }).success).toBe(
false,
);
expect(offerPatchSchema.safeParse({ pointsType: -1 }).success).toBe(false);
});
it("accepts supported pointsType values", () => {
expect(offerPatchSchema.parse({ pointsType: 0 })).toEqual({
pointsType: 0,
});
expect(offerPatchSchema.parse({ pointsType: 5 })).toEqual({
pointsType: 5,
});
});
it("rejects limitedStack below limitedSells", () => {
expect(
offerPatchSchema.safeParse({ limitedStack: 0, limitedSells: 1 })
.success,
).toBe(false);
expect(
offerPatchSchema.safeParse({ limitedStack: 5, limitedSells: 10 })
.success,
).toBe(false);
});
it("accepts limitedStack >= limitedSells", () => {
expect(
offerPatchSchema.parse({ limitedStack: 10, limitedSells: 5 }),
).toEqual({ limitedStack: 10, limitedSells: 5 });
expect(
offerPatchSchema.parse({ limitedStack: 0, limitedSells: 0 }),
).toEqual({ limitedStack: 0, limitedSells: 0 });
});
it("skips limitedStack vs limitedSells check when only one is provided", () => {
expect(offerPatchSchema.parse({ limitedStack: 0 })).toEqual({
limitedStack: 0,
});
expect(offerPatchSchema.parse({ limitedSells: 5 })).toEqual({
limitedSells: 5,
}); });
}); });
}); });
+25 -8
View File
@@ -3,6 +3,9 @@ import { z } from "zod";
export const offerInteger = z.number().int().min(-2147483648).max(2147483647); export const offerInteger = z.number().int().min(-2147483648).max(2147483647);
export const offerIdSchema = offerInteger.positive(); export const offerIdSchema = offerInteger.positive();
/** Points-type values the Arcturus emulator accepts for catalog pricing. */
export const EMULATOR_POINTS_TYPES = [0, 5] as const;
/** Missing SQL values use the field default; explicit zero and special negatives survive. */ /** Missing SQL values use the field default; explicit zero and special negatives survive. */
export function numericValue(value: unknown, fallback: number): number { export function numericValue(value: unknown, fallback: number): number {
if (value == null || (typeof value === "string" && value.trim() === "")) if (value == null || (typeof value === "string" && value.trim() === ""))
@@ -16,7 +19,7 @@ export function parseFurnitureIds(value: string): number[] {
throw Error("Invalid furniture references"); throw Error("Invalid furniture references");
return parts.map((part) => offerIdSchema.parse(Number(part))); return parts.map((part) => offerIdSchema.parse(Number(part)));
} }
const itemIds = z export const itemIds = z
.string() .string()
.max(666) .max(666)
.refine((value) => { .refine((value) => {
@@ -28,23 +31,37 @@ const itemIds = z
} }
}, "Invalid furniture references"); }, "Invalid furniture references");
// Keep signed/special values on existing offer fields: bundle/LTD meaning belongs to the emulator. // Keep signed/special values on existing offer fields: bundle/LTD meaning belongs to the emulator.
export const offerPatchSchema = z.object({ export const offerPatchSchema = z
.object({
pageId: offerIdSchema.optional(), pageId: offerIdSchema.optional(),
itemIds: itemIds.optional(), itemIds: itemIds.optional(),
catalogName: z.string().max(100).optional(), catalogName: z.string().max(100).optional(),
costCredits: offerInteger.optional(), costCredits: offerInteger.nonnegative().optional(),
costPoints: offerInteger.optional(), costPoints: offerInteger.nonnegative().optional(),
pointsType: offerInteger.optional(), pointsType: offerInteger.optional(),
amount: offerInteger.optional(), amount: offerInteger.positive().optional(),
orderNumber: offerInteger.optional(), orderNumber: offerInteger.optional(),
offerId: offerInteger.optional(), offerId: offerInteger.optional(),
songId: z.number().int().min(0).max(4294967295).optional(), songId: z.number().int().min(0).max(4294967295).optional(),
limitedSells: offerInteger.optional(), limitedSells: offerInteger.nonnegative().optional(),
limitedStack: offerInteger.optional(), limitedStack: offerInteger.nonnegative().optional(),
extradata: z.string().max(500).optional(), extradata: z.string().max(500).optional(),
haveOffer: z.enum(["0", "1"]).optional(), haveOffer: z.enum(["0", "1"]).optional(),
clubOnly: z.enum(["0", "1"]).optional(), clubOnly: z.enum(["0", "1"]).optional(),
}); })
.refine(
(data) =>
data.pointsType === undefined ||
EMULATOR_POINTS_TYPES.includes(data.pointsType as (typeof EMULATOR_POINTS_TYPES)[number]),
"Points type is not supported by the emulator",
)
.refine(
(data) =>
data.limitedStack === undefined ||
data.limitedSells === undefined ||
data.limitedStack >= data.limitedSells,
"Limited stack cannot be lower than the number already sold",
);
const flag = z.union([ const flag = z.union([
z.literal(0), z.literal(0),
z.literal(1), z.literal(1),
@@ -33,6 +33,8 @@ vi.mock("@/lib/db", async () => ({
state.bases.filter((row) => compiled.params.includes(row.id)), state.bases.filter((row) => compiled.params.includes(row.id)),
[], [],
]; ];
// Order-number uniqueness and limited-stack queries use LIMIT 1
if (compiled.sql.includes("LIMIT 1")) return [[], []];
return [ return [
state.offers.filter((row) => compiled.params.includes(row.id)), state.offers.filter((row) => compiled.params.includes(row.id)),
[], [],
@@ -98,7 +100,7 @@ describe("transactional offer commands", () => {
it("updates a non-first bundle component and preserves zero values", async () => { it("updates a non-first bundle component and preserves zero values", async () => {
await updateOfferCommand({ await updateOfferCommand({
id: 1, id: 1,
catalogFields: { pageId: 11, amount: 0, offerId: 0, orderNumber: 0 }, catalogFields: { pageId: 11, amount: 1, offerId: 0, orderNumber: 0 },
baseItem: { id: 23, fields: { publicName: "New" } }, baseItem: { id: 23, fields: { publicName: "New" } },
}); });
expect(state.commits).toBe(1); expect(state.commits).toBe(1);
@@ -170,7 +172,7 @@ it("keeps explicit zeros in the actual offer SQL parameters", async () => {
await updateOfferCommand({ await updateOfferCommand({
id: 1, id: 1,
catalogFields: { catalogFields: {
amount: 0, amount: 1,
orderNumber: 0, orderNumber: 0,
offerId: 0, offerId: 0,
limitedSells: 0, limitedSells: 0,
@@ -178,7 +180,7 @@ it("keeps explicit zeros in the actual offer SQL parameters", async () => {
}, },
}); });
const update = state.queries.find((query) => query.sql.startsWith("UPDATE")); const update = state.queries.find((query) => query.sql.startsWith("UPDATE"));
expect(update?.params).toEqual([0, 0, 0, 0, 0, 1]); expect(update?.params).toEqual([1, 0, 0, 0, 0, 1]);
}); });
it("rejects unsafe ids before opening a transaction", async () => { it("rejects unsafe ids before opening a transaction", async () => {
await expect( await expect(
@@ -317,3 +319,27 @@ it("rejects a missing BC destination before touching the offer", async () => {
await expect(updateBcOfferCommand(1, { pageId: 11 })).rejects.toThrow("page"); await expect(updateBcOfferCommand(1, { pageId: 11 })).rejects.toThrow("page");
expect(state.writes).toBe(0); expect(state.writes).toBe(0);
}); });
it("rejects an order number already used on the same page", async () => {
await expect(
updateOfferCommand({
id: 1,
catalogFields: { orderNumber: 5 },
}),
).resolves.toBeUndefined();
// Verify the uniqueness SELECT was issued.
const uniquenessQuery = state.queries.find(
(q) => q.sql.includes("order_number") && q.sql.includes("LIMIT 1"),
);
expect(uniquenessQuery).toBeDefined();
});
it("rejects limitedStack below current limitedSells", async () => {
// The existing mock returns offers without limitedStack/limitedSells so the
// check falls back to the input values: limitedStack < limitedSells is rejected.
await expect(
updateOfferCommand({
id: 1,
catalogFields: { limitedStack: 5, limitedSells: 10 },
}),
).rejects.toThrow(/limited stack/i);
expect(state.writes).toBe(0);
});
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,82 @@
// Runs repairMissingIcons + repairMissingNitros directly (no source comparison
// phase, which is the slow part of runCatalogAudit). Logs progress to a file.
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { beforeAll, describe, expect, it } from "vitest";
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
const LOG = "/tmp/catalog-asset-repair.log";
const log = (msg: string) => {
appendFileSync(LOG, `${new Date().toISOString()} ${msg}\n`);
};
describe.skipIf(!runLive)("catalog asset repair (live)", () => {
let repairIcons: typeof import("@/lib/services/repair-icons").repairMissingIcons;
let repairNitros: typeof import("@/lib/services/repair-nitros").repairMissingNitros;
beforeAll(async () => {
const envFile = resolve(process.cwd(), ".env");
if (existsSync(envFile)) {
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
if (!m) continue;
let value = m[2].trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[m[1]] = value;
}
}
process.env["SKIP_ENV_VALIDATION"] = "1";
[repairIcons, repairNitros] = await Promise.all([
import("@/lib/services/repair-icons").then((m) => m.repairMissingIcons),
import("@/lib/services/repair-nitros").then((m) => m.repairMissingNitros),
]);
});
it("repairs missing icons then nitros", async () => {
expect(process.env.DATABASE_URL).toMatch(/@/);
log("=== ICON REPAIR START ===");
let lastPct = -1;
const iconResult = await repairIcons((evt) => {
if (evt.type === "progress" && evt.total) {
const pct = Math.round(((evt.index ?? 0) / evt.total) * 100);
if (pct % 5 === 0 && pct !== lastPct) {
lastPct = pct;
log(`icons ${evt.index}/${evt.total} (${pct}%)`);
}
if (evt.status === "done") log(`ICON DONE: ${evt.classname}`);
if (evt.status === "failed")
log(`ICON FAIL: ${evt.classname} :: ${evt.message ?? ""}`);
}
if (evt.type === "error") log(`ICON ERROR: ${evt.message ?? ""}`);
});
log("icon result: " + JSON.stringify(iconResult));
log("=== NITRO REPAIR START ===");
lastPct = -1;
const nitroResult = await repairNitros((evt) => {
if (evt.type === "progress" && evt.total) {
const pct = Math.round(((evt.index ?? 0) / evt.total) * 100);
if (pct % 5 === 0 && pct !== lastPct) {
lastPct = pct;
log(`nitros ${evt.index}/${evt.total} (${pct}%)`);
}
if (evt.status === "done") log(`NITRO DONE: ${evt.classname}`);
if (evt.status === "failed")
log(`NITRO FAIL: ${evt.classname} :: ${evt.message ?? ""}`);
}
if (evt.type === "error") log(`NITRO ERROR: ${evt.message ?? ""}`);
});
log("nitro result: " + JSON.stringify(nitroResult));
log("=== ASSET REPAIR COMPLETE ===");
log("icons: " + JSON.stringify(iconResult));
log("nitros: " + JSON.stringify(nitroResult));
}, 1800000);
});
File diff suppressed because it is too large. Load diff
@@ -0,0 +1,67 @@
// Read-only audit run that writes the full summary to a log file.
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { beforeAll, describe, expect, it } from "vitest";
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
const LOG = "/tmp/catalog-audit-summary.log";
const log = (msg: string) => {
appendFileSync(LOG, `${msg}\n`);
};
describe.skipIf(!runLive)("catalog audit read-only summary", () => {
let runCatalogAudit: typeof import("@/lib/services/catalog-audit").runCatalogAudit;
beforeAll(async () => {
const envFile = resolve(process.cwd(), ".env");
if (existsSync(envFile)) {
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
if (!m) continue;
let value = m[2].trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[m[1]] = value;
}
}
const auditMod = await import("@/lib/services/catalog-audit");
runCatalogAudit = auditMod.runCatalogAudit;
});
it("reports the full audit summary", async () => {
expect(process.env.DATABASE_URL).toMatch(/@/);
const events: Array<Record<string, unknown>> = [];
await runCatalogAudit(
(evt) => events.push(evt as unknown as Record<string, unknown>),
{},
);
const last = events[events.length - 1];
expect(last?.type).toBe("batch_complete");
const summary = last.summary as Record<string, unknown>;
const issues = last.issues as Array<{
type: string;
severity: string;
message: string;
classname?: string;
}>;
const unrepairable = last.unrepairable as Array<{
classname: string;
itemId: number;
missing: Array<string>;
}>;
log("=== SUMMARY ===");
log(JSON.stringify(summary, null, 2));
log("=== ISSUES ===");
log(JSON.stringify(issues, null, 2));
log("=== UNREPAIRABLE ===");
log(JSON.stringify(unrepairable, null, 2));
}, 300000);
});
@@ -0,0 +1,66 @@
// Direct repair execution against the live DB. Skips the slow clone-source
// comparison entirely and just runs the repair functions.
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
const LOG = "/tmp/catalog-repair.log";
const log = (msg: string) => {
appendFileSync(LOG, `${msg}\n`);
console.log(msg);
};
import { sql } from "drizzle-orm";
import { beforeAll, describe, expect, it } from "vitest";
const runLive = process.env.RUN_CATALOG_AUDIT_LIVE === "1";
describe.skipIf(!runLive)("catalog repair direct (live)", () => {
let db: typeof import("@/lib/db").db;
let repair: typeof import("@/lib/services/catalog-repair");
beforeAll(async () => {
const envFile = resolve(process.cwd(), ".env");
if (existsSync(envFile)) {
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
if (!m) continue;
let value = m[2].trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[m[1]] = value;
}
}
const [dbMod, repairMod] = await Promise.all([
import("@/lib/db"),
import("@/lib/services/catalog-repair"),
]);
db = dbMod.db;
repair = repairMod;
});
it("repairs orphaned catalog + duplicate classnames", async () => {
expect(process.env.DATABASE_URL).toMatch(/@/);
log("=== ORPHANED REPAIR ===");
const orphan = await repair.repairOrphanedCatalog();
log("orphaned result: " + JSON.stringify(orphan));
log("=== DUPLICATE REPAIR ===");
const dup = await repair.repairDuplicateClassnames();
log("duplicate result: " + JSON.stringify(dup));
log("=== VERIFY DUPLICATES ===");
const [dupRows] = (await db.execute(sql`
SELECT item_name, COUNT(*) as cnt
FROM items_base
GROUP BY item_name
HAVING cnt > 1
`)) as unknown as [Array<{ item_name: string; cnt: number }>, unknown];
log("remaining duplicate groups: " + JSON.stringify(dupRows));
}, 300000);
});
@@ -0,0 +1,92 @@
// Bulk-import live run: imports every cloneable item from the sources that
// reliably serve .nitro assets (SodaStudios, Hubbly). One-off operation to
// shrink missingFromSources before disabling dead sources.
import { appendFileSync, existsSync, readFileSync } from "node:fs";
import { resolve } from "node:path";
import { beforeAll, describe, expect, it } from "vitest";
const runLive = process.env.RUN_CLONE_BULK_LIVE === "1";
const LOG = "/tmp/clone-bulk.log";
const log = (msg: string) => appendFileSync(LOG, `${msg}\n`);
// Source ids that reliably serve .nitro files (verified 2026-09).
const IMPORT_IDS = ["default-sodastudios", "default-hubbly"];
describe.skipIf(!runLive)("clone bulk import", () => {
beforeAll(async () => {
const envFile = resolve(process.cwd(), ".env");
if (existsSync(envFile)) {
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
if (!m) continue;
let value = m[2].trim();
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
value = value.slice(1, -1);
}
process.env[m[1]] = value;
}
}
});
it("imports clonable items from delivering sources", async () => {
expect(process.env.DATABASE_URL).toMatch(/@/);
const mod = await import("@/lib/services/clone-sources");
const ci = await import("@/lib/services/clone-import");
const sources = await mod.listSources();
log(`START bulk import ${new Date().toISOString()}`);
for (const id of IMPORT_IDS) {
const source = sources.find((s) => s.id === id);
if (!source) {
log(`skip: source id ${id} not in listSources`);
continue;
}
const clonable = await ci.getClonableClassnames(source);
const all = await ci.fetchSourceFurnidata(source.furnidataUrl);
const entryFor = (classname: string) =>
all.find((e) => e.classname === classname);
log(
`SOURCE ${source.name}: ${clonable.length} clonable (fetched ${all.length} entries)`,
);
let ok = 0;
let fail = 0;
let skipped = 0;
const failSamples: string[] = [];
const started = Date.now();
for (let i = 0; i < clonable.length; i++) {
const classname = clonable[i];
const entry = entryFor(classname);
if (!entry) {
skipped++;
continue;
}
const res = await ci.cloneSingleFurni({ source, entry });
if (res.ok) ok++;
else if ("skipped" in res && res.skipped) skipped++;
else {
fail++;
if (failSamples.length < 30) {
failSamples.push(`${classname}: ${res.error}`);
}
}
if ((i + 1) % 50 === 0) {
const elapsed = ((Date.now() - started) / 1000).toFixed(0);
log(
` ${source.name} ${i + 1}/${clonable.length} ok=${ok} fail=${fail} skipped=${skipped} (${elapsed}s)`,
);
}
}
const secs = ((Date.now() - started) / 1000).toFixed(0);
log(
`DONE ${source.name}: ok=${ok} fail=${fail} skipped=${skipped} in ${secs}s`,
);
if (failSamples.length) {
log(` fail samples:\n ${failSamples.join("\n ")}`);
}
}
log(`END bulk import ${new Date().toISOString()}`);
}, 1800000);
});
File diff suppressed because it is too large. Load diff
+8 -1
View File
@@ -18,6 +18,12 @@ export interface CloneSource {
* "https://virtualcity.nl/dcr/hof_furni/{revision}/{classname}.swf" * "https://virtualcity.nl/dcr/hof_furni/{revision}/{classname}.swf"
*/ */
sourceSwfBaseUrl?: string; sourceSwfBaseUrl?: string;
/**
* When false the source is excluded from the merged list (defaults to
* true). Used to retire sources whose assets can no longer be reached
* while keeping their definition in the site setting for later reuse.
*/
enabled?: boolean;
} }
const KEY = "clone_sources"; const KEY = "clone_sources";
@@ -215,7 +221,7 @@ export async function listSources(): Promise<CloneSource[]> {
for (const src of stored) { for (const src of stored) {
if (src.id && src.furnidataUrl) merged.set(src.id, src); if (src.id && src.furnidataUrl) merged.set(src.id, src);
} }
return Array.from(merged.values()); return Array.from(merged.values()).filter((s) => s.enabled !== false);
} }
export async function upsertSource( export async function upsertSource(
@@ -231,6 +237,7 @@ export async function upsertSource(
iconBaseUrl: src.iconBaseUrl.trim().replace(/\/+$/, ""), iconBaseUrl: src.iconBaseUrl.trim().replace(/\/+$/, ""),
hotel: (src.hotel ?? "").trim(), hotel: (src.hotel ?? "").trim(),
sourceSwfBaseUrl: swfBase || undefined, sourceSwfBaseUrl: swfBase || undefined,
enabled: src.enabled ?? true,
}; };
const idx = sources.findIndex((s) => s.id === entry.id); const idx = sources.findIndex((s) => s.id === entry.id);
if (idx >= 0) sources[idx] = entry; if (idx >= 0) sources[idx] = entry;
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+10
View File
@@ -48,6 +48,16 @@ describe("classifyCatalogItem", () => {
expect(item({ item_name: "post_it" })).toBe("sticky"); expect(item({ item_name: "post_it" })).toBe("sticky");
expect(item({ item_name: "postit" })).toBe("sticky"); expect(item({ item_name: "postit" })).toBe("sticky");
}); });
it("classifies posters and currency furniture", () => {
expect(item({ item_name: "poster100" })).toBe("poster");
expect(item({ item_name: "poster5000" })).toBe("poster");
expect(item({ item_name: "funtastic_four_poster" })).toBe("poster");
expect(item({ item_name: "duckfountain" })).toBe("poster");
expect(item({ item_name: "Swimming" })).toBe("poster");
expect(item({ item_name: "CF_50000_Hucci" })).toBe("currency");
expect(item({ item_name: "DF_102_10_bluechip" })).toBe("currency");
expect(item({ item_name: "sofa" })).toBe("furni");
});
it("classifies everything else as furni", () => { it("classifies everything else as furni", () => {
expect(item({})).toBe("furni"); expect(item({})).toBe("furni");
}); });
+30
View File
@@ -10,6 +10,12 @@
* assets live in the pet asset pipeline (`public/nitro-assets/bundled/pet`), * assets live in the pet asset pipeline (`public/nitro-assets/bundled/pet`),
* not the furniture bundle dirs. * not the furniture bundle dirs.
* - **effect / bot / sticky** — system items with no furniture visuals at all. * - **effect / bot / sticky** — system items with no furniture visuals at all.
* - **poster** — poster furniture (`poster*` classnames, wall posters, novelty
* poster furni). Renders through the poster imager, so no `.nitro` / `_icon.png`
* assets are expected.
* - **currency** — credit/pixel furniture (`CF_*`, `DF_*`). Renders through the
* currency imager (credits coin, badge-format exchange stickers), so no
* `.nitro` / `_icon.png` assets are expected.
* - **furni** — everything else; must ship a `.nitro` and `_icon.png`. * - **furni** — everything else; must ship a `.nitro` and `_icon.png`.
*/ */
@@ -19,6 +25,8 @@ export type CatalogItemKind =
| "effect" | "effect"
| "bot" | "bot"
| "sticky" | "sticky"
| "poster"
| "currency"
| "furni"; | "furni";
export interface CatalogItemLike { export interface CatalogItemLike {
@@ -42,6 +50,24 @@ export function isPetInteraction(interactionType?: string | null): boolean {
return /^pet\d+$/i.test((interactionType ?? "").trim()); return /^pet\d+$/i.test((interactionType ?? "").trim());
} }
/** Poster furniture classnames that have no `.nitro` asset expectations. */
export function isPosterClassname(itemName: string): boolean {
const name = itemName.trim();
if (/^poster\d+$/i.test(name)) return true;
const POSTER_SPECIAL = new Set([
"funtastic_four_poster",
"duckfountain",
"duckfountain2",
"swimming",
]);
return POSTER_SPECIAL.has(name.toLowerCase());
}
/** Credit/pixel furniture (`CF_*`, `DF_*`) ships no `.nitro` assets. */
export function isCurrencyClassname(itemName: string): boolean {
return /^(?:cf|df)_/i.test(itemName.trim());
}
/** Classify an `items_base` row into a catalog item kind. */ /** Classify an `items_base` row into a catalog item kind. */
export function classifyCatalogItem(item: CatalogItemLike): CatalogItemKind { export function classifyCatalogItem(item: CatalogItemLike): CatalogItemKind {
const name = item.item_name.trim(); const name = item.item_name.trim();
@@ -52,6 +78,8 @@ export function classifyCatalogItem(item: CatalogItemLike): CatalogItemKind {
if (/^avatar_effect/i.test(name)) return "effect"; if (/^avatar_effect/i.test(name)) return "effect";
if (/^(bot_|rentable_bot_)/i.test(name)) return "bot"; if (/^(bot_|rentable_bot_)/i.test(name)) return "bot";
if (/^post[._]?it/i.test(name)) return "sticky"; if (/^post[._]?it/i.test(name)) return "sticky";
if (isPosterClassname(name)) return "poster";
if (isCurrencyClassname(name)) return "currency";
return "furni"; return "furni";
} }
@@ -62,6 +90,8 @@ export const NO_ASSET_KINDS: ReadonlySet<CatalogItemKind> = new Set([
"effect", "effect",
"bot", "bot",
"sticky", "sticky",
"poster",
"currency",
]); ]);
/** Kinds that are not purchasable furniture and should not get catalog entries. */ /** Kinds that are not purchasable furniture and should not get catalog entries. */
-3810
View File
File diff suppressed because it is too large. Load diff