feat(catalog): prevent and repair Arcturus emulator data errors
Block invalid catalog_items writes at the API level (points currency allowlist, non-negative prices, positive amount, limited stack >= sold count, unique sibling order numbers) and auto-assign unique order numbers on bulk create. Add a catalog-maintenance scan + transactional repair that fixes pre-existing rows: resets unsupported points_type, clamps negative costs, sets amount to 1, raises limited_stack, renumbers duplicate orders and deletes offers with missing page/item references. Surface the issue count and a fix button in the admin maintenance panel. Also: add enabled/retired flag to clone sources, classify poster and currency furniture in item-kind, and remove the obsolete update-Nitrov3.sh.
This commit is contained in:
1 parent
ad65d80d41
commit
3e69b72513
26 files changed
+17651
-3844
No files matched your search
@@ -0,0 +1,139 @@
|
||||
// Feasibility probe: splits the audit's missing-from-sources list per clone
|
||||
// source, marks which sources can deliver .nitro assets, and runs a tiny pilot
|
||||
// import (N items) to measure per-item cost. Writes a report to /tmp.
|
||||
import { appendFileSync, existsSync, readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { beforeAll, describe, expect, it } from "vitest";
|
||||
|
||||
const runLive = process.env.RUN_CLONE_FEASIBILITY_LIVE === "1";
|
||||
const LOG = "/tmp/clone-feasibility.log";
|
||||
const log = (msg: string) => appendFileSync(LOG, `${msg}\n`);
|
||||
|
||||
describe.skipIf(!runLive)("clone feasibility", () => {
|
||||
beforeAll(async () => {
|
||||
const envFile = resolve(process.cwd(), ".env");
|
||||
if (existsSync(envFile)) {
|
||||
for (const line of readFileSync(envFile, "utf8").split(/\r?\n/)) {
|
||||
const m = line.match(/^\s*([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*?)\s*$/);
|
||||
if (!m) continue;
|
||||
let value = m[2].trim();
|
||||
if (
|
||||
(value.startsWith('"') && value.endsWith('"')) ||
|
||||
(value.startsWith("'") && value.endsWith("'"))
|
||||
) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
process.env[m[1]] = value;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it("reports per-source clonable counts + pilot", async () => {
|
||||
expect(process.env.DATABASE_URL).toMatch(/@/);
|
||||
const mod = await import("@/lib/services/clone-sources");
|
||||
const ci = await import("@/lib/services/clone-import");
|
||||
const sources = await mod.listSources();
|
||||
log(`sources: ${sources.length}`);
|
||||
const report: Array<Record<string, unknown>> = [];
|
||||
for (const source of sources) {
|
||||
const start = Date.now();
|
||||
try {
|
||||
const clonable = await ci.getClonableClassnames(source);
|
||||
const secs = ((Date.now() - start) / 1000).toFixed(1);
|
||||
log(
|
||||
`${source.name}: haveNitro=${Boolean(source.nitroBaseUrl)} clonable=${clonable.length} (${secs}s)`,
|
||||
);
|
||||
report.push({
|
||||
source: source.name,
|
||||
clonable: clonable.length,
|
||||
hasNitro: Boolean(source.nitroBaseUrl),
|
||||
secs: Number(secs),
|
||||
});
|
||||
} catch (err) {
|
||||
log(` source ${source.name} FAILED: ${(err as Error).message}`);
|
||||
report.push({ source: source.name, error: (err as Error).message });
|
||||
}
|
||||
}
|
||||
const withNitro = report
|
||||
.filter((r) => r.hasNitro && typeof r.clonable === "number")
|
||||
.reduce((a, r) => a + (r.clonable as number), 0);
|
||||
const totalClonable = report
|
||||
.filter((r) => typeof r.clonable === "number")
|
||||
.reduce((a, r) => a + (r.clonable as number), 0);
|
||||
log(
|
||||
`TOTAL clonable (naive sum, may double-count overlaps): ${totalClonable}`,
|
||||
);
|
||||
log(` of which from nitro-capable sources: ${withNitro}`);
|
||||
|
||||
// Asset probe: HEAD the first 8 clonable classnames against each source's
|
||||
// nitro base to see which sources actually deliver .nitro files.
|
||||
for (const source of sources) {
|
||||
if (!source.nitroBaseUrl) continue;
|
||||
try {
|
||||
const clonable = await ci.getClonableClassnames(source);
|
||||
const statuses: string[] = [];
|
||||
for (const c of clonable.slice(0, 8)) {
|
||||
const url = `${source.nitroBaseUrl}/${c}.nitro`;
|
||||
try {
|
||||
const r = await fetch(url, {
|
||||
method: "HEAD",
|
||||
signal: AbortSignal.timeout(8000),
|
||||
});
|
||||
statuses.push(`${c}:${r.status}`);
|
||||
} catch {
|
||||
try {
|
||||
const r = await fetch(url, {
|
||||
signal: AbortSignal.timeout(8000),
|
||||
});
|
||||
statuses.push(`${c}:${r.status}`);
|
||||
} catch {
|
||||
statuses.push(`${c}:ERR`);
|
||||
}
|
||||
}
|
||||
}
|
||||
log(
|
||||
`PROBE ${source.name} (${clonable.length} clonable): ${JSON.stringify(statuses)}`,
|
||||
);
|
||||
} catch (err) {
|
||||
log(`PROBE ${source.name}: fetch fail ${(err as Error).message}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Pilot: import up to 5 items from the first nitro-capable source that
|
||||
// has clonable items, to measure per-item cost.
|
||||
const pilotSource: (typeof sources)[number] | null =
|
||||
sources.find((s) => s.nitroBaseUrl) ?? null;
|
||||
if (pilotSource) {
|
||||
const clonable = await ci.getClonableClassnames(pilotSource);
|
||||
const all = await ci.fetchSourceFurnidata(pilotSource.furnidataUrl);
|
||||
log(`PILOT source: ${pilotSource.name} clonable=${clonable.length}`);
|
||||
const sample = clonable.slice(0, 5);
|
||||
const start = Date.now();
|
||||
let ok = 0;
|
||||
let fail = 0;
|
||||
for (const classname of sample) {
|
||||
const entry = all.find((e) => e.classname === classname);
|
||||
if (!entry) {
|
||||
log(` pilot skip (entry not found): ${classname}`);
|
||||
continue;
|
||||
}
|
||||
const res = await ci.cloneSingleFurni({ source: pilotSource, entry });
|
||||
log(
|
||||
` pilot ${res.ok ? "OK" : "FAIL"} ${classname} -> ${
|
||||
res.error ?? "done"
|
||||
} ${res.warnings?.join(";") ?? ""}`,
|
||||
);
|
||||
if (res.ok) ok++;
|
||||
else fail++;
|
||||
}
|
||||
const secs = Date.now() - start;
|
||||
log(
|
||||
`PILOT done: ${ok} ok / ${fail} fail, ${(secs / 1000).toFixed(1)}s for ${sample.length} items`,
|
||||
);
|
||||
log(JSON.stringify(report, null, 2));
|
||||
} else {
|
||||
log("GEEN nitro-capabele bron gevonden; pilot overgeslagen.");
|
||||
log(JSON.stringify(report, null, 2));
|
||||
}
|
||||
}, 600000);
|
||||
});
|
||||
Reference in new issue
Block a user