feat: parallelize audit repair and add nitro/SQL repair options
- Parallelize icon and nitro downloads in the audit repair with a sliding-window worker pool (6 concurrent) to speed up large catalogs - Add repairMissingNitros: fetch missing .nitro bundles from configured nitro sources (Wibbo default), validating each bundle before writing - Add catalog-repair service: generate/apply catalog_items SQL for furni missing a catalog entry and repair FurnitureData.json (add missing + dedupe classnames) - Wire all options through the audit API and client UI with live progress and result stats (icons, nitros, SQL, furnidata) - Add Wibbo as default nitro source alongside existing icon sources - Ignore runtime furni assets downloaded into public/ during repair
This commit is contained in:
1 parent
0c8e62357f
commit
5308ce6a12
8 files changed
+1166
-58
No files matched your search
@@ -24,5 +24,10 @@ gitea
|
|||||||
# Runtime uploaded media (persistent, outside public/)
|
# Runtime uploaded media (persistent, outside public/)
|
||||||
storage/
|
storage/
|
||||||
|
|
||||||
|
# Runtime downloaded furni assets (mirrored from gamedata / audit repair)
|
||||||
|
public/nitro-assets/bundled/furniture/
|
||||||
|
public/swf/dcr/
|
||||||
|
public/tmp/
|
||||||
|
|
||||||
# Test coverage reports
|
# Test coverage reports
|
||||||
coverage/
|
coverage/
|
||||||
@@ -2,6 +2,7 @@
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
AlertCircle,
|
AlertCircle,
|
||||||
|
Check,
|
||||||
CheckCircle2,
|
CheckCircle2,
|
||||||
ClipboardList,
|
ClipboardList,
|
||||||
ExternalLink,
|
ExternalLink,
|
||||||
@@ -39,6 +40,21 @@ interface AuditSummary {
|
|||||||
missingIcon: number;
|
missingIcon: number;
|
||||||
duplicates: number;
|
duplicates: number;
|
||||||
missingFromSourcesTotal: number;
|
missingFromSourcesTotal: number;
|
||||||
|
iconsRepaired: number;
|
||||||
|
iconsStillMissing: number;
|
||||||
|
nitrosRepaired: number;
|
||||||
|
nitrosStillMissing: number;
|
||||||
|
catalogSqlCount: number;
|
||||||
|
catalogSqlApplied: number;
|
||||||
|
furniDataAdded: number;
|
||||||
|
furniDataDuplicatesRemoved: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface RepairStats {
|
||||||
|
succeeded: number;
|
||||||
|
failed: number;
|
||||||
|
skipped: number;
|
||||||
|
stillMissing: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface AuditEvent {
|
interface AuditEvent {
|
||||||
@@ -50,6 +66,13 @@ interface AuditEvent {
|
|||||||
issues?: AuditIssue[];
|
issues?: AuditIssue[];
|
||||||
missingFromSources?: MissingFromSource[];
|
missingFromSources?: MissingFromSource[];
|
||||||
summary?: AuditSummary;
|
summary?: AuditSummary;
|
||||||
|
repair?: RepairStats;
|
||||||
|
repairNitro?: RepairStats;
|
||||||
|
sql?: string;
|
||||||
|
sqlCount?: number;
|
||||||
|
applied?: number;
|
||||||
|
sqlFailed?: number;
|
||||||
|
furniData?: { added: number; removedDuplicates: number };
|
||||||
}
|
}
|
||||||
|
|
||||||
const TYPE_LABELS: Record<string, string> = {
|
const TYPE_LABELS: Record<string, string> = {
|
||||||
@@ -61,16 +84,72 @@ const TYPE_LABELS: Record<string, string> = {
|
|||||||
source_fetch_failed: "Source fetch failed",
|
source_fetch_failed: "Source fetch failed",
|
||||||
};
|
};
|
||||||
|
|
||||||
|
function AuditCheckbox({
|
||||||
|
checked,
|
||||||
|
disabled,
|
||||||
|
onChange,
|
||||||
|
children,
|
||||||
|
}: {
|
||||||
|
checked: boolean;
|
||||||
|
disabled?: boolean;
|
||||||
|
onChange: (value: boolean) => void;
|
||||||
|
children: React.ReactNode;
|
||||||
|
}) {
|
||||||
|
return (
|
||||||
|
// biome-ignore lint/a11y/useSemanticElements: custom checkbox so the checked state shows a visible checkmark on every admin theme
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
role="checkbox"
|
||||||
|
aria-checked={checked}
|
||||||
|
disabled={disabled}
|
||||||
|
onClick={() => onChange(!checked)}
|
||||||
|
className={`flex items-center gap-2 text-sm text-muted-foreground cursor-pointer select-none ${
|
||||||
|
disabled ? "opacity-50 cursor-not-allowed" : ""
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
<span
|
||||||
|
aria-hidden
|
||||||
|
className={`inline-flex h-4 w-4 shrink-0 items-center justify-center rounded-[3px] border-2 transition-colors ${
|
||||||
|
checked
|
||||||
|
? "border-[var(--admin-accent)] bg-[var(--admin-accent)]"
|
||||||
|
: "border-[var(--admin-border)] bg-transparent"
|
||||||
|
}`}
|
||||||
|
>
|
||||||
|
{checked && <Check className="h-3 w-3 text-[var(--admin-surface)]" />}
|
||||||
|
</span>
|
||||||
|
<span className="pointer-events-none">{children}</span>
|
||||||
|
</button>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export function AuditClient() {
|
export function AuditClient() {
|
||||||
const [loading, setLoading] = useState(false);
|
const [loading, setLoading] = useState(false);
|
||||||
const [progress, setProgress] = useState("");
|
const [progress, setProgress] = useState("");
|
||||||
const [activeTab, setActiveTab] = useState<"issues" | "sources">("issues");
|
const [activeTab, setActiveTab] = useState<"issues" | "sources">("issues");
|
||||||
|
const [repairEnabled, setRepairEnabled] = useState(true);
|
||||||
|
const [repairNitros, setRepairNitros] = useState(true);
|
||||||
|
const [generateSql, setGenerateSql] = useState(false);
|
||||||
|
const [applySql, setApplySql] = useState(false);
|
||||||
|
const [repairFurniData, setRepairFurniData] = useState(false);
|
||||||
|
|
||||||
const [issues, setIssues] = useState<AuditIssue[]>([]);
|
const [issues, setIssues] = useState<AuditIssue[]>([]);
|
||||||
const [missingFromSources, setMissingFromSources] = useState<
|
const [missingFromSources, setMissingFromSources] = useState<
|
||||||
MissingFromSource[]
|
MissingFromSource[]
|
||||||
>([]);
|
>([]);
|
||||||
const [summary, setSummary] = useState<AuditSummary | null>(null);
|
const [summary, setSummary] = useState<AuditSummary | null>(null);
|
||||||
|
const [repairStats, setRepairStats] = useState<RepairStats | null>(null);
|
||||||
|
const [nitroRepairStats, setNitroRepairStats] = useState<RepairStats | null>(
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
const [sqlText, setSqlText] = useState<string | null>(null);
|
||||||
|
const [sqlApplied, setSqlApplied] = useState<{
|
||||||
|
applied: number;
|
||||||
|
failed: number;
|
||||||
|
} | null>(null);
|
||||||
|
const [furniDataResult, setFurniDataResult] = useState<{
|
||||||
|
added: number;
|
||||||
|
removedDuplicates: number;
|
||||||
|
} | null>(null);
|
||||||
|
|
||||||
const [checkProgress, setCheckProgress] = useState<{
|
const [checkProgress, setCheckProgress] = useState<{
|
||||||
section: string;
|
section: string;
|
||||||
@@ -86,6 +165,11 @@ export function AuditClient() {
|
|||||||
setIssues([]);
|
setIssues([]);
|
||||||
setMissingFromSources([]);
|
setMissingFromSources([]);
|
||||||
setSummary(null);
|
setSummary(null);
|
||||||
|
setRepairStats(null);
|
||||||
|
setNitroRepairStats(null);
|
||||||
|
setSqlText(null);
|
||||||
|
setSqlApplied(null);
|
||||||
|
setFurniDataResult(null);
|
||||||
setCheckProgress(null);
|
setCheckProgress(null);
|
||||||
setActiveTab("issues");
|
setActiveTab("issues");
|
||||||
|
|
||||||
@@ -96,6 +180,14 @@ export function AuditClient() {
|
|||||||
const res = await adminFetch("/api/admin/import/audit", {
|
const res = await adminFetch("/api/admin/import/audit", {
|
||||||
method: "POST",
|
method: "POST",
|
||||||
signal: abort.signal,
|
signal: abort.signal,
|
||||||
|
headers: { "Content-Type": "application/json" },
|
||||||
|
body: JSON.stringify({
|
||||||
|
repair: repairEnabled,
|
||||||
|
repairNitros,
|
||||||
|
sql: generateSql || applySql,
|
||||||
|
applySql,
|
||||||
|
repairFurniData,
|
||||||
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
if (!res.body) {
|
if (!res.body) {
|
||||||
@@ -155,6 +247,26 @@ export function AuditClient() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if (evt.type === "repair_complete") {
|
||||||
|
if (evt.repair) setRepairStats(evt.repair);
|
||||||
|
if (evt.repairNitro) setNitroRepairStats(evt.repairNitro);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (evt.type === "sql_ready") {
|
||||||
|
setSqlText(evt.sql ?? null);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (evt.type === "sql_applied") {
|
||||||
|
setSqlApplied({
|
||||||
|
applied: evt.applied ?? 0,
|
||||||
|
failed: evt.sqlFailed ?? 0,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
if (evt.type === "furnidata_repair_complete") {
|
||||||
|
if (evt.furniData) setFurniDataResult(evt.furniData);
|
||||||
|
}
|
||||||
|
|
||||||
if (evt.type === "batch_complete") {
|
if (evt.type === "batch_complete") {
|
||||||
setIssues(evt.issues ?? []);
|
setIssues(evt.issues ?? []);
|
||||||
setMissingFromSources(evt.missingFromSources ?? []);
|
setMissingFromSources(evt.missingFromSources ?? []);
|
||||||
@@ -197,11 +309,22 @@ export function AuditClient() {
|
|||||||
setLoading(false);
|
setLoading(false);
|
||||||
abortRef.current = null;
|
abortRef.current = null;
|
||||||
}
|
}
|
||||||
}, []);
|
}, [repairEnabled, repairNitros, generateSql, applySql, repairFurniData]);
|
||||||
|
|
||||||
const errors = issues.filter((i) => i.severity === "error");
|
const errors = issues.filter((i) => i.severity === "error");
|
||||||
const warnings = issues.filter((i) => i.severity === "warning");
|
const warnings = issues.filter((i) => i.severity === "warning");
|
||||||
|
|
||||||
|
const downloadSql = () => {
|
||||||
|
if (!sqlText) return;
|
||||||
|
const blob = new Blob([sqlText], { type: "text/sql" });
|
||||||
|
const url = URL.createObjectURL(blob);
|
||||||
|
const a = document.createElement("a");
|
||||||
|
a.href = url;
|
||||||
|
a.download = `catalog-repair-${new Date().toISOString().slice(0, 10)}.sql`;
|
||||||
|
a.click();
|
||||||
|
URL.revokeObjectURL(url);
|
||||||
|
};
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<div className="space-y-6">
|
<div className="space-y-6">
|
||||||
{!summary && (
|
{!summary && (
|
||||||
@@ -214,6 +337,41 @@ export function AuditClient() {
|
|||||||
)}
|
)}
|
||||||
{loading ? progress || "Running…" : "Run Full Catalog Audit"}
|
{loading ? progress || "Running…" : "Run Full Catalog Audit"}
|
||||||
</Button>
|
</Button>
|
||||||
|
<AuditCheckbox
|
||||||
|
checked={repairEnabled}
|
||||||
|
disabled={loading}
|
||||||
|
onChange={setRepairEnabled}
|
||||||
|
>
|
||||||
|
Repair missing icons
|
||||||
|
</AuditCheckbox>
|
||||||
|
<AuditCheckbox
|
||||||
|
checked={repairNitros}
|
||||||
|
disabled={loading}
|
||||||
|
onChange={setRepairNitros}
|
||||||
|
>
|
||||||
|
Repair missing .nitro
|
||||||
|
</AuditCheckbox>
|
||||||
|
<AuditCheckbox
|
||||||
|
checked={generateSql}
|
||||||
|
disabled={loading}
|
||||||
|
onChange={setGenerateSql}
|
||||||
|
>
|
||||||
|
Generate catalog SQL
|
||||||
|
</AuditCheckbox>
|
||||||
|
<AuditCheckbox
|
||||||
|
checked={applySql}
|
||||||
|
disabled={loading}
|
||||||
|
onChange={setApplySql}
|
||||||
|
>
|
||||||
|
Apply SQL to database
|
||||||
|
</AuditCheckbox>
|
||||||
|
<AuditCheckbox
|
||||||
|
checked={repairFurniData}
|
||||||
|
disabled={loading}
|
||||||
|
onChange={setRepairFurniData}
|
||||||
|
>
|
||||||
|
Repair FurnitureData.json (add missing + dedupe)
|
||||||
|
</AuditCheckbox>
|
||||||
{loading && (
|
{loading && (
|
||||||
<Button
|
<Button
|
||||||
variant="outline"
|
variant="outline"
|
||||||
@@ -328,6 +486,121 @@ export function AuditClient() {
|
|||||||
sources
|
sources
|
||||||
</span>
|
</span>
|
||||||
</div>
|
</div>
|
||||||
|
{repairStats && (
|
||||||
|
<div className="flex items-center gap-2 col-span-full">
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
Icon repair:{" "}
|
||||||
|
<span className="text-[var(--admin-success)] font-medium">
|
||||||
|
{repairStats.succeeded} fixed
|
||||||
|
</span>
|
||||||
|
{repairStats.failed > 0 && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-error)] font-medium">
|
||||||
|
{repairStats.failed} failed
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{repairStats.skipped > 0 && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-text-muted)]">
|
||||||
|
{repairStats.skipped} skipped
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{repairStats.stillMissing > 0 && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-warning)] font-medium">
|
||||||
|
{repairStats.stillMissing} still missing
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{nitroRepairStats && (
|
||||||
|
<div className="flex items-center gap-2 col-span-full">
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
Nitro repair:{" "}
|
||||||
|
<span className="text-[var(--admin-success)] font-medium">
|
||||||
|
{nitroRepairStats.succeeded} fixed
|
||||||
|
</span>
|
||||||
|
{nitroRepairStats.failed > 0 && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-error)] font-medium">
|
||||||
|
{nitroRepairStats.failed} failed
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{nitroRepairStats.skipped > 0 && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-text-muted)]">
|
||||||
|
{nitroRepairStats.skipped} skipped
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
{nitroRepairStats.stillMissing > 0 && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-warning)] font-medium">
|
||||||
|
{nitroRepairStats.stillMissing} still missing
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{sqlText && (
|
||||||
|
<div className="flex items-center gap-2 col-span-full">
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
Catalog SQL:{" "}
|
||||||
|
<span className="font-medium">
|
||||||
|
{summary.catalogSqlCount} statement(s) generated
|
||||||
|
</span>
|
||||||
|
{sqlApplied && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-success)] font-medium">
|
||||||
|
{sqlApplied.applied} applied
|
||||||
|
</span>
|
||||||
|
{sqlApplied.failed > 0 && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-error)] font-medium">
|
||||||
|
{sqlApplied.failed} failed
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
<Button variant="outline" size="sm" onClick={downloadSql}>
|
||||||
|
Download .sql
|
||||||
|
</Button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{furniDataResult && (
|
||||||
|
<div className="flex items-center gap-2 col-span-full">
|
||||||
|
<span className="text-xs text-muted-foreground">
|
||||||
|
FurnitureData repair:{" "}
|
||||||
|
<span className="text-[var(--admin-success)] font-medium">
|
||||||
|
{furniDataResult.added} added
|
||||||
|
</span>
|
||||||
|
{furniDataResult.removedDuplicates > 0 && (
|
||||||
|
<>
|
||||||
|
{" · "}
|
||||||
|
<span className="text-[var(--admin-warning)] font-medium">
|
||||||
|
{furniDataResult.removedDuplicates} duplicates removed
|
||||||
|
</span>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
{/* Tabs */}
|
{/* Tabs */}
|
||||||
@@ -455,6 +728,11 @@ export function AuditClient() {
|
|||||||
setSummary(null);
|
setSummary(null);
|
||||||
setIssues([]);
|
setIssues([]);
|
||||||
setMissingFromSources([]);
|
setMissingFromSources([]);
|
||||||
|
setRepairStats(null);
|
||||||
|
setNitroRepairStats(null);
|
||||||
|
setSqlText(null);
|
||||||
|
setSqlApplied(null);
|
||||||
|
setFurniDataResult(null);
|
||||||
setProgress("");
|
setProgress("");
|
||||||
setCheckProgress(null);
|
setCheckProgress(null);
|
||||||
}}
|
}}
|
||||||
|
|||||||
@@ -6,41 +6,72 @@ function sseEvent(data: unknown): string {
|
|||||||
return `data: ${JSON.stringify(data)}\n\n`;
|
return `data: ${JSON.stringify(data)}\n\n`;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const POST = withAdmin({ permission: PERMS.ASSETS_IMPORT }, async () => {
|
export const POST = withAdmin(
|
||||||
const encoder = new TextEncoder();
|
{ permission: PERMS.ASSETS_IMPORT },
|
||||||
|
async (request) => {
|
||||||
|
const encoder = new TextEncoder();
|
||||||
|
|
||||||
const stream = new ReadableStream({
|
let repair = false;
|
||||||
async start(controller) {
|
let repairNitros = false;
|
||||||
const send = (data: unknown) => {
|
let sql = false;
|
||||||
try {
|
let applySql = false;
|
||||||
controller.enqueue(encoder.encode(sseEvent(data)));
|
let repairFurniData = false;
|
||||||
} catch {
|
try {
|
||||||
/* stream closed */
|
const body = (await request.json().catch(() => ({}))) as {
|
||||||
}
|
repair?: unknown;
|
||||||
|
repairNitros?: unknown;
|
||||||
|
sql?: unknown;
|
||||||
|
applySql?: unknown;
|
||||||
|
repairFurniData?: unknown;
|
||||||
};
|
};
|
||||||
|
repair = body.repair === true;
|
||||||
|
repairNitros = body.repairNitros === true;
|
||||||
|
sql = body.sql === true;
|
||||||
|
applySql = body.applySql === true;
|
||||||
|
repairFurniData = body.repairFurniData === true;
|
||||||
|
} catch {
|
||||||
|
/* no body */
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
const stream = new ReadableStream({
|
||||||
await runCatalogAudit(send);
|
async start(controller) {
|
||||||
} catch (err) {
|
const send = (data: unknown) => {
|
||||||
send({
|
try {
|
||||||
type: "error",
|
controller.enqueue(encoder.encode(sseEvent(data)));
|
||||||
message: `Audit threw: ${(err as Error).message}`,
|
} catch {
|
||||||
});
|
/* stream closed */
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
try {
|
try {
|
||||||
controller.close();
|
await runCatalogAudit(send, {
|
||||||
} catch {
|
repair,
|
||||||
/* ignore */
|
repairNitros,
|
||||||
}
|
sql,
|
||||||
},
|
applySql,
|
||||||
});
|
repairFurniData,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
send({
|
||||||
|
type: "error",
|
||||||
|
message: `Audit threw: ${(err as Error).message}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
return new Response(stream, {
|
try {
|
||||||
headers: {
|
controller.close();
|
||||||
"Content-Type": "text/event-stream",
|
} catch {
|
||||||
"Cache-Control": "no-cache",
|
/* ignore */
|
||||||
Connection: "keep-alive",
|
}
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
});
|
|
||||||
|
return new Response(stream, {
|
||||||
|
headers: {
|
||||||
|
"Content-Type": "text/event-stream",
|
||||||
|
"Cache-Control": "no-cache",
|
||||||
|
Connection: "keep-alive",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
},
|
||||||
|
);
|
||||||
@@ -5,7 +5,15 @@ import { db } from "@/lib/db";
|
|||||||
import { fetchSourceFurnidata } from "@/lib/services/clone-import";
|
import { fetchSourceFurnidata } from "@/lib/services/clone-import";
|
||||||
import { listSources } from "@/lib/services/clone-sources";
|
import { listSources } from "@/lib/services/clone-sources";
|
||||||
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
|
||||||
|
import {
|
||||||
|
applyCatalogSql,
|
||||||
|
generateCatalogSql,
|
||||||
|
type MissingCatalogEntry,
|
||||||
|
repairFurniData,
|
||||||
|
} from "./catalog-repair";
|
||||||
import { ensureDirectories } from "./furni-import";
|
import { ensureDirectories } from "./furni-import";
|
||||||
|
import { repairMissingIcons } from "./repair-icons";
|
||||||
|
import { repairMissingNitros } from "./repair-nitros";
|
||||||
|
|
||||||
export interface AuditEvent {
|
export interface AuditEvent {
|
||||||
type:
|
type:
|
||||||
@@ -13,6 +21,10 @@ export interface AuditEvent {
|
|||||||
| "items_loaded"
|
| "items_loaded"
|
||||||
| "catalog_loaded"
|
| "catalog_loaded"
|
||||||
| "checking"
|
| "checking"
|
||||||
|
| "repair_complete"
|
||||||
|
| "sql_ready"
|
||||||
|
| "sql_applied"
|
||||||
|
| "furnidata_repair_complete"
|
||||||
| "batch_complete"
|
| "batch_complete"
|
||||||
| "error";
|
| "error";
|
||||||
message?: string;
|
message?: string;
|
||||||
@@ -22,6 +34,20 @@ export interface AuditEvent {
|
|||||||
issues?: AuditIssue[];
|
issues?: AuditIssue[];
|
||||||
missingFromSources?: MissingFromSource[];
|
missingFromSources?: MissingFromSource[];
|
||||||
summary?: AuditSummary;
|
summary?: AuditSummary;
|
||||||
|
repair?: RepairStats;
|
||||||
|
repairNitro?: RepairStats;
|
||||||
|
sql?: string;
|
||||||
|
sqlCount?: number;
|
||||||
|
applied?: number;
|
||||||
|
sqlFailed?: number;
|
||||||
|
furniData?: { added: number; removedDuplicates: number };
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RepairStats {
|
||||||
|
succeeded: number;
|
||||||
|
failed: number;
|
||||||
|
skipped: number;
|
||||||
|
stillMissing: number;
|
||||||
}
|
}
|
||||||
|
|
||||||
export interface AuditIssue {
|
export interface AuditIssue {
|
||||||
@@ -50,12 +76,29 @@ export interface AuditSummary {
|
|||||||
missingIcon: number;
|
missingIcon: number;
|
||||||
duplicates: number;
|
duplicates: number;
|
||||||
missingFromSourcesTotal: number;
|
missingFromSourcesTotal: number;
|
||||||
|
iconsRepaired: number;
|
||||||
|
iconsStillMissing: number;
|
||||||
|
nitrosRepaired: number;
|
||||||
|
nitrosStillMissing: number;
|
||||||
|
catalogSqlCount: number;
|
||||||
|
catalogSqlApplied: number;
|
||||||
|
furniDataAdded: number;
|
||||||
|
furniDataDuplicatesRemoved: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CatalogAuditOptions {
|
||||||
|
repair?: boolean;
|
||||||
|
repairNitros?: boolean;
|
||||||
|
sql?: boolean;
|
||||||
|
applySql?: boolean;
|
||||||
|
repairFurniData?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function runCatalogAudit(
|
export async function runCatalogAudit(
|
||||||
onEvent?: (evt: AuditEvent) => void,
|
onEvent?: (evt: AuditEvent) => void,
|
||||||
|
options?: CatalogAuditOptions,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
const issues: AuditIssue[] = [];
|
let issues: AuditIssue[] = [];
|
||||||
const missingFromSources: MissingFromSource[] = [];
|
const missingFromSources: MissingFromSource[] = [];
|
||||||
|
|
||||||
try {
|
try {
|
||||||
@@ -164,6 +207,7 @@ export async function runCatalogAudit(
|
|||||||
total: items.length,
|
total: items.length,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const missingCatalogEntries: MissingCatalogEntry[] = [];
|
||||||
for (let i = 0; i < items.length; i++) {
|
for (let i = 0; i < items.length; i++) {
|
||||||
const item = items[i];
|
const item = items[i];
|
||||||
if (!itemIdsWithCatalog.has(item.id)) {
|
if (!itemIdsWithCatalog.has(item.id)) {
|
||||||
@@ -174,6 +218,15 @@ export async function runCatalogAudit(
|
|||||||
itemId: item.id,
|
itemId: item.id,
|
||||||
message: `"${item.item_name}" (ID ${item.id}) has no catalog_items entry — not purchasable`,
|
message: `"${item.item_name}" (ID ${item.id}) has no catalog_items entry — not purchasable`,
|
||||||
});
|
});
|
||||||
|
// Only include items that actually have a .nitro asset — a catalog
|
||||||
|
// entry for a furni without a bundle would be un-placeable in-game.
|
||||||
|
if (existsSync(path.join(nitroDir, `${item.item_name}.nitro`))) {
|
||||||
|
missingCatalogEntries.push({
|
||||||
|
classname: item.item_name,
|
||||||
|
itemId: item.id,
|
||||||
|
itemType: item.type,
|
||||||
|
});
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if (i % 100 === 0) {
|
if (i % 100 === 0) {
|
||||||
onEvent?.({
|
onEvent?.({
|
||||||
@@ -233,10 +286,12 @@ export async function runCatalogAudit(
|
|||||||
total: items.length,
|
total: items.length,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const missingNitroClassnames: string[] = [];
|
||||||
for (let i = 0; i < items.length; i++) {
|
for (let i = 0; i < items.length; i++) {
|
||||||
const item = items[i];
|
const item = items[i];
|
||||||
const nitroPath = path.join(nitroDir, `${item.item_name}.nitro`);
|
const nitroPath = path.join(nitroDir, `${item.item_name}.nitro`);
|
||||||
if (!existsSync(nitroPath)) {
|
if (!existsSync(nitroPath)) {
|
||||||
|
missingNitroClassnames.push(item.item_name);
|
||||||
issues.push({
|
issues.push({
|
||||||
type: "missing_nitro",
|
type: "missing_nitro",
|
||||||
severity: "error",
|
severity: "error",
|
||||||
@@ -257,10 +312,12 @@ export async function runCatalogAudit(
|
|||||||
|
|
||||||
onEvent?.({ type: "checking", section: "missing_icon", total: items.length });
|
onEvent?.({ type: "checking", section: "missing_icon", total: items.length });
|
||||||
|
|
||||||
|
const missingIconClassnames: string[] = [];
|
||||||
for (let i = 0; i < items.length; i++) {
|
for (let i = 0; i < items.length; i++) {
|
||||||
const item = items[i];
|
const item = items[i];
|
||||||
const iconPath = path.join(iconDir, `${item.item_name}_icon.png`);
|
const iconPath = path.join(iconDir, `${item.item_name}_icon.png`);
|
||||||
if (!existsSync(iconPath)) {
|
if (!existsSync(iconPath)) {
|
||||||
|
missingIconClassnames.push(item.item_name);
|
||||||
issues.push({
|
issues.push({
|
||||||
type: "missing_icon",
|
type: "missing_icon",
|
||||||
severity: "warning",
|
severity: "warning",
|
||||||
@@ -279,6 +336,177 @@ export async function runCatalogAudit(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let repairStats: RepairStats | null = null;
|
||||||
|
if (options?.repair && missingIconClassnames.length > 0) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "progress",
|
||||||
|
message: `Repairing ${missingIconClassnames.length} missing icons…`,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await repairMissingIcons((evt) => {
|
||||||
|
if (evt.type === "progress") {
|
||||||
|
onEvent?.({
|
||||||
|
type: "checking",
|
||||||
|
section: "repair_icons",
|
||||||
|
current: evt.index,
|
||||||
|
total: evt.total,
|
||||||
|
message:
|
||||||
|
evt.status === "done"
|
||||||
|
? `Repaired ${evt.classname}`
|
||||||
|
: evt.status === "failed"
|
||||||
|
? `Failed: ${evt.classname}`
|
||||||
|
: undefined,
|
||||||
|
});
|
||||||
|
} else if (evt.type === "error") {
|
||||||
|
onEvent?.({ type: "error", message: evt.message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Re-check which of the previously-missing icons are still missing
|
||||||
|
// and drop the ones the repair fixed from the issue list.
|
||||||
|
const stillMissing: string[] = [];
|
||||||
|
for (const classname of missingIconClassnames) {
|
||||||
|
if (!existsSync(path.join(iconDir, `${classname}_icon.png`))) {
|
||||||
|
stillMissing.push(classname);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
issues = issues.filter(
|
||||||
|
(i) =>
|
||||||
|
i.type !== "missing_icon" || stillMissing.includes(i.classname ?? ""),
|
||||||
|
);
|
||||||
|
|
||||||
|
repairStats = {
|
||||||
|
succeeded: result.succeeded,
|
||||||
|
failed: result.failed,
|
||||||
|
skipped: result.skipped,
|
||||||
|
stillMissing: stillMissing.length,
|
||||||
|
};
|
||||||
|
onEvent?.({ type: "repair_complete", repair: repairStats });
|
||||||
|
} catch (err) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "error",
|
||||||
|
message: `Icon repair failed: ${(err as Error).message}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let nitroRepairStats: RepairStats | null = null;
|
||||||
|
if (options?.repairNitros && missingNitroClassnames.length > 0) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "progress",
|
||||||
|
message: `Repairing ${missingNitroClassnames.length} missing .nitro files…`,
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const result = await repairMissingNitros((evt) => {
|
||||||
|
if (evt.type === "progress") {
|
||||||
|
onEvent?.({
|
||||||
|
type: "checking",
|
||||||
|
section: "repair_nitros",
|
||||||
|
current: evt.index,
|
||||||
|
total: evt.total,
|
||||||
|
message:
|
||||||
|
evt.status === "done"
|
||||||
|
? `Repaired ${evt.classname}`
|
||||||
|
: evt.status === "failed"
|
||||||
|
? `Failed: ${evt.classname}`
|
||||||
|
: undefined,
|
||||||
|
});
|
||||||
|
} else if (evt.type === "error") {
|
||||||
|
onEvent?.({ type: "error", message: evt.message });
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
// Re-check which of the previously-missing nitros are still missing
|
||||||
|
// and drop the ones the repair fixed from the issue list.
|
||||||
|
const stillMissing: string[] = [];
|
||||||
|
for (const classname of missingNitroClassnames) {
|
||||||
|
if (!existsSync(path.join(nitroDir, `${classname}.nitro`))) {
|
||||||
|
stillMissing.push(classname);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
issues = issues.filter(
|
||||||
|
(i) =>
|
||||||
|
i.type !== "missing_nitro" ||
|
||||||
|
stillMissing.includes(i.classname ?? ""),
|
||||||
|
);
|
||||||
|
|
||||||
|
nitroRepairStats = {
|
||||||
|
succeeded: result.succeeded,
|
||||||
|
failed: result.failed,
|
||||||
|
skipped: result.skipped,
|
||||||
|
stillMissing: stillMissing.length,
|
||||||
|
};
|
||||||
|
onEvent?.({
|
||||||
|
type: "repair_complete",
|
||||||
|
repairNitro: nitroRepairStats,
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "error",
|
||||||
|
message: `.nitro repair failed: ${(err as Error).message}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Catalog SQL generation / apply ────────────────────────────────
|
||||||
|
let catalogSqlCount = 0;
|
||||||
|
let catalogSqlApplied = 0;
|
||||||
|
if ((options?.sql || options?.applySql) && missingCatalogEntries.length > 0) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "progress",
|
||||||
|
message: `Generating catalog SQL for ${missingCatalogEntries.length} items…`,
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
const result = await generateCatalogSql(missingCatalogEntries);
|
||||||
|
catalogSqlCount = result.count;
|
||||||
|
onEvent?.({
|
||||||
|
type: "sql_ready",
|
||||||
|
sql: result.sql,
|
||||||
|
sqlCount: result.count,
|
||||||
|
});
|
||||||
|
if (options.applySql) {
|
||||||
|
const applyResult = await applyCatalogSql(result.statements);
|
||||||
|
catalogSqlApplied = applyResult.applied;
|
||||||
|
onEvent?.({
|
||||||
|
type: "sql_applied",
|
||||||
|
applied: applyResult.applied,
|
||||||
|
sqlFailed: applyResult.failed,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} catch (err) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "error",
|
||||||
|
message: `Catalog SQL failed: ${(err as Error).message}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── FurnitureData.json repair (add missing + dedupe) ──────────────
|
||||||
|
let furniDataAdded = 0;
|
||||||
|
let furniDataDuplicatesRemoved = 0;
|
||||||
|
if (options?.repairFurniData) {
|
||||||
|
onEvent?.({ type: "progress", message: "Repairing FurnitureData.json…" });
|
||||||
|
try {
|
||||||
|
const result = await repairFurniData(items);
|
||||||
|
furniDataAdded = result.added;
|
||||||
|
furniDataDuplicatesRemoved = result.removedDuplicates;
|
||||||
|
onEvent?.({
|
||||||
|
type: "furnidata_repair_complete",
|
||||||
|
furniData: {
|
||||||
|
added: result.added,
|
||||||
|
removedDuplicates: result.removedDuplicates,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (err) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "error",
|
||||||
|
message: `FurnitureData repair failed: ${(err as Error).message}`,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
onEvent?.({ type: "progress", message: "Comparing with clone sources…" });
|
onEvent?.({ type: "progress", message: "Comparing with clone sources…" });
|
||||||
|
|
||||||
const sources = await listSources();
|
const sources = await listSources();
|
||||||
@@ -319,6 +547,16 @@ export async function runCatalogAudit(
|
|||||||
missingIcon: issues.filter((i) => i.type === "missing_icon").length,
|
missingIcon: issues.filter((i) => i.type === "missing_icon").length,
|
||||||
duplicates: issues.filter((i) => i.type === "duplicate_classname").length,
|
duplicates: issues.filter((i) => i.type === "duplicate_classname").length,
|
||||||
missingFromSourcesTotal: missingFromSources.length,
|
missingFromSourcesTotal: missingFromSources.length,
|
||||||
|
iconsRepaired: repairStats?.succeeded ?? 0,
|
||||||
|
iconsStillMissing:
|
||||||
|
repairStats?.stillMissing ?? missingIconClassnames.length,
|
||||||
|
nitrosRepaired: nitroRepairStats?.succeeded ?? 0,
|
||||||
|
nitrosStillMissing:
|
||||||
|
nitroRepairStats?.stillMissing ?? missingNitroClassnames.length,
|
||||||
|
catalogSqlCount,
|
||||||
|
catalogSqlApplied,
|
||||||
|
furniDataAdded,
|
||||||
|
furniDataDuplicatesRemoved,
|
||||||
};
|
};
|
||||||
|
|
||||||
onEvent?.({
|
onEvent?.({
|
||||||
|
|||||||
@@ -0,0 +1,205 @@
|
|||||||
|
import { sql } from "drizzle-orm";
|
||||||
|
import { db } from "@/lib/db";
|
||||||
|
import {
|
||||||
|
readFurniData,
|
||||||
|
withFurniDataLock,
|
||||||
|
writeFurniData,
|
||||||
|
} from "@/lib/services/furni-data";
|
||||||
|
import {
|
||||||
|
allocateCatalogItemId,
|
||||||
|
autoPriceFurni,
|
||||||
|
getOrCreateCategoryPage,
|
||||||
|
} from "@/lib/services/furni-import";
|
||||||
|
|
||||||
|
export interface MissingCatalogEntry {
|
||||||
|
classname: string;
|
||||||
|
itemId: number;
|
||||||
|
itemType: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface CatalogSqlResult {
|
||||||
|
statements: string[];
|
||||||
|
sql: string;
|
||||||
|
count: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function escSqlLiteral(value: string): string {
|
||||||
|
return value.replace(/'/g, "''");
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build standalone `catalog_items` INSERT statements for items_base entries
|
||||||
|
* that exist (with assets) but have no catalog entry yet. Ids and prices are
|
||||||
|
* resolved the same way the app's own imports do it (autoPriceFurni + category
|
||||||
|
* page + next catalog id), so running the SQL matches an app-managed import.
|
||||||
|
*/
|
||||||
|
export async function generateCatalogSql(
|
||||||
|
entries: MissingCatalogEntry[],
|
||||||
|
): Promise<CatalogSqlResult> {
|
||||||
|
const statements: string[] = [
|
||||||
|
"-- AtomCMS catalog repair",
|
||||||
|
`-- Generated ${new Date().toISOString()}`,
|
||||||
|
`-- ${entries.length} item(s) to add`,
|
||||||
|
"START TRANSACTION;",
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const entry of entries) {
|
||||||
|
let pageId: number;
|
||||||
|
try {
|
||||||
|
pageId = await getOrCreateCategoryPage(entry.classname, entry.itemType);
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const price = autoPriceFurni(entry.classname);
|
||||||
|
let nextId: number;
|
||||||
|
try {
|
||||||
|
nextId = await allocateCatalogItemId(async (id) => id);
|
||||||
|
} catch {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
statements.push(
|
||||||
|
`INSERT INTO catalog_items (id, page_id, item_ids, catalog_name, cost_credits, cost_points, points_type, amount, order_number, offer_id, extradata) VALUES (${nextId}, ${pageId}, ${entry.itemId}, '${escSqlLiteral(entry.classname)}', ${price.credits}, ${price.points}, ${price.pointsType}, 1, 1, '-1', '');`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
statements.push("COMMIT;");
|
||||||
|
return { statements, sql: statements.join("\n"), count: entries.length };
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Execute generated INSERT statements directly against the database. */
|
||||||
|
export async function applyCatalogSql(
|
||||||
|
statements: string[],
|
||||||
|
): Promise<{ applied: number; failed: number }> {
|
||||||
|
let applied = 0;
|
||||||
|
let failed = 0;
|
||||||
|
for (const statement of statements) {
|
||||||
|
const trimmed = statement.trim();
|
||||||
|
if (
|
||||||
|
!trimmed ||
|
||||||
|
trimmed.startsWith("--") ||
|
||||||
|
trimmed === "START TRANSACTION;" ||
|
||||||
|
trimmed === "COMMIT;"
|
||||||
|
) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
await db.execute(sql.raw(trimmed));
|
||||||
|
applied++;
|
||||||
|
} catch {
|
||||||
|
failed++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { applied, failed };
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface FurniDataRepairResult {
|
||||||
|
added: number;
|
||||||
|
removedDuplicates: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildMinimalFurniEntry(item: {
|
||||||
|
id: number;
|
||||||
|
item_name: string;
|
||||||
|
public_name: string;
|
||||||
|
}): Record<string, unknown> {
|
||||||
|
return {
|
||||||
|
id: item.id,
|
||||||
|
classname: item.item_name,
|
||||||
|
revision: 0,
|
||||||
|
category: "unknown",
|
||||||
|
defaultdir: 0,
|
||||||
|
xdim: 1,
|
||||||
|
ydim: 1,
|
||||||
|
partcolors: { color: [] },
|
||||||
|
name: item.public_name || item.item_name,
|
||||||
|
description: "",
|
||||||
|
adurl: "",
|
||||||
|
offerid: item.id,
|
||||||
|
buyout: false,
|
||||||
|
rentofferid: -1,
|
||||||
|
rentbuyout: false,
|
||||||
|
bc: false,
|
||||||
|
excludeddynamic: false,
|
||||||
|
customparams: "",
|
||||||
|
specialtype: 0,
|
||||||
|
canstandon: false,
|
||||||
|
cansiton: false,
|
||||||
|
canlayon: false,
|
||||||
|
furniline: "",
|
||||||
|
environment: "",
|
||||||
|
rare: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Add FurnitureData.json entries for items_base rows that are missing from the
|
||||||
|
* file, and remove duplicate classnames (keeping the first occurrence).
|
||||||
|
*/
|
||||||
|
export async function repairFurniData(
|
||||||
|
items: Array<{
|
||||||
|
id: number;
|
||||||
|
item_name: string;
|
||||||
|
public_name: string;
|
||||||
|
type: string;
|
||||||
|
}>,
|
||||||
|
): Promise<FurniDataRepairResult> {
|
||||||
|
if (items.length === 0) return { added: 0, removedDuplicates: 0 };
|
||||||
|
|
||||||
|
return withFurniDataLock(async () => {
|
||||||
|
const furniData = (await readFurniData()) as Record<
|
||||||
|
string,
|
||||||
|
{ furnitype: Array<Record<string, unknown>> }
|
||||||
|
>;
|
||||||
|
|
||||||
|
// 1) Deduplicate classnames per section (keep the first occurrence).
|
||||||
|
let removedDuplicates = 0;
|
||||||
|
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||||
|
const sec = furniData[section];
|
||||||
|
if (!sec?.furnitype) continue;
|
||||||
|
const seen = new Set<string>();
|
||||||
|
const deduped = sec.furnitype.filter((e) => {
|
||||||
|
const classname = typeof e.classname === "string" ? e.classname : "";
|
||||||
|
if (!classname) return true;
|
||||||
|
if (seen.has(classname)) {
|
||||||
|
removedDuplicates++;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
seen.add(classname);
|
||||||
|
return true;
|
||||||
|
});
|
||||||
|
if (deduped.length !== sec.furnitype.length) {
|
||||||
|
furniData[section].furnitype = deduped;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Add entries for items_base rows missing from the file.
|
||||||
|
const existing = new Set<string>();
|
||||||
|
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||||
|
for (const e of furniData[section]?.furnitype ?? []) {
|
||||||
|
if (typeof e.classname === "string") existing.add(e.classname);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const added: Array<{ entry: Record<string, unknown>; itemType: string }> =
|
||||||
|
[];
|
||||||
|
for (const item of items) {
|
||||||
|
if (existing.has(item.item_name)) continue;
|
||||||
|
added.push({
|
||||||
|
entry: buildMinimalFurniEntry(item),
|
||||||
|
itemType: item.type,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3) Write once if anything changed.
|
||||||
|
if (removedDuplicates > 0 || added.length > 0) {
|
||||||
|
for (const { entry, itemType } of added) {
|
||||||
|
const sectionKey = itemType === "i" ? "wallitemtypes" : "roomitemtypes";
|
||||||
|
if (!furniData[sectionKey]) {
|
||||||
|
furniData[sectionKey] = { furnitype: [] };
|
||||||
|
}
|
||||||
|
furniData[sectionKey].furnitype.push(entry);
|
||||||
|
}
|
||||||
|
await writeFurniData(furniData as Record<string, unknown>);
|
||||||
|
}
|
||||||
|
return { added: added.length, removedDuplicates };
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -35,6 +35,16 @@ export const DEFAULT_ICON_SOURCES: CloneSource[] = [
|
|||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
export const DEFAULT_NITRO_SOURCES: CloneSource[] = [
|
||||||
|
{
|
||||||
|
id: "default-wibbo",
|
||||||
|
name: "Wibbo",
|
||||||
|
furnidataUrl: "",
|
||||||
|
nitroBaseUrl: "https://assets.wibbo.org/bundled/furniture",
|
||||||
|
iconBaseUrl: "",
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
export async function listSources(): Promise<CloneSource[]> {
|
export async function listSources(): Promise<CloneSource[]> {
|
||||||
const raw = (await siteSettings.get(KEY, "[]")) ?? "[]";
|
const raw = (await siteSettings.get(KEY, "[]")) ?? "[]";
|
||||||
try {
|
try {
|
||||||
|
|||||||
@@ -16,6 +16,28 @@ import {
|
|||||||
import { downloadFile } from "@/lib/services/import/core/download";
|
import { downloadFile } from "@/lib/services/import/core/download";
|
||||||
import { ensureDirectories } from "./furni-import";
|
import { ensureDirectories } from "./furni-import";
|
||||||
|
|
||||||
|
const REPAIR_CONCURRENCY = 6;
|
||||||
|
|
||||||
|
/** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */
|
||||||
|
async function runPool<T>(
|
||||||
|
items: T[],
|
||||||
|
concurrency: number,
|
||||||
|
worker: (item: T, index: number) => Promise<void>,
|
||||||
|
): Promise<void> {
|
||||||
|
const limit = Math.min(Math.max(concurrency, 1), items.length);
|
||||||
|
if (limit === 0) return;
|
||||||
|
let next = 0;
|
||||||
|
await Promise.allSettled(
|
||||||
|
Array.from({ length: limit }, async () => {
|
||||||
|
while (true) {
|
||||||
|
const i = next++;
|
||||||
|
if (i >= items.length) break;
|
||||||
|
await worker(items[i], i);
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
function uniqueDirs(dirs: string[]): string[] {
|
function uniqueDirs(dirs: string[]): string[] {
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const out: string[] = [];
|
const out: string[] = [];
|
||||||
@@ -135,8 +157,7 @@ export async function repairMissingIcons(
|
|||||||
const total = items.length;
|
const total = items.length;
|
||||||
onEvent?.({ type: "started", total });
|
onEvent?.({ type: "started", total });
|
||||||
|
|
||||||
for (let i = 0; i < items.length; i++) {
|
const processItem = async (classname: string) => {
|
||||||
const { item_name: classname } = items[i];
|
|
||||||
const fileName = `${classname}_icon.png`;
|
const fileName = `${classname}_icon.png`;
|
||||||
|
|
||||||
// A dir may already have the icon (e.g. the full gamedata) while the
|
// A dir may already have the icon (e.g. the full gamedata) while the
|
||||||
@@ -147,15 +168,7 @@ export async function repairMissingIcons(
|
|||||||
const missingDirs = iconDirs.filter((dir) => !presentDirs.includes(dir));
|
const missingDirs = iconDirs.filter((dir) => !presentDirs.includes(dir));
|
||||||
|
|
||||||
if (missingDirs.length === 0) {
|
if (missingDirs.length === 0) {
|
||||||
skipped++;
|
return { status: "skipped" } as const;
|
||||||
onEvent?.({
|
|
||||||
type: "progress",
|
|
||||||
classname,
|
|
||||||
status: "skipped",
|
|
||||||
index: i,
|
|
||||||
total,
|
|
||||||
});
|
|
||||||
continue;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let ok = false;
|
let ok = false;
|
||||||
@@ -303,6 +316,29 @@ export async function repairMissingIcons(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if (ok) {
|
if (ok) {
|
||||||
|
return { status: "done", warnings } as const;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
message: warnings[0] || "could not obtain icon",
|
||||||
|
} as const;
|
||||||
|
};
|
||||||
|
|
||||||
|
await runPool(items, REPAIR_CONCURRENCY, async (item, i) => {
|
||||||
|
const classname = item.item_name;
|
||||||
|
const result = await processItem(classname);
|
||||||
|
if (result.status === "skipped") {
|
||||||
|
skipped++;
|
||||||
|
onEvent?.({
|
||||||
|
type: "progress",
|
||||||
|
classname,
|
||||||
|
status: "skipped",
|
||||||
|
index: i,
|
||||||
|
total,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (result.status === "done") {
|
||||||
succeeded++;
|
succeeded++;
|
||||||
onEvent?.({
|
onEvent?.({
|
||||||
type: "progress",
|
type: "progress",
|
||||||
@@ -310,20 +346,20 @@ export async function repairMissingIcons(
|
|||||||
status: "done",
|
status: "done",
|
||||||
index: i,
|
index: i,
|
||||||
total,
|
total,
|
||||||
warnings: warnings.length ? warnings : undefined,
|
warnings: result.warnings.length ? result.warnings : undefined,
|
||||||
});
|
|
||||||
} else {
|
|
||||||
failed++;
|
|
||||||
onEvent?.({
|
|
||||||
type: "progress",
|
|
||||||
classname,
|
|
||||||
status: "failed",
|
|
||||||
index: i,
|
|
||||||
total,
|
|
||||||
message: warnings[0] || "could not obtain icon",
|
|
||||||
});
|
});
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
}
|
failed++;
|
||||||
|
onEvent?.({
|
||||||
|
type: "progress",
|
||||||
|
classname,
|
||||||
|
status: "failed",
|
||||||
|
index: i,
|
||||||
|
total,
|
||||||
|
message: result.message,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
onEvent?.({
|
onEvent?.({
|
||||||
type: "batch_complete",
|
type: "batch_complete",
|
||||||
|
|||||||
@@ -0,0 +1,305 @@
|
|||||||
|
import { existsSync, promises as fs } from "node:fs";
|
||||||
|
import os from "node:os";
|
||||||
|
import path from "node:path";
|
||||||
|
import { sql } from "drizzle-orm";
|
||||||
|
import { db } from "@/lib/db";
|
||||||
|
import {
|
||||||
|
type CloneSource,
|
||||||
|
DEFAULT_NITRO_SOURCES,
|
||||||
|
listSources,
|
||||||
|
} from "@/lib/services/clone-sources";
|
||||||
|
import {
|
||||||
|
type FurniAssetWriteTargets,
|
||||||
|
getFurniAssetWriteTargets,
|
||||||
|
} from "@/lib/services/furni-asset-dirs";
|
||||||
|
import { downloadFile } from "@/lib/services/import/core/download";
|
||||||
|
import { parseNitroBundle } from "@/lib/services/swf/nitro-builder";
|
||||||
|
import { ensureDirectories } from "./furni-import";
|
||||||
|
|
||||||
|
const REPAIR_CONCURRENCY = 6;
|
||||||
|
|
||||||
|
/** Sliding-window worker pool: keeps up to `concurrency` tasks in flight. */
|
||||||
|
async function runPool<T>(
|
||||||
|
items: T[],
|
||||||
|
concurrency: number,
|
||||||
|
worker: (item: T, index: number) => Promise<void>,
|
||||||
|
): Promise<void> {
|
||||||
|
const limit = Math.min(Math.max(concurrency, 1), items.length);
|
||||||
|
if (limit === 0) return;
|
||||||
|
let next = 0;
|
||||||
|
await Promise.allSettled(
|
||||||
|
Array.from({ length: limit }, async () => {
|
||||||
|
while (true) {
|
||||||
|
const i = next++;
|
||||||
|
if (i >= items.length) break;
|
||||||
|
await worker(items[i], i);
|
||||||
|
}
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function uniqueDirs(dirs: string[]): string[] {
|
||||||
|
const seen = new Set<string>();
|
||||||
|
const out: string[] = [];
|
||||||
|
for (const dir of dirs) {
|
||||||
|
const key = process.platform === "win32" ? dir.toLowerCase() : dir;
|
||||||
|
if (seen.has(key)) continue;
|
||||||
|
seen.add(key);
|
||||||
|
out.push(dir);
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeNitroToDirs(
|
||||||
|
nitro: Buffer,
|
||||||
|
dirs: string[],
|
||||||
|
fileName: string,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const errors: string[] = [];
|
||||||
|
for (const dir of dirs) {
|
||||||
|
try {
|
||||||
|
await fs.mkdir(dir, { recursive: true });
|
||||||
|
await fs.writeFile(path.join(dir, fileName), nitro);
|
||||||
|
} catch (err) {
|
||||||
|
errors.push(`write .nitro to ${dir} failed: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function copyNitroAcrossDirs(
|
||||||
|
srcDir: string,
|
||||||
|
destDirs: string[],
|
||||||
|
fileName: string,
|
||||||
|
): Promise<string[]> {
|
||||||
|
const errors: string[] = [];
|
||||||
|
const src = path.join(srcDir, fileName);
|
||||||
|
for (const dir of destDirs) {
|
||||||
|
if (dir === srcDir) continue;
|
||||||
|
try {
|
||||||
|
await fs.mkdir(dir, { recursive: true });
|
||||||
|
await fs.copyFile(src, path.join(dir, fileName));
|
||||||
|
} catch (err) {
|
||||||
|
errors.push(`copy .nitro to ${dir} failed: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return errors;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface RepairNitroEvent {
|
||||||
|
type: "started" | "progress" | "done" | "batch_complete" | "error";
|
||||||
|
classname?: string;
|
||||||
|
status?: string;
|
||||||
|
index?: number;
|
||||||
|
total?: number;
|
||||||
|
message?: string;
|
||||||
|
warnings?: string[];
|
||||||
|
succeeded?: number;
|
||||||
|
failed?: number;
|
||||||
|
skipped?: number;
|
||||||
|
duration?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function repairMissingNitros(
|
||||||
|
onEvent?: (evt: RepairNitroEvent) => void,
|
||||||
|
): Promise<{ succeeded: number; failed: number; skipped: number }> {
|
||||||
|
let succeeded = 0;
|
||||||
|
let failed = 0;
|
||||||
|
let skipped = 0;
|
||||||
|
|
||||||
|
try {
|
||||||
|
await ensureDirectories();
|
||||||
|
} catch (err) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "error",
|
||||||
|
message: `ensureDirectories failed: ${(err as Error).message}`,
|
||||||
|
});
|
||||||
|
return { succeeded: 0, failed: 0, skipped: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
let targets: FurniAssetWriteTargets;
|
||||||
|
try {
|
||||||
|
targets = await getFurniAssetWriteTargets();
|
||||||
|
} catch (err) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "error",
|
||||||
|
message: `getFurniAssetWriteTargets failed: ${(err as Error).message}`,
|
||||||
|
});
|
||||||
|
return { succeeded: 0, failed: 0, skipped: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
// Every location the app mirrors furni assets into — the webroot (public/)
|
||||||
|
// plus the live gamedata (e.g. /var/www/Gamedata). Repair writes to all of
|
||||||
|
// them so both the CMS and the client see the .nitro.
|
||||||
|
const nitroDirs = uniqueDirs([
|
||||||
|
targets.nitroDir,
|
||||||
|
...targets.mirrorDirs.map((d) => d.nitroDir),
|
||||||
|
]);
|
||||||
|
|
||||||
|
let items: Array<{ id: number; item_name: string }>;
|
||||||
|
try {
|
||||||
|
const [rows] = (await db.execute(sql`
|
||||||
|
SELECT id, item_name FROM items_base ORDER BY id
|
||||||
|
`)) as unknown as [Array<{ id: number; item_name: string }>, unknown];
|
||||||
|
items = rows;
|
||||||
|
} catch (err) {
|
||||||
|
onEvent?.({
|
||||||
|
type: "error",
|
||||||
|
message: `Database query failed: ${(err as Error).message}`,
|
||||||
|
});
|
||||||
|
return { succeeded: 0, failed: 0, skipped: 0 };
|
||||||
|
}
|
||||||
|
|
||||||
|
const total = items.length;
|
||||||
|
onEvent?.({ type: "started", total });
|
||||||
|
|
||||||
|
const processItem = async (classname: string) => {
|
||||||
|
const fileName = `${classname}.nitro`;
|
||||||
|
|
||||||
|
const presentDirs = nitroDirs.filter((dir) =>
|
||||||
|
existsSync(path.join(dir, fileName)),
|
||||||
|
);
|
||||||
|
const missingDirs = nitroDirs.filter((dir) => !presentDirs.includes(dir));
|
||||||
|
|
||||||
|
if (missingDirs.length === 0) {
|
||||||
|
return { status: "skipped" } as const;
|
||||||
|
}
|
||||||
|
|
||||||
|
let ok = false;
|
||||||
|
const warnings: string[] = [];
|
||||||
|
|
||||||
|
// 1) Copy across mirrors — the .nitro already exists in one target (e.g.
|
||||||
|
// gamedata), so just fill the ones that are missing it.
|
||||||
|
if (presentDirs.length > 0) {
|
||||||
|
try {
|
||||||
|
const copyErrors = await copyNitroAcrossDirs(
|
||||||
|
presentDirs[0],
|
||||||
|
missingDirs,
|
||||||
|
fileName,
|
||||||
|
);
|
||||||
|
warnings.push(...copyErrors);
|
||||||
|
ok = copyErrors.length === 0;
|
||||||
|
} catch (err) {
|
||||||
|
warnings.push(
|
||||||
|
`copy across .nitro dirs failed: ${(err as Error).message}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2) Download from configured or default nitro sources.
|
||||||
|
if (!ok) {
|
||||||
|
let sources: CloneSource[] = [];
|
||||||
|
try {
|
||||||
|
sources = (await listSources()).filter((s) => s.nitroBaseUrl);
|
||||||
|
} catch (err) {
|
||||||
|
warnings.push(`listSources failed: ${(err as Error).message}`);
|
||||||
|
}
|
||||||
|
if (sources.length === 0) sources = DEFAULT_NITRO_SOURCES;
|
||||||
|
|
||||||
|
const nitroCandidates = [
|
||||||
|
`${classname}.nitro`,
|
||||||
|
// Variant classnames (e.g. `table_silo_med*10`) often only exist
|
||||||
|
// on sources under their base name (e.g. `table_silo_med.nitro`)
|
||||||
|
// or with the `*` replaced by `_` (e.g. `table_silo_med_10.nitro`).
|
||||||
|
...(classname.includes("*")
|
||||||
|
? [
|
||||||
|
`${classname.split("*")[0]}.nitro`,
|
||||||
|
`${classname.replace(/\*/g, "_")}.nitro`,
|
||||||
|
]
|
||||||
|
: []),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const source of sources) {
|
||||||
|
for (const nitroName of nitroCandidates) {
|
||||||
|
const tmpPath = path.join(
|
||||||
|
os.tmpdir(),
|
||||||
|
`.repair_nitro_${classname}_${Math.random().toString(36).slice(2, 8)}.nitro`,
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
const dl = await downloadFile(
|
||||||
|
`${source.nitroBaseUrl}/${nitroName}`,
|
||||||
|
tmpPath,
|
||||||
|
{ maxRetries: 1 },
|
||||||
|
);
|
||||||
|
if (dl.ok) {
|
||||||
|
// Validate it is a real Nitro bundle before writing it
|
||||||
|
// anywhere (mirrors clone-import.ts).
|
||||||
|
const buffer = await fs.readFile(tmpPath);
|
||||||
|
parseNitroBundle(buffer);
|
||||||
|
const writeErrors = await writeNitroToDirs(
|
||||||
|
buffer,
|
||||||
|
missingDirs,
|
||||||
|
fileName,
|
||||||
|
);
|
||||||
|
warnings.push(...writeErrors);
|
||||||
|
ok = writeErrors.length === 0;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
warnings.push(`${nitroName} not found on ${source.name}`);
|
||||||
|
} catch (err) {
|
||||||
|
warnings.push(
|
||||||
|
`download from ${source.name} failed: ${(err as Error).message}`,
|
||||||
|
);
|
||||||
|
} finally {
|
||||||
|
await fs.unlink(tmpPath).catch(() => {});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (ok) break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ok) {
|
||||||
|
return { status: "done", warnings } as const;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
status: "failed",
|
||||||
|
message: warnings[0] || "could not obtain .nitro",
|
||||||
|
} as const;
|
||||||
|
};
|
||||||
|
|
||||||
|
await runPool(items, REPAIR_CONCURRENCY, async (item, i) => {
|
||||||
|
const classname = item.item_name;
|
||||||
|
const result = await processItem(classname);
|
||||||
|
if (result.status === "skipped") {
|
||||||
|
skipped++;
|
||||||
|
onEvent?.({
|
||||||
|
type: "progress",
|
||||||
|
classname,
|
||||||
|
status: "skipped",
|
||||||
|
index: i,
|
||||||
|
total,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (result.status === "done") {
|
||||||
|
succeeded++;
|
||||||
|
onEvent?.({
|
||||||
|
type: "progress",
|
||||||
|
classname,
|
||||||
|
status: "done",
|
||||||
|
index: i,
|
||||||
|
total,
|
||||||
|
warnings: result.warnings.length ? result.warnings : undefined,
|
||||||
|
});
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
failed++;
|
||||||
|
onEvent?.({
|
||||||
|
type: "progress",
|
||||||
|
classname,
|
||||||
|
status: "failed",
|
||||||
|
index: i,
|
||||||
|
total,
|
||||||
|
message: result.message,
|
||||||
|
});
|
||||||
|
});
|
||||||
|
|
||||||
|
onEvent?.({
|
||||||
|
type: "batch_complete",
|
||||||
|
succeeded,
|
||||||
|
failed,
|
||||||
|
skipped,
|
||||||
|
});
|
||||||
|
|
||||||
|
return { succeeded, failed, skipped };
|
||||||
|
}
|
||||||
Reference in new issue
Block a user