feat: parallelize audit repair and add nitro/SQL repair options
- Parallelize icon and nitro downloads in the audit repair with a sliding-window worker pool (6 concurrent) to speed up large catalogs - Add repairMissingNitros: fetch missing .nitro bundles from configured nitro sources (Wibbo default), validating each bundle before writing - Add catalog-repair service: generate/apply catalog_items SQL for furni missing a catalog entry and repair FurnitureData.json (add missing + dedupe classnames) - Wire all options through the audit API and client UI with live progress and result stats (icons, nitros, SQL, furnidata) - Add Wibbo as default nitro source alongside existing icon sources - Ignore runtime furni assets downloaded into public/ during repair
This commit is contained in:
1 parent
0c8e62357f
commit
5308ce6a12
8 files changed
+1166
-58
No files matched your search
@@ -0,0 +1,205 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { db } from "@/lib/db";
|
||||
import {
|
||||
readFurniData,
|
||||
withFurniDataLock,
|
||||
writeFurniData,
|
||||
} from "@/lib/services/furni-data";
|
||||
import {
|
||||
allocateCatalogItemId,
|
||||
autoPriceFurni,
|
||||
getOrCreateCategoryPage,
|
||||
} from "@/lib/services/furni-import";
|
||||
|
||||
export interface MissingCatalogEntry {
|
||||
classname: string;
|
||||
itemId: number;
|
||||
itemType: string;
|
||||
}
|
||||
|
||||
export interface CatalogSqlResult {
|
||||
statements: string[];
|
||||
sql: string;
|
||||
count: number;
|
||||
}
|
||||
|
||||
function escSqlLiteral(value: string): string {
|
||||
return value.replace(/'/g, "''");
|
||||
}
|
||||
|
||||
/**
|
||||
* Build standalone `catalog_items` INSERT statements for items_base entries
|
||||
* that exist (with assets) but have no catalog entry yet. Ids and prices are
|
||||
* resolved the same way the app's own imports do it (autoPriceFurni + category
|
||||
* page + next catalog id), so running the SQL matches an app-managed import.
|
||||
*/
|
||||
export async function generateCatalogSql(
|
||||
entries: MissingCatalogEntry[],
|
||||
): Promise<CatalogSqlResult> {
|
||||
const statements: string[] = [
|
||||
"-- AtomCMS catalog repair",
|
||||
`-- Generated ${new Date().toISOString()}`,
|
||||
`-- ${entries.length} item(s) to add`,
|
||||
"START TRANSACTION;",
|
||||
];
|
||||
|
||||
for (const entry of entries) {
|
||||
let pageId: number;
|
||||
try {
|
||||
pageId = await getOrCreateCategoryPage(entry.classname, entry.itemType);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
const price = autoPriceFurni(entry.classname);
|
||||
let nextId: number;
|
||||
try {
|
||||
nextId = await allocateCatalogItemId(async (id) => id);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
statements.push(
|
||||
`INSERT INTO catalog_items (id, page_id, item_ids, catalog_name, cost_credits, cost_points, points_type, amount, order_number, offer_id, extradata) VALUES (${nextId}, ${pageId}, ${entry.itemId}, '${escSqlLiteral(entry.classname)}', ${price.credits}, ${price.points}, ${price.pointsType}, 1, 1, '-1', '');`,
|
||||
);
|
||||
}
|
||||
|
||||
statements.push("COMMIT;");
|
||||
return { statements, sql: statements.join("\n"), count: entries.length };
|
||||
}
|
||||
|
||||
/** Execute generated INSERT statements directly against the database. */
|
||||
export async function applyCatalogSql(
|
||||
statements: string[],
|
||||
): Promise<{ applied: number; failed: number }> {
|
||||
let applied = 0;
|
||||
let failed = 0;
|
||||
for (const statement of statements) {
|
||||
const trimmed = statement.trim();
|
||||
if (
|
||||
!trimmed ||
|
||||
trimmed.startsWith("--") ||
|
||||
trimmed === "START TRANSACTION;" ||
|
||||
trimmed === "COMMIT;"
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
try {
|
||||
await db.execute(sql.raw(trimmed));
|
||||
applied++;
|
||||
} catch {
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
return { applied, failed };
|
||||
}
|
||||
|
||||
export interface FurniDataRepairResult {
|
||||
added: number;
|
||||
removedDuplicates: number;
|
||||
}
|
||||
|
||||
function buildMinimalFurniEntry(item: {
|
||||
id: number;
|
||||
item_name: string;
|
||||
public_name: string;
|
||||
}): Record<string, unknown> {
|
||||
return {
|
||||
id: item.id,
|
||||
classname: item.item_name,
|
||||
revision: 0,
|
||||
category: "unknown",
|
||||
defaultdir: 0,
|
||||
xdim: 1,
|
||||
ydim: 1,
|
||||
partcolors: { color: [] },
|
||||
name: item.public_name || item.item_name,
|
||||
description: "",
|
||||
adurl: "",
|
||||
offerid: item.id,
|
||||
buyout: false,
|
||||
rentofferid: -1,
|
||||
rentbuyout: false,
|
||||
bc: false,
|
||||
excludeddynamic: false,
|
||||
customparams: "",
|
||||
specialtype: 0,
|
||||
canstandon: false,
|
||||
cansiton: false,
|
||||
canlayon: false,
|
||||
furniline: "",
|
||||
environment: "",
|
||||
rare: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Add FurnitureData.json entries for items_base rows that are missing from the
|
||||
* file, and remove duplicate classnames (keeping the first occurrence).
|
||||
*/
|
||||
export async function repairFurniData(
|
||||
items: Array<{
|
||||
id: number;
|
||||
item_name: string;
|
||||
public_name: string;
|
||||
type: string;
|
||||
}>,
|
||||
): Promise<FurniDataRepairResult> {
|
||||
if (items.length === 0) return { added: 0, removedDuplicates: 0 };
|
||||
|
||||
return withFurniDataLock(async () => {
|
||||
const furniData = (await readFurniData()) as Record<
|
||||
string,
|
||||
{ furnitype: Array<Record<string, unknown>> }
|
||||
>;
|
||||
|
||||
// 1) Deduplicate classnames per section (keep the first occurrence).
|
||||
let removedDuplicates = 0;
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
const sec = furniData[section];
|
||||
if (!sec?.furnitype) continue;
|
||||
const seen = new Set<string>();
|
||||
const deduped = sec.furnitype.filter((e) => {
|
||||
const classname = typeof e.classname === "string" ? e.classname : "";
|
||||
if (!classname) return true;
|
||||
if (seen.has(classname)) {
|
||||
removedDuplicates++;
|
||||
return false;
|
||||
}
|
||||
seen.add(classname);
|
||||
return true;
|
||||
});
|
||||
if (deduped.length !== sec.furnitype.length) {
|
||||
furniData[section].furnitype = deduped;
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Add entries for items_base rows missing from the file.
|
||||
const existing = new Set<string>();
|
||||
for (const section of ["roomitemtypes", "wallitemtypes"] as const) {
|
||||
for (const e of furniData[section]?.furnitype ?? []) {
|
||||
if (typeof e.classname === "string") existing.add(e.classname);
|
||||
}
|
||||
}
|
||||
const added: Array<{ entry: Record<string, unknown>; itemType: string }> =
|
||||
[];
|
||||
for (const item of items) {
|
||||
if (existing.has(item.item_name)) continue;
|
||||
added.push({
|
||||
entry: buildMinimalFurniEntry(item),
|
||||
itemType: item.type,
|
||||
});
|
||||
}
|
||||
|
||||
// 3) Write once if anything changed.
|
||||
if (removedDuplicates > 0 || added.length > 0) {
|
||||
for (const { entry, itemType } of added) {
|
||||
const sectionKey = itemType === "i" ? "wallitemtypes" : "roomitemtypes";
|
||||
if (!furniData[sectionKey]) {
|
||||
furniData[sectionKey] = { furnitype: [] };
|
||||
}
|
||||
furniData[sectionKey].furnitype.push(entry);
|
||||
}
|
||||
await writeFurniData(furniData as Record<string, unknown>);
|
||||
}
|
||||
return { added: added.length, removedDuplicates };
|
||||
});
|
||||
}
|
||||
Reference in new issue
Block a user