EpicNext-Cms/src/lib/services/catalog-audit.ts

import { readdir } from "node:fs/promises";
import { sql } from "drizzle-orm";
import { db } from "@/lib/db";
import { fetchSourceFurnidata } from "@/lib/services/clone-import";
import { listSources } from "@/lib/services/clone-sources";
import { getFurniAssetDirs } from "@/lib/services/furni-asset-dirs";
import {
	applyCatalogSql,
	generateCatalogSql,
	type MissingCatalogEntry,
	repairDuplicateClassnames,
	repairFurniData,
	repairOrphanedCatalog,
} from "./catalog-repair";
import { ensureDirectories } from "./furni-import";
import { repairMissingIcons } from "./repair-icons";
import { repairMissingNitros } from "./repair-nitros";

export interface AuditEvent {
	type:
		| "progress"
		| "items_loaded"
		| "catalog_loaded"
		| "checking"
		| "repair_complete"
		| "sql_ready"
		| "sql_applied"
		| "furnidata_repair_complete"
		| "structure_repair_complete"
		| "batch_complete"
		| "error";
	message?: string;
	current?: number;
	total?: number;
	section?: string;
	issues?: AuditIssue[];
	missingFromSources?: MissingFromSource[];
	summary?: AuditSummary;
	repair?: RepairStats;
	repairNitro?: RepairStats;
	sql?: string;
	sqlCount?: number;
	applied?: number;
	sqlFailed?: number;
	furniData?: { added: number; removedDuplicates: number };
	structure?: {
		orphanedRemoved: number;
		orphanedCleaned: number;
		duplicatesMerged: number;
		duplicateRowsRemoved: number;
		remappedReferences: number;
	};
}

export interface RepairStats {
	succeeded: number;
	failed: number;
	skipped: number;
	stillMissing: number;
}

export interface AuditIssue {
	type: string;
	severity: "error" | "warning" | "info";
	classname?: string;
	itemId?: number;
	catalogId?: number;
	message: string;
	sourceName?: string;
}

export interface MissingFromSource {
	sourceName: string;
	sourceId: string;
	classname: string;
	itemName: string;
}

export interface AuditSummary {
	totalItemsBase: number;
	totalCatalogItems: number;
	missingCatalog: number;
	orphanedCatalog: number;
	missingNitro: number;
	missingIcon: number;
	duplicates: number;
	missingFromSourcesTotal: number;
	iconsRepaired: number;
	iconsStillMissing: number;
	nitrosRepaired: number;
	nitrosStillMissing: number;
	catalogSqlCount: number;
	catalogSqlApplied: number;
	furniDataAdded: number;
	furniDataDuplicatesRemoved: number;
	orphanedRemoved: number;
	orphanedCleaned: number;
	duplicatesMerged: number;
	duplicateRowsRemoved: number;
	remappedReferences: number;
}

export interface CatalogAuditOptions {
	repair?: boolean;
	repairNitros?: boolean;
	sql?: boolean;
	applySql?: boolean;
	repairFurniData?: boolean;
	repairStructure?: boolean;
}

export async function runCatalogAudit(
	onEvent?: (evt: AuditEvent) => void,
	options?: CatalogAuditOptions,
): Promise<void> {
	let issues: AuditIssue[] = [];
	const missingFromSources: MissingFromSource[] = [];

	try {
		await ensureDirectories();
	} catch (err) {
		onEvent?.({
			type: "error",
			message: `ensureDirectories failed: ${(err as Error).message}`,
		});
		return;
	}

	let iconDir: string;
	let nitroDir: string;
	try {
		const dirs = await getFurniAssetDirs();
		iconDir = dirs.iconDir;
		nitroDir = dirs.nitroDir;
	} catch (err) {
		onEvent?.({
			type: "error",
			message: `getFurniAssetDirs failed: ${(err as Error).message}`,
		});
		return;
	}

	const [iconFiles, nitroFiles] = await Promise.all([
		readdir(iconDir).catch(() => [] as string[]),
		readdir(nitroDir).catch(() => [] as string[]),
	]);
	const iconSet = new Set(iconFiles);
	const nitroSet = new Set(nitroFiles);

	onEvent?.({ type: "progress", message: "Loading items_base…" });

	let items: Array<{
		id: number;
		item_name: string;
		public_name: string;
		type: string;
	}>;
	try {
		const [rows] = (await db.execute(sql`
			SELECT id, item_name, public_name, type FROM items_base ORDER BY id
		`)) as unknown as [
			Array<{
				id: number;
				item_name: string;
				public_name: string;
				type: string;
			}>,
			unknown,
		];
		items = rows;
	} catch (err) {
		onEvent?.({
			type: "error",
			message: `items_base query failed: ${(err as Error).message}`,
		});
		return;
	}

	onEvent?.({ type: "items_loaded", total: items.length });

	onEvent?.({ type: "progress", message: "Loading catalog_items…" });

	let catalogItems: Array<{
		id: number;
		item_ids: string;
		catalog_name: string;
	}>;
	try {
		const [rows] = (await db.execute(sql`
			SELECT id, item_ids, catalog_name FROM catalog_items ORDER BY id
		`)) as unknown as [
			Array<{ id: number; item_ids: string; catalog_name: string }>,
			unknown,
		];
		catalogItems = rows;
	} catch (err) {
		onEvent?.({
			type: "error",
			message: `catalog_items query failed: ${(err as Error).message}`,
		});
		return;
	}

	onEvent?.({ type: "catalog_loaded", total: catalogItems.length });

	const itemIdsWithCatalog = new Set<number>();
	for (const ci of catalogItems) {
		const ids = ci.item_ids.split(";").map(Number).filter(Boolean);
		for (const id of ids) {
			itemIdsWithCatalog.add(id);
		}
	}

	const itemById = new Map<number, (typeof items)[0]>();
	const itemByName = new Map<string, (typeof items)[0]>();
	const classnameCount = new Map<string, number>();
	for (const item of items) {
		itemById.set(item.id, item);
		itemByName.set(item.item_name, item);
		classnameCount.set(
			item.item_name,
			(classnameCount.get(item.item_name) ?? 0) + 1,
		);
	}

	onEvent?.({
		type: "checking",
		section: "missing_catalog",
		total: items.length,
	});

	const missingCatalogEntries: MissingCatalogEntry[] = [];
	for (let i = 0; i < items.length; i++) {
		const item = items[i];
		if (!itemIdsWithCatalog.has(item.id)) {
			issues.push({
				type: "missing_catalog",
				severity: "warning",
				classname: item.item_name,
				itemId: item.id,
				message: `"${item.item_name}" (ID ${item.id}) has no catalog_items entry — not purchasable`,
			});
			// Badge items use .gif files, not .nitro bundles.
			if (
				!item.item_name.startsWith("badge_") &&
				nitroSet.has(`${item.item_name}.nitro`)
			) {
				missingCatalogEntries.push({
					classname: item.item_name,
					itemId: item.id,
					itemType: item.type,
				});
			}
		}
		if (i % 100 === 0) {
			onEvent?.({
				type: "checking",
				section: "missing_catalog",
				current: i,
				total: items.length,
			});
		}
	}

	onEvent?.({ type: "checking", section: "duplicates" });

	for (const [name, count] of classnameCount) {
		if (count > 1) {
			issues.push({
				type: "duplicate_classname",
				severity: "error",
				classname: name,
				message: `"${name}" appears ${count} times in items_base`,
			});
		}
	}

	onEvent?.({
		type: "checking",
		section: "orphaned_catalog",
		total: catalogItems.length,
	});

	for (let i = 0; i < catalogItems.length; i++) {
		const ci = catalogItems[i];
		const ids = ci.item_ids.split(";").map(Number).filter(Boolean);
		for (const id of ids) {
			if (!itemById.has(id)) {
				issues.push({
					type: "orphaned_catalog",
					severity: "error",
					catalogId: ci.id,
					message: `catalog_items ID ${ci.id} ("${ci.catalog_name}") references items_base ID ${id} which does not exist`,
				});
			}
		}
		if (i % 200 === 0) {
			onEvent?.({
				type: "checking",
				section: "orphaned_catalog",
				current: i,
				total: catalogItems.length,
			});
		}
	}

	onEvent?.({
		type: "checking",
		section: "missing_nitro",
		total: items.length,
	});

	const missingNitroClassnames: string[] = [];
	for (let i = 0; i < items.length; i++) {
		const item = items[i];
		// Badge items use .gif files, not .nitro bundles.
		if (item.item_name.startsWith("badge_")) continue;
		if (!nitroSet.has(`${item.item_name}.nitro`)) {
			missingNitroClassnames.push(item.item_name);
			issues.push({
				type: "missing_nitro",
				severity: "error",
				classname: item.item_name,
				itemId: item.id,
				message: `"${item.item_name}" (ID ${item.id}) has no .nitro file`,
			});
		}
		if (i % 100 === 0) {
			onEvent?.({
				type: "checking",
				section: "missing_nitro",
				current: i,
				total: items.length,
			});
		}
	}

	onEvent?.({ type: "checking", section: "missing_icon", total: items.length });

	const missingIconClassnames: string[] = [];
	for (let i = 0; i < items.length; i++) {
		const item = items[i];
		// Badge items use .gif icons stored as badge_<code>_icon.png,
		// not <code>_icon.png like regular furni.
		const iconFileName = `${item.item_name}_icon.png`;
		const badgeIconFileName = `badge_${item.item_name}_icon.png`;
		if (!iconSet.has(iconFileName) && !iconSet.has(badgeIconFileName)) {
			missingIconClassnames.push(item.item_name);
			issues.push({
				type: "missing_icon",
				severity: "warning",
				classname: item.item_name,
				itemId: item.id,
				message: `"${item.item_name}" (ID ${item.id}) has no icon file`,
			});
		}
		if (i % 100 === 0) {
			onEvent?.({
				type: "checking",
				section: "missing_icon",
				current: i,
				total: items.length,
			});
		}
	}

	let repairStats: RepairStats | null = null;
	if (options?.repair && missingIconClassnames.length > 0) {
		onEvent?.({
			type: "progress",
			message: `Repairing ${missingIconClassnames.length} missing icons…`,
		});

		try {
			const result = await repairMissingIcons((evt) => {
				if (evt.type === "progress") {
					onEvent?.({
						type: "checking",
						section: "repair_icons",
						current: evt.index,
						total: evt.total,
						message:
							evt.status === "done"
								? `Repaired ${evt.classname}`
								: evt.status === "failed"
									? `Failed: ${evt.classname}`
									: undefined,
					});
				} else if (evt.type === "error") {
					onEvent?.({ type: "error", message: evt.message });
				}
			});

			// Re-check which of the previously-missing icons are still missing
			// and drop the ones the repair fixed from the issue list.
			const stillMissing: string[] = [];
			for (const classname of missingIconClassnames) {
				if (!iconSet.has(`${classname}_icon.png`)) {
					stillMissing.push(classname);
				}
			}
			issues = issues.filter(
				(i) =>
					i.type !== "missing_icon" || stillMissing.includes(i.classname ?? ""),
			);

			repairStats = {
				succeeded: result.succeeded,
				failed: result.failed,
				skipped: result.skipped,
				stillMissing: stillMissing.length,
			};
			onEvent?.({ type: "repair_complete", repair: repairStats });
		} catch (err) {
			onEvent?.({
				type: "error",
				message: `Icon repair failed: ${(err as Error).message}`,
			});
		}
	}

	let nitroRepairStats: RepairStats | null = null;
	if (options?.repairNitros && missingNitroClassnames.length > 0) {
		onEvent?.({
			type: "progress",
			message: `Repairing ${missingNitroClassnames.length} missing .nitro files…`,
		});

		try {
			const result = await repairMissingNitros((evt) => {
				if (evt.type === "progress") {
					onEvent?.({
						type: "checking",
						section: "repair_nitros",
						current: evt.index,
						total: evt.total,
						message:
							evt.status === "done"
								? `Repaired ${evt.classname}`
								: evt.status === "failed"
									? `Failed: ${evt.classname}`
									: undefined,
					});
				} else if (evt.type === "error") {
					onEvent?.({ type: "error", message: evt.message });
				}
			});

			// Re-check which of the previously-missing nitros are still missing
			// and drop the ones the repair fixed from the issue list.
			const stillMissing: string[] = [];
			for (const classname of missingNitroClassnames) {
				if (!nitroSet.has(`${classname}.nitro`)) {
					stillMissing.push(classname);
				}
			}
			issues = issues.filter(
				(i) =>
					i.type !== "missing_nitro" ||
					stillMissing.includes(i.classname ?? ""),
			);

			nitroRepairStats = {
				succeeded: result.succeeded,
				failed: result.failed,
				skipped: result.skipped,
				stillMissing: stillMissing.length,
			};
			onEvent?.({
				type: "repair_complete",
				repairNitro: nitroRepairStats,
			});
		} catch (err) {
			onEvent?.({
				type: "error",
				message: `.nitro repair failed: ${(err as Error).message}`,
			});
		}
	}

	// ── Structural repair: orphaned catalog refs + duplicate classnames ──
	let orphanedRemoved = 0;
	let orphanedCleaned = 0;
	let duplicatesMerged = 0;
	let duplicateRowsRemoved = 0;
	let remappedReferences = 0;
	if (options?.repairStructure) {
		onEvent?.({
			type: "progress",
			message: "Repairing structural issues (orphaned refs + duplicates)…",
		});
		try {
			const orphanResult = await repairOrphanedCatalog();
			orphanedRemoved = orphanResult.removed;
			orphanedCleaned = orphanResult.cleaned;

			// Re-derive orphaned_catalog issues from the fixed data.
			const [afterCatalog] = (await db.execute(sql`
				SELECT id, item_ids, catalog_name FROM catalog_items ORDER BY id
			`)) as unknown as [
				Array<{ id: number; item_ids: string; catalog_name: string }>,
				unknown,
			];
			const orphanedCatalogIds = new Set<number>();
			for (const ci of afterCatalog) {
				const ids = ci.item_ids.split(";").map(Number).filter(Boolean);
				for (const id of ids) {
					if (!itemById.has(id)) orphanedCatalogIds.add(ci.id);
				}
			}
			issues = issues.filter(
				(i) =>
					i.type !== "orphaned_catalog" ||
					orphanedCatalogIds.has(i.catalogId ?? -1),
			);

			const duplicateResult = await repairDuplicateClassnames();
			duplicatesMerged = duplicateResult.merged;
			duplicateRowsRemoved = duplicateResult.rowsRemoved;
			remappedReferences = duplicateResult.remapped;

			// Re-derive duplicate_classname issues from the fixed data.
			const [afterItems] = (await db.execute(sql`
				SELECT item_name FROM items_base
			`)) as unknown as [Array<{ item_name: string }>, unknown];
			const afterCount = new Map<string, number>();
			for (const row of afterItems) {
				afterCount.set(row.item_name, (afterCount.get(row.item_name) ?? 0) + 1);
			}
			const dupNames = new Set(
				[...afterCount].filter(([, count]) => count > 1).map(([name]) => name),
			);
			issues = issues.filter(
				(i) =>
					i.type !== "duplicate_classname" || dupNames.has(i.classname ?? ""),
			);

			onEvent?.({
				type: "structure_repair_complete",
				structure: {
					orphanedRemoved,
					orphanedCleaned,
					duplicatesMerged,
					duplicateRowsRemoved,
					remappedReferences,
				},
			});
		} catch (err) {
			onEvent?.({
				type: "error",
				message: `Structural repair failed: ${(err as Error).message}`,
			});
		}
	}

	// ── Catalog SQL generation / apply ────────────────────────────────
	let catalogSqlCount = 0;
	let catalogSqlApplied = 0;
	if ((options?.sql || options?.applySql) && missingCatalogEntries.length > 0) {
		onEvent?.({
			type: "progress",
			message: `Generating catalog SQL for ${missingCatalogEntries.length} items…`,
		});
		try {
			const result = await generateCatalogSql(missingCatalogEntries);
			catalogSqlCount = result.count;
			onEvent?.({
				type: "sql_ready",
				sql: result.sql,
				sqlCount: result.count,
			});
			if (options.applySql) {
				const applyResult = await applyCatalogSql(result.statements);
				catalogSqlApplied = applyResult.applied;
				onEvent?.({
					type: "sql_applied",
					applied: applyResult.applied,
					sqlFailed: applyResult.failed,
				});
			}
		} catch (err) {
			onEvent?.({
				type: "error",
				message: `Catalog SQL failed: ${(err as Error).message}`,
			});
		}
	}

	// ── FurnitureData.json repair (add missing + dedupe) ──────────────
	let furniDataAdded = 0;
	let furniDataDuplicatesRemoved = 0;
	if (options?.repairFurniData) {
		onEvent?.({ type: "progress", message: "Repairing FurnitureData.json…" });
		try {
			const result = await repairFurniData(items);
			furniDataAdded = result.added;
			furniDataDuplicatesRemoved = result.removedDuplicates;
			onEvent?.({
				type: "furnidata_repair_complete",
				furniData: {
					added: result.added,
					removedDuplicates: result.removedDuplicates,
				},
			});
		} catch (err) {
			onEvent?.({
				type: "error",
				message: `FurnitureData repair failed: ${(err as Error).message}`,
			});
		}
	}

	onEvent?.({ type: "progress", message: "Comparing with clone sources…" });

	const sources = await listSources();
	const sourceResults = await Promise.all(
		sources
			.filter((s) => s.furnidataUrl)
			.map(async (source) => {
				try {
					const entries = await fetchSourceFurnidata(source.furnidataUrl);
					return {
						source,
						entries,
						error: null,
					} as const;
				} catch {
					return {
						source,
						entries: [] as Array<{ classname: string; name: string }>,
						error: true,
					} as const;
				}
			}),
	);

	for (const result of sourceResults) {
		if (result.error) {
			issues.push({
				type: "source_fetch_failed",
				severity: "error",
				sourceName: result.source.name,
				message: `Failed to fetch furnidata from "${result.source.name}"`,
			});
			continue;
		}
		for (const entry of result.entries) {
			if (!itemByName.has(entry.classname)) {
				missingFromSources.push({
					sourceName: result.source.name,
					sourceId: result.source.id,
					classname: entry.classname,
					itemName: entry.name,
				});
			}
		}
	}

	issues.sort((a, b) => {
		const sev = { error: 0, warning: 1, info: 2 };
		return (sev[a.severity] ?? 0) - (sev[b.severity] ?? 0);
	});

	const summary: AuditSummary = {
		totalItemsBase: items.length,
		totalCatalogItems: catalogItems.length,
		missingCatalog: issues.filter((i) => i.type === "missing_catalog").length,
		orphanedCatalog: issues.filter((i) => i.type === "orphaned_catalog").length,
		missingNitro: issues.filter((i) => i.type === "missing_nitro").length,
		missingIcon: issues.filter((i) => i.type === "missing_icon").length,
		duplicates: issues.filter((i) => i.type === "duplicate_classname").length,
		missingFromSourcesTotal: missingFromSources.length,
		iconsRepaired: repairStats?.succeeded ?? 0,
		iconsStillMissing:
			repairStats?.stillMissing ?? missingIconClassnames.length,
		nitrosRepaired: nitroRepairStats?.succeeded ?? 0,
		nitrosStillMissing:
			nitroRepairStats?.stillMissing ?? missingNitroClassnames.length,
		catalogSqlCount,
		catalogSqlApplied,
		furniDataAdded,
		furniDataDuplicatesRemoved,
		orphanedRemoved,
		orphanedCleaned,
		duplicatesMerged,
		duplicateRowsRemoved,
		remappedReferences,
	};

	onEvent?.({
		type: "batch_complete",
		issues,
		missingFromSources,
		summary,
	});
}