EpicNext-Cms/scripts/catalog-sync.ts

// Command-line driver for the catalog audit/repair pipeline.
//
// The same engine the admin "Catalog audit" UI streams from (`runCatalogAudit`),
// for the terminal: audit the hotel catalog against one or more clone sources
// (Habbo and the retros), fix the furniture that is broken, and bring missing
// items into the catalog — optionally writing and applying the catalog SQL.
//
// Usage:
//   pnpm catalog:sync                        # interactive menu (scripts/catalog-sync.sh)
//   pnpm exec tsx scripts/catalog-sync.ts --list-sources
//   pnpm exec tsx scripts/catalog-sync.ts --sources="Habbo (COM)"    # audit only
//   pnpm exec tsx scripts/catalog-sync.ts --all --sources=default-habbo-com
//   pnpm exec tsx scripts/catalog-sync.ts --all --sql --apply-sql   # + write/apply SQL
//
// Flags:
//   --list-sources     print the configured clone sources and exit
//   --tsv              with the above, print `id\tname\tfurnidataUrl` rows
//   --sources=<a,b>    compare against these sources (id or name, comma-separated)
//   --icons            re-download missing furniture icons from the sources
//   --nitros           re-download missing .nitro / swf bundles
//   --furnidata        repair FurnitureData.json (add missing, drop duplicates)
//   --structure        repair orphaned / duplicate catalog rows
//   --check-ids        check FurnitureData.json for spriteId conflicts
//   --sql              generate the catalog SQL for items missing an offer
//   --apply-sql        apply the generated SQL (implies --sql)
//   --organize-sql     order the generated SQL by catalog page (implies --sql)
//   --all              --icons --nitros --furnidata --structure --check-ids
//   --json             print one machine-readable JSON report
//   --fail-on-errors   exit 3 when error-severity issues remain
//
// Audit is read-only by default: nothing is written until one of the repair
// flags is passed, and `--apply-sql` is the only flag that changes the catalog.

import "./load-env";
import { mkdir, writeFile } from "node:fs/promises";
import { join } from "node:path";
import { pathToFileURL } from "node:url";
import type { AuditEvent, AuditSummary } from "@/lib/services/catalog-audit";

/** Where generated catalog SQL is written (`storage/` is gitignored). */
function sqlOutputDir(): string {
	return join(process.cwd(), "storage", "catalog-sync");
}

export type SourceRef = { id: string; name: string; furnidataUrl: string };

export type AuditIssue = NonNullable<AuditEvent["issues"]>[number];
export type MissingFromSource = NonNullable<
	AuditEvent["missingFromSources"]
>[number];

export interface AuditReport {
	sources: SourceRef[];
	summary?: AuditSummary;
	issues: AuditIssue[];
	missingFromSources: MissingFromSource[];
	sqlFile?: string;
}

export interface CatalogSyncFlags {
	listSources: boolean;
	tsv: boolean;
	sourceIds: string[];
	repairIcons: boolean;
	repairNitros: boolean;
	repairFurniData: boolean;
	repairStructure: boolean;
	checkFurniDataIds: boolean;
	generateSql: boolean;
	applySql: boolean;
	organizeSql: boolean;
	json: boolean;
	failOnErrors: boolean;
}

export const DEFAULT_CATALOG_SYNC_FLAGS: Omit<CatalogSyncFlags, "sourceIds"> = {
	listSources: false,
	tsv: false,
	repairIcons: false,
	repairNitros: false,
	repairFurniData: false,
	repairStructure: false,
	checkFurniDataIds: false,
	generateSql: false,
	applySql: false,
	organizeSql: false,
	json: false,
	failOnErrors: false,
};

const BOOLEAN_FLAGS: Record<string, keyof CatalogSyncFlags> = {
	"--list-sources": "listSources",
	"--tsv": "tsv",
	"--icons": "repairIcons",
	"--repair-icons": "repairIcons",
	"--nitros": "repairNitros",
	"--repair-nitros": "repairNitros",
	"--furnidata": "repairFurniData",
	"--repair-furnidata": "repairFurniData",
	"--structure": "repairStructure",
	"--repair-structure": "repairStructure",
	"--check-ids": "checkFurniDataIds",
	"--sql": "generateSql",
	"--apply-sql": "applySql",
	"--organize-sql": "organizeSql",
	"--json": "json",
	"--fail-on-errors": "failOnErrors",
};

export function parseCatalogSyncArgs(raw: string[]): CatalogSyncFlags {
	const flags: CatalogSyncFlags = {
		...DEFAULT_CATALOG_SYNC_FLAGS,
		sourceIds: [],
	};

	for (const arg of raw) {
		if (arg === "--") continue;

		if (arg === "--all") {
			flags.repairIcons = true;
			flags.repairNitros = true;
			flags.repairFurniData = true;
			flags.repairStructure = true;
			flags.checkFurniDataIds = true;
			continue;
		}

		const separator = arg.indexOf("=");
		const name = separator >= 0 ? arg.slice(0, separator) : arg;

		const booleanFlag = BOOLEAN_FLAGS[name];
		if (booleanFlag) {
			flags[booleanFlag] = true as never;
			continue;
		}

		if (name === "--sources" || name === "--source") {
			if (separator < 0)
				throw new Error(
					`${name} needs a value, e.g. --sources=default-habbo-com`,
				);
			const value = arg.slice(separator + 1);
			for (const part of value
				.split(",")
				.map((entry) => entry.trim())
				.filter(Boolean))
				flags.sourceIds.push(part);
			continue;
		}

		throw new Error(
			`Unknown flag: ${arg} (run with --help for the supported flags)`,
		);
	}

	// Applying SQL implies generating it, and organising it implies generating.
	if (flags.applySql) flags.generateSql = true;
	if (flags.organizeSql) flags.generateSql = true;
	return flags;
}

/** Human-readable list of what the run will do. */
export function describeActions(flags: CatalogSyncFlags): string {
	const actions = ["audit"];
	if (flags.repairIcons) actions.push("icons");
	if (flags.repairNitros) actions.push("nitros");
	if (flags.repairFurniData) actions.push("furnidata");
	if (flags.repairStructure) actions.push("structure");
	if (flags.checkFurniDataIds) actions.push("id-check");
	if (flags.applySql) actions.push("apply-sql");
	else if (flags.generateSql) actions.push("generate-sql");
	if (flags.organizeSql) actions.push("organised");
	return actions.join(" + ");
}

/** True when the file is being executed, not imported by a test. */
function isEntryPoint(): boolean {
	const entry = process.argv[1];
	return (
		Boolean(entry) && import.meta.url === pathToFileURL(String(entry)).href
	);
}

async function loadSourceCatalogue(): Promise<SourceRef[]> {
	const { listSources } = await import("@/lib/services/clone-sources");
	const sources = await listSources();
	return sources
		.filter((source) => source.furnidataUrl)
		.map((source) => ({
			id: source.id,
			name: source.name,
			furnidataUrl: source.furnidataUrl,
		}));
}

async function commandListSources(flags: CatalogSyncFlags): Promise<number> {
	const sources = await loadSourceCatalogue();

	if (flags.tsv) {
		for (const source of sources)
			console.log([source.id, source.name, source.furnidataUrl].join("\t"));
		return 0;
	}

	console.log(`Configured clone sources (${sources.length}):`);
	for (const source of sources)
		console.log(`  ${source.id}\t${source.name}\t${source.furnidataUrl}`);
	console.log(
		"\nCompare with: --sources=<id> (comma-separated; display names work too)",
	);
	return 0;
}

/** Persist the generated SQL so it can be reviewed or replayed by hand. */
export async function writeGeneratedSql(sql: string): Promise<string> {
	await mkdir(sqlOutputDir(), { recursive: true });
	const stamp = new Date().toISOString().replace(/[:.]/g, "-");
	const file = join(sqlOutputDir(), `catalog-${stamp}.sql`);
	await writeFile(file, sql.endsWith("\n") ? sql : `${sql}\n`, "utf8");
	return file.replace(`${process.cwd()}/`, "");
}

export async function commandRun(flags: CatalogSyncFlags): Promise<number> {
	const catalogue = await loadSourceCatalogue();

	// Resolve the selection up-front so a typo fails before anything is written.
	const known = new Set<string>();
	for (const source of catalogue) {
		known.add(source.id);
		known.add(source.name);
	}
	const unknown = flags.sourceIds.filter((id) => !known.has(id));
	if (unknown.length > 0) {
		console.error(
			`Unknown source: ${unknown.join(", ")}\nRun --list-sources to see the configured sites.`,
		);
		return 1;
	}

	const selected = flags.sourceIds.length
		? catalogue.filter(
				(source) =>
					flags.sourceIds.includes(source.id) ||
					flags.sourceIds.includes(source.name),
			)
		: catalogue;

	const { runCatalogAudit } = await import("@/lib/services/catalog-audit");

	const report: AuditReport = {
		sources: selected,
		issues: [],
		missingFromSources: [],
	};
	const quiet = flags.json;
	const log = (line: string) => {
		if (!quiet) console.log(line);
	};
	const started = Date.now();
	let pendingSql: string | undefined;

	log(
		`Catalog sync — ${selected.length} source(s), ${describeActions(flags)}` +
			(flags.applySql ? "  [SQL WILL BE APPLIED]" : ""),
	);

	await runCatalogAudit(
		(event) => {
			switch (event.type) {
				case "progress":
					log(`· ${event.message ?? ""}`);
					break;
				case "items_loaded":
					log(`· items_base: ${event.total} rows`);
					break;
				case "catalog_loaded":
					log(`· catalog_items: ${event.total} rows`);
					break;
				case "sql_ready":
					log(`· catalog SQL generated: ${event.sqlCount ?? 0} statement(s)`);
					pendingSql = event.sql;
					break;
				case "sql_applied":
					log(
						`· catalog SQL applied: ${event.applied ?? 0} (failed ${event.sqlFailed ?? 0})`,
					);
					break;
				case "furnidata_repair_complete":
					log(
						`· FurnitureData.json: +${event.furniData?.added ?? 0} entries, ${event.furniData?.removedDuplicates ?? 0} duplicate(s) removed`,
					);
					break;
				case "structure_repair_complete":
					log(
						`· structure: ${event.structure?.orphanedRemoved ?? 0} orphaned removed, ${event.structure?.duplicatesMerged ?? 0} duplicate(s) merged, ${event.structure?.remappedReferences ?? 0} reference(s) remapped`,
					);
					break;
				case "repair_complete":
					log(
						`· asset repair: ${event.repair?.succeeded ?? 0} ok, ${event.repair?.failed ?? 0} failed, ${event.repair?.skipped ?? 0} skipped`,
					);
					break;
				case "batch_complete":
					report.issues = event.issues ?? [];
					report.missingFromSources = event.missingFromSources ?? [];
					report.summary = event.summary;
					break;
				case "error":
					console.error(`! ${event.message ?? "audit error"}`);
					break;
				default:
					break;
			}
		},
		{
			sourceIds: flags.sourceIds,
			repair: flags.repairIcons,
			repairNitros: flags.repairNitros,
			sql: flags.generateSql,
			applySql: flags.applySql,
			organizeSql: flags.organizeSql,
			repairFurniData: flags.repairFurniData,
			repairStructure: flags.repairStructure,
			checkFurniDataIds: flags.checkFurniDataIds,
		},
	);

	if (pendingSql?.trim()) report.sqlFile = await writeGeneratedSql(pendingSql);

	if (flags.json) {
		console.log(
			JSON.stringify(
				{ flags, elapsedMs: Date.now() - started, ...report },
				null,
				2,
			),
		);
	} else {
		printSummary(report, started);
		if (report.sqlFile) console.log(`Catalog SQL: ${report.sqlFile}`);
	}

	const errors = report.issues.filter((issue) => issue.severity === "error");
	if (flags.failOnErrors && errors.length > 0) {
		if (!quiet) console.error(`${errors.length} error(s) remain unresolved.`);
		return 3;
	}
	return 0;
}

function printSummary(report: AuditReport, started: number): void {
	const summary = report.summary;
	const seconds = ((Date.now() - started) / 1000).toFixed(1);

	console.log(`\nDone in ${seconds}s`);
	if (!summary) {
		console.log("No summary was produced (the audit stopped early).");
		return;
	}

	console.log(`  items_base             ${summary.totalItemsBase}`);
	console.log(`  catalog_items          ${summary.totalCatalogItems}`);
	console.log(`  missing catalog entry  ${summary.missingCatalog}`);
	console.log(`  orphaned catalog rows  ${summary.orphanedCatalog}`);
	console.log(`  duplicate classnames   ${summary.duplicates}`);
	console.log(
		`  missing icons          ${summary.missingIcon} (repaired ${summary.iconsRepaired}, still missing ${summary.iconsStillMissing})`,
	);
	console.log(
		`  missing nitro bundles  ${summary.missingNitro} (repaired ${summary.nitrosRepaired}, still missing ${summary.nitrosStillMissing})`,
	);
	console.log(
		`  catalog SQL            ${summary.catalogSqlCount} stmt, applied ${summary.catalogSqlApplied}`,
	);
	console.log(`  furnidata added        ${summary.furniDataAdded}`);
	console.log(
		`  not restorable         ${summary.unrepairableCount} (no icon/bundle on any compared source)`,
	);

	if (report.missingFromSources.length > 0) {
		const bySource = new Map<string, number>();
		for (const entry of report.missingFromSources)
			bySource.set(entry.sourceName, (bySource.get(entry.sourceName) ?? 0) + 1);
		console.log(
			`\n  on compared sources but not in this hotel: ${report.missingFromSources.length}`,
		);
		for (const [name, count] of bySource)
			console.log(`    ${name.padEnd(24)} ${count}`);
	}

	const errors = report.issues.filter((issue) => issue.severity === "error");
	if (errors.length > 0) {
		console.log(`\n  unresolved errors: ${errors.length}`);
		for (const issue of errors.slice(0, 20))
			console.log(`    ${issue.classname ?? issue.type}: ${issue.message}`);
		if (errors.length > 20) console.log(`    … and ${errors.length - 20} more`);
	}
}

async function main(): Promise<number> {
	const flags = parseCatalogSyncArgs(process.argv.slice(2));
	try {
		if (flags.listSources) return await commandListSources(flags);
		return await commandRun(flags);
	} catch (error) {
		console.error(
			`Catalog sync failed: ${error instanceof Error ? error.message : String(error)}`,
		);
		return 1;
	}
}

if (isEntryPoint()) {
	main().then(
		(code) => {
			process.exit(code);
		},
		(error) => {
			console.error(error);
			process.exit(1);
		},
	);
}