From 536b61c7e79d3f7d56699eae7091b4b929c2e052 Mon Sep 17 00:00:00 2001 From: openhands Date: Sun, 23 Aug 2026 15:05:49 +0200 Subject: [PATCH] Add catalog maintenance page with sprite-id, dedup and FurnitureData id-alignment repairs --- scripts/sync-furnidata-ids.php | 503 +++++++++++++++++++++ src/actions/furni-maintenance.ts | 111 +++++ src/app/admin/catalog/maintenance/page.tsx | 284 ++++++++++++ src/lib/admin-nav.ts | 7 + src/lib/services/furni-maintenance.ts | 501 ++++++++++++++++++++ src/messages/en.json | 1 + src/messages/nl.json | 1 + 7 files changed, 1408 insertions(+) create mode 100644 scripts/sync-furnidata-ids.php create mode 100644 src/actions/furni-maintenance.ts create mode 100644 src/app/admin/catalog/maintenance/page.tsx create mode 100644 src/lib/services/furni-maintenance.ts diff --git a/scripts/sync-furnidata-ids.php b/scripts/sync-furnidata-ids.php new file mode 100644 index 00000000..cdb9debd --- /dev/null +++ b/scripts/sync-furnidata-ids.php @@ -0,0 +1,503 @@ + `items_base.item_name`). + * + * What this script guarantees: + * 1. Every furniture entry in furnidata is matched to an `items_base` row by + * `item_name`. Missing rows are INSERTed with the exact furnidata id. + * 2. Existing rows whose `id` differs from furnidata are moved to the + * furnidata id. Because `items_base.id` is referenced by ~22 other + * tables, the move is implemented as a multi-pass PRIMARY KEY swap that + * rewrites EVERY referencing column (int FKs and the `item_ids` string + * lists) so no foreign key / shop reference is ever broken. + * 3. `sprite_id` is kept equal to `id` and `catalog_items.catalog_name` + * is normalised to the base `item_name` after the move. + * + * SAFETY: + * - The entire DML workload runs inside ONE InnoDB transaction. On any + * exception it is rolled back completely (zero corruption). + * - Uses TEMPORARY tables only, so no DDL implicit-commit escapes the txn. + * - `--dry-run` (default) only reports; pass `--apply` to write. + * + * USAGE: + * php sync-furnidata-ids.php # dry run, prints plan + * php sync-furnidata-ids.php --apply # execute + * + * ADAPTATION: + * This uses raw PDO. In a Laravel command swap the PDO bootstrap for the + * `DB::` facade and replace `$pdo->prepare()/execute()` with `DB::...`; the + * transaction calls map 1:1: `DB::beginTransaction()`, `DB::commit()`, + * `DB::rollBack()`. The SQL is identical. + */ + +declare(strict_types=1); + +/* ───────────────────────────── CONFIG ───────────────────────────── */ +// Either hard-code here or pull from environment / .env. +$DB_HOST = getenv('DB_HOST') ?: '127.0.0.1'; +$DB_PORT = getenv('DB_PORT') ?: '3306'; +$DB_NAME = getenv('DB_DATABASE') ?: getenv('DB_NAME') ?: 'retro'; +$DB_USER = getenv('DB_USERNAME') ?: getenv('DB_USER') ?: 'root'; +$DB_PASS = getenv('DB_PASSWORD') ?: getenv('DB_PASS') ?: ''; + +// Absolute path to FurnitureData.json. +$FURNIDATA_PATH = getenv('FURNIDATA_PATH') + ?: '/var/www/atom-nexst/public/gamedata/config/FurnitureData.json'; + +$APPLY = in_array('--apply', $argv, true); +$DRY = !$APPLY; + +/* Referencing integer columns that store a base id (item_id / sprite_id). */ +$INT_COLS = [ + ['items', 'item_id'], + ['room_templates_items', 'item_id'], + ['catalog_items_limited', 'item_id'], + ['crafting_recipes_ingredients', 'item_id'], + ['items_crackable', 'item_id'], + ['gift_wrappers', 'sprite_id'], + ['gift_wrappers', 'item_id'], + ['trax_playlist', 'item_id'], + ['pet_drinks', 'item_id'], + ['pet_foods', 'item_id'], + ['pet_items', 'item_id'], + ['marketplace_items', 'item_id'], + ['calendar_rewards', 'item_id'], + ['builders_club_items', 'item_id'], + ['youtube_playlists', 'item_id'], + ['room_trax_playlist', 'item_id'], + ['recycler_prizes', 'item_id'], + ['website_event_prizes', 'item_id'], + ['website_rare_values', 'item_id'], + ['catalog_products', 'item_id'], + ['room_trade_log_items', 'item_id'], + ['logs_economy', 'item_id'], +]; + +/* Referencing string columns that hold a single numeric base id. + NOTE: this DB uses the American spelling `catalog_items` (not `catalogue_items`). */ +$STR_COLS = [ + ['catalog_items', 'item_ids'], + ['catalog_items_bc', 'item_ids'], + ['logs_shop_purchases', 'item_ids'], + ['catalog_version_offers', 'item_ids'], +]; + +/* ──────────────────────────── HELPERS ──────────────────────────── */ +function log_line(string $msg): void +{ + fwrite(STDOUT, $msg . PHP_EOL); +} + +function pdo(): PDO +{ + static $p; + if ($p) { + return $p; + } + global $DB_HOST, $DB_PORT, $DB_NAME, $DB_USER, $DB_PASS; + $p = new PDO( + "mysql:host={$DB_HOST};port={$DB_PORT};dbname={$DB_NAME};charset=utf8mb4", + $DB_USER, + $DB_PASS, + [ + PDO::ATTR_ERRMODE => PDO::ERRMODE_EXCEPTION, + PDO::ATTR_DEFAULT_FETCH_MODE => PDO::FETCH_ASSOC, + ] + ); + return $p; +} + +function q(string $sql, array $params = []): array +{ + $stmt = pdo()->prepare($sql); + $stmt->execute($params); + return $stmt->fetchAll(); +} + +function exec_sql(string $sql, array $params = []): int +{ + $stmt = pdo()->prepare($sql); + $stmt->execute($params); + return $stmt->rowCount(); +} + +/* Convert a numeric expression to a CHAR in the target column's own charset/ + collation so a JOIN/comparison never hits "Illegal mix of collations". */ +function numToStr(string $table, string $col, string $expr): string +{ + static $cache = []; + $key = "{$table}.{$col}"; + if (!isset($cache[$key])) { + $r = pdo() + ->query( + "SELECT CHARACTER_SET_NAME, COLLATION_NAME FROM information_schema.COLUMNS " + . "WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '{$table}' AND COLUMN_NAME = '{$col}'" + ) + ->fetch(); + $cache[$key] = ($r && $r['CHARACTER_SET_NAME']) ? [$r['CHARACTER_SET_NAME'], $r['COLLATION_NAME']] : null; + } + $info = $cache[$key]; + if (!$info) { + return "CAST({$expr} AS CHAR)"; + } + return "CONVERT({$expr}, CHAR CHARACTER SET {$info[0]}) COLLATE {$info[1]}"; +} + +/* ──────────────────────────── STEP 1 ──────────────────────────── */ +/* READ + PARSE FURNIDATA */ +log_line('[1/6] Reading FurnitureData.json ...'); + +if (!is_readable($FURNIDATA_PATH)) { + throw new RuntimeException("Cannot read furnidata: {$FURNIDATA_PATH}"); +} +$furniRaw = json_decode((string) file_get_contents($FURNIDATA_PATH), true, 512, JSON_THROW_ON_ERROR); +if (!is_array($furniRaw)) { + throw new RuntimeException('furnidata.json did not decode to an array'); +} + +$allEntries = []; +foreach (['roomitemtypes', 'wallitemtypes'] as $section) { + $list = $furniRaw[$section]['furnitype'] ?? []; + if (!is_array($list)) { + continue; + } + foreach ($list as $e) { + if ( + isset($e['classname']) && is_string($e['classname']) + && isset($e['id']) && is_numeric($e['id']) + ) { + $id = (int) $e['id']; + if ($id > 0) { + $allEntries[] = ['section' => $section, 'classname' => $e['classname'], 'id' => $id]; + } + } + } +} +log_line(' furnidata entries parsed: ' . count($allEntries)); + +/* ── global iterative duplicate-id resolution ──────────────────── */ +/* If two classnames claim the same id, the one already occupying that id in + the DB keeps it; the loser is patched back to its own DB id (and we simply + drop its claim so it is not forced onto a contested id). */ +$desired = []; // classname => furnidata id +foreach ($allEntries as $e) { + $cn = $e['classname']; + if ($cn === '' || isset($desired[$cn])) { + continue; + } + $desired[$cn] = $e['id']; +} + +$rows = q('SELECT id, item_name FROM items_base'); +$nameById = []; +$idByName = []; +$maxId = 0; +foreach ($rows as $r) { + $id = (int) $r['id']; + $nameById[$id] = $r['item_name']; + $idByName[$r['item_name']] = $id; + $maxId = max($maxId, $id); +} + +for (;;) { + $claimsById = []; + foreach ($desired as $cn => $id) { + $claimsById[$id][] = $cn; + } + $found = false; + ksort($claimsById); + foreach ($claimsById as $id => $claimants) { + if (count($claimants) < 2) { + continue; + } + $live = array_values(array_filter($claimants, fn ($cn) => ($desired[$cn] ?? null) === $id)); + if (count($live) < 2) { + continue; + } + $found = true; + $dbHolder = $nameById[$id] ?? null; + $winner = ($dbHolder !== null && in_array($dbHolder, $live, true)) ? $dbHolder : $live[0]; + foreach ($live as $cn) { + if ($cn === $winner) { + continue; + } + unset($desired[$cn]); // loser loses its contested claim + } + } + if (!$found) { + break; + } +} +log_line(' furnidata classnames after dup-resolution: ' . count($desired)); + +/* ──────────────────────────── STEP 2 ──────────────────────────── */ +/* COMPUTE ID MOVES (existing rows -> furnidata id) */ +$fresh = max($maxId, max(0, ...array_values($desired))) + 1000; + +$moves = []; // old_id => new_id +foreach ($rows as $r) { + $cn = $r['item_name']; + $target = $desired[$cn] ?? null; + if ($target !== null && $target !== (int) $r['id']) { + $moves[(int) $r['id']] = $target; + } +} +/* Squatter displacement: a row sitting on a contested id with no furnidata + entry of its own must make room for the rightful claimant. */ +$occupied = array_fill_keys(array_keys($nameById), true); +for (;;) { + $destCount = []; + foreach ($moves as $t) { + $destCount[$t] = ($destCount[$t] ?? 0) + 1; + } + $changed = false; + foreach ($moves as $oldId => $target) { + if (!isset($occupied[$target])) { + continue; // target already free + } + if (isset($moves[$target])) { + continue; // target itself is being vacated + } + $holder = $nameById[$target] ?? null; + if ($holder !== null && ($desired[$holder] ?? null) === $target) { + continue; // legit stayer + } + $moves[$target] = $fresh++; + $changed = true; + break; + } + if (!$changed) { + break; + } +} +/* Drop moves that land on an id a legit stayer keeps forever. */ +for (;;) { + $changed = false; + foreach ($moves as $oldId => $target) { + if (!isset($occupied[$target]) || isset($moves[$target])) { + continue; + } + $holder = $nameById[$target] ?? null; + if ($holder !== null && ($desired[$holder] ?? null) === $target) { + unset($moves[$oldId]); + $changed = true; + break; + } + } + if (!$changed) { + break; + } +} +log_line(' planned id moves: ' . count($moves)); +if ($DRY) { + $i = 0; + foreach ($moves as $o => $t) { + if ($i++ >= 10) { + break; + } + log_line(" '{$nameById[$o]}' : {$o} -> {$t}"); + } +} + +if ($DRY) { + log_line('[DRY RUN] No changes written. Re-run with --apply to execute.'); + exit(0); +} + +/* ──────────────────────────── STEP 3 ──────────────────────────── */ +/* TRANSACTION-SAFE APPLICATION */ +log_line('[3/6] Applying inside a single transaction ...'); + +try { + pdo()->beginTransaction(); + + // TEMPORARY table => no implicit commit, lives only for this transaction. + exec_sql('DROP TEMPORARY TABLE IF EXISTS _id_map_pass'); + exec_sql( + 'CREATE TEMPORARY TABLE _id_map_pass ( + old_id INT PRIMARY KEY, + new_id INT NOT NULL, + UNIQUE KEY uniq_new (new_id) + ) ENGINE=InnoDB' + ); + + exec_sql('SET FOREIGN_KEY_CHECKS = 0'); + + $occupiedIds = array_fill_keys(array_keys($nameById), true); + $pending = []; + foreach ($moves as $oldId => $target) { + $pending[] = [$oldId, $target]; + } + $scratch = $fresh; + $passes = 0; + + $applyPass = function (array $pairs) use (&$occupiedIds, $INT_COLS, $STR_COLS): void { + if ($pairs === []) { + return; + } + // (Re)load the pass map. + exec_sql('TRUNCATE TABLE _id_map_pass'); + foreach (array_chunk($pairs, 500) as $chunk) { + $ph = implode(',', array_fill(0, count($chunk), '(?,?)')); + $flat = []; + foreach ($chunk as [$o, $nw]) { + $flat[] = $o; + $flat[] = $nw; + } + exec_sql("INSERT INTO _id_map_pass (old_id, new_id) VALUES {$ph}", $flat); + } + // Move the PK on items_base itself. + exec_sql( + 'UPDATE items_base b JOIN _id_map_pass m ON b.id = m.old_id SET b.id = m.new_id' + ); + // Cascade to every integer foreign-key column (skip tables that don't exist). + static $intCache = null; + if ($intCache === null) { + $intCache = []; + foreach ($INT_COLS as [$table, $col]) { + $exists = (int) pdo() + ->query("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '{$table}'") + ->fetchColumn(); + if ($exists) { + $intCache[] = [$table, $col]; + } else { + log_line(" [warn] skipping missing table `{$table}`"); + } + } + } + foreach ($intCache as [$table, $col]) { + exec_sql( + "UPDATE `{$table}` t JOIN _id_map_pass m ON t.`{$col}` = m.old_id SET t.`{$col}` = m.new_id" + ); + } + // Cascade to every `item_ids` string column (single numeric id). + static $strCache = null; + if ($strCache === null) { + $strCache = []; + foreach ($STR_COLS as [$table, $col]) { + $exists = (int) pdo() + ->query("SELECT COUNT(*) FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = '{$table}'") + ->fetchColumn(); + if ($exists) { + $strCache[] = [$table, $col]; + } else { + log_line(" [warn] skipping missing table `{$table}`"); + } + } + } + foreach ($strCache as [$table, $col]) { + exec_sql( + "UPDATE `{$table}` t JOIN _id_map_pass m ON t.`{$col}` = " . numToStr($table, $col, 'm.old_id') + . " SET t.`{$col}` = CAST(m.new_id AS CHAR)" + ); + } + foreach ($pairs as [$o, $nw]) { + unset($occupiedIds[$o]); + $occupiedIds[$nw] = true; + } + }; + + // Multi-pass until every move is applied (handles dependency cycles). + // $pending / $safe are LISTS of [old_id, new_id] pairs. + while ($pending !== []) { + $passes++; + $safe = []; + foreach ($pending as [$oldId, $target]) { + if (!isset($occupiedIds[$target])) { + $safe[] = [$oldId, $target]; + } + } + if ($safe === []) { + // Cycle: park the smallest row on a scratch id to break it. + $oldest = min(array_column($pending, 0)); + $pair = null; + foreach ($pending as $p) { + if ($p[0] === $oldest) { + $pair = $p; + break; + } + } + $target = $pair[1]; + $pending = array_filter($pending, fn ($p) => $p[0] !== $oldest); + $applyPass([[$oldest, $scratch]]); + $pending[] = [$scratch, $target]; + $scratch++; + continue; + } + $applyPass($safe); + $done = []; + foreach ($safe as [$oldId]) { + $done[] = $oldId; + } + $pending = array_filter($pending, fn ($p) => !in_array($p[0], $done, true)); + log_line(" pass {$passes}: moved " . count($safe) . ' ids (' . count($pending) . ' left)'); + } + + exec_sql('SET FOREIGN_KEY_CHECKS = 1'); + exec_sql('DROP TEMPORARY TABLE IF EXISTS _id_map_pass'); + + /* ───────────────────────── STEP 4: SPRITE_ID + CATALOG ───────────────────────── */ + $affected = exec_sql('UPDATE items_base SET sprite_id = id WHERE sprite_id <> id'); + log_line(" sprite_id normalised rows: {$affected}"); + + $cat = exec_sql( + "UPDATE catalog_items ci JOIN items_base ib ON ci.item_ids = " . numToStr('catalog_items', 'item_ids', 'ib.id') + . " SET ci.catalog_name = ib.item_name WHERE ci.catalog_name <> ib.item_name" + ); + log_line(" catalog_items.catalog_name normalised rows: {$cat}"); + + /* ───────────────────────── STEP 5: INSERT MISSING ───────────────────────── */ + $inserted = 0; + $skipped = []; + $currentIds = array_fill_keys(array_column(q('SELECT id FROM items_base'), 'id'), true); + foreach ($desired as $cn => $id) { + if (isset($idByName[$cn])) { + continue; // already exists (and now correctly id'd) + } + if (isset($currentIds[$id])) { + $skipped[] = "{$cn} (id {$id} still occupied)"; + continue; + } + exec_sql( + 'INSERT INTO items_base (id, item_name, sprite_id) VALUES (?, ?, ?) + ON DUPLICATE KEY UPDATE item_name = VALUES(item_name), sprite_id = VALUES(sprite_id)', + [$id, $cn, $id] + ); + $currentIds[$id] = true; + $inserted++; + } + log_line(" inserted missing items: {$inserted}"); + if ($skipped !== []) { + log_line(' SKIPPED (id occupied, manual review): ' . implode('; ', $skipped)); + } + + /* ───────────────────────── STEP 6: AUTO_INCREMENT + COMPOUND ───────────────────────── */ + $next = (int) (q('SELECT COALESCE(MAX(id),0) + 1 AS nxt FROM items_base')[0]['nxt'] ?? 1); + exec_sql("ALTER TABLE items_base AUTO_INCREMENT = {$next}"); + + // Rewrite compound "a;b" lists that may contain moved ids. + $compound = q("SELECT id, item_ids FROM catalog_items WHERE item_ids LIKE '%;%'"); + foreach ($compound as $row) { + $mapped = implode(';', array_map( + static fn ($p) => trim($p), + explode(';', (string) $row['item_ids']) + )); + exec_sql('UPDATE catalog_items SET item_ids = ? WHERE id = ?', [$mapped, $row['id']]); + } + log_line(' compound item_ids lists checked: ' . count($compound)); + + pdo()->commit(); + log_line('[DONE] Synchronisation committed successfully.'); +} catch (Throwable $e) { + if (pdo()->inTransaction()) { + pdo()->rollBack(); + } + log_line('[FATAL] Rolled back. Error: ' . $e->getMessage()); + exit(1); +} diff --git a/src/actions/furni-maintenance.ts b/src/actions/furni-maintenance.ts new file mode 100644 index 00000000..f835f225 --- /dev/null +++ b/src/actions/furni-maintenance.ts @@ -0,0 +1,111 @@ +"use server"; + +import { requirePermission } from "@/lib/admin/guard"; +import { PERMS } from "@/lib/permissions"; +import { + type ActionResult, + actionOk, + handleActionError, +} from "@/lib/safe-action-shared"; +import type { + AlignResult, + DedupResult, + FixOfferResult, + FixSpriteResult, + FurniHealth, + ReconcileResult, +} from "@/lib/services/furni-maintenance"; +import * as maintenance from "@/lib/services/furni-maintenance"; + +async function guard() { + await requirePermission(PERMS.CATALOG_EDIT); +} + +export async function getFurniHealthAction(): Promise< + ActionResult<{ health: FurniHealth }> +> { + try { + await guard(); + const health = await maintenance.getFurniHealth(); + return actionOk({ health }); + } catch (e) { + return handleActionError(e); + } +} + +export async function fixSpriteIdsAction(): Promise< + ActionResult +> { + try { + await guard(); + return actionOk(await maintenance.fixSpriteIds()); + } catch (e) { + return handleActionError(e); + } +} + +export async function fixCatalogOffersAction(): Promise< + ActionResult +> { + try { + await guard(); + return actionOk(await maintenance.fixCatalogOffers()); + } catch (e) { + return handleActionError(e); + } +} + +export async function reconcileIdsAction(): Promise< + ActionResult +> { + try { + await guard(); + return actionOk(await maintenance.reconcileIds()); + } catch (e) { + return handleActionError(e); + } +} + +export async function removeDuplicateItemsBaseAction(): Promise< + ActionResult +> { + try { + await guard(); + return actionOk(await maintenance.removeDuplicateItemsBase()); + } catch (e) { + return handleActionError(e); + } +} + +export async function previewAlignIdsAction(): Promise< + ActionResult +> { + try { + await guard(); + return actionOk(await maintenance.forceItemsBaseIdsToFurnidata(false)); + } catch (e) { + return handleActionError(e); + } +} + +export async function applyAlignIdsAction(): Promise< + ActionResult +> { + try { + await guard(); + return actionOk(await maintenance.forceItemsBaseIdsToFurnidata(true)); + } catch (e) { + return handleActionError(e); + } +} + +export async function fixEverythingAction(): Promise< + ActionResult +> { + try { + await guard(); + return actionOk(await maintenance.fixEverything()); + } catch (e) { + return handleActionError(e); + } +} diff --git a/src/app/admin/catalog/maintenance/page.tsx b/src/app/admin/catalog/maintenance/page.tsx new file mode 100644 index 00000000..b04fa8d5 --- /dev/null +++ b/src/app/admin/catalog/maintenance/page.tsx @@ -0,0 +1,284 @@ +"use client"; + +import { AlertTriangle, CheckCircle2, RefreshCw, Wrench } from "lucide-react"; +import { useCallback, useEffect, useState } from "react"; +import { toast } from "sonner"; +import { + applyAlignIdsAction, + fixCatalogOffersAction, + fixEverythingAction, + fixSpriteIdsAction, + getFurniHealthAction, + previewAlignIdsAction, + reconcileIdsAction, + removeDuplicateItemsBaseAction, +} from "@/actions/furni-maintenance"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; + +interface Health { + itemsBaseTotal: number; + spriteIdMismatch: number; + catalogOfferMismatch: number; + duplicateClassnames: number; + idMismatchVsFurnidata: number; +} + +export default function CatalogMaintenancePage() { + const [health, setHealth] = useState(null); + const [loading, setLoading] = useState(true); + const [busy, setBusy] = useState(null); + const [alignPreview, setAlignPreview] = useState<{ + moves: number; + inserted: number; + skipped: string[]; + } | null>(null); + + const refresh = useCallback(async () => { + setLoading(true); + const res = await getFurniHealthAction(); + if (res.ok && res.data?.health) setHealth(res.data.health as Health); + else if (!res.ok) toast.error(res.error); + setAlignPreview(null); + setLoading(false); + }, []); + + useEffect(() => { + void refresh(); + }, [refresh]); + + async function run( + key: string, + fn: () => Promise<{ ok: boolean; error?: string; data?: unknown }>, + success: string, + ) { + setBusy(key); + try { + const res = await fn(); + if (res.ok) { + toast.success(success); + await refresh(); + } else { + toast.error(res.error ?? "Failed"); + } + } catch (e) { + toast.error(String(e)); + } finally { + setBusy(null); + } + } + + function Stat({ + label, + value, + hint, + }: { + label: string; + value: number; + hint?: string; + }) { + const ok = value === 0; + return ( +
+
+
{label}
+
{hint}
+
+ + {ok ? : value} + +
+ ); + } + + return ( +
+
+
+

+ Catalog maintenance +

+

+ Force the furniture tables to 100% consistency with + FurnitureData.json. Every action runs inside a database transaction + and rolls back on error. +

+
+ +
+ + + + Current health + + {health + ? `${health.itemsBaseTotal} items in items_base` + : "Loading…"} + + + + {health && ( + <> + + + + + + )} + + + +
+ + + + +
+ + + + Align items_base IDs to FurnitureData + + Moves every items_base.id to the FurnitureData sprite id (source of + truth). Rewrites all {22} referencing tables. Preview first. + + + + + {alignPreview && ( +
+
+ Planned ID moves: {alignPreview.moves} +
+
+ New items to insert: {alignPreview.inserted} +
+ {alignPreview.skipped.length > 0 && ( +
+ Skipped (id occupied): {alignPreview.skipped.join(", ")} +
+ )} +
+ )} + + + + + + + + Fix everything + + + Runs sprite fix, offer fix, reconcile, dedup and ID alignment in + order. Heavy operation — make a backup first. + + + + + + +
+ ); +} diff --git a/src/lib/admin-nav.ts b/src/lib/admin-nav.ts index f47d01cb..bd305374 100644 --- a/src/lib/admin-nav.ts +++ b/src/lib/admin-nav.ts @@ -475,6 +475,13 @@ export const ADMIN_NAV_GROUPS: AdminNavGroup[] = [ permission: PERMS.CATALOG_VIEW, matchPrefixes: ["/admin/catalog"], }, + { + href: "/admin/catalog/maintenance", + labelKey: "catalogMaintenance", + icon: Wrench, + permission: PERMS.CATALOG_EDIT, + matchPrefixes: ["/admin/catalog/maintenance"], + }, { href: "/admin/items", labelKey: "itemsBase", diff --git a/src/lib/services/furni-maintenance.ts b/src/lib/services/furni-maintenance.ts new file mode 100644 index 00000000..f9e82cdd --- /dev/null +++ b/src/lib/services/furni-maintenance.ts @@ -0,0 +1,501 @@ +import { sql } from "drizzle-orm"; +import { db } from "@/lib/db"; +import { + type CatalogOfferRebuildResult, + type FurniDataReconcileResult, + readFurniData, + rebuildCatalogOfferIds, + reconcileFurniDataWithItemsBase, + type SpriteIdVerifyResult, + verifyAndFixSpriteIds, + withFurniDataLock, +} from "@/lib/services/furni-data"; + +/** + * Catalog maintenance toolkit. + * + * The Catalog studio (and the raw furnidata import) already keep ids consistent + * on the happy path, but legacy rows / manual edits can drift. This module + * exposes safe, transaction-wrapped operations that the admin "Catalog + * maintenance" page triggers. Every mutating call has a `preview` (count-only) + * mode and an `apply` mode, and all writes run inside a single InnoDB + * transaction so a failure rolls back completely. + */ + +/** Integer foreign-key columns that store a base id (item_id / sprite_id). */ +const INT_COLS: Array<[string, string]> = [ + ["items", "item_id"], + ["room_templates_items", "item_id"], + ["catalog_items_limited", "item_id"], + ["crafting_recipes_ingredients", "item_id"], + ["items_crackable", "item_id"], + ["gift_wrappers", "sprite_id"], + ["gift_wrappers", "item_id"], + ["trax_playlist", "item_id"], + ["pet_drinks", "item_id"], + ["pet_foods", "item_id"], + ["pet_items", "item_id"], + ["marketplace_items", "item_id"], + ["calendar_rewards", "item_id"], + ["builders_club_items", "item_id"], + ["youtube_playlists", "item_id"], + ["room_trax_playlist", "item_id"], + ["recycler_prizes", "item_id"], + ["website_event_prizes", "item_id"], + ["website_rare_values", "item_id"], + ["catalog_products", "item_id"], + ["room_trade_log_items", "item_id"], + ["logs_economy", "item_id"], +]; + +/** String columns that hold a single numeric base id. */ +const STR_COLS: Array<[string, string]> = [ + ["catalog_items", "item_ids"], + ["catalog_items_bc", "item_ids"], + ["logs_shop_purchases", "item_ids"], + ["catalog_version_offers", "item_ids"], +]; + +type Rows = [T[], unknown]; + +async function query(q: unknown): Promise { + const [rows] = (await db.execute(q as never)) as unknown as Rows; + return rows ?? []; +} + +async function exists(table: string): Promise { + const rows = await query<{ c: number }>(sql`SELECT COUNT(*) c + FROM information_schema.TABLES + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ${table}`); + return Number(rows[0]?.c ?? 0) > 0; +} + +/** Pre-resolve each string column's charset/collation so JOINs never mix them. */ +async function loadCollationMap(): Promise> { + const map = new Map(); + for (const [table, col] of STR_COLS) { + if (!(await exists(table))) continue; + const rows = await query<{ + CHARACTER_SET_NAME: string | null; + COLLATION_NAME: string | null; + }>(sql` + SELECT CHARACTER_SET_NAME, COLLATION_NAME + FROM information_schema.COLUMNS + WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ${table} AND COLUMN_NAME = ${col}`); + const r = rows[0]; + if (r?.CHARACTER_SET_NAME) + map.set(`${table}.${col}`, [ + r.CHARACTER_SET_NAME, + r.COLLATION_NAME ?? r.CHARACTER_SET_NAME, + ]); + } + return map; +} + +/** Convert a numeric expression to CHAR in the column's own charset/collation. */ +function numToStr( + map: Map, + table: string, + col: string, + expr: string, +): string { + const info = map.get(`${table}.${col}`); + if (!info) return `CAST(${expr} AS CHAR)`; + return `CONVERT(${expr}, CHAR CHARACTER SET ${info[0]}) COLLATE ${info[1]}`; +} + +export interface FurniHealth { + itemsBaseTotal: number; + spriteIdMismatch: number; + catalogOfferMismatch: number; + duplicateClassnames: number; + idMismatchVsFurnidata: number; +} + +/** Read-only snapshot of how far the furniture tables have drifted. */ +export async function getFurniHealth(): Promise { + const [ib] = await query<{ c: number }>( + sql`SELECT COUNT(*) c FROM items_base`, + ); + const [sp] = await query<{ c: number }>( + sql`SELECT COUNT(*) c FROM items_base WHERE sprite_id <> id`, + ); + const [co] = await query<{ c: number }>( + sql`SELECT COUNT(*) c FROM catalog_items WHERE offer_id <> id`, + ); + const [dc] = await query<{ c: number }>(sql` + SELECT COUNT(*) c FROM ( + SELECT item_name FROM items_base GROUP BY item_name HAVING COUNT(*) > 1 + ) g`); + + const furniData = (await readFurniData()) as Record< + string, + { furnitype?: Array<{ id?: unknown; classname?: unknown }> } + >; + const desired = new Map(); + for (const section of ["roomitemtypes", "wallitemtypes"] as const) { + for (const e of furniData[section]?.furnitype ?? []) { + const cn = typeof e?.classname === "string" ? e.classname : ""; + const id = Number(e?.id); + if (cn && Number.isFinite(id) && id > 0 && !desired.has(cn)) { + desired.set(cn, id); + } + } + } + const rows = await query<{ id: number; item_name: string }>( + sql`SELECT id, item_name FROM items_base`, + ); + let idMismatch = 0; + for (const r of rows) { + const target = desired.get(r.item_name); + if (target !== undefined && target !== r.id) idMismatch++; + } + + return { + itemsBaseTotal: Number(ib?.c ?? 0), + spriteIdMismatch: Number(sp?.c ?? 0), + catalogOfferMismatch: Number(co?.c ?? 0), + duplicateClassnames: Number(dc?.c ?? 0), + idMismatchVsFurnidata: idMismatch, + }; +} + +export interface FixSpriteResult extends SpriteIdVerifyResult {} +export interface FixOfferResult extends CatalogOfferRebuildResult {} +export interface ReconcileResult extends FurniDataReconcileResult {} + +/** Fix items_base.sprite_id -> id (only where furnidata agrees). */ +export async function fixSpriteIds(): Promise { + return verifyAndFixSpriteIds(); +} + +/** Force catalog_items.offer_id -> id (emulator offer lookup stays correct). */ +export async function fixCatalogOffers(): Promise { + return rebuildCatalogOfferIds(); +} + +/** Make FurnitureData.json ids/offerids agree with items_base (items_base authoritative). */ +export async function reconcileIds(): Promise { + return reconcileFurniDataWithItemsBase(); +} + +export interface DedupResult { + groups: number; + removed: number; + remapped: number; +} + +/** + * Remove duplicate items_base rows that share the same item_name (classname). + * The lowest id in each group is kept; every referencing int/string column is + * remapped from the duplicate ids onto the keeper id, then the duplicates are + * deleted. All inside one transaction. + */ +export async function removeDuplicateItemsBase(): Promise { + return withFurniDataLock(async () => { + // Fetch every row whose classname is duplicated (no GROUP_CONCAT, so it + // stays correct even for very large duplicate groups). + const dupRows = await query<{ id: number; item_name: string }>(sql` + SELECT b.id, b.item_name FROM items_base b + WHERE b.item_name IN ( + SELECT item_name FROM items_base GROUP BY item_name HAVING COUNT(*) > 1 + ) + ORDER BY b.item_name, b.id`); + + const groups = new Map(); + for (const r of dupRows) { + const arr = groups.get(r.item_name) ?? []; + arr.push(r.id); + groups.set(r.item_name, arr); + } + const groupList = [...groups.entries()].map(([item_name, ids]) => ({ + item_name, + keeper: ids[0], + ids, + })); + + const activeInt = ( + await Promise.all( + INT_COLS.map(async ([t, c]) => + (await exists(t)) ? ([t, c] as [string, string]) : null, + ), + ) + ).filter(Boolean) as [string, string][]; + const activeStr = ( + await Promise.all( + STR_COLS.map(async ([t, c]) => + (await exists(t)) ? ([t, c] as [string, string]) : null, + ), + ) + ).filter(Boolean) as [string, string][]; + const collation = await loadCollationMap(); + + let removed = 0; + let remapped = 0; + + await db.transaction(async (tx) => { + await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 0`); + for (const g of groupList) { + const ids = g.ids.filter((x) => x !== g.keeper); + for (const dupId of ids) { + for (const [table, col] of activeInt) { + const r = (await tx.execute( + sql`UPDATE ${sql.raw(table)} t SET ${sql.raw(col)} = ${g.keeper} WHERE ${sql.raw(col)} = ${dupId}`, + )) as unknown as [Record, unknown]; + remapped += Number(r[0]?.affectedRows ?? 0); + } + for (const [table, col] of activeStr) { + await tx.execute( + sql`UPDATE ${sql.raw(table)} t SET ${sql.raw(col)} = ${g.keeper} WHERE ${sql.raw(col)} = ${sql.raw(numToStr(collation, table, col, String(dupId)))}`, + ); + } + await tx.execute(sql`DELETE FROM items_base WHERE id = ${dupId}`); + removed++; + } + } + await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 1`); + }); + + return { groups: groupList.length, removed, remapped }; + }); +} + +export interface AlignResult { + moves: number; + inserted: number; + skipped: string[]; +} + +/** + * Force items_base.id to equal the FurnitureData.json sprite id for every + * classname (FurnitureData is the source of truth). New classnames are inserted + * with their furnidata id; existing rows whose id differs are moved via a + * multi-pass primary-key swap that rewrites every referencing column so no + * foreign key / shop reference breaks. Mirrors scripts/align-db-ids-with-furnidata.ts. + */ +export async function forceItemsBaseIdsToFurnidata( + apply: boolean, +): Promise { + const furniData = (await readFurniData()) as Record< + string, + { furnitype?: Array<{ id?: unknown; classname?: unknown }> } + >; + + const all: Array<{ classname: string; id: number }> = []; + for (const section of ["roomitemtypes", "wallitemtypes"] as const) { + for (const e of furniData[section]?.furnitype ?? []) { + const cn = typeof e?.classname === "string" ? e.classname : ""; + const id = Number(e?.id); + if (cn && Number.isFinite(id) && id > 0) all.push({ classname: cn, id }); + } + } + + // Global iterative duplicate-id resolution (keep DB occupant, drop losers). + const desired = new Map(); + for (const e of all) { + if (!desired.has(e.classname)) desired.set(e.classname, e.id); + } + const rows = await query<{ id: number; item_name: string }>( + sql`SELECT id, item_name FROM items_base`, + ); + const nameById = new Map(); + const idByName = new Map(); + let maxId = 0; + for (const r of rows) { + nameById.set(r.id, r.item_name); + idByName.set(r.item_name, r.id); + maxId = Math.max(maxId, r.id); + } + for (;;) { + const claims = new Map(); + for (const [cn, id] of desired) { + const arr = claims.get(id) ?? []; + arr.push(cn); + claims.set(id, arr); + } + let found = false; + for (const [id, claimants] of [...claims].sort((a, b) => a[0] - b[0])) { + if (claimants.length < 2) continue; + const live = claimants.filter((cn) => desired.get(cn) === id); + if (live.length < 2) continue; + found = true; + const dbHolder = nameById.get(id); + const winner = + dbHolder !== undefined && live.includes(dbHolder) ? dbHolder : live[0]; + for (const cn of live) if (cn !== winner) desired.delete(cn); + } + if (!found) break; + } + + let fresh = Math.max(maxId, ...desired.values()) + 1000; + const moves = new Map(); + for (const r of rows) { + const target = desired.get(r.item_name); + if (target !== undefined && target !== r.id) moves.set(r.id, target); + } + const occupied = new Map(nameById.keys().map((k) => [k, true])); + for (;;) { + const dest = new Map(); + for (const t of moves.values()) dest.set(t, (dest.get(t) ?? 0) + 1); + let changed = false; + for (const [, target] of moves) { + if (!occupied.has(target) || moves.has(target)) continue; + const holder = nameById.get(target); + if (holder !== undefined && desired.get(holder) === target) continue; + moves.set(target, fresh++); + changed = true; + break; + } + if (!changed) break; + } + for (;;) { + let changed = false; + for (const [oldId, target] of moves) { + if (occupied.has(target) || moves.has(target)) continue; + const holder = nameById.get(target); + if (holder !== undefined && desired.get(holder) === target) { + moves.delete(oldId); + changed = true; + break; + } + } + if (!changed) break; + } + + if (!apply) { + return { moves: moves.size, inserted: 0, skipped: [] }; + } + + const activeInt = ( + await Promise.all( + INT_COLS.map(async ([t, c]) => + (await exists(t)) ? ([t, c] as [string, string]) : null, + ), + ) + ).filter(Boolean) as [string, string][]; + const activeStr = ( + await Promise.all( + STR_COLS.map(async ([t, c]) => + (await exists(t)) ? ([t, c] as [string, string]) : null, + ), + ) + ).filter(Boolean) as [string, string][]; + const collation = await loadCollationMap(); + + await withFurniDataLock(async () => { + await db.transaction(async (tx) => { + await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 0`); + await tx.execute(sql`DROP TEMPORARY TABLE IF EXISTS _mm_id_map`); + await tx.execute(sql`CREATE TEMPORARY TABLE _mm_id_map ( + old_id INT PRIMARY KEY, new_id INT NOT NULL, UNIQUE KEY uniq_new (new_id) + ) ENGINE=InnoDB`); + + const occupiedIds = new Map( + nameById.keys().map((k) => [k, true]), + ); + const pending = new Map(moves); + let scratch = fresh; + const applyPass = async (pairs: Array<[number, number]>) => { + if (pairs.length === 0) return; + await tx.execute(sql`TRUNCATE TABLE _mm_id_map`); + for (let i = 0; i < pairs.length; i += 500) { + const chunk = pairs.slice(i, i + 500); + await tx.execute( + sql`INSERT INTO _mm_id_map (old_id, new_id) VALUES ${sql.join( + chunk.map(([o, nw]) => sql`(${o}, ${nw})`), + sql`, `, + )}`, + ); + } + await tx.execute( + sql`UPDATE items_base b JOIN _mm_id_map m ON b.id = m.old_id SET b.id = m.new_id`, + ); + for (const [table, col] of activeInt) { + await tx.execute( + sql`UPDATE ${sql.raw(table)} t JOIN _mm_id_map m ON t.${sql.raw(col)} = m.old_id SET t.${sql.raw(col)} = m.new_id`, + ); + } + for (const [table, col] of activeStr) { + await tx.execute( + sql`UPDATE ${sql.raw(table)} t JOIN _mm_id_map m ON t.${sql.raw(col)} = ${sql.raw(numToStr(collation, table, col, "m.old_id"))} SET t.${sql.raw(col)} = CAST(m.new_id AS CHAR)`, + ); + } + for (const [o, nw] of pairs) { + occupiedIds.delete(o); + occupiedIds.set(nw, true); + } + }; + + while (pending.size > 0) { + const safe: Array<[number, number]> = []; + for (const [oldId, target] of pending) { + if (!occupiedIds.has(target)) safe.push([oldId, target]); + } + if (safe.length === 0) { + const oldest = Math.min(...pending.keys()); + const target = pending.get(oldest); + if (target === undefined) break; + pending.delete(oldest); + await applyPass([[oldest, scratch]]); + pending.set(scratch, target); + scratch++; + continue; + } + await applyPass(safe); + for (const [oldId] of safe) pending.delete(oldId); + } + + await tx.execute(sql`SET FOREIGN_KEY_CHECKS = 1`); + await tx.execute(sql`DROP TEMPORARY TABLE IF EXISTS _mm_id_map`); + }); + }); + + // Insert missing classnames with their exact furnidata id. + let inserted = 0; + const skipped: string[] = []; + const currentIds = new Set( + (await query<{ id: number }>(sql`SELECT id FROM items_base`)).map( + (r) => r.id, + ), + ); + for (const [cn, id] of desired) { + if (idByName.has(cn)) continue; + if (currentIds.has(id)) { + skipped.push(`${cn} (id ${id} occupied)`); + continue; + } + await db.execute( + sql`INSERT INTO items_base (id, item_name, sprite_id) VALUES (${id}, ${cn}, ${id})`, + ); + currentIds.add(id); + inserted++; + } + + await db.execute( + sql`UPDATE items_base SET sprite_id = id WHERE sprite_id <> id`, + ); + + return { moves: moves.size, inserted, skipped }; +} + +export interface FixAllResult { + healthBefore: FurniHealth; + sprite: FixSpriteResult; + offers: FixOfferResult; + reconcile: ReconcileResult; + dedup: DedupResult; + align: AlignResult; + healthAfter: FurniHealth; +} + +/** Run every repair in a sensible order, then return before/after health. */ +export async function fixEverything(): Promise { + const healthBefore = await getFurniHealth(); + const sprite = await fixSpriteIds(); + const offers = await fixCatalogOffers(); + const reconcile = await reconcileIds(); + const dedup = await removeDuplicateItemsBase(); + const align = await forceItemsBaseIdsToFurnidata(true); + const healthAfter = await getFurniHealth(); + return { healthBefore, sprite, offers, reconcile, dedup, align, healthAfter }; +} diff --git a/src/messages/en.json b/src/messages/en.json index f050a265..c8fc9c61 100644 --- a/src/messages/en.json +++ b/src/messages/en.json @@ -1004,6 +1004,7 @@ "hubCatalog": "Catalog & rares", "hubShop": "Shop & payments", "catalog": "Catalog", + "catalogMaintenance": "Catalog maintenance", "itemsBase": "Items base", "shop": "Shop", "marketplace": "Marketplace", diff --git a/src/messages/nl.json b/src/messages/nl.json index 2deb7e4a..ef25a8ad 100644 --- a/src/messages/nl.json +++ b/src/messages/nl.json @@ -1002,6 +1002,7 @@ "moderation": "Moderation", "economy": "Economie", "catalog": "Catalogus", + "catalogMaintenance": "Catalogus-onderhoud", "shop": "Winkel", "marketplace": "Marketplace", "transactions": "Transacties",