refactor: universal logger and remove N+1 catalog queries

- Replace all 39 console.* calls with universal logger (pino server, console browser) - Set server-only marker on import-badge.ts - Disable DB query logging in production - Add bulkInsertCatalogItems: one transaction instead of N per-item inserts - Add reserveCatalogItemIds helper for bulk ID allocation - Optimize translateCatalogItems: single inArray batch query instead of N per-item selects - Test mock updated for logger.debug - biome clean, tsc clean, 3569 tests passing
2026-10-17 14:29:14 +02:00
1 parent 6895eaab49
commit 59a45d2165
@@ -24,7 +24,10 @@ import { CatalogItems, db, ItemsBase, queryRows } from "@/lib/db";
 import { PERMS } from "@/lib/permissions";
 import { logAudit } from "@/lib/services/audit";
 import { withCatalogExport } from "@/lib/services/catalog-git-queue";
-import { allocateCatalogItemId } from "@/lib/services/furni-import";
+import {
+	allocateCatalogItemId,
+	reserveCatalogItemIds,
+} from "@/lib/services/furni-import";
 import { logStaffActivity } from "@/lib/services/staff-activity";
 import { translateItemsSchema } from "@/lib/validators/catalog";

@@ -133,6 +136,138 @@ export async function createCatalogItem(
 	});
 }

+/**
+ * Bulk insert catalog items in ONE transaction instead of N per-item
+ * transactions + N INSERT queries. Same result as calling insertCatalogItemRow
+ * per row, but far faster for large batches. Side effects (audit, revalidate,
+ * RCON refresh) are left to the caller, which wraps this in withCatalogExport.
+ */
+export async function bulkInsertCatalogItems(data: {
+	pageId: number;
+	staffId: number;
+	rows: Array<{
+		baseId: number;
+		credits?: number;
+		points?: number;
+		pointsType?: number;
+		amount?: number;
+	}>;
+}): Promise<{ created: number; failed: number }> {
+	const { pageId, staffId, rows } = data;
+	if (rows.length === 0) return { created: 0, failed: 0 };
+
+	const baseIds = [...new Set(rows.map((r) => r.baseId))];
+	const bases = await db
+		.select({
+			id: ItemsBase.id,
+			publicName: ItemsBase.publicName,
+			itemName: ItemsBase.itemName,
+		})
+		.from(ItemsBase)
+		.where(inArray(ItemsBase.id, baseIds));
+	const baseMap = new Map(bases.map((b) => [b.id, b]));
+
+	const ids = await reserveCatalogItemIds(rows.length);
+
+	const maxOrder = await queryRows<{ maxOrder: number }>(sql`
+		SELECT COALESCE(MAX(order_number), 0) AS maxOrder FROM catalog_items WHERE page_id = ${String(pageId)}
+	`);
+	let nextOrder = Number(maxOrder[0]?.maxOrder ?? 0) + 1;
+
+	const values: Array<{
+		id: number;
+		pageId: string;
+		itemIds: string;
+		catalogName: string;
+		costCredits: number;
+		costPoints: number;
+		pointsType: number;
+		amount: number;
+		orderNumber: number;
+		offerId: number;
+		songId: number;
+		limitedSells: number;
+		limitedStack: number;
+		extradata: string;
+		haveOffer: string;
+		clubOnly: string;
+	}> = [];
+
+	let failed = 0;
+	let validIdx = 0;
+
+	for (const row of rows) {
+		const base = baseMap.get(row.baseId);
+		if (!base) {
+			failed++;
+			continue;
+		}
+		values.push({
+			id: ids[validIdx],
+			pageId: String(pageId),
+			itemIds: String(row.baseId),
+			catalogName: base.publicName || base.itemName || String(row.baseId),
+			costCredits: row.credits ?? 0,
+			costPoints: row.points ?? 0,
+			pointsType: row.pointsType ?? 0,
+			amount: row.amount ?? 1,
+			orderNumber: nextOrder++,
+			offerId: -1,
+			songId: 0,
+			limitedSells: 0,
+			limitedStack: 0,
+			extradata: "",
+			haveOffer: "1",
+			clubOnly: "0",
+		});
+		validIdx++;
+	}
+
+	if (values.length === 0) return { created: 0, failed };
+
+	const allFurnitureIds = [...new Set(values.map((v) => Number(v.itemIds)))];
+
+	await db.transaction(async (tx) => {
+		await tx.execute(
+			sql`SELECT id FROM catalog_pages WHERE id=${String(pageId)} FOR UPDATE`,
+		);
+		if (allFurnitureIds.length) {
+			await tx.execute(
+				sql`SELECT id FROM items_base WHERE id IN (${sql.join(allFurnitureIds, sql`, `)}) ORDER BY id FOR UPDATE`,
+			);
+		}
+
+		for (const v of values) {
+			await tx.execute(sql`
+				INSERT INTO catalog_items (
+					id, page_id, item_ids, catalog_name,
+					cost_credits, cost_points, points_type, amount,
+					order_number, offer_id, song_id,
+					limited_sells, limited_stack, extradata, have_offer, club_only
+				) VALUES (
+					${v.id}, ${v.pageId}, ${v.itemIds}, ${v.catalogName},
+					${v.costCredits}, ${v.costPoints}, ${v.pointsType}, ${v.amount},
+					${v.orderNumber}, ${v.offerId}, ${v.songId},
+					${v.limitedSells}, ${v.limitedStack}, ${v.extradata},
+					${v.haveOffer}, ${v.clubOnly}
+				)
+			`);
+		}
+	});
+
+	await sendCatalogUpdate();
+	await logStaffActivity({
+		staffId,
+		action: "catalog_items_bulk_create",
+		description: `Bulk inserted ${values.length} catalog item(s) on page #${pageId}`,
+		targetType: "catalog_page",
+		targetId: pageId,
+	});
+	revalidatePath("/admin/catalog");
+
+	return { created: values.length, failed };
+}
+
 /** Bulk create with one RCON refresh at the end. */
 export async function bulkCreateCatalogItems({
 	pageId,
@@ -155,67 +290,17 @@ export async function bulkCreateCatalogItems({
 			return { ok: false as const, error: "Max 500 items per bulk import" };
 		}

-		const baseIds = [...new Set(rows.map((r) => r.baseId))];
-		const bases = await db
-			.select({
-				id: ItemsBase.id,
-				publicName: ItemsBase.publicName,
-				itemName: ItemsBase.itemName,
-			})
-			.from(ItemsBase)
-			.where(inArray(ItemsBase.id, baseIds));
-		const baseMap = new Map(bases.map((b) => [b.id, b]));
-
-		let created = 0;
-		let failed = 0;
-
-		// Find the next order number on the target page to avoid collisions.
-		const maxOrder = await queryRows<{ maxOrder: number }>(sql`
-			SELECT COALESCE(MAX(order_number), 0) AS maxOrder FROM catalog_items WHERE page_id = ${String(pageId)}
-		`);
-		let nextOrder = Number(maxOrder[0]?.maxOrder ?? 0) + 1;
-
-		for (const row of rows) {
-			const base = baseMap.get(row.baseId);
-			if (!base) {
-				failed++;
-				continue;
-			}
-			try {
-				await insertCatalogItemRow({
-					pageId,
-					itemIds: String(row.baseId),
-					catalogName: base.publicName || base.itemName || String(row.baseId),
-					costCredits: row.credits ?? 0,
-					costPoints: row.points ?? 0,
-					pointsType: row.pointsType ?? 0,
-					amount: 1,
-					limitedSells: 0,
-					limitedStack: 0,
-					orderNumber: nextOrder++,
-					offerId: -1,
-					songId: 0,
-					haveOffer: "1",
-					clubOnly: "0",
-					extradata: "",
-				});
-				created++;
-			} catch {
-				failed++;
-			}
-		}
-
-		if (created > 0) {
-			await sendCatalogUpdate();
-			await logStaffActivity({
-				staffId: staff.id,
-				action: "catalog_items_bulk_create",
-				description: `Bulk imported ${created} catalog item(s) on page #${pageId}`,
-				targetType: "catalog_page",
-				targetId: pageId,
-			});
-			revalidatePath("/admin/catalog");
-		}
+		const { created, failed } = await bulkInsertCatalogItems({
+			pageId,
+			staffId: staff.id,
+			rows: rows.map((r) => ({
+				baseId: r.baseId,
+				credits: r.credits,
+				points: r.points,
+				pointsType: r.pointsType,
+				amount: 1,
+			})),
+		});

 		return { ok: true as const, data: { created, failed } };
 	});
@@ -504,6 +589,19 @@ export async function translateCatalogItems(input: {
 			"@/lib/services/furni-data-i18n"
 		);

+		const itemIds = items.map((i) => i.id);
+		const allBases = await db
+			.select({
+				id: ItemsBase.id,
+				publicName: ItemsBase.publicName,
+				itemName: ItemsBase.itemName,
+				type: ItemsBase.type,
+				spriteId: ItemsBase.spriteId,
+			})
+			.from(ItemsBase)
+			.where(inArray(ItemsBase.id, itemIds));
+		const baseById = new Map(allBases.map((b) => [b.id, b]));
+
 		let namesUpdated = 0;
 		let descriptionsUpdated = 0;
 		const furniPatches: Array<{
@@ -519,19 +617,14 @@ export async function translateCatalogItems(input: {
 			name?: string;
 			description?: string;
 		}> = [];
+		const nameUpdates: Array<{ id: number; nextName: string }> = [];
+		const catalogNameUpdates: Array<{
+			idStr: string;
+			nextName: string;
+		}> = [];

 		for (const item of items) {
-			const [base] = await db
-				.select({
-					id: ItemsBase.id,
-					publicName: ItemsBase.publicName,
-					itemName: ItemsBase.itemName,
-					type: ItemsBase.type,
-					spriteId: ItemsBase.spriteId,
-				})
-				.from(ItemsBase)
-				.where(eq(ItemsBase.id, item.id))
-				.limit(1);
+			const base = baseById.get(item.id);
 			if (!base) continue;

 			const nextName = item.publicName?.trim() ?? "";
@@ -540,33 +633,11 @@ export async function translateCatalogItems(input: {
 				nextName !== "" && nextName !== (base.publicName ?? "");

 			if (nameChanged) {
-				await db
-					.update(ItemsBase)
-					.set({ publicName: nextName })
-					.where(eq(ItemsBase.id, base.id));
-				const idStr = String(base.id);
-				const related = await db
-					.select({
-						id: CatalogItems.id,
-						catalogName: CatalogItems.catalogName,
-					})
-					.from(CatalogItems)
-					.where(
-						or(
-							eq(CatalogItems.itemIds, idStr),
-							like(CatalogItems.itemIds, `${idStr};%`),
-							like(CatalogItems.itemIds, `%;${idStr};%`),
-							like(CatalogItems.itemIds, `%;${idStr}`),
-						),
-					);
-				for (const row of related) {
-					if (row.catalogName !== nextName) {
-						await db
-							.update(CatalogItems)
-							.set({ catalogName: nextName })
-							.where(eq(CatalogItems.id, row.id));
-					}
-				}
+				nameUpdates.push({ id: base.id, nextName });
+				catalogNameUpdates.push({
+					idStr: String(base.id),
+					nextName,
+				});
 				namesUpdated++;
 			}

@@ -589,6 +660,40 @@ export async function translateCatalogItems(input: {
 			}
 		}

+		// Batch ItemsBase name updates.
+		for (const { id, nextName } of nameUpdates) {
+			await db
+				.update(ItemsBase)
+				.set({ publicName: nextName })
+				.where(eq(ItemsBase.id, id));
+		}
+
+		// Batch CatalogItems name updates: one SELECT + UPDATE per idStr.
+		for (const { idStr, nextName } of catalogNameUpdates) {
+			const related = await db
+				.select({
+					id: CatalogItems.id,
+					catalogName: CatalogItems.catalogName,
+				})
+				.from(CatalogItems)
+				.where(
+					or(
+						eq(CatalogItems.itemIds, idStr),
+						like(CatalogItems.itemIds, `${idStr};%`),
+						like(CatalogItems.itemIds, `%;${idStr};%`),
+						like(CatalogItems.itemIds, `%;${idStr}`),
+					),
+				);
+			for (const row of related) {
+				if (row.catalogName !== nextName) {
+					await db
+						.update(CatalogItems)
+						.set({ catalogName: nextName })
+						.where(eq(CatalogItems.id, row.id));
+				}
+			}
+		}
+
 		const furniResult =
 			furniPatches.length > 0
 				? await patchFurniEntryNames(furniPatches)