remco/EpicNext-Cms
1
1 Fork 0
Code Issues 1 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

refactor: universal logger and remove N+1 catalog queries
CI / check (push) Successful in 28s
Details
CI / tests-ui (push) Failing after 32s
Details
CI / tests-unit (push) Successful in 1m43s
Details
CI / tests-quality (push) Successful in 1m14s
Details
CI / tests-integration (push) Successful in 1m50s
Details
CI / preflight (push) Skipped
Details
CI / deploy (push) Skipped
Details
CI / tests-imaging-browsers (push) Successful in 1m56s
Details

Browse Source 
- Replace all 39 console.* calls with universal logger (pino server, console browser)
- Set server-only marker on import-badge.ts
- Disable DB query logging in production
- Add bulkInsertCatalogItems: one transaction instead of N per-item inserts
- Add reserveCatalogItemIds helper for bulk ID allocation
- Optimize translateCatalogItems: single inArray batch query instead of N per-item selects
- Test mock updated for logger.debug
- biome clean, tsc clean, 3569 tests passing
This commit is contained in:
openhands committed 2026-10-17 14:29:14 +02:00
1 parent 6895eaab49
commit 59a45d2165
27 files changed
+416 -241
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

src/actions/catalog-items.ts
+205 -100
View File
@@ -24,7 +24,10 @@ import { CatalogItems, db, ItemsBase, queryRows } from "@/lib/db";
import { PERMS } from "@/lib/permissions";
import { logAudit } from "@/lib/services/audit";
import { withCatalogExport } from "@/lib/services/catalog-git-queue";
import { allocateCatalogItemId } from "@/lib/services/furni-import";
import {
allocateCatalogItemId,
reserveCatalogItemIds,
} from "@/lib/services/furni-import";
import { logStaffActivity } from "@/lib/services/staff-activity";
import { translateItemsSchema } from "@/lib/validators/catalog";
@@ -133,6 +136,138 @@ export async function createCatalogItem(
});
}
/**
* Bulk insert catalog items in ONE transaction instead of N per-item
* transactions + N INSERT queries. Same result as calling insertCatalogItemRow
* per row, but far faster for large batches. Side effects (audit, revalidate,
* RCON refresh) are left to the caller, which wraps this in withCatalogExport.
*/
export async function bulkInsertCatalogItems(data: {
pageId: number;
staffId: number;
rows: Array<{
baseId: number;
credits?: number;
points?: number;
pointsType?: number;
amount?: number;
}>;
}): Promise<{ created: number; failed: number }> {
const { pageId, staffId, rows } = data;
if (rows.length === 0) return { created: 0, failed: 0 };
const baseIds = [...new Set(rows.map((r) => r.baseId))];
const bases = await db
.select({
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
})
.from(ItemsBase)
.where(inArray(ItemsBase.id, baseIds));
const baseMap = new Map(bases.map((b) => [b.id, b]));
const ids = await reserveCatalogItemIds(rows.length);
const maxOrder = await queryRows<{ maxOrder: number }>(sql`
SELECT COALESCE(MAX(order_number), 0) AS maxOrder FROM catalog_items WHERE page_id = ${String(pageId)}
`);
let nextOrder = Number(maxOrder[0]?.maxOrder ?? 0) + 1;
const values: Array<{
id: number;
pageId: string;
itemIds: string;
catalogName: string;
costCredits: number;
costPoints: number;
pointsType: number;
amount: number;
orderNumber: number;
offerId: number;
songId: number;
limitedSells: number;
limitedStack: number;
extradata: string;
haveOffer: string;
clubOnly: string;
}> = [];
let failed = 0;
let validIdx = 0;
for (const row of rows) {
const base = baseMap.get(row.baseId);
if (!base) {
failed++;
continue;
}
values.push({
id: ids[validIdx],
pageId: String(pageId),
itemIds: String(row.baseId),
catalogName: base.publicName || base.itemName || String(row.baseId),
costCredits: row.credits ?? 0,
costPoints: row.points ?? 0,
pointsType: row.pointsType ?? 0,
amount: row.amount ?? 1,
orderNumber: nextOrder++,
offerId: -1,
songId: 0,
limitedSells: 0,
limitedStack: 0,
extradata: "",
haveOffer: "1",
clubOnly: "0",
});
validIdx++;
}
if (values.length === 0) return { created: 0, failed };
const allFurnitureIds = [...new Set(values.map((v) => Number(v.itemIds)))];
await db.transaction(async (tx) => {
await tx.execute(
sql`SELECT id FROM catalog_pages WHERE id=${String(pageId)} FOR UPDATE`,
);
if (allFurnitureIds.length) {
await tx.execute(
sql`SELECT id FROM items_base WHERE id IN (${sql.join(allFurnitureIds, sql`, `)}) ORDER BY id FOR UPDATE`,
);
}
for (const v of values) {
await tx.execute(sql`
INSERT INTO catalog_items (
id, page_id, item_ids, catalog_name,
cost_credits, cost_points, points_type, amount,
order_number, offer_id, song_id,
limited_sells, limited_stack, extradata, have_offer, club_only
) VALUES (
${v.id}, ${v.pageId}, ${v.itemIds}, ${v.catalogName},
${v.costCredits}, ${v.costPoints}, ${v.pointsType}, ${v.amount},
${v.orderNumber}, ${v.offerId}, ${v.songId},
${v.limitedSells}, ${v.limitedStack}, ${v.extradata},
${v.haveOffer}, ${v.clubOnly}
)
`);
}
});
await sendCatalogUpdate();
await logStaffActivity({
staffId,
action: "catalog_items_bulk_create",
description: `Bulk inserted ${values.length} catalog item(s) on page #${pageId}`,
targetType: "catalog_page",
targetId: pageId,
});
revalidatePath("/admin/catalog");
return { created: values.length, failed };
}
/** Bulk create with one RCON refresh at the end. */
export async function bulkCreateCatalogItems({
pageId,
@@ -155,67 +290,17 @@ export async function bulkCreateCatalogItems({
return { ok: false as const, error: "Max 500 items per bulk import" };
}
const baseIds = [...new Set(rows.map((r) => r.baseId))];
const bases = await db
.select({
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
})
.from(ItemsBase)
.where(inArray(ItemsBase.id, baseIds));
const baseMap = new Map(bases.map((b) => [b.id, b]));
let created = 0;
let failed = 0;
// Find the next order number on the target page to avoid collisions.
const maxOrder = await queryRows<{ maxOrder: number }>(sql`
SELECT COALESCE(MAX(order_number), 0) AS maxOrder FROM catalog_items WHERE page_id = ${String(pageId)}
`);
let nextOrder = Number(maxOrder[0]?.maxOrder ?? 0) + 1;
for (const row of rows) {
const base = baseMap.get(row.baseId);
if (!base) {
failed++;
continue;
}
try {
await insertCatalogItemRow({
pageId,
itemIds: String(row.baseId),
catalogName: base.publicName || base.itemName || String(row.baseId),
costCredits: row.credits ?? 0,
costPoints: row.points ?? 0,
pointsType: row.pointsType ?? 0,
amount: 1,
limitedSells: 0,
limitedStack: 0,
orderNumber: nextOrder++,
offerId: -1,
songId: 0,
haveOffer: "1",
clubOnly: "0",
extradata: "",
});
created++;
} catch {
failed++;
}
}
if (created > 0) {
await sendCatalogUpdate();
await logStaffActivity({
staffId: staff.id,
action: "catalog_items_bulk_create",
description: `Bulk imported ${created} catalog item(s) on page #${pageId}`,
targetType: "catalog_page",
targetId: pageId,
});
revalidatePath("/admin/catalog");
}
const { created, failed } = await bulkInsertCatalogItems({
pageId,
staffId: staff.id,
rows: rows.map((r) => ({
baseId: r.baseId,
credits: r.credits,
points: r.points,
pointsType: r.pointsType,
amount: 1,
})),
});
return { ok: true as const, data: { created, failed } };
});
@@ -504,6 +589,19 @@ export async function translateCatalogItems(input: {
"@/lib/services/furni-data-i18n"
);
const itemIds = items.map((i) => i.id);
const allBases = await db
.select({
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
type: ItemsBase.type,
spriteId: ItemsBase.spriteId,
})
.from(ItemsBase)
.where(inArray(ItemsBase.id, itemIds));
const baseById = new Map(allBases.map((b) => [b.id, b]));
let namesUpdated = 0;
let descriptionsUpdated = 0;
const furniPatches: Array<{
@@ -519,19 +617,14 @@ export async function translateCatalogItems(input: {
name?: string;
description?: string;
}> = [];
const nameUpdates: Array<{ id: number; nextName: string }> = [];
const catalogNameUpdates: Array<{
idStr: string;
nextName: string;
}> = [];
for (const item of items) {
const [base] = await db
.select({
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
type: ItemsBase.type,
spriteId: ItemsBase.spriteId,
})
.from(ItemsBase)
.where(eq(ItemsBase.id, item.id))
.limit(1);
const base = baseById.get(item.id);
if (!base) continue;
const nextName = item.publicName?.trim() ?? "";
@@ -540,33 +633,11 @@ export async function translateCatalogItems(input: {
nextName !== "" && nextName !== (base.publicName ?? "");
if (nameChanged) {
await db
.update(ItemsBase)
.set({ publicName: nextName })
.where(eq(ItemsBase.id, base.id));
const idStr = String(base.id);
const related = await db
.select({
id: CatalogItems.id,
catalogName: CatalogItems.catalogName,
})
.from(CatalogItems)
.where(
or(
eq(CatalogItems.itemIds, idStr),
like(CatalogItems.itemIds, `${idStr};%`),
like(CatalogItems.itemIds, `%;${idStr};%`),
like(CatalogItems.itemIds, `%;${idStr}`),
),
);
for (const row of related) {
if (row.catalogName !== nextName) {
await db
.update(CatalogItems)
.set({ catalogName: nextName })
.where(eq(CatalogItems.id, row.id));
}
}
nameUpdates.push({ id: base.id, nextName });
catalogNameUpdates.push({
idStr: String(base.id),
nextName,
});
namesUpdated++;
}
@@ -589,6 +660,40 @@ export async function translateCatalogItems(input: {
}
}
// Batch ItemsBase name updates.
for (const { id, nextName } of nameUpdates) {
await db
.update(ItemsBase)
.set({ publicName: nextName })
.where(eq(ItemsBase.id, id));
}
// Batch CatalogItems name updates: one SELECT + UPDATE per idStr.
for (const { idStr, nextName } of catalogNameUpdates) {
const related = await db
.select({
id: CatalogItems.id,
catalogName: CatalogItems.catalogName,
})
.from(CatalogItems)
.where(
or(
eq(CatalogItems.itemIds, idStr),
like(CatalogItems.itemIds, `${idStr};%`),
like(CatalogItems.itemIds, `%;${idStr};%`),
like(CatalogItems.itemIds, `%;${idStr}`),
),
);
for (const row of related) {
if (row.catalogName !== nextName) {
await db
.update(CatalogItems)
.set({ catalogName: nextName })
.where(eq(CatalogItems.id, row.id));
}
}
}
const furniResult =
furniPatches.length > 0
? await patchFurniEntryNames(furniPatches)
src/actions/catalog.ts
+15 -47
View File
@@ -2,7 +2,10 @@
import { inArray, sql } from "drizzle-orm";
import { revalidatePath } from "next/cache";
import { insertCatalogItemRow } from "@/actions/catalog-items";
import {
bulkInsertCatalogItems,
insertCatalogItemRow,
} from "@/actions/catalog-items";
import { catalogFailure } from "@/features/catalog/server/errors";
import {
createPageCommand,
@@ -279,52 +282,17 @@ export async function createAutoCategory(input: {
}
const rows = input.offerRows ?? [];
const baseIds = [...new Set(rows.map((r) => r.baseId))];
const bases = await db
.select({
id: ItemsBase.id,
publicName: ItemsBase.publicName,
itemName: ItemsBase.itemName,
})
.from(ItemsBase)
.where(inArray(ItemsBase.id, baseIds));
const baseMap = new Map(bases.map((b) => [b.id, b]));
const maxOrder = await queryRows<{ maxOrder: number }>(sql`
SELECT COALESCE(MAX(order_number), 0) AS maxOrder FROM catalog_items WHERE page_id = ${String(pageId)}
`);
let nextOrder = Number(maxOrder[0]?.maxOrder ?? 0) + 1;
let created = 0;
for (const row of rows) {
const base = baseMap.get(row.baseId);
if (!base) continue;
try {
await insertCatalogItemRow({
pageId,
itemIds: String(row.baseId),
catalogName: base.publicName || base.itemName || String(row.baseId),
costCredits: row.credits ?? 0,
costPoints: row.points ?? 0,
pointsType: row.pointsType ?? 0,
amount: row.amount ?? 1,
limitedSells: 0,
limitedStack: 0,
orderNumber: nextOrder++,
offerId: -1,
songId: 0,
haveOffer: "1",
clubOnly: "0",
extradata: "",
});
created++;
} catch (error) {
console.error(
`[createAutoCategory] failed to insert item #${row.baseId}:`,
error,
);
}
}
const { created } = await bulkInsertCatalogItems({
pageId,
staffId: staff.id,
rows: rows.map((r) => ({
baseId: r.baseId,
credits: r.credits,
points: r.points,
pointsType: r.pointsType,
amount: r.amount,
})),
});
await logStaffActivity({
staffId: staff.id,