feat(admin): add production error center and refresh CMS dependencies

src/lib/api-handler.ts

@@ -122,12 +122,12 @@ export function withAdmin(
 			return response;
 		} catch (error) {
 			await finishExport?.().catch(() => undefined);
-			logServerError("admin.api_failed", error, {
+			const errorId = logServerError("admin.api_failed", error, {
 				path: request.nextUrl.pathname,
 				userId: context.session.user.id,
 			});
 			return NextResponse.json(
-				{ ok: false, error: "Internal server error" },
+				{ ok: false, error: "Internal server error", errorId },
 				{ status: 500 },
 			);
 		}

src/lib/browser-errors.test.ts

@@ -0,0 +1,28 @@
+import { afterEach, expect, it, vi } from "vitest";
+import { reportBrowserError } from "./browser-errors";
+
+afterEach(() => vi.unstubAllGlobals());
+it("returns a reusable reference, deduplicates reports and omits URL parameters", () => {
+	const fetchMock = vi.fn().mockResolvedValue({ ok: true });
+	vi.stubGlobal("window", {});
+	vi.stubGlobal("location", {
+		pathname: "/admin/studio",
+		search: "?token=secret",
+	});
+	vi.stubGlobal("fetch", fetchMock);
+	const error = new Error("browser-test-unique");
+	const id = reportBrowserError(error);
+	expect(reportBrowserError(error)).toBe(id);
+	expect(fetchMock).toHaveBeenCalledTimes(1);
+	const payload = JSON.parse(fetchMock.mock.calls[0][1].body);
+	expect(payload.clientReference).toBe(id);
+	expect(payload.path).toBe("/admin/studio");
+	expect(JSON.stringify(payload)).not.toContain("token=secret");
+});
+it("never throws again when reporting fails on the network", async () => {
+	vi.stubGlobal("window", {});
+	vi.stubGlobal("location", { pathname: "/" });
+	vi.stubGlobal("fetch", vi.fn().mockRejectedValue(new Error("offline")));
+	expect(reportBrowserError(new Error("offline-test"))).toBeTruthy();
+	await Promise.resolve();
+});

src/lib/browser-errors.ts

@@ -0,0 +1,34 @@
+const seen = new Map<string, { at: number; id: string }>();
+export function reportBrowserError(
+	error: unknown,
+	event = "browser.exception",
+) {
+	if (typeof window === "undefined") return;
+	const candidate =
+		error instanceof Error
+			? error
+			: new Error(typeof error === "string" ? error : "Unknown browser error");
+	const key = `${event}:${location.pathname}:${"digest" in candidate ? candidate.digest : ""}:${candidate.message}`;
+	const now = Date.now();
+	const prior = seen.get(key);
+	if (prior && now - prior.at < 60000) return prior.id;
+	const reference = crypto.randomUUID();
+	if (seen.size >= 100) seen.clear();
+	seen.set(key, { at: now, id: reference });
+	const digest = "digest" in candidate ? String(candidate.digest) : undefined;
+	void fetch("/api/diagnostics/errors", {
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		keepalive: true,
+		body: JSON.stringify({
+			event,
+			clientReference: reference,
+			message: candidate.message.slice(0, 1000),
+			stack: candidate.stack?.slice(0, 6000),
+			path: location.pathname,
+			digest,
+			release: process.env.NEXT_PUBLIC_CMS_RELEASE,
+		}),
+	}).catch(() => {});
+	return reference;
+}

src/lib/error-monitor.test.ts

@@ -0,0 +1,56 @@
+import { mkdtemp, rm } from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+import { expect, it } from "vitest";
+import { createErrorRecord, ErrorStore } from "./error-monitor";
+
+it("redacts secrets in messages, stack and context while retaining a reference", () => {
+	const r = createErrorRecord(
+		"server",
+		"payment.failed",
+		new Error("https://user:password@host/a?token=abc password=hello"),
+		{
+			token: "secret",
+			path: "/a?session=hidden",
+			nested: { password: "never" },
+		},
+	);
+	const text = JSON.stringify(r);
+	for (const secret of ["password@", "token=abc", "hello", "hidden", "never"])
+		expect(text).not.toContain(secret);
+	expect(r.id).toBeTruthy();
+	expect(r.stack).toContain("Error");
+});
+it("persists across instances and groups repeated errors without losing references", async () => {
+	const dir = await mkdtemp(path.join(os.tmpdir(), "cms-errors-"));
+	try {
+		const store = new ErrorStore(dir);
+		const a = createErrorRecord("server", "test", new Error("failed"));
+		await store.append(a);
+		await store.append({ ...a, id: "second" });
+		const result = await new ErrorStore(dir).read();
+		expect(result.records).toHaveLength(2);
+		expect(result.records[0].fingerprint).toBe(result.records[1].fingerprint);
+	} finally {
+		await rm(dir, { recursive: true, force: true });
+	}
+});
+it("resolves a group and reopens it when a later occurrence arrives", async () => {
+	const dir = await mkdtemp(path.join(os.tmpdir(), "cms-errors-"));
+	try {
+		const store = new ErrorStore(dir);
+		const a = createErrorRecord("server", "test", new Error("failed"));
+		await store.append(a);
+		await store.resolve(a.fingerprint);
+		expect((await store.read()).records[0].resolved).toBe(true);
+		await store.append({
+			...a,
+			id: "later",
+			at: new Date(Date.now() + 1000).toISOString(),
+		});
+		expect((await store.read()).records[0].resolved).toBe(false);
+		await expect(store.resolve("../escape")).rejects.toThrow();
+	} finally {
+		await rm(dir, { recursive: true, force: true });
+	}
+});

src/lib/error-monitor.ts

@@ -0,0 +1,209 @@
+import { createHash, randomUUID } from "node:crypto";
+import {
+	appendFile,
+	mkdir,
+	readdir,
+	readFile,
+	stat,
+	unlink,
+	writeFile,
+} from "node:fs/promises";
+import path from "node:path";
+
+export function redactErrorText(value: string): string {
+	return value
+		.slice(0, 12000)
+		.replace(
+			/(https?:\/\/|mysql:\/\/|redis:\/\/)[^\s/@]+:[^\s/@]+@/gi,
+			"$1[REDACTED]@",
+		)
+		.replace(/([?&])[^\s#)]+/g, "$1[REDACTED]")
+		.replace(
+			/((?:password|secret|token|authorization|cookie|api[_-]?key)\s*[:=]\s*)(?:"[^"]*"|'[^']*'|[^\s,;]+)/gi,
+			"$1[REDACTED]",
+		)
+		.replace(/Bearer\s+[^\s,;]+/gi, "Bearer [REDACTED]");
+}
+export interface CmsErrorRecord {
+	id: string;
+	resolved?: boolean;
+	at: string;
+	source: "server" | "browser";
+	event: string;
+	message: string;
+	stack: string;
+	release: string;
+	fingerprint: string;
+	context: Record<string, string | number | boolean | null>;
+}
+export function createErrorRecord(
+	source: CmsErrorRecord["source"],
+	event: string,
+	error: unknown,
+	context: Record<string, unknown> = {},
+): CmsErrorRecord {
+	const message = redactErrorText(
+		error instanceof Error
+			? error.message
+			: typeof error === "string"
+				? error
+				: "Unknown error",
+	);
+	const stack =
+		error instanceof Error ? redactErrorText(error.stack ?? "") : "";
+	const safeContext = Object.fromEntries(
+		Object.entries(context)
+			.slice(0, 24)
+			.map(([key, value]) => [
+				key,
+				/password|secret|token|cookie|authorization|body|query|email|api[_-]?key/i.test(
+					key,
+				)
+					? "[REDACTED]"
+					: typeof value === "string"
+						? redactErrorText(value).slice(0, 1000)
+						: typeof value === "number" ||
+								typeof value === "boolean" ||
+								value === null
+							? value
+							: "[NON_SCALAR]",
+			]),
+	);
+	return {
+		id: randomUUID(),
+		at: new Date().toISOString(),
+		source,
+		event: redactErrorText(event).slice(0, 160),
+		message,
+		stack,
+		release: process.env.NEXT_PUBLIC_CMS_RELEASE ?? "unknown",
+		fingerprint: createHash("sha256")
+			.update(`${source}:${event}:${message}:${stack.split("\n")[1] ?? ""}`)
+			.digest("hex")
+			.slice(0, 16),
+		context: safeContext,
+	};
+}
+const DAY_LIMIT = 10 * 1024 * 1024;
+const RETENTION_DAYS = 7;
+export class ErrorStore {
+	private static queue: Promise<void> = Promise.resolve();
+	private static pending = 0;
+	constructor(
+		private directory = path.join(process.cwd(), "storage", "cms-errors"),
+	) {}
+	async append(record: CmsErrorRecord) {
+		if (ErrorStore.pending >= 100) throw new Error("Error storage queue full");
+		ErrorStore.pending++;
+		const write = ErrorStore.queue.then(() => this.writeRecord(record));
+		ErrorStore.queue = write.catch(() => {});
+		try {
+			await write;
+		} finally {
+			ErrorStore.pending--;
+		}
+	}
+	private async writeRecord(record: CmsErrorRecord) {
+		await mkdir(this.directory, { recursive: true });
+		const files = await readdir(this.directory);
+		const cutoff = new Date(Date.now() - RETENTION_DAYS * 86400000)
+			.toISOString()
+			.slice(0, 10);
+		for (const name of files)
+			if (
+				/^\d{4}-\d{2}-\d{2}(\.jsonl|\.[a-f0-9]{16}\.resolved)$/.test(name) &&
+				name.slice(0, 10) < cutoff
+			)
+				await unlink(path.join(this.directory, name)).catch(() => {});
+		const file = path.join(this.directory, record.at.slice(0, 10) + ".jsonl");
+		const size = await stat(file)
+			.then((s) => s.size)
+			.catch(() => 0);
+		if (size >= DAY_LIMIT) throw new Error("Daily error storage limit reached");
+		await appendFile(file, JSON.stringify(record) + "\n", { mode: 0o600 });
+	}
+	async resolve(fingerprint: string) {
+		if (!/^[a-f0-9]{16}$/.test(fingerprint))
+			throw new Error("Invalid fingerprint");
+		const { records } = await this.read();
+		if (!records.some((r) => r.fingerprint === fingerprint))
+			throw new Error("Error group no longer available");
+		const now = new Date().toISOString();
+		await writeFile(
+			path.join(this.directory, `${now.slice(0, 10)}.${fingerprint}.resolved`),
+			now,
+			{ mode: 0o600 },
+		);
+	}
+	async read(): Promise<{ records: CmsErrorRecord[]; truncated: boolean }> {
+		const files = await readdir(this.directory).catch(
+			(e: NodeJS.ErrnoException) => {
+				if (e.code === "ENOENT") return [];
+				throw e;
+			},
+		);
+		const resolved = new Map<string, string>();
+		for (const name of files
+			.filter((f) => /^\d{4}-\d{2}-\d{2}\.[a-f0-9]{16}\.resolved$/.test(f))
+			.sort()) {
+			const at = await readFile(path.join(this.directory, name), "utf8");
+			resolved.set(name.split(".")[1], at);
+		}
+		const records: CmsErrorRecord[] = [];
+		const cutoff = new Date(Date.now() - RETENTION_DAYS * 86400000)
+			.toISOString()
+			.slice(0, 10);
+		for (const name of files
+			.filter(
+				(f) => /^\d{4}-\d{2}-\d{2}\.jsonl$/.test(f) && f.slice(0, 10) >= cutoff,
+			)
+			.sort()
+			.reverse()) {
+			const file = path.join(this.directory, name);
+			if ((await stat(file)).size > DAY_LIMIT + 100000) continue;
+			const lines = (await readFile(file, "utf8")).trim().split("\n").reverse();
+			for (const line of lines) {
+				try {
+					const r = JSON.parse(line);
+					if (r.id && r.fingerprint)
+						records.push({
+							...r,
+							resolved: (resolved.get(r.fingerprint) ?? "") >= r.at,
+						});
+				} catch {
+					/* Ignore an interrupted final write. */
+				}
+				if (records.length >= 1000) return { records, truncated: true };
+			}
+		}
+		return { records, truncated: false };
+	}
+}
+const store = new ErrorStore();
+let pending = 0;
+let lastFailure = 0;
+export function captureCmsError(
+	source: CmsErrorRecord["source"],
+	event: string,
+	error: unknown,
+	context: Record<string, unknown> = {},
+) {
+	const record = createErrorRecord(source, event, error, context);
+	if (process.env.NODE_ENV === "test" || process.env.VITEST || pending >= 100)
+		return record.id;
+	pending++;
+	void store
+		.append(record)
+		.catch(() => {
+			if (Date.now() - lastFailure > 60000) {
+				lastFailure = Date.now();
+				process.stderr.write(
+					"CMS error monitor could not persist an event; check storage permissions or daily quota.\n",
+				);
+			}
+		})
+		.finally(() => {
+			pending--;
+		});
+	return record.id;
+}

src/lib/logger.ts

@@ -1,5 +1,6 @@
 import pino from "pino";
 import { env } from "@/env";
+import { captureCmsError } from "./error-monitor";
 
 type LogLevel = "debug" | "info" | "warn" | "error";
 
@@ -41,7 +42,21 @@ export const logger = {
 	warn(message: string, meta: Record<string, unknown> = {}): void {
 		pinoLogger.warn(meta, message);
 	},
-	error(message: string, meta: Record<string, unknown> = {}): void {
-		pinoLogger.error(meta, message);
+	error(message: string, meta: Record<string, unknown> = {}): string {
+		const error =
+			meta.err instanceof Error
+				? meta.err
+				: meta.error instanceof Error
+					? meta.error
+					: new Error(
+							typeof meta.err === "string"
+								? meta.err
+								: typeof meta.error === "string"
+									? meta.error
+									: message,
+						);
+		const errorId = captureCmsError("server", message, error, meta);
+		pinoLogger.error({ ...meta, errorId }, message);
+		return errorId;
 	},
 };

src/lib/observability-contract.test.ts

@@ -1,10 +1,14 @@
-import { createRequire } from "node:module";
-import { describe, expect, it } from "vitest";
+import { expect, it } from "vitest";
+import { createErrorRecord } from "./error-monitor";
 
-describe("observability dependency contract", () => {
-	it("does not ship the Sentry SDK", () => {
-		const require = createRequire(import.meta.url);
-
-		expect(() => require.resolve("@sentry/nextjs")).toThrow();
-	});
+it("captures diagnostic data without serializing arbitrary request bodies", () => {
+	const record = createErrorRecord(
+		"server",
+		"request.failed",
+		new Error("failure"),
+		{ body: { password: "secret" }, path: "/admin/studio" },
+	);
+	expect(record.context.body).toBe("[REDACTED]");
+	expect(record.context.path).toBe("/admin/studio");
+	expect(record.stack).toContain("failure");
 });

src/lib/report-error.ts

@@ -5,9 +5,8 @@ import { logger } from "@/lib/logger";
  * Domain errors (validation, auth, etc.) should NOT go through here.
  */
 export function reportError(error: unknown, context = "Unhandled error"): void {
-	const message = error instanceof Error ? error.message : String(error);
 	logger.error(context, {
-		err: message,
+		err: error,
 		name: error instanceof Error ? error.name : undefined,
 	});
 }

src/lib/server-log.ts

@@ -38,8 +38,11 @@ export function logServerError(
 	event: string,
 	error: unknown,
 	context: LogContext = {},
-): void {
-	logger.error(JSON.stringify(serverErrorRecord(event, error, context)));
+): string {
+	return logger.error(event, {
+		...serverErrorRecord(event, error, context).context,
+		err: error,
+	});
 }
 
 function logScalar(value: unknown): LogScalar {

src/lib/services/furni-job-worker.ts

@@ -17,7 +17,9 @@ let running: Promise<void> | undefined;
 export function drainFurnitureImports(): Promise<void> {
 	if (running) return running;
 	running = drain()
-		.catch((error) => logServerError("furni.worker_failed", error))
+		.catch((error) => {
+			logServerError("furni.worker_failed", error);
+		})
 		.finally(() => {
 			running = undefined;
 		});