feat(ops): health-fail alerts, optional DB backup, admin UX polish

Wire jobs-worker health probes to Discord/email alerts with cooldown, optional mysqldump, rate-limit /api/health, mark-all-read alerts, ConfirmDialog on destructive admin actions, and raise coverage floors.

Co-authored-by: Cursor <[email protected]>

.env.example

@@ -39,6 +39,11 @@ BADGE_UPLOAD_DIR=./public/assets/images/badges
 EMULATOR_JAR_PATH=./emulator/Arcturus.jar
 EMULATOR_BACKUP_DIR=./backups/emulator
 EMULATOR_BACKUP_KEEP=7
+# Optional mysqldump (jobs-worker daily 03:30). Requires mysqldump on PATH.
+DB_BACKUP_DIR=
+DB_BACKUP_KEEP=7
+# Minutes between repeat health-fail alerts from jobs-worker (default 15).
+HEALTH_ALERT_COOLDOWN_MIN=15
 
 # --- RCON (Low Latency Loop) ---
 RCON_HOST=127.0.0.1

scripts/jobs-worker.ts

@@ -1,9 +1,12 @@
 import * as Sentry from "@sentry/nextjs";
 import { Cron } from "croner";
-import { lt } from "drizzle-orm";
+import { lt, sql } from "drizzle-orm";
 import { env } from "../src/env";
 import { db, PasswordReset, WebsiteLoginLogs } from "../src/lib/db";
 import { logger } from "../src/lib/logger";
+import { redis } from "../src/lib/redis";
+import { emulatorOffline, healthDegraded } from "../src/lib/services/alert";
+import { rcon } from "../src/lib/services/rcon";
 
 function initWorkerSentry(): void {
 	const dsn = process.env.SENTRY_DSN;
@@ -28,6 +31,71 @@ function captureWorkerError(err: unknown, context: string): void {
 	}
 }
 
+/** In-process cooldown so a flapping probe does not spam Discord/email. */
+const alertCooldownMs = (env.HEALTH_ALERT_COOLDOWN_MIN ?? 15) * 60_000;
+const lastHealthAlertAt = new Map<string, number>();
+
+function canAlert(key: string): boolean {
+	const now = Date.now();
+	const prev = lastHealthAlertAt.get(key) ?? 0;
+	if (now - prev < alertCooldownMs) return false;
+	lastHealthAlertAt.set(key, now);
+	return true;
+}
+
+async function probeHealth(): Promise<{
+	database: boolean;
+	redis: boolean | null;
+	emulator: boolean;
+}> {
+	const database = await db
+		.execute(sql`SELECT 1`)
+		.then(() => true)
+		.catch(() => false);
+
+	let redisOk: boolean | null = null;
+	if (env.REDIS_URL) {
+		if (!redis) {
+			redisOk = false;
+		} else {
+			try {
+				redisOk = (await redis.ping()) === "PONG";
+			} catch {
+				redisOk = false;
+			}
+		}
+	}
+
+	const emulator = await rcon.send("ping", null).catch(() => false);
+	return {
+		database,
+		redis: redisOk,
+		emulator: Boolean(emulator),
+	};
+}
+
+async function checkOpsHealth(): Promise<void> {
+	try {
+		const health = await probeHealth();
+		const degraded =
+			!health.database || health.redis === false || !health.emulator;
+		if (!degraded) return;
+
+		if (!health.emulator && health.database && health.redis !== false) {
+			if (canAlert("emulator")) {
+				await emulatorOffline("jobs-worker RCON ping failed");
+			}
+			return;
+		}
+
+		if (canAlert("health")) {
+			await healthDegraded(health);
+		}
+	} catch (err) {
+		captureWorkerError(err, "Health probe failed");
+	}
+}
+
 async function backupEmulatorJar(): Promise<void> {
 	if (!env.EMULATOR_JAR_PATH || !env.EMULATOR_BACKUP_DIR) return;
 
@@ -70,6 +138,82 @@ async function backupEmulatorJar(): Promise<void> {
 	}
 }
 
+/** Optional mysqldump when DB_BACKUP_DIR is set (host must have mysqldump on PATH). */
+async function backupDatabase(): Promise<void> {
+	if (!env.DB_BACKUP_DIR || !env.DATABASE_URL) return;
+
+	const { mkdirSync, readdirSync, unlinkSync, existsSync, createWriteStream } =
+		await import("node:fs");
+	const { resolve } = await import("node:path");
+	const { spawn } = await import("node:child_process");
+
+	let parsed: URL;
+	try {
+		parsed = new URL(env.DATABASE_URL);
+	} catch {
+		logger.error("Invalid DATABASE_URL for DB backup", { module: "jobs" });
+		return;
+	}
+
+	if (!existsSync(env.DB_BACKUP_DIR)) {
+		mkdirSync(env.DB_BACKUP_DIR, { recursive: true });
+	}
+
+	const timestamp = new Date().toISOString().slice(0, 19).replace(/[T:]/g, "-");
+	const dbName =
+		decodeURIComponent(parsed.pathname.replace(/^\//, "")) || "cms";
+	const outFile = resolve(env.DB_BACKUP_DIR, `db-${dbName}-${timestamp}.sql`);
+
+	const args = [
+		`-h${parsed.hostname}`,
+		`-P${parsed.port || "3306"}`,
+		`-u${decodeURIComponent(parsed.username)}`,
+		`--single-transaction`,
+		`--routines`,
+		`--databases`,
+		dbName,
+	];
+	if (parsed.password) {
+		args.splice(3, 0, `-p${decodeURIComponent(parsed.password)}`);
+	}
+
+	await new Promise<void>((resolvePromise) => {
+		const child = spawn("mysqldump", args, {
+			stdio: ["ignore", "pipe", "pipe"],
+		});
+		const out = createWriteStream(outFile);
+		child.stdout.pipe(out);
+		let stderr = "";
+		child.stderr.on("data", (chunk: Buffer) => {
+			stderr += chunk.toString();
+		});
+		child.on("error", (err) => {
+			captureWorkerError(err, "mysqldump spawn failed (is it on PATH?)");
+			resolvePromise();
+		});
+		child.on("close", (code) => {
+			out.end();
+			if (code !== 0) {
+				captureWorkerError(
+					new Error(stderr || `mysqldump exit ${code}`),
+					"DB backup failed",
+				);
+			} else {
+				logger.info("Backed up database", { module: "jobs", outFile });
+				const keep = env.DB_BACKUP_KEEP ?? 7;
+				const files = readdirSync(env.DB_BACKUP_DIR)
+					.filter((f) => f.startsWith("db-") && f.endsWith(".sql"))
+					.sort()
+					.reverse();
+				for (let i = keep; i < files.length; i++) {
+					unlinkSync(resolve(env.DB_BACKUP_DIR, files[i]));
+				}
+			}
+			resolvePromise();
+		});
+	});
+}
+
 async function cleanupOldLogs(): Promise<void> {
 	try {
 		const cutoff = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
@@ -107,6 +251,15 @@ async function main() {
 		});
 	}
 
+	if (env.DB_BACKUP_DIR) {
+		new Cron("30 3 * * *", () => {
+			backupDatabase().catch((e) => captureWorkerError(e, "DB backup error"));
+		});
+		logger.info("Scheduled: mysqldump DB backup (daily 03:30)", {
+			module: "jobs",
+		});
+	}
+
 	new Cron("0 4 * * *", () => {
 		Promise.all([cleanupOldLogs(), cleanupOldSessions()]).catch((e) =>
 			captureWorkerError(e, "Cleanup error"),
@@ -114,10 +267,16 @@ async function main() {
 	});
 	logger.info("Scheduled: old data cleanup (daily 04:00)", { module: "jobs" });
 
+	new Cron("*/5 * * * *", () => {
+		checkOpsHealth().catch((e) => captureWorkerError(e, "Health check error"));
+	});
+	logger.info("Scheduled: ops health probe (every 5 min)", { module: "jobs" });
+
 	await Promise.all([
 		backupEmulatorJar(),
 		cleanupOldLogs(),
 		cleanupOldSessions(),
+		checkOpsHealth(),
 	]);
 }
 

src/actions/admin-alerts.ts

@@ -1,7 +1,9 @@
 "use server";
 
+import { eq } from "drizzle-orm";
 import { revalidatePath } from "next/cache";
 import { requirePermission } from "@/lib/admin/guard";
+import { AlertLogs, db } from "@/lib/db";
 import { PERMS } from "@/lib/permissions";
 import { rcon } from "@/lib/services/rcon";
 
@@ -29,3 +31,17 @@ export async function sendHotelAlert(formData: FormData): Promise<void> {
 
 	revalidatePath("/admin/alerts");
 }
+
+/** Mark every unread ops alert as read. */
+export async function markAllAlertsRead(): Promise<void> {
+	await requirePermission(PERMS.NOTIFICATIONS_VIEW);
+	try {
+		await db
+			.update(AlertLogs)
+			.set({ isRead: true, updatedAt: new Date() })
+			.where(eq(AlertLogs.isRead, false));
+	} catch {
+		/* ignore */
+	}
+	revalidatePath("/admin/alerts");
+}

src/app/admin/alerts/loading.tsx

@@ -0,0 +1 @@
+export { default } from "../loading";

src/app/admin/alerts/page.tsx

@@ -1,7 +1,7 @@
 import { desc } from "drizzle-orm";
 import { redirect } from "next/navigation";
 import { getTranslations } from "next-intl/server";
-import { sendHotelAlert } from "@/actions/admin-alerts";
+import { markAllAlertsRead, sendHotelAlert } from "@/actions/admin-alerts";
 import { StatusCard } from "@/components/admin/dashboard";
 import { Button } from "@/components/ui/button";
 import { AlertLogs, db } from "@/lib/db";
@@ -125,7 +125,16 @@ export default async function AdminAlerts() {
 			</form>
 
 			<section className="admin-card">
-				<h3 className="mt-0">{t("recentAlerts", { count: alerts.length })}</h3>
+				<div className="mb-3 flex flex-wrap items-center justify-between gap-2">
+					<h3 className="m-0">{t("recentAlerts", { count: alerts.length })}</h3>
+					{unread > 0 ? (
+						<form action={markAllAlertsRead}>
+							<Button type="submit" variant="outline" size="sm">
+								{t("markAllRead")}
+							</Button>
+						</form>
+					) : null}
+				</div>
 				{alerts.length === 0 ? (
 					<div className="admin-empty">{t("noAlerts")}</div>
 				) : (

src/app/admin/commandocentrum/loading.tsx

@@ -0,0 +1 @@
+export { default } from "../loading";

src/app/admin/devops/loading.tsx

@@ -0,0 +1 @@
+export { default } from "../loading";

src/app/admin/users/[id]/user-quick-actions.tsx

@@ -19,6 +19,7 @@ import {
 	sendCredits,
 	unmuteUser,
 } from "@/actions/users";
+import { useConfirmDialog } from "@/components/admin/confirm-dialog";
 import { CurrencyIcon } from "@/components/shared/currency-icon";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
@@ -46,6 +47,7 @@ export function UserQuickActions({
 	isOnline,
 }: UserQuickActionsProps) {
 	const { run, isPending } = useServerAction();
+	const { confirm, dialog: confirmDialog } = useConfirmDialog();
 
 	// Reset password state
 	const [newPassword, setNewPassword] = useState<string | null>(null);
@@ -60,13 +62,14 @@ export function UserQuickActions({
 	// Credits state
 	const [creditAmount, setCreditAmount] = useState("1000");
 
-	function handleResetPassword() {
+	async function handleResetPassword() {
-		if (
+		const ok = await confirm({
-			!confirm(
+			title: "Reset password",
-				`Reset password for ${username}? A new random password will be generated.`,
+			description: `Reset password for ${username}? A new random password will be generated.`,
-			)
+			confirmLabel: "Reset",
-		)
+			variant: "danger",
-			return;
+		});
+		if (!ok) return;
 		run(() => resetPassword({ userId }), {
 			successMessage: "Password reset successfully.",
 			onSuccess: (data) => {
@@ -121,6 +124,7 @@ export function UserQuickActions({
 
 	return (
 		<>
+			{confirmDialog}
 			<Card>
 				<CardHeader>
 					<CardTitle>Quick Actions</CardTitle>

src/app/api/health/route.ts

@@ -1,7 +1,9 @@
 import { sql } from "drizzle-orm";
+import { NextResponse } from "next/server";
 import { env } from "@/env";
 import { apiJson } from "@/lib/api";
 import { db } from "@/lib/db";
+import { clientIp, rateLimit } from "@/lib/rate-limit";
 import { redis } from "@/lib/redis";
 import { rcon } from "@/lib/services/rcon";
 
@@ -11,9 +13,21 @@ export const dynamic = "force-dynamic";
  * Ops health probe: database reachability, Redis (when configured), emulator
  * RCON, SMTP (when configured), and runtime info. Returns HTTP 200 always
  * (read the `status`/`database` fields), so it's safe for uptime monitors that
- * only care about reachability.
+ * only care about reachability. Rate-limited per client IP.
  */
 export async function GET() {
+	const ip = await clientIp();
+	const limit = await rateLimit(`health:${ip}`, 60, 60_000);
+	if (!limit.ok) {
+		return NextResponse.json(
+			{ status: "rate_limited", retryAfter: limit.retryAfter },
+			{
+				status: 429,
+				headers: { "Retry-After": String(limit.retryAfter) },
+			},
+		);
+	}
+
 	const database = await db
 		.execute(sql`SELECT 1`)
 		.then(() => true)

src/components/admin/article-card.tsx

@@ -1,8 +1,11 @@
 "use client";
 
 import Link from "next/link";
+import { useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
+import { useTransition } from "react";
 import { deleteArticle } from "@/actions/admin-articles";
+import { useConfirmDialog } from "@/components/admin/confirm-dialog";
 
 export function ArticleCard({
 	id,
@@ -18,9 +21,29 @@ export function ArticleCard({
 	author?: string;
 }) {
 	const t = useTranslations("pages.admin.articles");
+	const router = useRouter();
+	const [pending, startTransition] = useTransition();
+	const { confirm, dialog: confirmDialog } = useConfirmDialog();
+
+	async function handleDelete() {
+		const ok = await confirm({
+			title: t("delete"),
+			description: t("confirmDelete", { title }),
+			confirmLabel: t("delete"),
+			variant: "danger",
+		});
+		if (!ok) return;
+		startTransition(async () => {
+			const fd = new FormData();
+			fd.set("id", id);
+			await deleteArticle(fd);
+			router.refresh();
+		});
+	}
 
 	return (
 		<div className="admin-card overflow-hidden flex flex-col">
+			{confirmDialog}
 			<div className="relative w-full aspect-[16/9] bg-[var(--admin-canvas)]">
 				{/* eslint-disable-next-line @next/next/no-img-element */}
 				<img
@@ -56,20 +79,14 @@ export function ArticleCard({
 					>
 						{t("edit")}
 					</Link>
-					<form action={deleteArticle} className="m-0">
+					<button
-						<input type="hidden" name="id" value={id} />
+						type="button"
-						<button
+						className="btn btn-danger"
-							type="submit"
+						disabled={pending}
-							className="btn btn-danger"
+						onClick={handleDelete}
-							onClick={(e) => {
+					>
-								if (!window.confirm(t("confirmDelete", { title }))) {
+						{t("delete")}
-									e.preventDefault();
+					</button>
-								}
-							}}
-						>
-							{t("delete")}
-						</button>
-					</form>
 				</div>
 			</div>
 		</div>

src/components/admin/media-grid.tsx

@@ -3,6 +3,7 @@
 import { useTranslations } from "next-intl";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { deleteMedia, uploadMedia } from "@/actions/admin-media";
+import { useConfirmDialog } from "@/components/admin/confirm-dialog";
 import { formatDate } from "@/lib/format-date";
 
 type MediaFile = {
@@ -32,6 +33,7 @@ function extOf(name: string): string {
 
 export function AdminMediaGrid() {
 	const t = useTranslations("pages.admin.media");
+	const { confirm, dialog: confirmDialog } = useConfirmDialog();
 	const [files, setFiles] = useState<MediaFile[]>([]);
 	const [loading, setLoading] = useState(true);
 	const [uploading, setUploading] = useState(false);
@@ -103,7 +105,13 @@ export function AdminMediaGrid() {
 	}
 
 	async function remove(name: string) {
-		if (!window.confirm(t("confirmDelete"))) return;
+		const ok = await confirm({
+			title: t("delete"),
+			description: t("confirmDelete"),
+			confirmLabel: t("delete"),
+			variant: "danger",
+		});
+		if (!ok) return;
 		try {
 			await deleteMedia(name);
 			setFiles((prev) => prev.filter((f) => f.name !== name));
@@ -115,6 +123,7 @@ export function AdminMediaGrid() {
 
 	return (
 		<div className="admin-card">
+			{confirmDialog}
 			{/* Toolbar */}
 			<div className="flex flex-wrap items-center gap-3 mb-4">
 				<label className="btn btn-primary cursor-pointer text-sm">

src/env.ts

@@ -65,6 +65,11 @@ const schema = z
 		EMULATOR_JAR_PATH: z.string().optional(),
 		EMULATOR_BACKUP_DIR: z.string().optional(),
 		EMULATOR_BACKUP_KEEP: z.coerce.number().int().positive().optional(),
+		// Optional mysqldump backup (jobs-worker); requires mysqldump on PATH.
+		DB_BACKUP_DIR: z.string().optional(),
+		DB_BACKUP_KEEP: z.coerce.number().int().positive().optional(),
+		// Minutes between repeat health-fail Discord/email alerts (jobs-worker).
+		HEALTH_ALERT_COOLDOWN_MIN: z.coerce.number().int().positive().optional(),
 		// Optional AI content moderation (comments / guestbook).
 		OPENAI_API_KEY: z.string().optional(),
 		// Optional alerting (jobs worker / alert service).

src/lib/services/alert.ts

@@ -232,6 +232,33 @@ export function emulatorOffline(detail?: string): Promise<SendAlertResult> {
 	});
 }
 
+/**
+ * Raise an ERROR alert when ops health is degraded (DB / Redis / emulator).
+ */
+export function healthDegraded(parts: {
+	database: boolean;
+	redis: boolean | null;
+	emulator: boolean;
+}): Promise<SendAlertResult> {
+	const failed: string[] = [];
+	if (!parts.database) failed.push("database");
+	if (parts.redis === false) failed.push("redis");
+	if (!parts.emulator) failed.push("emulator");
+	return sendAlert({
+		type: "health",
+		severity: parts.database ? "error" : "critical",
+		message:
+			failed.length > 0
+				? `Ops health degraded: ${failed.join(", ")} unavailable.`
+				: "Ops health degraded.",
+		context: {
+			database: parts.database,
+			redis: parts.redis,
+			emulator: parts.emulator,
+		},
+	});
+}
+
 /**
  * Raise a WARNING alert that a possible DDoS / abusive request pattern was
  * detected from a single IP (count = requests seen in the sampling window).

src/lib/staff-smoke-contract.test.ts

@@ -179,6 +179,23 @@ describe("staff smoke contract", () => {
 		);
 	});
 
+	it("ships ops health alerts, optional DB backup, and health rate limit", () => {
+		const worker = readFileSync("scripts/jobs-worker.ts", "utf8");
+		expect(worker).toContain("checkOpsHealth");
+		expect(worker).toContain("healthDegraded");
+		expect(worker).toContain("backupDatabase");
+		expect(readFileSync("src/lib/services/alert.ts", "utf8")).toContain(
+			"healthDegraded",
+		);
+		expect(readFileSync("src/app/api/health/route.ts", "utf8")).toContain(
+			"rateLimit",
+		);
+		expect(readFileSync("src/actions/admin-alerts.ts", "utf8")).toContain(
+			"markAllAlertsRead",
+		);
+		expect(readFileSync("src/env.ts", "utf8")).toContain("DB_BACKUP_DIR");
+	});
+
 	it("documents local-only photo file purge", () => {
 		expect(readFileSync("src/app/admin/photos/page.tsx", "utf8")).toContain(
 			"purgeHint",

src/messages/en.json

@@ -2416,15 +2416,22 @@
 			},
 			"alerts": {
 				"title": "Alerts",
-				"subtitle": "Send hotel-wide alerts and view recent alerts",
+				"subtitle": "Send hotel-wide alerts and view recent ops alerts",
 				"sendAlert": "Send hotel alert",
-				"recentAlerts": "Recent alerts",
+				"recentAlerts": "Recent alerts ({count})",
+				"noAlerts": "No recent alerts",
 				"noRecentAlerts": "No recent alerts",
+				"alertsShown": "Shown",
+				"critical": "Critical",
+				"warnings": "Warnings",
+				"unread": "Unread",
+				"markAllRead": "Mark all read",
 				"form": {
 					"title": "Alert title",
 					"titlePlaceholder": "e.g. Server maintenance in 10 minutes",
 					"message": "Alert message",
 					"messagePlaceholder": "Write the alert message…",
+					"description": "Broadcasts via RCON <code>hotelalert</code> to online users.",
 					"type": "Type",
 					"typeHotel": "Hotel-wide alert",
 					"typeStaff": "Staff only",
@@ -2437,6 +2444,9 @@
 				"colSentBy": "Sent by",
 				"colType": "Type",
 				"colDate": "Date",
+				"colWhen": "When",
+				"colSeverity": "Severity",
+				"colMessage": "Message",
 				"colActions": "Actions"
 			},
 			"logs": {

vitest.config.ts

@@ -24,10 +24,10 @@ export default defineConfig({
 			provider: "v8",
 			reporter: ["text", "lcov", "html"],
 			thresholds: {
-				statements: 6,
+				statements: 7,
-				branches: 4,
+				branches: 5,
-				functions: 5,
+				functions: 6,
-				lines: 6,
+				lines: 7,
 			},
 			include: ["src/**/*.{ts,tsx}"],
 			exclude: [