feat(ops): health-fail alerts, optional DB backup, admin UX polish

Wire jobs-worker health probes to Discord/email alerts with cooldown, optional mysqldump, rate-limit /api/health, mark-all-read alerts, ConfirmDialog on destructive admin actions, and raise coverage floors.

Co-authored-by: Cursor <[email protected]>

.env.example

@@ -39,6 +39,11 @@ BADGE_UPLOAD_DIR=./public/assets/images/badges
 EMULATOR_JAR_PATH=./emulator/Arcturus.jar
 EMULATOR_BACKUP_DIR=./backups/emulator
 EMULATOR_BACKUP_KEEP=7
+# Optional mysqldump (jobs-worker daily 03:30). Requires mysqldump on PATH.
+DB_BACKUP_DIR=
+DB_BACKUP_KEEP=7
+# Minutes between repeat health-fail alerts from jobs-worker (default 15).
+HEALTH_ALERT_COOLDOWN_MIN=15
 
 # --- RCON (Low Latency Loop) ---
 RCON_HOST=127.0.0.1

src/actions/admin-alerts.ts

@@ -1,7 +1,9 @@
 "use server";
 
+import { eq } from "drizzle-orm";
 import { revalidatePath } from "next/cache";
 import { requirePermission } from "@/lib/admin/guard";
+import { AlertLogs, db } from "@/lib/db";
 import { PERMS } from "@/lib/permissions";
 import { rcon } from "@/lib/services/rcon";
 
@@ -29,3 +31,17 @@ export async function sendHotelAlert(formData: FormData): Promise<void> {
 
 	revalidatePath("/admin/alerts");
 }
+
+/** Mark every unread ops alert as read. */
+export async function markAllAlertsRead(): Promise<void> {
+	await requirePermission(PERMS.NOTIFICATIONS_VIEW);
+	try {
+		await db
+			.update(AlertLogs)
+			.set({ isRead: true, updatedAt: new Date() })
+			.where(eq(AlertLogs.isRead, false));
+	} catch {
+		/* ignore */
+	}
+	revalidatePath("/admin/alerts");
+}

src/app/admin/alerts/loading.tsx

@@ -0,0 +1 @@
+export { default } from "../loading";

src/app/admin/alerts/page.tsx

@@ -1,7 +1,7 @@
 import { desc } from "drizzle-orm";
 import { redirect } from "next/navigation";
 import { getTranslations } from "next-intl/server";
-import { sendHotelAlert } from "@/actions/admin-alerts";
+import { markAllAlertsRead, sendHotelAlert } from "@/actions/admin-alerts";
 import { StatusCard } from "@/components/admin/dashboard";
 import { Button } from "@/components/ui/button";
 import { AlertLogs, db } from "@/lib/db";
@@ -125,7 +125,16 @@ export default async function AdminAlerts() {
 			</form>
 
 			<section className="admin-card">
-				<h3 className="mt-0">{t("recentAlerts", { count: alerts.length })}</h3>
+				<div className="mb-3 flex flex-wrap items-center justify-between gap-2">
+					<h3 className="m-0">{t("recentAlerts", { count: alerts.length })}</h3>
+					{unread > 0 ? (
+						<form action={markAllAlertsRead}>
+							<Button type="submit" variant="outline" size="sm">
+								{t("markAllRead")}
+							</Button>
+						</form>
+					) : null}
+				</div>
 				{alerts.length === 0 ? (
 					<div className="admin-empty">{t("noAlerts")}</div>
 				) : (

src/app/admin/commandocentrum/loading.tsx

@@ -0,0 +1 @@
+export { default } from "../loading";

src/app/admin/devops/loading.tsx

@@ -0,0 +1 @@
+export { default } from "../loading";

src/app/admin/users/[id]/user-quick-actions.tsx

@@ -19,6 +19,7 @@ import {
 	sendCredits,
 	unmuteUser,
 } from "@/actions/users";
+import { useConfirmDialog } from "@/components/admin/confirm-dialog";
 import { CurrencyIcon } from "@/components/shared/currency-icon";
 import { Badge } from "@/components/ui/badge";
 import { Button } from "@/components/ui/button";
@@ -46,6 +47,7 @@ export function UserQuickActions({
 	isOnline,
 }: UserQuickActionsProps) {
 	const { run, isPending } = useServerAction();
+	const { confirm, dialog: confirmDialog } = useConfirmDialog();
 
 	// Reset password state
 	const [newPassword, setNewPassword] = useState<string | null>(null);
@@ -60,13 +62,14 @@ export function UserQuickActions({
 	// Credits state
 	const [creditAmount, setCreditAmount] = useState("1000");
 
-	function handleResetPassword() {
+	async function handleResetPassword() {
-		if (
+		const ok = await confirm({
-			!confirm(
+			title: "Reset password",
-				`Reset password for ${username}? A new random password will be generated.`,
+			description: `Reset password for ${username}? A new random password will be generated.`,
-			)
+			confirmLabel: "Reset",
-		)
+			variant: "danger",
-			return;
+		});
+		if (!ok) return;
 		run(() => resetPassword({ userId }), {
 			successMessage: "Password reset successfully.",
 			onSuccess: (data) => {
@@ -121,6 +124,7 @@ export function UserQuickActions({
 
 	return (
 		<>
+			{confirmDialog}
 			<Card>
 				<CardHeader>
 					<CardTitle>Quick Actions</CardTitle>

src/app/api/health/route.ts

@@ -1,7 +1,9 @@
 import { sql } from "drizzle-orm";
+import { NextResponse } from "next/server";
 import { env } from "@/env";
 import { apiJson } from "@/lib/api";
 import { db } from "@/lib/db";
+import { clientIp, rateLimit } from "@/lib/rate-limit";
 import { redis } from "@/lib/redis";
 import { rcon } from "@/lib/services/rcon";
 
@@ -11,9 +13,21 @@ export const dynamic = "force-dynamic";
  * Ops health probe: database reachability, Redis (when configured), emulator
  * RCON, SMTP (when configured), and runtime info. Returns HTTP 200 always
  * (read the `status`/`database` fields), so it's safe for uptime monitors that
- * only care about reachability.
+ * only care about reachability. Rate-limited per client IP.
  */
 export async function GET() {
+	const ip = await clientIp();
+	const limit = await rateLimit(`health:${ip}`, 60, 60_000);
+	if (!limit.ok) {
+		return NextResponse.json(
+			{ status: "rate_limited", retryAfter: limit.retryAfter },
+			{
+				status: 429,
+				headers: { "Retry-After": String(limit.retryAfter) },
+			},
+		);
+	}
+
 	const database = await db
 		.execute(sql`SELECT 1`)
 		.then(() => true)

src/components/admin/article-card.tsx

@@ -1,8 +1,11 @@
 "use client";
 
 import Link from "next/link";
+import { useRouter } from "next/navigation";
 import { useTranslations } from "next-intl";
+import { useTransition } from "react";
 import { deleteArticle } from "@/actions/admin-articles";
+import { useConfirmDialog } from "@/components/admin/confirm-dialog";
 
 export function ArticleCard({
 	id,
@@ -18,9 +21,29 @@ export function ArticleCard({
 	author?: string;
 }) {
 	const t = useTranslations("pages.admin.articles");
+	const router = useRouter();
+	const [pending, startTransition] = useTransition();
+	const { confirm, dialog: confirmDialog } = useConfirmDialog();
+
+	async function handleDelete() {
+		const ok = await confirm({
+			title: t("delete"),
+			description: t("confirmDelete", { title }),
+			confirmLabel: t("delete"),
+			variant: "danger",
+		});
+		if (!ok) return;
+		startTransition(async () => {
+			const fd = new FormData();
+			fd.set("id", id);
+			await deleteArticle(fd);
+			router.refresh();
+		});
+	}
 
 	return (
 		<div className="admin-card overflow-hidden flex flex-col">
+			{confirmDialog}
 			<div className="relative w-full aspect-[16/9] bg-[var(--admin-canvas)]">
 				{/* eslint-disable-next-line @next/next/no-img-element */}
 				<img
@@ -56,20 +79,14 @@ export function ArticleCard({
 					>
 						{t("edit")}
 					</Link>
-					<form action={deleteArticle} className="m-0">
-						<input type="hidden" name="id" value={id} />
 					<button
-							type="submit"
+						type="button"
 						className="btn btn-danger"
-							onClick={(e) => {
+						disabled={pending}
-								if (!window.confirm(t("confirmDelete", { title }))) {
+						onClick={handleDelete}
-									e.preventDefault();
-								}
-							}}
 					>
 						{t("delete")}
 					</button>
-					</form>
 				</div>
 			</div>
 		</div>

src/components/admin/media-grid.tsx

@@ -3,6 +3,7 @@
 import { useTranslations } from "next-intl";
 import { useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { deleteMedia, uploadMedia } from "@/actions/admin-media";
+import { useConfirmDialog } from "@/components/admin/confirm-dialog";
 import { formatDate } from "@/lib/format-date";
 
 type MediaFile = {
@@ -32,6 +33,7 @@ function extOf(name: string): string {
 
 export function AdminMediaGrid() {
 	const t = useTranslations("pages.admin.media");
+	const { confirm, dialog: confirmDialog } = useConfirmDialog();
 	const [files, setFiles] = useState<MediaFile[]>([]);
 	const [loading, setLoading] = useState(true);
 	const [uploading, setUploading] = useState(false);
@@ -103,7 +105,13 @@ export function AdminMediaGrid() {
 	}
 
 	async function remove(name: string) {
-		if (!window.confirm(t("confirmDelete"))) return;
+		const ok = await confirm({
+			title: t("delete"),
+			description: t("confirmDelete"),
+			confirmLabel: t("delete"),
+			variant: "danger",
+		});
+		if (!ok) return;
 		try {
 			await deleteMedia(name);
 			setFiles((prev) => prev.filter((f) => f.name !== name));
@@ -115,6 +123,7 @@ export function AdminMediaGrid() {
 
 	return (
 		<div className="admin-card">
+			{confirmDialog}
 			{/* Toolbar */}
 			<div className="flex flex-wrap items-center gap-3 mb-4">
 				<label className="btn btn-primary cursor-pointer text-sm">

src/env.ts

@@ -65,6 +65,11 @@ const schema = z
 		EMULATOR_JAR_PATH: z.string().optional(),
 		EMULATOR_BACKUP_DIR: z.string().optional(),
 		EMULATOR_BACKUP_KEEP: z.coerce.number().int().positive().optional(),
+		// Optional mysqldump backup (jobs-worker); requires mysqldump on PATH.
+		DB_BACKUP_DIR: z.string().optional(),
+		DB_BACKUP_KEEP: z.coerce.number().int().positive().optional(),
+		// Minutes between repeat health-fail Discord/email alerts (jobs-worker).
+		HEALTH_ALERT_COOLDOWN_MIN: z.coerce.number().int().positive().optional(),
 		// Optional AI content moderation (comments / guestbook).
 		OPENAI_API_KEY: z.string().optional(),
 		// Optional alerting (jobs worker / alert service).

src/lib/services/alert.ts

@@ -232,6 +232,33 @@ export function emulatorOffline(detail?: string): Promise<SendAlertResult> {
 	});
 }
 
+/**
+ * Raise an ERROR alert when ops health is degraded (DB / Redis / emulator).
+ */
+export function healthDegraded(parts: {
+	database: boolean;
+	redis: boolean | null;
+	emulator: boolean;
+}): Promise<SendAlertResult> {
+	const failed: string[] = [];
+	if (!parts.database) failed.push("database");
+	if (parts.redis === false) failed.push("redis");
+	if (!parts.emulator) failed.push("emulator");
+	return sendAlert({
+		type: "health",
+		severity: parts.database ? "error" : "critical",
+		message:
+			failed.length > 0
+				? `Ops health degraded: ${failed.join(", ")} unavailable.`
+				: "Ops health degraded.",
+		context: {
+			database: parts.database,
+			redis: parts.redis,
+			emulator: parts.emulator,
+		},
+	});
+}
+
 /**
  * Raise a WARNING alert that a possible DDoS / abusive request pattern was
  * detected from a single IP (count = requests seen in the sampling window).

src/lib/staff-smoke-contract.test.ts

@@ -179,6 +179,23 @@ describe("staff smoke contract", () => {
 		);
 	});
 
+	it("ships ops health alerts, optional DB backup, and health rate limit", () => {
+		const worker = readFileSync("scripts/jobs-worker.ts", "utf8");
+		expect(worker).toContain("checkOpsHealth");
+		expect(worker).toContain("healthDegraded");
+		expect(worker).toContain("backupDatabase");
+		expect(readFileSync("src/lib/services/alert.ts", "utf8")).toContain(
+			"healthDegraded",
+		);
+		expect(readFileSync("src/app/api/health/route.ts", "utf8")).toContain(
+			"rateLimit",
+		);
+		expect(readFileSync("src/actions/admin-alerts.ts", "utf8")).toContain(
+			"markAllAlertsRead",
+		);
+		expect(readFileSync("src/env.ts", "utf8")).toContain("DB_BACKUP_DIR");
+	});
+
 	it("documents local-only photo file purge", () => {
 		expect(readFileSync("src/app/admin/photos/page.tsx", "utf8")).toContain(
 			"purgeHint",

src/messages/en.json

@@ -2416,15 +2416,22 @@
 			},
 			"alerts": {
 				"title": "Alerts",
-				"subtitle": "Send hotel-wide alerts and view recent alerts",
+				"subtitle": "Send hotel-wide alerts and view recent ops alerts",
 				"sendAlert": "Send hotel alert",
-				"recentAlerts": "Recent alerts",
+				"recentAlerts": "Recent alerts ({count})",
+				"noAlerts": "No recent alerts",
 				"noRecentAlerts": "No recent alerts",
+				"alertsShown": "Shown",
+				"critical": "Critical",
+				"warnings": "Warnings",
+				"unread": "Unread",
+				"markAllRead": "Mark all read",
 				"form": {
 					"title": "Alert title",
 					"titlePlaceholder": "e.g. Server maintenance in 10 minutes",
 					"message": "Alert message",
 					"messagePlaceholder": "Write the alert message…",
+					"description": "Broadcasts via RCON <code>hotelalert</code> to online users.",
 					"type": "Type",
 					"typeHotel": "Hotel-wide alert",
 					"typeStaff": "Staff only",
@@ -2437,6 +2444,9 @@
 				"colSentBy": "Sent by",
 				"colType": "Type",
 				"colDate": "Date",
+				"colWhen": "When",
+				"colSeverity": "Severity",
+				"colMessage": "Message",
 				"colActions": "Actions"
 			},
 			"logs": {

vitest.config.ts

@@ -24,10 +24,10 @@ export default defineConfig({
 			provider: "v8",
 			reporter: ["text", "lcov", "html"],
 			thresholds: {
-				statements: 6,
+				statements: 7,
-				branches: 4,
+				branches: 5,
-				functions: 5,
+				functions: 6,
-				lines: 6,
+				lines: 7,
 			},
 			include: ["src/**/*.{ts,tsx}"],
 			exclude: [