feat(ops): health-fail alerts, optional DB backup, admin UX polish

Wire jobs-worker health probes to Discord/email alerts with cooldown, optional mysqldump, rate-limit /api/health, mark-all-read alerts, ConfirmDialog on destructive admin actions, and raise coverage floors.

Co-authored-by: Cursor <[email protected]>

scripts/jobs-worker.ts

@@ -1,9 +1,12 @@
 import * as Sentry from "@sentry/nextjs";
 import { Cron } from "croner";
-import { lt } from "drizzle-orm";
+import { lt, sql } from "drizzle-orm";
 import { env } from "../src/env";
 import { db, PasswordReset, WebsiteLoginLogs } from "../src/lib/db";
 import { logger } from "../src/lib/logger";
+import { redis } from "../src/lib/redis";
+import { emulatorOffline, healthDegraded } from "../src/lib/services/alert";
+import { rcon } from "../src/lib/services/rcon";
 
 function initWorkerSentry(): void {
 	const dsn = process.env.SENTRY_DSN;
@@ -28,6 +31,71 @@ function captureWorkerError(err: unknown, context: string): void {
 	}
 }
 
+/** In-process cooldown so a flapping probe does not spam Discord/email. */
+const alertCooldownMs = (env.HEALTH_ALERT_COOLDOWN_MIN ?? 15) * 60_000;
+const lastHealthAlertAt = new Map<string, number>();
+
+function canAlert(key: string): boolean {
+	const now = Date.now();
+	const prev = lastHealthAlertAt.get(key) ?? 0;
+	if (now - prev < alertCooldownMs) return false;
+	lastHealthAlertAt.set(key, now);
+	return true;
+}
+
+async function probeHealth(): Promise<{
+	database: boolean;
+	redis: boolean | null;
+	emulator: boolean;
+}> {
+	const database = await db
+		.execute(sql`SELECT 1`)
+		.then(() => true)
+		.catch(() => false);
+
+	let redisOk: boolean | null = null;
+	if (env.REDIS_URL) {
+		if (!redis) {
+			redisOk = false;
+		} else {
+			try {
+				redisOk = (await redis.ping()) === "PONG";
+			} catch {
+				redisOk = false;
+			}
+		}
+	}
+
+	const emulator = await rcon.send("ping", null).catch(() => false);
+	return {
+		database,
+		redis: redisOk,
+		emulator: Boolean(emulator),
+	};
+}
+
+async function checkOpsHealth(): Promise<void> {
+	try {
+		const health = await probeHealth();
+		const degraded =
+			!health.database || health.redis === false || !health.emulator;
+		if (!degraded) return;
+
+		if (!health.emulator && health.database && health.redis !== false) {
+			if (canAlert("emulator")) {
+				await emulatorOffline("jobs-worker RCON ping failed");
+			}
+			return;
+		}
+
+		if (canAlert("health")) {
+			await healthDegraded(health);
+		}
+	} catch (err) {
+		captureWorkerError(err, "Health probe failed");
+	}
+}
+
 async function backupEmulatorJar(): Promise<void> {
 	if (!env.EMULATOR_JAR_PATH || !env.EMULATOR_BACKUP_DIR) return;
 
@@ -70,6 +138,82 @@ async function backupEmulatorJar(): Promise<void> {
 	}
 }
 
+/** Optional mysqldump when DB_BACKUP_DIR is set (host must have mysqldump on PATH). */
+async function backupDatabase(): Promise<void> {
+	if (!env.DB_BACKUP_DIR || !env.DATABASE_URL) return;
+
+	const { mkdirSync, readdirSync, unlinkSync, existsSync, createWriteStream } =
+		await import("node:fs");
+	const { resolve } = await import("node:path");
+	const { spawn } = await import("node:child_process");
+
+	let parsed: URL;
+	try {
+		parsed = new URL(env.DATABASE_URL);
+	} catch {
+		logger.error("Invalid DATABASE_URL for DB backup", { module: "jobs" });
+		return;
+	}
+
+	if (!existsSync(env.DB_BACKUP_DIR)) {
+		mkdirSync(env.DB_BACKUP_DIR, { recursive: true });
+	}
+
+	const timestamp = new Date().toISOString().slice(0, 19).replace(/[T:]/g, "-");
+	const dbName =
+		decodeURIComponent(parsed.pathname.replace(/^\//, "")) || "cms";
+	const outFile = resolve(env.DB_BACKUP_DIR, `db-${dbName}-${timestamp}.sql`);
+
+	const args = [
+		`-h${parsed.hostname}`,
+		`-P${parsed.port || "3306"}`,
+		`-u${decodeURIComponent(parsed.username)}`,
+		`--single-transaction`,
+		`--routines`,
+		`--databases`,
+		dbName,
+	];
+	if (parsed.password) {
+		args.splice(3, 0, `-p${decodeURIComponent(parsed.password)}`);
+	}
+
+	await new Promise<void>((resolvePromise) => {
+		const child = spawn("mysqldump", args, {
+			stdio: ["ignore", "pipe", "pipe"],
+		});
+		const out = createWriteStream(outFile);
+		child.stdout.pipe(out);
+		let stderr = "";
+		child.stderr.on("data", (chunk: Buffer) => {
+			stderr += chunk.toString();
+		});
+		child.on("error", (err) => {
+			captureWorkerError(err, "mysqldump spawn failed (is it on PATH?)");
+			resolvePromise();
+		});
+		child.on("close", (code) => {
+			out.end();
+			if (code !== 0) {
+				captureWorkerError(
+					new Error(stderr || `mysqldump exit ${code}`),
+					"DB backup failed",
+				);
+			} else {
+				logger.info("Backed up database", { module: "jobs", outFile });
+				const keep = env.DB_BACKUP_KEEP ?? 7;
+				const files = readdirSync(env.DB_BACKUP_DIR)
+					.filter((f) => f.startsWith("db-") && f.endsWith(".sql"))
+					.sort()
+					.reverse();
+				for (let i = keep; i < files.length; i++) {
+					unlinkSync(resolve(env.DB_BACKUP_DIR, files[i]));
+				}
+			}
+			resolvePromise();
+		});
+	});
+}
+
 async function cleanupOldLogs(): Promise<void> {
 	try {
 		const cutoff = new Date(Date.now() - 30 * 24 * 60 * 60 * 1000);
@@ -107,6 +251,15 @@ async function main() {
 		});
 	}
 
+	if (env.DB_BACKUP_DIR) {
+		new Cron("30 3 * * *", () => {
+			backupDatabase().catch((e) => captureWorkerError(e, "DB backup error"));
+		});
+		logger.info("Scheduled: mysqldump DB backup (daily 03:30)", {
+			module: "jobs",
+		});
+	}
+
 	new Cron("0 4 * * *", () => {
 		Promise.all([cleanupOldLogs(), cleanupOldSessions()]).catch((e) =>
 			captureWorkerError(e, "Cleanup error"),
@@ -114,10 +267,16 @@ async function main() {
 	});
 	logger.info("Scheduled: old data cleanup (daily 04:00)", { module: "jobs" });
 
+	new Cron("*/5 * * * *", () => {
+		checkOpsHealth().catch((e) => captureWorkerError(e, "Health check error"));
+	});
+	logger.info("Scheduled: ops health probe (every 5 min)", { module: "jobs" });
+
 	await Promise.all([
 		backupEmulatorJar(),
 		cleanupOldLogs(),
 		cleanupOldSessions(),
+		checkOpsHealth(),
 	]);
 }