feat(security): ops alerts, shared backoff, atomic quota and daily stats for CrowdSec

Add an alerting/stats layer over the existing CrowdSec integration:

- New crowdsec-alerts.ts: cooldown-gated ops alerts (Redis NX lock, TTL from
  HEALTH_ALERT_COOLDOWN_MIN) fanning out through the app's sendAlert service.
  Raised for daily quota exhaustion, block bursts (5-min window past
  CROWDSEC_ALERT_BLOCK_BURST), and signal-push failures.
- New crowdsec-stats.ts: daily counters (lookups/blocks/reports/report_fail)
  in Redis with a 14-day reader for the admin panel.
- Shared 403/429 backoff: the pause marker now lives in Redis
  (crowdsec:backoff-until) so every instance honours it, not just the process
  that hit the limit.
- Atomic quota reservation: INCR-before-call with self-rollback on overshoot,
  so concurrent instances can never slip calls past the daily ceiling.
- Admin anti-DDoS page gains a last-14-days activity table next to the quota bar.

.env.example

@@ -93,6 +93,9 @@ CROWDSEC_CTI_BASE_URL=https://cti.api.crowdsec.net/v2
 # counter reaches it, reputation lookups pause until tomorrow so a spread
 # DDoS cannot silently burn the whole quota. 0 = unlimited.
 CROWDSEC_CTI_DAILY_QUOTA=10000
+# How many new community-reputation blocks within a 5-minute window justify an
+# ops alert (quota/backoff/report alerts all use HEALTH_ALERT_COOLDOWN_MIN).
+CROWDSEC_ALERT_BLOCK_BURST=10
 
 # --- CROWDSEC SIGNAL PUSH (share our blocks back, optional) ---
 # Opt-in: pushes blocked IPs + behaviors to the CrowdSec Central API (CAPI) so

src/app/admin/devops/antiddos/page.tsx

@@ -45,6 +45,7 @@ import {
 	crowdsecReportEnabled,
 	getLastCrowdsecReport,
 } from "@/lib/crowdsec-report";
+import { getCrowdsecStats } from "@/lib/crowdsec-stats";
 import { db, WebsiteSetting } from "@/lib/db";
 import { canAccess, getAdminContext, PERMS } from "@/lib/permissions";
 import { redis } from "@/lib/redis";
@@ -147,6 +148,7 @@ export default async function AdminAntiDdosPage() {
 	const crowdsecUsage = redisOk ? await getCrowdsecQuotaUsage() : null;
 	const reportingEnabled = await crowdsecReportEnabled();
 	const lastReport = await getLastCrowdsecReport();
+	const crowdsecStats = redisOk ? await getCrowdsecStats(14) : [];
 
 	return (
 		<div className="space-y-6">
@@ -723,6 +725,62 @@ export default async function AdminAntiDdosPage() {
 						</div>
 					)}
 
+					{crowdsecStats.length > 0 && (
+						<div className="rounded-md border p-3">
+							<p className="text-xs font-medium mb-2">
+								Daily activity (last {crowdsecStats.length} days)
+							</p>
+							<div className="max-h-40 overflow-y-auto">
+								<table className="w-full text-xs">
+									<thead>
+										<tr className="text-left text-muted-foreground">
+											<th className="pb-1 pr-2 font-medium">Date</th>
+											<th className="pb-1 pr-2 font-medium text-right">
+												Lookups
+											</th>
+											<th className="pb-1 pr-2 font-medium text-right">
+												Blocks
+											</th>
+											<th className="pb-1 pr-2 font-medium text-right">
+												Reports
+											</th>
+											<th className="pb-1 font-medium text-right">Failures</th>
+										</tr>
+									</thead>
+									<tbody>
+										{crowdsecStats.map((row) => (
+											<tr key={row.date} className="border-t">
+												<td className="py-1 pr-2 text-muted-foreground">
+													{row.date === new Date().toISOString().slice(0, 10)
+														? "Today"
+														: row.date.slice(5)}
+												</td>
+												<td className="py-1 pr-2 text-right">
+													{row.lookups.toLocaleString()}
+												</td>
+												<td className="py-1 pr-2 text-right">
+													{row.blocks.toLocaleString()}
+												</td>
+												<td className="py-1 pr-2 text-right">
+													{row.reports.toLocaleString()}
+												</td>
+												<td className="py-1 text-right">
+													{row.reportFailures > 0 ? (
+														<span className="text-destructive">
+															{row.reportFailures.toLocaleString()}
+														</span>
+													) : (
+														"–"
+													)}
+												</td>
+											</tr>
+										))}
+									</tbody>
+								</table>
+							</div>
+						</div>
+					)}
+
 					<div className="rounded-md border p-3">
 						<p className="text-xs font-medium mb-1">Community signal push</p>
 						<div className="flex flex-wrap items-center gap-3">

src/env.ts

@@ -182,6 +182,9 @@ const schema = z
 		// it, so a spread DDoS can never silently burn the whole quota; 0
 		// disables the guard.
 		CROWDSEC_CTI_DAILY_QUOTA: z.coerce.number().int().min(0).default(10_000),
+		// How many new community-reputation blocks within a 5-minute window
+		// justify an ops alert (cooldown-gated via HEALTH_ALERT_COOLDOWN_MIN).
+		CROWDSEC_ALERT_BLOCK_BURST: z.coerce.number().int().min(1).default(10),
 		// Share our own detections back into the CrowdSec community blocklist
 		// (signal push over the Central API). Opt-in: flipping this on publicly
 		// shares blocked IPs + behaviors, so it defaults to off.

src/lib/crowdsec-alerts.ts

@@ -0,0 +1,66 @@
+import "server-only";
+
+import { env } from "@/env";
+import { logger } from "@/lib/logger";
+import { redis } from "@/lib/redis";
+import { type SendAlertInput, sendAlert } from "@/lib/services/alert";
+
+// === CrowdSec operational alerts ===========================================
+//
+// Thin, fire-and-forget wrapper around the app's alert service for the
+// reputation pipeline. Every raise is cooldown-gated through a Redis NX lock
+// (key crowdsec:alert:{key}, TTL = HEALTH_ALERT_COOLDOWN_MIN), so N instances
+// and flapping conditions surface exactly one alert per window instead of
+// spamming Discord/email/alert_logs. Falls back to alerting anyway when Redis
+// is unreachable — a silent quota blowout is worse than one duplicate alert.
+
+const ALERT_PREFIX = "crowdsec:alert:";
+
+function cooldownSeconds(): number {
+	const raw = Number(env.HEALTH_ALERT_COOLDOWN_MIN ?? 15);
+	return Math.ceil((Number.isFinite(raw) && raw > 0 ? raw : 15) * 60);
+}
+
+/**
+ * Raise an alert unless the cooldown window is still active. Returns the
+ * sendAlert promise when the alert was actually raised, or false when it was
+ * suppressed. Never throws; the caller may `void` the result on hot paths.
+ */
+export async function raiseCrowdsecAlert(
+	key: string,
+	input: {
+		type?: string;
+		severity: SendAlertInput["severity"];
+		message: string;
+		context?: SendAlertInput["context"];
+	},
+): Promise<false | Awaited<ReturnType<typeof sendAlert>>> {
+	if (redis) {
+		try {
+			const acquired = await redis.set(
+				`${ALERT_PREFIX}${key}`,
+				String(Date.now()),
+				"EX",
+				cooldownSeconds(),
+				"NX",
+			);
+			if (acquired !== "OK") return false;
+		} catch {
+			// Cooldown bookkeeping failed — alert anyway rather than silently drop.
+		}
+	}
+	try {
+		return await sendAlert({
+			type: "ddos",
+			severity: input.severity,
+			message: input.message,
+			context: input.context,
+		});
+	} catch (error) {
+		logger.error("[crowdsec-alert] sendAlert raised an unexpected error", {
+			key,
+			err: error,
+		});
+		return false;
+	}
+}

src/lib/crowdsec-report.test.ts

@@ -11,7 +11,10 @@ import {
 
 // The signal-push watcher is tested against a deterministic in-memory Redis
 // fake (NX lock + token cache) and a mocked fetch that routes the CAPI paths.
-const state = vi.hoisted(() => ({ map: new Map<string, string>() }));
+const state = vi.hoisted(() => ({
+	map: new Map<string, string>(),
+	sendAlert: vi.fn(),
+}));
 
 vi.mock("@/lib/redis", () => ({
 	redis: {
@@ -31,6 +34,12 @@ vi.mock("@/lib/redis", () => ({
 			for (const key of keys) state.map.delete(key);
 			return keys.length;
 		},
+		incr: async (key: string) => {
+			const next = (Number(state.map.get(key)) || 0) + 1;
+			state.map.set(key, String(next));
+			return next;
+		},
+		expire: async () => 1,
 	},
 	__esModule: true,
 }));
@@ -44,6 +53,13 @@ vi.mock("@/lib/logger", () => ({
 	},
 }));
 
+vi.mock("@/lib/services/alert", () => ({
+	sendAlert: state.sendAlert,
+	ddosDetected: vi.fn(),
+}));
+
+const tick = () => new Promise((resolve) => setTimeout(resolve, 20));
+
 const CAPI = "https://capi.example.test/v3";
 const MACHINE = "m".repeat(48);
 const PASSWORD = "Strong!1P@ssw0rdStrong!1P@ssw0rd";
@@ -111,6 +127,7 @@ describe("crowdsec-report", () => {
 		vi.unstubAllGlobals();
 		vi.unstubAllEnvs();
 		state.map.clear();
+		state.sendAlert.mockReset();
 		resetCrowdsecReportCache();
 		fetchMock = vi.fn();
 		vi.stubGlobal("fetch", fetchMock);
@@ -256,12 +273,56 @@ describe("crowdsec-report", () => {
 		});
 
 		await expect(reportCrowdsecSignal(signalInput())).resolves.toBeUndefined();
-		await new Promise((resolve) => setTimeout(resolve, 20));
+		await tick();
 		const last: CrowdsecReportStatus | null = await getLastCrowdsecReport();
 		expect(last?.ok).toBe(false);
 		expect(last?.message).toContain("signal push rejected");
 	});
 
+	it("counts a failed push and raises a cooldown-gated ops alert", async () => {
+		fetchMock.mockImplementation((url: string) => {
+			const path = String(url).replace(CAPI, "");
+			if (path === "/watchers/login") {
+				return Promise.resolve(
+					jsonResponse({
+						token: "jwt-xyz",
+						expire: new Date(Date.now() + 3_600_000).toISOString(),
+					}),
+				);
+			}
+			if (path === "/signals") {
+				return Promise.resolve(jsonResponse({ message: "boom" }, 500));
+			}
+			return Promise.resolve(jsonResponse({}));
+		});
+
+		await reportCrowdsecSignal(signalInput("198.51.100.20"));
+		await tick();
+		const today = new Date().toISOString().slice(0, 10);
+		expect(state.map.get(`crowdsec:stat:report_fail:${today}`)).toBe("1");
+		expect(state.sendAlert).toHaveBeenCalledTimes(1);
+		const [input] = state.sendAlert.mock.calls[0];
+		expect(input.type).toBe("ddos");
+		expect(input.severity).toBe("warning");
+		expect(input.context).toMatchObject({ ip: "198.51.100.20" });
+
+		// A second failed push inside the cooldown window stays silent.
+		await reportCrowdsecSignal(signalInput("198.51.100.21"));
+		await tick();
+		expect(state.sendAlert).toHaveBeenCalledTimes(1);
+		expect(state.map.get(`crowdsec:stat:report_fail:${today}`)).toBe("2");
+	});
+
+	it("tallies successful pushes into the daily stats histogram", async () => {
+		routeCapi();
+		await reportCrowdsecSignal(signalInput("198.51.100.30"));
+		await reportCrowdsecSignal(signalInput("198.51.100.31"));
+		await tick();
+		const today = new Date().toISOString().slice(0, 10);
+		expect(state.map.get(`crowdsec:stat:reports:${today}`)).toBe("2");
+		expect(state.sendAlert).not.toHaveBeenCalled();
+	});
+
 	it("verifies the watcher channel end to end", async () => {
 		routeCapi();
 		const status = await verifyCrowdsecReporting();

src/lib/crowdsec-report.ts

@@ -2,11 +2,13 @@ import "server-only";
 
 import { createHash, randomBytes } from "node:crypto";
 import { env } from "@/env";
+import { raiseCrowdsecAlert } from "@/lib/crowdsec-alerts";
 import type {
 	CrowdsecBlockMeta,
 	CrowdsecConnectionStatus,
 	CrowdsecVerdict,
 } from "@/lib/crowdsec-api";
+import { bumpCrowdsecStat } from "@/lib/crowdsec-stats";
 import { logger } from "@/lib/logger";
 import { redis } from "@/lib/redis";
 import { UNKNOWN_CLIENT_IP } from "./client-ip";
@@ -390,6 +392,7 @@ async function pushSignal(input: {
 		}
 		const status: CrowdsecReportStatus = { ok: true, at: Date.now() };
 		await setLastCrowdsecReport(status);
+		void bumpCrowdsecStat("reports");
 		logger.info("[crowdsec-report] Detection shared with the community", {
 			ip: input.ip,
 			category: input.category,
@@ -404,6 +407,19 @@ async function pushSignal(input: {
 			message: String(error),
 		};
 		await setLastCrowdsecReport(status);
+		void bumpCrowdsecStat("report_fail");
+		// Ops alert, cooldown-gated: a silently broken channel means the
+		// community never learns about the blocks we keep sharing.
+		void raiseCrowdsecAlert("report", {
+			type: "ddos",
+			severity: "warning",
+			message:
+				"CrowdSec signal push failed — detections are not reaching the community.",
+			context: {
+				ip: input.ip,
+				detail: String(error).slice(0, 300),
+			},
+		});
 		logger.error("[crowdsec-report] Signal push failed", {
 			ip: input.ip,
 			err: error,