feat(security): ops alerts, shared backoff, atomic quota and daily stats for CrowdSec

Add an alerting/stats layer over the existing CrowdSec integration:

- New crowdsec-alerts.ts: cooldown-gated ops alerts (Redis NX lock, TTL from
  HEALTH_ALERT_COOLDOWN_MIN) fanning out through the app's sendAlert service.
  Raised for daily quota exhaustion, block bursts (5-min window past
  CROWDSEC_ALERT_BLOCK_BURST), and signal-push failures.
- New crowdsec-stats.ts: daily counters (lookups/blocks/reports/report_fail)
  in Redis with a 14-day reader for the admin panel.
- Shared 403/429 backoff: the pause marker now lives in Redis
  (crowdsec:backoff-until) so every instance honours it, not just the process
  that hit the limit.
- Atomic quota reservation: INCR-before-call with self-rollback on overshoot,
  so concurrent instances can never slip calls past the daily ceiling.
- Admin anti-DDoS page gains a last-14-days activity table next to the quota bar.

src/lib/crowdsec-api.test.ts

@@ -14,11 +14,20 @@ import {
 	verdictIsMalicious,
 	verifyCrowdsecConnection,
 } from "./crowdsec-api";
+import { type CrowdsecDailyStat, getCrowdsecStats } from "./crowdsec-stats";
 
 // Unit-test the CTI client in isolation: a deterministic in-memory Redis fake
 // and a silenced logger, so fetch calls count only CrowdSec lookups. CrowdSec
 // deliberately never touches Cloudflare, so no Cloudflare surface is stubbed.
-const state = vi.hoisted(() => ({ map: new Map<string, string>() }));
+const state = vi.hoisted(() => ({
+	map: new Map<string, string>(),
+	sendAlert: vi.fn(),
+}));
+
+vi.mock("@/lib/services/alert", () => ({
+	sendAlert: state.sendAlert,
+	ddosDetected: vi.fn(),
+}));
 
 vi.mock("@/lib/redis", () => ({
 	redis: {
@@ -43,6 +52,11 @@ vi.mock("@/lib/redis", () => ({
 			state.map.set(key, String(next));
 			return next;
 		},
+		decr: async (key: string) => {
+			const next = (Number(state.map.get(key)) || 0) - 1;
+			state.map.set(key, String(next));
+			return next;
+		},
 		expire: async () => 1,
 		pttl: async () => 60_000,
 	},
@@ -58,6 +72,8 @@ vi.mock("@/lib/logger", () => ({
 	},
 }));
 
+const tick = () => new Promise((resolve) => setTimeout(resolve, 20));
+
 function jsonResponse(body: unknown, status = 200): Response {
 	return new Response(JSON.stringify(body), {
 		status,
@@ -109,6 +125,7 @@ describe("crowdsec-api", () => {
 		vi.unstubAllGlobals();
 		vi.unstubAllEnvs();
 		state.map.clear();
+		state.sendAlert.mockReset();
 		resetCrowdsecCache();
 		fetchMock = vi.fn();
 		vi.stubGlobal("fetch", fetchMock);
@@ -352,6 +369,34 @@ describe("crowdsec-api", () => {
 		expect(fetchMock).toHaveBeenCalledTimes(1);
 	});
 
+	it("publishes the backoff to shared Redis so every instance respects it", async () => {
+		vi.stubEnv("CROWDSEC_API_KEY", "cs_key");
+		fetchMock.mockResolvedValue(jsonResponse({ message: "Invalid key" }, 403));
+
+		await maybeAutoBlockCrowdsec({
+			ip: blockIp(),
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		// The shared marker exists and points into the future.
+		const until = Number(state.map.get("crowdsec:backoff-until"));
+		expect(Number.isFinite(until)).toBe(true);
+		expect(until).toBeGreaterThan(Date.now());
+
+		// A fresh instance (reset in-process state) still honours the marker.
+		resetCrowdsecCache();
+		await maybeAutoBlockCrowdsec({
+			ip: "203.0.113.44",
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		expect(fetchMock).toHaveBeenCalledTimes(1);
+	});
+
 	it("backs off after a 429 rate limit as well", async () => {
 		vi.stubEnv("CROWDSEC_API_KEY", "cs_key");
 		fetchMock.mockResolvedValue(jsonResponse({ message: "rate limited" }, 429));
@@ -525,4 +570,125 @@ describe("crowdsec-api", () => {
 		}
 		expect(getMemoryVerdictCacheSize()).toBe(2000);
 	});
+
+	it("raises an ops alert when the daily quota is exhausted", async () => {
+		vi.stubEnv("CROWDSEC_API_KEY", "cs_key");
+		vi.stubEnv("CROWDSEC_CTI_DAILY_QUOTA", "1");
+		fetchMock.mockImplementation(() =>
+			Promise.resolve(jsonResponse(maliciousItem(blockIp()))),
+		);
+
+		await maybeAutoBlockCrowdsec({
+			ip: blockIp(),
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		await maybeAutoBlockCrowdsec({
+			ip: "198.51.100.2",
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		await tick();
+		expect(fetchMock).toHaveBeenCalledTimes(1);
+		expect(state.sendAlert).toHaveBeenCalledTimes(1);
+		const [input] = state.sendAlert.mock.calls[0];
+		expect(input.type).toBe("ddos");
+		expect(input.severity).toBe("warning");
+		expect(input.message).toContain("quota exhausted");
+		expect(input.context).toMatchObject({ quota: 1 });
+	});
+
+	it("floods once per cooldown window when blocks burst past the threshold", async () => {
+		vi.stubEnv("CROWDSEC_API_KEY", "cs_key");
+		vi.stubEnv("CROWDSEC_ALERT_BLOCK_BURST", "2");
+		vi.stubEnv("CROWDSEC_CTI_DAILY_QUOTA", "0");
+		fetchMock.mockImplementation((url: string | URL) =>
+			Promise.resolve(
+				jsonResponse(maliciousItem(String(url).split("/").pop() ?? "ip")),
+			),
+		);
+
+		await maybeAutoBlockCrowdsec({
+			ip: "198.51.100.71",
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		await maybeAutoBlockCrowdsec({
+			ip: "198.51.100.72",
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		// Third block in the same window: threshold crossed, but the alert is
+		// cooldown-gated so it still fires exactly once.
+		await maybeAutoBlockCrowdsec({
+			ip: "198.51.100.73",
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		await tick();
+		expect(state.sendAlert).toHaveBeenCalledTimes(1);
+		const [input] = state.sendAlert.mock.calls[0];
+		expect(input.type).toBe("ddos");
+		expect(input.context).toMatchObject({ blocks: 2, threshold: 2 });
+		expect(state.map.get(`antiddos:block:198.51.100.72`)).toBe("crowdsec");
+	});
+
+	it("tallies lookups and blocks into the daily stats histogram", async () => {
+		vi.stubEnv("CROWDSEC_API_KEY", "cs_key");
+		vi.stubEnv("CROWDSEC_CTI_DAILY_QUOTA", "0");
+		fetchMock.mockImplementation((url: string | URL) => {
+			const ip = String(url).split("/").pop() ?? "ip";
+			return Promise.resolve(
+				jsonResponse(
+					ip === "198.51.100.83" ? suspiciousItem(ip, 3) : maliciousItem(ip),
+				),
+			);
+		});
+
+		await maybeAutoBlockCrowdsec({
+			ip: "198.51.100.81",
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		await maybeAutoBlockCrowdsec({
+			ip: "198.51.100.82",
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 4,
+			enabled: true,
+		});
+		await maybeAutoBlockCrowdsec({
+			ip: "198.51.100.83",
+			category: "api",
+			ttlSeconds: 600,
+			scoreThreshold: 7, // suspicious/known verdicts below threshold: lookup only
+			enabled: true,
+		});
+		await tick();
+
+		const stats = await getCrowdsecStats(1);
+		const today: CrowdsecDailyStat | undefined = stats.find(
+			(row) => row.date === new Date().toISOString().slice(0, 10),
+		);
+		expect(today?.lookups).toBe(3);
+		expect(today?.blocks).toBe(2);
+		expect(today?.reportFailures).toBe(0);
+		expect(
+			state.map.get(
+				`crowdsec:stat:lookups:${new Date().toISOString().slice(0, 10)}`,
+			),
+		).toBe("3");
+	});
 });