feat(security): recovery alerts, gate-block sharing, rolling-window burst and admin breakdown for CrowdSec

src/lib/crowdsec-api.ts

@@ -1,9 +1,13 @@
 import "server-only";
 
+import { randomUUID } from "node:crypto";
 import { env } from "@/env";
 import { raiseCrowdsecAlert } from "@/lib/crowdsec-alerts";
 import { reportCrowdsecSignal } from "@/lib/crowdsec-report";
-import { bumpCrowdsecStat } from "@/lib/crowdsec-stats";
+import {
+	bumpCrowdsecBreakdownStat,
+	bumpCrowdsecStat,
+} from "@/lib/crowdsec-stats";
 import { logger } from "@/lib/logger";
 import { redis } from "@/lib/redis";
 import { UNKNOWN_CLIENT_IP } from "./client-ip";
@@ -81,7 +85,7 @@ export interface CrowdsecConnectionStatus {
 
 /** Why a CrowdSec-sourced block exists — persisted next to the block key. */
 export interface CrowdsecBlockMeta {
-	source: typeof CROWDSEC_BLOCK_SOURCE;
+	source: typeof CROWDSEC_BLOCK_SOURCE | "gate";
 	category: string;
 	reputation: CrowdsecReputation | null;
 	score: number;
@@ -117,7 +121,7 @@ const QUOTA_PREFIX = "crowdsec:usage:";
 const QUOTA_KEY_TTL_SECONDS = 48 * 3_600;
 /** Shared 403/429 pause marker, so every instance respects the backoff. */
 const BACKOFF_KEY = "crowdsec:backoff-until";
-/** Short-window block burst counter: crowdsec:burst:{unix-5min-bucket}. */
+/** Short-window block burst counter: crowdsec:burst:recent (ZSET of timestamps). */
 const BURST_PREFIX = "crowdsec:burst:";
 const BURST_WINDOW_SECONDS = 300;
 /** In-process verdict cache cap so a flood of distinct IPs cannot grow it forever. */
@@ -319,6 +323,7 @@ async function acquireLookupLock(ip: string): Promise<boolean> {
 
 let backoffUntil = 0;
 let quotaWarnedDate: string | null = null;
+let quotaExhaustedDate: string | null = null;
 
 /**
  * Next moment (epoch ms) the CTI API may be called again — the max of the
@@ -418,6 +423,7 @@ async function reserveQuota(): Promise<boolean> {
 			// back and refuse: the budget would be spent the very next call
 			// anyway, so stopping here is both safe and quota-exact.
 			await redis.decr(key);
+			quotaExhaustedDate = date;
 			logger.warn(
 				"[crowdsec-api] CTI daily quota exhausted — pausing lookups until tomorrow",
 				{ quota },
@@ -437,6 +443,17 @@ async function reserveQuota(): Promise<boolean> {
 				quota,
 			});
 		}
+		if (quotaExhaustedDate) {
+			// A reserve just succeeded after an exhaustion day (counter was
+			// reset or the calendar rolled over) — say so, once per cooldown.
+			void raiseCrowdsecAlert("quota-restored", {
+				type: "ddos",
+				severity: "info",
+				message: `CrowdSec reputation quota is available again (${used} of ${quota} used today) — lookups resumed.`,
+				context: { used, quota, date },
+			});
+			quotaExhaustedDate = null;
+		}
 		return true;
 	} catch {
 		// Redis hiccup at a moment we could not count — allow the call rather
@@ -452,23 +469,27 @@ function dailyBlockBurstThreshold(): number {
 }
 
 /**
- * A burst of new community-reputation blocks is usually an automated attack
- * wave. Count blocks into a rolling 5-minute bucket and alert once per
- * cooldown window when they cross CROWDSEC_ALERT_BLOCK_BURST. Fire-and-forget.
+ * A burst of new blocks is usually an automated attack wave. Track block
+ * timestamps in a rolling window (Redis sorted set, 5 minutes) so a burst that
+ * straddles a bucket boundary is still counted together, and alert once per
+ * cooldown window when the count crosses CROWDSEC_ALERT_BLOCK_BURST.
+ * Fire-and-forget.
  */
 async function trackBlockBurst(): Promise<void> {
 	if (!redis) return;
-	const bucket = Math.floor(Date.now() / 1000 / BURST_WINDOW_SECONDS);
-	const key = `${BURST_PREFIX}${bucket}`;
+	const now = Date.now();
+	const key = `${BURST_PREFIX}recent`;
 	const threshold = dailyBlockBurstThreshold();
 	try {
-		const count = await redis.incr(key);
+		await redis.zadd(key, now, randomUUID());
+		await redis.zremrangebyscore(key, 0, now - BURST_WINDOW_SECONDS * 1000);
+		const count = await redis.zcard(key);
 		await redis.expire(key, BURST_WINDOW_SECONDS * 2);
 		if (count >= threshold) {
 			void raiseCrowdsecAlert("block-burst", {
 				type: "ddos",
 				severity: "warning",
-				message: `CrowdSec community reputation blocked ${count} IPs in the last ${BURST_WINDOW_SECONDS / 60} minutes — likely an automated attack wave.`,
+				message: `Anti-DDoS auto-block created ${count} blocks in the last ${BURST_WINDOW_SECONDS / 60} minutes — likely an automated attack wave.`,
 				context: {
 					blocks: count,
 					windowSeconds: BURST_WINDOW_SECONDS,
@@ -531,9 +552,18 @@ export async function lookupCrowdsecVerdict(
 			return verdict;
 		}
 		if (response.status === 403) {
+			const detail = await errorDetail(response);
 			await setBackoff(AUTH_BACKOFF_MS);
+			// A rejected key paralyses the whole reputation pipeline — surface
+			// it once (cooldown-gated) so rotating the key is an ops priority.
+			void raiseCrowdsecAlert("cti-auth", {
+				type: "ddos",
+				severity: "critical",
+				message: `CrowdSec CTI API key rejected (HTTP 403): ${detail} — reputation lookups are paused for ${Math.round(AUTH_BACKOFF_MS / 60_000)} minutes. Rotate CROWDSEC_API_KEY.`,
+				context: { status: 403, detail, backoffMs: AUTH_BACKOFF_MS },
+			});
 			throw new CrowdsecApiError(
-				`CrowdSec API key rejected (HTTP 403): ${await errorDetail(response)}`,
+				`CrowdSec API key rejected (HTTP 403): ${detail}`,
 			);
 		}
 		if (response.status === 429) {
@@ -542,6 +572,12 @@ export async function lookupCrowdsecVerdict(
 				ip,
 				backoffMs: RATE_LIMIT_BACKOFF_MS,
 			});
+			void raiseCrowdsecAlert("cti-ratelimit", {
+				type: "ddos",
+				severity: "warning",
+				message: `CrowdSec CTI API rate limited — all instances backed off for ${Math.round(RATE_LIMIT_BACKOFF_MS / 1000)}s.`,
+				context: { status: 429, backoffMs: RATE_LIMIT_BACKOFF_MS },
+			});
 			return null;
 		}
 		if (!response.ok) {
@@ -635,6 +671,10 @@ export async function maybeAutoBlockCrowdsec(input: {
 		void reportCrowdsecSignal({ ip, category, ttlSeconds, verdict, meta });
 		// Daily histogram + burst detection (cooldown-gated ops alert).
 		void bumpCrowdsecStat("blocks");
+		void bumpCrowdsecBreakdownStat("category", category);
+		if (verdict.reputation) {
+			void bumpCrowdsecBreakdownStat("reputation", verdict.reputation);
+		}
 		void trackBlockBurst();
 	} catch (error) {
 		logger.error("[crowdsec-api] Automatic IP block failed", {
@@ -744,5 +784,6 @@ export function resetCrowdsecCache(): void {
 	memoryVerdicts.clear();
 	backoffUntil = 0;
 	quotaWarnedDate = null;
+	quotaExhaustedDate = null;
 	lastVerifyMemory = null;
 }