From 9562a7537897890054e30eecae2b7efba8659ffb Mon Sep 17 00:00:00 2001 From: openhands Date: Thu, 24 Sep 2026 18:39:38 +0200 Subject: [PATCH] feat(security): external IP blocklist sync for the local CrowdSec engine --- .env.example | 12 ++ README.md | 30 ++++ cms | 2 +- docs/operations/docker-installation.md | 2 + scripts/blocklists-sync.sh | 210 +++++++++++++++++++++++++ scripts/crowdsec-setup.sh | 9 +- 6 files changed, 263 insertions(+), 2 deletions(-) create mode 100644 scripts/blocklists-sync.sh diff --git a/.env.example b/.env.example index 83ce1f69..837020c1 100644 --- a/.env.example +++ b/.env.example @@ -126,6 +126,18 @@ CROWDSEC_LAPI_PORT=18080 CROWDSEC_LAPI_URL=http://127.0.0.1:18080 # Generated by `bash cms security`; keep in .env, never commit a value. CROWDSEC_LAPI_API_KEY= +# IP blocklist sync (`bash cms security blocklists`): space-separated URLs, by +# default Spamhaus DROP/EDROP, DShield, CINS, Greensnow, StopForumSpam, +# blocklist.de, Emerging Threats, abuse.ch Feodo/SSLBL/URLhaus, IPsum, +# Firehol ipsets and Tor exit nodes. Requires internet to fetch; detection and +# blocking stay local. +#CROWDSEC_BLOCKLIST_SOURCES=https://www.spamhaus.org/drop/drop.txt https://example.org/list.txt +# Expiration for each blocklist decision (re-synced keeps them fresh). +#CROWDSEC_BLOCKLIST_DURATION=24h +# Combined cap per sync (safety valve against excessive decisions). +#CROWDSEC_BLOCKLIST_MAX_DECISIONS=250000 +# Comma-separated IPs/CIDRs that a sync must always skip (allowlist). +#CROWDSEC_BLOCKLIST_ALLOW=1.2.3.4,10.0.0.0/8 # --- PATHS --- BADGE_UPLOAD_DIR=./public/assets/images/badges diff --git a/README.md b/README.md index e0419ba2..e6b74731 100644 --- a/README.md +++ b/README.md @@ -827,6 +827,36 @@ bash cms security disable Stops the container and sets `CROWDSEC_LOCAL_ENABLED=false`. Volumes and the `.env` key are kept. +### Step 5 — Load external IP blocklists (optional) + +The engine has no built-in lists, so provide your own via a one-shot sync +(fetches the sources, replaces every previous `cscli-import` decision): + +```bash +bash cms security blocklists +``` + +Defaults: Spamhaus DROP/EDROP, DShield, CINS, Greensnow, StopForumSpam, +Binary Defense, blocklist.de, Emerging Threats, BruteForceBlocker, abuse.ch +Feodo/SSLBL/URLhaus, Botvrij, IPsum, Firehol ipsets and Tor exit nodes +(26 sources). The largest commercial/crowdsourced lists (AbuseIPDB, MaxMind, +Cisco Talos, AlienVault OTX) are not included because they require an account +or API key; IPsum already aggregates ~30 additional feeds. No account is +needed, but internet access is — only for fetching; detection and blocking +remain local. Sync hourly as a cron job: + +```bash +bash cms security blocklists-install-cron +``` + +Removal: `bash cms security blocklists-uninstall-cron`. Dry-run without +touching LAPI: `bash cms security blocklists --dry-run`. Override the sources, +duration, a combined cap or an allowlist in `.env` +(`CROWDSEC_BLOCKLIST_SOURCES`, `CROWDSEC_BLOCKLIST_DURATION`, +`CROWDSEC_BLOCKLIST_MAX_DECISIONS`, `CROWDSEC_BLOCKLIST_ALLOW`). Existing +`cscli-import` decisions are replaced on every sync, so removed entries +expire. + ### Environment variables | Variable | Default | Purpose | diff --git a/cms b/cms index a2a510ea..45b94923 100644 --- a/cms +++ b/cms @@ -5,6 +5,6 @@ case "${1:-help}" in install) shift; exec bash "$DIR/scripts/docker-install.sh" "$@" ;; update) shift; exec bash "$DIR/scripts/docker-update.sh" "$@" ;; security) shift; exec bash "$DIR/scripts/crowdsec-setup.sh" "$@" ;; - help|--help|-h) printf '%s\n' 'bash cms install Configure and install on a Linux Docker host' 'bash cms update Update using saved settings; --skip-pull uses checked-out release' 'bash cms security Configure the opt-in local CrowdSec stack (enable|status|disable)' ;; + help|--help|-h) printf '%s\n' 'bash cms install Configure and install on a Linux Docker host' 'bash cms update Update using saved settings; --skip-pull uses checked-out release' 'bash cms security Configure the opt-in local CrowdSec stack (enable|status|disable|blocklists)' ;; *) echo "Unknown command. Use: bash cms install | update | security" >&2; exit 1 ;; esac diff --git a/docs/operations/docker-installation.md b/docs/operations/docker-installation.md index ee83a773..5d837f48 100644 --- a/docs/operations/docker-installation.md +++ b/docs/operations/docker-installation.md @@ -101,6 +101,8 @@ This bouncer is application-layer: it sheds known-bad IPs at the CMS process and The running CMS loads the new env values on its next restart or deployment. For a CI-managed `epicnext-cms-app`, the next deploy (which sources `.env`) applies them; for a clone, `bash cms update --skip-pull` restarts it. `.env` now holds the LAPI key — keep its permissions restrictive. +External IP blocklists (Spamhaus, DShield, CINS, blocklist.de, abuse.ch, IPsum, Firehol, Tor exit nodes, …) can be synced into the local LAPI with `bash cms security blocklists`, and hourly with `bash cms security blocklists-install-cron` (no account, but internet to fetch). Configure via `CROWDSEC_BLOCKLIST_*`. + ## Routine and selected-release updates ```sh diff --git a/scripts/blocklists-sync.sh b/scripts/blocklists-sync.sh new file mode 100644 index 00000000..94bcc7d3 --- /dev/null +++ b/scripts/blocklists-sync.sh @@ -0,0 +1,210 @@ +#!/usr/bin/env bash +set -Eeuo pipefail +DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$DIR" +ENV_FILE="$DIR/.env" +COMPOSE_FILE="deployment/crowdsec/compose.crowdsec.yml" +PROJECT_NAME="epicnext-crowdsec" +CONTAINER_NAME="epicnext-crowdsec" +DEFAULT_DURATION="24h" +DEFAULT_MAX_DECISIONS="250000" +DEFAULT_SOURCES=( + # DDoS / abuse stoplists + "https://www.spamhaus.org/drop/drop.txt" + "https://www.spamhaus.org/drop/edrop.txt" + "https://www.dshield.org/block.txt" + "https://cinsscore.com/list/ci-badguys.txt" + "https://blocklist.greensnow.co/greensnow.txt" + "https://www.stopforumspam.com/downloads/toxic_ip_cidr.txt" + "https://www.binarydefense.com/banlist.txt" + # Brute force / credential stuffing + "https://lists.blocklist.de/lists/all.txt" + "https://lists.blocklist.de/lists/ssh.txt" + "https://lists.blocklist.de/lists/apache.txt" + "https://rules.emergingthreats.net/blockrules/compromised-ips.txt" + "https://danger.rulez.sk/projects/bruteforceblocker/blist.php" + # Malware C2 / botnets + "https://feodotracker.abuse.ch/downloads/ipblocklist.txt" + "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt" + "https://urlhaus.abuse.ch/downloads/text_online/" + "https://www.botvrij.eu/data/ioclist.ip-dst.raw" + # Aggregated threat intel + "https://raw.githubusercontent.com/stamparm/ipsum/master/levels/3.txt" + "https://raw.githubusercontent.com/stamparm/ipsum/master/levels/2.txt" + # Firehol ipsets (security scanners, abusers, proxies, anonymous) + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset" + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset" + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_abusers_1d.netset" + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_abusers_30d.netset" + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_proxies.netset" + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_anonymous.netset" + "https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset" + # Tor exit nodes + "https://check.torproject.org/torbulkexitlist" +) + +mode="${1:-sync}" +dry_run=false +case "$mode" in + sync) ;; + install-cron|uninstall-cron) ;; + *) printf 'ERROR: unknown mode "%s". Modes: sync [--dry-run] | install-cron | uninstall-cron\n' "$mode" >&2; exit 1 ;; +esac +[[ "${2:-}" = --dry-run ]] && dry_run=true + +umask 077 +fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; } +for command in docker curl flock; do command -v "$command" >/dev/null || fail "Required command: $command"; done +docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin required." +exec 9>"$DIR/.deploy.lock" +flock -w 30 9 || fail "Another installation, update or sync is running." +[[ -f "$ENV_FILE" ]] || fail "Create .env first (bash cms install)." + +env_get() { + local key="$1" line + while IFS= read -r line || [[ -n "$line" ]]; do + case "$line" in + "$key="*) line="${line#*=}"; line="${line%\"}"; line="${line#\"}"; printf '%s' "$line"; return 0 ;; + esac + done < "$ENV_FILE" + return 1 +} + +compose_cmd() { + docker compose --project-name "$PROJECT_NAME" --env-file "$ENV_FILE" -f "$COMPOSE_FILE" --profile security "$@" +} + +container_running() { + [[ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || true)" = true ]] +} + +cscli_exec() { + compose_cmd exec -T "$CONTAINER_NAME" cscli "$@" +} + +fetch_sources() { + local target="$1" + local sources=( "${DEFAULT_SOURCES[@]}" ) + IFS=' ' read -r -a parsed <<< "${CROWDSEC_BLOCKLIST_SOURCES:-}" + [[ "${#parsed[@]}" -gt 0 ]] && sources=( "${parsed[@]}" ) + local index=0 url + for url in "${sources[@]}"; do + [[ -n "$url" ]] || continue + index=$((index + 1)) + if ! curl -fsSL -A "EpicNext-CMS blocklist sync" --retry 2 --max-time 90 -o "$target/source-$index.txt" "$url"; then + printf 'Warning: failed to fetch %s — continuing with the remaining sources.\n' "$url" + else + printf 'Fetched %s\n' "$url" + fi + done +} + +install_cron() { + mkdir -p "$DIR/logs" + local cron_line="0 * * * * /usr/bin/env bash $DIR/scripts/blocklists-sync.sh >> $DIR/logs/blocklists-sync.log 2>&1" + if crontab -l 2>/dev/null | grep -Fq "$DIR/scripts/blocklists-sync.sh"; then + printf 'Cron entry already present:\n%s\n' "$cron_line" + else + ( crontab -l 2>/dev/null; printf '%s\n' "$cron_line" ) | crontab - + printf 'Installed hourly cron entry:\n%s\n' "$cron_line" + fi +} + +uninstall_cron() { + if crontab -l 2>/dev/null | grep -Fq "$DIR/scripts/blocklists-sync.sh"; then + crontab -l 2>/dev/null | grep -Fv "$DIR/scripts/blocklists-sync.sh" | crontab - + printf 'Removed cron entry matching %s.\n' "$DIR/scripts/blocklists-sync.sh" + else + printf 'No cron entry to remove.\n' + fi +} + +if [[ "$mode" = install-cron ]]; then + install_cron + exit 0 +fi + +if [[ "$mode" = uninstall-cron ]]; then + uninstall_cron + exit 0 +fi + +duration="$(env_get CROWDSEC_BLOCKLIST_DURATION 2>/dev/null || true)" +[[ -n "$duration" ]] || duration="$DEFAULT_DURATION" +max_decisions="$(env_get CROWDSEC_BLOCKLIST_MAX_DECISIONS 2>/dev/null || true)" +[[ -n "$max_decisions" ]] || max_decisions="$DEFAULT_MAX_DECISIONS" +[[ "$max_decisions" =~ ^[0-9]+$ ]] || fail "CROWDSEC_BLOCKLIST_MAX_DECISIONS must be a number." +allowlist="${CROWDSEC_BLOCKLIST_ALLOW:-$(env_get CROWDSEC_BLOCKLIST_ALLOW 2>/dev/null || true)}" + +work="$(mktemp -d "$DIR/.blocklists.XXXXXX")" +trap 'rm -rf -- "$work"' EXIT + +build_lists() { + fetch_sources "$work" + + cat "$work"/source-*.txt 2>/dev/null | awk '{print $1}' \ + | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){2,}[0-9a-fA-F:]+(/[0-9]+)?' \ + | awk ' + { + if (index($0, "/") > 0) { + n = split($0, seg, "/") + if (n != 2 || seg[2] !~ /^[0-9]+$/) next + if (index(seg[1], ":") > 0) { if (seg[2] + 0 <= 128) print; next } + if (seg[2] + 0 <= 32) print + next + } + n = split($0, part, ".") + if (n != 4) next + ok = 1 + for (i = 1; i <= 4; i++) { + if (part[i] !~ /^[0-9]+$/ || part[i] + 0 > 255) { ok = 0; break } + if (length(part[i]) > 1 && part[i] ~ /^0/) { ok = 0; break } + } + if (ok) print + }' \ + | sort -u > "$work/candidates.txt" + + if [[ -n "$allowlist" ]]; then + printf '%s\n' "$allowlist" | tr ',' '\n' | while IFS= read -r line; do printf '%s\n' "$line"; done | sort -u > "$work/allow.txt" + comm -23 "$work/candidates.txt" "$work/allow.txt" > "$work/final.txt" + else + cp "$work/candidates.txt" "$work/final.txt" + fi + + if [[ "$(wc -l < "$work/final.txt" | tr -d ' ')" -gt "$max_decisions" ]]; then + sort -u "$work/final.txt" | head -n "$max_decisions" > "$work/final.limited.txt" || true + mv "$work/final.limited.txt" "$work/final.txt" + printf 'Note: capped the combined list at %s decisions (CROWDSEC_BLOCKLIST_MAX_DECISIONS).\n' "$max_decisions" + fi + + count_total=$(wc -l < "$work/final.txt" | tr -d ' ') + count_ip=$(grep -cv '/' "$work/final.txt" || true) + count_range=$(grep -c '/' "$work/final.txt" || true) + if [[ "$count_total" -lt 1 ]]; then + fail "No valid addresses could be parsed from the configured sources. Configure CROWDSEC_BLOCKLIST_SOURCES." + fi +} + +build_lists + +printf 'Parsed %s targets (%s IPs, %s ranges).\n' "$count_total" "$count_ip" "$count_range" + +if $dry_run; then + printf 'Dry run: would replace the cscli-import decisions with these %s targets.\n' "$count_total" + exit 0 +fi + +container_running || fail "The CrowdSec engine is not running. Start it first with: bash cms security" + +printf 'Removing previous cscli-import decisions...\n' +cscli_exec decisions delete --origin cscli-import >/dev/null 2>&1 || true + +{ + printf 'duration,scope,value\n' + awk -v d="$duration" '{ if (index($0, "/") > 0) printf "%s,range,%s\n", d, $0; else printf "%s,ip,%s\n", d, $0 }' "$work/final.txt" +} > "$work/import.csv" + +printf 'Importing %s decisions into the local LAPI (duration %s)...\n' "$count_total" "$duration" +cscli_exec decisions import -i "$work/import.csv" --format csv --batch 1000 + +printf 'Done. The app bouncer picks these up within a few seconds.\n' \ No newline at end of file diff --git a/scripts/crowdsec-setup.sh b/scripts/crowdsec-setup.sh index 8c6f31ab..8c9bbc30 100644 --- a/scripts/crowdsec-setup.sh +++ b/scripts/crowdsec-setup.sh @@ -13,7 +13,8 @@ case "$mode" in enable|--enable) ;; status|--status) ;; disable|--disable) ;; - *) echo "Usage: bash cms security [enable|status|disable]" >&2; exit 1 ;; + blocklists|blocklists-install-cron|blocklists-uninstall-cron) ;; + *) echo "Usage: bash cms security [enable|status|disable|blocklists|blocklists-install-cron|blocklists-uninstall-cron]" >&2; exit 1 ;; esac umask 077 @@ -25,6 +26,12 @@ exec 9>"$DIR/.deploy.lock" flock -w 30 9 || fail "Another installation or update is running." [[ -f "$ENV_FILE" ]] || fail "Create .env first (bash cms install)." +case "$mode" in + blocklists) exec bash "$DIR/scripts/blocklists-sync.sh" sync "${2:-}" ;; + blocklists-install-cron) exec bash "$DIR/scripts/blocklists-sync.sh" install-cron ;; + blocklists-uninstall-cron) exec bash "$DIR/scripts/blocklists-sync.sh" uninstall-cron ;; +esac + env_get() { local key="$1" line while IFS= read -r line || [[ -n "$line" ]]; do