258 lines
10 KiB
Bash
258 lines
10 KiB
Bash
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$DIR"
|
|
ENV_FILE="$DIR/.env"
|
|
COMPOSE_FILE="deployment/crowdsec/compose.crowdsec.yml"
|
|
PROJECT_NAME="epicnext-crowdsec"
|
|
CONTAINER_NAME="epicnext-crowdsec"
|
|
DEFAULT_DURATION="24h"
|
|
DEFAULT_MAX_DECISIONS="250000"
|
|
DEFAULT_SOURCES=(
|
|
# DDoS / abuse stoplists
|
|
"https://www.spamhaus.org/drop/drop.txt"
|
|
"https://www.spamhaus.org/drop/edrop.txt"
|
|
"https://www.dshield.org/block.txt"
|
|
"https://cinsscore.com/list/ci-badguys.txt"
|
|
"https://blocklist.greensnow.co/greensnow.txt"
|
|
"https://www.stopforumspam.com/downloads/toxic_ip_cidr.txt"
|
|
"https://www.binarydefense.com/banlist.txt"
|
|
# Brute force / credential stuffing
|
|
"https://lists.blocklist.de/lists/all.txt"
|
|
"https://lists.blocklist.de/lists/ssh.txt"
|
|
"https://lists.blocklist.de/lists/apache.txt"
|
|
"https://rules.emergingthreats.net/blockrules/compromised-ips.txt"
|
|
"https://danger.rulez.sk/projects/bruteforceblocker/blist.php"
|
|
# Malware C2 / botnets
|
|
"https://feodotracker.abuse.ch/downloads/ipblocklist.txt"
|
|
"https://sslbl.abuse.ch/blacklist/sslipblacklist.txt"
|
|
"https://www.botvrij.eu/data/ioclist.ip-dst.raw"
|
|
# Live SSH/spam attackers (last 48h), bare IPs only
|
|
"https://www.darklist.de/raw.php"
|
|
# Aggregated threat intel
|
|
"https://raw.githubusercontent.com/stamparm/ipsum/master/levels/3.txt"
|
|
"https://raw.githubusercontent.com/stamparm/ipsum/master/levels/2.txt"
|
|
# Firehol ipsets (security scanners, abusers, proxies, anonymous)
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_abusers_1d.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_abusers_30d.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_proxies.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_anonymous.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset"
|
|
# Tor exit nodes
|
|
"https://check.torproject.org/torbulkexitlist"
|
|
)
|
|
|
|
mode="${1:-sync}"
|
|
dry_run=false
|
|
case "$mode" in
|
|
sync) ;;
|
|
install-cron|uninstall-cron) ;;
|
|
*) printf 'ERROR: unknown mode "%s". Modes: sync [--dry-run] | install-cron | uninstall-cron\n' "$mode" >&2; exit 1 ;;
|
|
esac
|
|
[[ "${2:-}" = --dry-run ]] && dry_run=true
|
|
|
|
umask 077
|
|
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
|
for command in docker curl flock; do command -v "$command" >/dev/null || fail "Required command: $command"; done
|
|
docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin required."
|
|
exec 9>"$DIR/.deploy.lock"
|
|
flock -w 30 9 || fail "Another installation, update or sync is running."
|
|
[[ -f "$ENV_FILE" ]] || fail "Create .env first (bash cms install)."
|
|
|
|
env_get() {
|
|
local key="$1" line
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
case "$line" in
|
|
"$key="*) line="${line#*=}"; line="${line%\"}"; line="${line#\"}"; printf '%s' "$line"; return 0 ;;
|
|
esac
|
|
done < "$ENV_FILE"
|
|
return 1
|
|
}
|
|
|
|
compose_cmd() {
|
|
docker compose --project-name "$PROJECT_NAME" --env-file "$ENV_FILE" -f "$COMPOSE_FILE" --profile security "$@"
|
|
}
|
|
|
|
container_running() {
|
|
[[ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || true)" = true ]]
|
|
}
|
|
|
|
cscli_exec() {
|
|
compose_cmd exec -T crowdsec cscli "$@"
|
|
}
|
|
|
|
fetch_sources() {
|
|
local target="$1"
|
|
local sources=( "${DEFAULT_SOURCES[@]}" )
|
|
IFS=' ' read -r -a parsed <<< "${CROWDSEC_BLOCKLIST_SOURCES:-}"
|
|
[[ "${#parsed[@]}" -gt 0 ]] && sources=( "${parsed[@]}" )
|
|
local index=0 url
|
|
for url in "${sources[@]}"; do
|
|
[[ -n "$url" ]] || continue
|
|
index=$((index + 1))
|
|
if ! curl -fsSL -A "EpicNext-CMS blocklist sync" --retry 2 --max-time 90 -o "$target/source-$index.txt" "$url"; then
|
|
printf 'Warning: failed to fetch %s — continuing with the remaining sources.\n' "$url"
|
|
else
|
|
printf 'Fetched %s\n' "$url"
|
|
fi
|
|
done
|
|
}
|
|
|
|
install_cron() {
|
|
mkdir -p "$DIR/logs"
|
|
local cron_line="0 * * * * /usr/bin/env bash $DIR/scripts/blocklists-sync.sh >> $DIR/logs/blocklists-sync.log 2>&1"
|
|
if crontab -l 2>/dev/null | grep -Fq "$DIR/scripts/blocklists-sync.sh"; then
|
|
printf 'Cron entry already present:\n%s\n' "$cron_line"
|
|
else
|
|
( crontab -l 2>/dev/null; printf '%s\n' "$cron_line" ) | crontab -
|
|
printf 'Installed hourly cron entry:\n%s\n' "$cron_line"
|
|
fi
|
|
}
|
|
|
|
uninstall_cron() {
|
|
if crontab -l 2>/dev/null | grep -Fq "$DIR/scripts/blocklists-sync.sh"; then
|
|
crontab -l 2>/dev/null | grep -Fv "$DIR/scripts/blocklists-sync.sh" | crontab -
|
|
printf 'Removed cron entry matching %s.\n' "$DIR/scripts/blocklists-sync.sh"
|
|
else
|
|
printf 'No cron entry to remove.\n'
|
|
fi
|
|
}
|
|
|
|
if [[ "$mode" = install-cron ]]; then
|
|
install_cron
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$mode" = uninstall-cron ]]; then
|
|
uninstall_cron
|
|
exit 0
|
|
fi
|
|
|
|
duration="$(env_get CROWDSEC_BLOCKLIST_DURATION 2>/dev/null || true)"
|
|
[[ -n "$duration" ]] || duration="$DEFAULT_DURATION"
|
|
max_decisions="$(env_get CROWDSEC_BLOCKLIST_MAX_DECISIONS 2>/dev/null || true)"
|
|
[[ -n "$max_decisions" ]] || max_decisions="$DEFAULT_MAX_DECISIONS"
|
|
[[ "$max_decisions" =~ ^[0-9]+$ ]] || fail "CROWDSEC_BLOCKLIST_MAX_DECISIONS must be a number."
|
|
allowlist="${CROWDSEC_BLOCKLIST_ALLOW:-$(env_get CROWDSEC_BLOCKLIST_ALLOW 2>/dev/null || true)}"
|
|
|
|
work="$(mktemp -d "$DIR/.blocklists.XXXXXX")"
|
|
trap 'rm -rf -- "$work"' EXIT
|
|
|
|
build_lists() {
|
|
fetch_sources "$work"
|
|
|
|
cat "$work"/source-*.txt 2>/dev/null | awk '{print $1}' \
|
|
| grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$|^([0-9a-fA-F]{1,4}:){2,}[0-9a-fA-F:]*[0-9a-fA-F](/[0-9]{1,3})?$' \
|
|
| awk '
|
|
# Only globally routable attacker space may become a decision. Reserved,
|
|
# private, loopback, link-local, CGNAT, test and multicast ranges never
|
|
# represent an external attacker and must not be imported (they could
|
|
# otherwise block the origin itself or internal traffic).
|
|
function isReserved4(prefix, a, b, c) {
|
|
if (a == 0 || a == 127 || a >= 224) return 1
|
|
if (a == 10) return 1
|
|
if (a == 100 && (prefix < 10 || (prefix >= 10 && b >= 64 && b <= 127))) return 1
|
|
if (a == 169 && (prefix < 16 || (prefix >= 16 && b == 254))) return 1
|
|
if (a == 172 && (prefix < 12 || (prefix >= 12 && b >= 16 && b <= 31))) return 1
|
|
if (a == 192 && b == 168) return 1
|
|
if (a == 192 && b == 0) return 1
|
|
if ((a == 198 && (b == 18 || b == 19)) || (a == 198 && b == 51 && c == 100)) return 1
|
|
if (a == 203 && b == 0 && c == 113) return 1
|
|
return 0
|
|
}
|
|
function isReserved6(line, prefix, first, h) {
|
|
if (prefix < 32) return 1
|
|
if (line ~ /^::/) return 1
|
|
first = tolower(line); sub(/^::?/, "", first); sub(/:.*/, "", first)
|
|
h = "0x" substr(first, 1, 2)
|
|
if (h >= 252) return 1 # ULA fc00::/7, link-local fe80::/10, multicast ff00::/8
|
|
return 0
|
|
}
|
|
{
|
|
if (index($0, "/") > 0) {
|
|
n = split($0, seg, "/")
|
|
if (n != 2 || seg[2] !~ /^[0-9]+$/) next
|
|
if (index(seg[1], ":") > 0) {
|
|
pref = seg[2] + 0
|
|
if (pref < 32 || pref > 128) next
|
|
if (isReserved6(seg[1], pref)) next
|
|
print
|
|
next
|
|
}
|
|
pref = seg[2] + 0
|
|
if (pref < 8 || pref > 32) next
|
|
split(seg[1], oct, ".")
|
|
ok = 1
|
|
for (i = 1; i <= 4; i++) {
|
|
if (oct[i] !~ /^[0-9]+$/ || oct[i] + 0 > 255) { ok = 0; break }
|
|
if (length(oct[i]) > 1 && oct[i] ~ /^0/) { ok = 0; break }
|
|
}
|
|
if (!ok) next
|
|
if (isReserved4(pref, oct[1] + 0, oct[2] + 0, oct[3] + 0)) next
|
|
print
|
|
next
|
|
}
|
|
if (index($0, ":") > 0) {
|
|
if (isReserved6($0, 128)) next
|
|
print
|
|
next
|
|
}
|
|
n = split($0, part, ".")
|
|
if (n != 4) next
|
|
ok = 1
|
|
for (i = 1; i <= 4; i++) {
|
|
if (part[i] !~ /^[0-9]+$/ || part[i] + 0 > 255) { ok = 0; break }
|
|
if (length(part[i]) > 1 && part[i] ~ /^0/) { ok = 0; break }
|
|
}
|
|
if (!ok) next
|
|
if (isReserved4(32, part[1] + 0, part[2] + 0, part[3] + 0)) next
|
|
print
|
|
}' \
|
|
| sort -u > "$work/candidates.txt"
|
|
|
|
if [[ -n "$allowlist" ]]; then
|
|
printf '%s\n' "$allowlist" | tr ',' '\n' | while IFS= read -r line; do printf '%s\n' "$line"; done | sort -u > "$work/allow.txt"
|
|
comm -23 "$work/candidates.txt" "$work/allow.txt" > "$work/final.txt"
|
|
else
|
|
cp "$work/candidates.txt" "$work/final.txt"
|
|
fi
|
|
|
|
if [[ "$(wc -l < "$work/final.txt" | tr -d ' ')" -gt "$max_decisions" ]]; then
|
|
sort -u "$work/final.txt" | head -n "$max_decisions" > "$work/final.limited.txt" || true
|
|
mv "$work/final.limited.txt" "$work/final.txt"
|
|
printf 'Note: capped the combined list at %s decisions (CROWDSEC_BLOCKLIST_MAX_DECISIONS).\n' "$max_decisions"
|
|
fi
|
|
|
|
count_total=$(wc -l < "$work/final.txt" | tr -d ' ')
|
|
count_ip=$(grep -cv '/' "$work/final.txt" || true)
|
|
count_range=$(grep -c '/' "$work/final.txt" || true)
|
|
if [[ "$count_total" -lt 1 ]]; then
|
|
fail "No valid addresses could be parsed from the configured sources. Configure CROWDSEC_BLOCKLIST_SOURCES."
|
|
fi
|
|
}
|
|
|
|
build_lists
|
|
|
|
printf 'Parsed %s targets (%s IPs, %s ranges).\n' "$count_total" "$count_ip" "$count_range"
|
|
|
|
if $dry_run; then
|
|
printf 'Dry run: would replace the cscli-import decisions with these %s targets.\n' "$count_total"
|
|
exit 0
|
|
fi
|
|
|
|
container_running || fail "The CrowdSec engine is not running. Start it first with: bash cms security"
|
|
|
|
printf 'Removing previous cscli-import decisions...\n'
|
|
cscli_exec decisions delete --origin cscli-import >/dev/null 2>&1 || true
|
|
|
|
{
|
|
printf 'duration,scope,value\n'
|
|
awk -v d="$duration" '{ if (index($0, "/") > 0) printf "%s,range,%s\n", d, $0; else printf "%s,ip,%s\n", d, $0 }' "$work/final.txt"
|
|
} > "$work/import.csv"
|
|
|
|
printf 'Importing %s decisions into the local LAPI (duration %s)...\n' "$count_total" "$duration"
|
|
cscli_exec decisions import -i - --format csv --batch 1000 < "$work/import.csv"
|
|
|
|
printf 'Done. The app bouncer picks these up within a few seconds.\n' |