openhands
9562a75378
Gitea Actions Runner Test / test-job (push) Successful in 0s
CI / check (push) Successful in 30s
CI / tests-integration (push) Successful in 1m42s
CI / tests-unit (push) Successful in 1m43s
CI / tests-ui (push) Successful in 2m35s
CI / preflight (push) Skipped
CI / deploy (push) Successful in 2m3s
210 lines
8.0 KiB
Bash
210 lines
8.0 KiB
Bash
#!/usr/bin/env bash
|
|
set -Eeuo pipefail
|
|
DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
|
cd "$DIR"
|
|
ENV_FILE="$DIR/.env"
|
|
COMPOSE_FILE="deployment/crowdsec/compose.crowdsec.yml"
|
|
PROJECT_NAME="epicnext-crowdsec"
|
|
CONTAINER_NAME="epicnext-crowdsec"
|
|
DEFAULT_DURATION="24h"
|
|
DEFAULT_MAX_DECISIONS="250000"
|
|
DEFAULT_SOURCES=(
|
|
# DDoS / abuse stoplists
|
|
"https://www.spamhaus.org/drop/drop.txt"
|
|
"https://www.spamhaus.org/drop/edrop.txt"
|
|
"https://www.dshield.org/block.txt"
|
|
"https://cinsscore.com/list/ci-badguys.txt"
|
|
"https://blocklist.greensnow.co/greensnow.txt"
|
|
"https://www.stopforumspam.com/downloads/toxic_ip_cidr.txt"
|
|
"https://www.binarydefense.com/banlist.txt"
|
|
# Brute force / credential stuffing
|
|
"https://lists.blocklist.de/lists/all.txt"
|
|
"https://lists.blocklist.de/lists/ssh.txt"
|
|
"https://lists.blocklist.de/lists/apache.txt"
|
|
"https://rules.emergingthreats.net/blockrules/compromised-ips.txt"
|
|
"https://danger.rulez.sk/projects/bruteforceblocker/blist.php"
|
|
# Malware C2 / botnets
|
|
"https://feodotracker.abuse.ch/downloads/ipblocklist.txt"
|
|
"https://sslbl.abuse.ch/blacklist/sslipblacklist.txt"
|
|
"https://urlhaus.abuse.ch/downloads/text_online/"
|
|
"https://www.botvrij.eu/data/ioclist.ip-dst.raw"
|
|
# Aggregated threat intel
|
|
"https://raw.githubusercontent.com/stamparm/ipsum/master/levels/3.txt"
|
|
"https://raw.githubusercontent.com/stamparm/ipsum/master/levels/2.txt"
|
|
# Firehol ipsets (security scanners, abusers, proxies, anonymous)
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level1.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level2.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_abusers_1d.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_abusers_30d.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_proxies.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_anonymous.netset"
|
|
"https://raw.githubusercontent.com/firehol/blocklist-ipsets/master/firehol_level3.netset"
|
|
# Tor exit nodes
|
|
"https://check.torproject.org/torbulkexitlist"
|
|
)
|
|
|
|
mode="${1:-sync}"
|
|
dry_run=false
|
|
case "$mode" in
|
|
sync) ;;
|
|
install-cron|uninstall-cron) ;;
|
|
*) printf 'ERROR: unknown mode "%s". Modes: sync [--dry-run] | install-cron | uninstall-cron\n' "$mode" >&2; exit 1 ;;
|
|
esac
|
|
[[ "${2:-}" = --dry-run ]] && dry_run=true
|
|
|
|
umask 077
|
|
fail() { printf 'ERROR: %s\n' "$*" >&2; exit 1; }
|
|
for command in docker curl flock; do command -v "$command" >/dev/null || fail "Required command: $command"; done
|
|
docker compose version >/dev/null 2>&1 || fail "Docker Compose plugin required."
|
|
exec 9>"$DIR/.deploy.lock"
|
|
flock -w 30 9 || fail "Another installation, update or sync is running."
|
|
[[ -f "$ENV_FILE" ]] || fail "Create .env first (bash cms install)."
|
|
|
|
env_get() {
|
|
local key="$1" line
|
|
while IFS= read -r line || [[ -n "$line" ]]; do
|
|
case "$line" in
|
|
"$key="*) line="${line#*=}"; line="${line%\"}"; line="${line#\"}"; printf '%s' "$line"; return 0 ;;
|
|
esac
|
|
done < "$ENV_FILE"
|
|
return 1
|
|
}
|
|
|
|
compose_cmd() {
|
|
docker compose --project-name "$PROJECT_NAME" --env-file "$ENV_FILE" -f "$COMPOSE_FILE" --profile security "$@"
|
|
}
|
|
|
|
container_running() {
|
|
[[ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER_NAME" 2>/dev/null || true)" = true ]]
|
|
}
|
|
|
|
cscli_exec() {
|
|
compose_cmd exec -T "$CONTAINER_NAME" cscli "$@"
|
|
}
|
|
|
|
fetch_sources() {
|
|
local target="$1"
|
|
local sources=( "${DEFAULT_SOURCES[@]}" )
|
|
IFS=' ' read -r -a parsed <<< "${CROWDSEC_BLOCKLIST_SOURCES:-}"
|
|
[[ "${#parsed[@]}" -gt 0 ]] && sources=( "${parsed[@]}" )
|
|
local index=0 url
|
|
for url in "${sources[@]}"; do
|
|
[[ -n "$url" ]] || continue
|
|
index=$((index + 1))
|
|
if ! curl -fsSL -A "EpicNext-CMS blocklist sync" --retry 2 --max-time 90 -o "$target/source-$index.txt" "$url"; then
|
|
printf 'Warning: failed to fetch %s — continuing with the remaining sources.\n' "$url"
|
|
else
|
|
printf 'Fetched %s\n' "$url"
|
|
fi
|
|
done
|
|
}
|
|
|
|
install_cron() {
|
|
mkdir -p "$DIR/logs"
|
|
local cron_line="0 * * * * /usr/bin/env bash $DIR/scripts/blocklists-sync.sh >> $DIR/logs/blocklists-sync.log 2>&1"
|
|
if crontab -l 2>/dev/null | grep -Fq "$DIR/scripts/blocklists-sync.sh"; then
|
|
printf 'Cron entry already present:\n%s\n' "$cron_line"
|
|
else
|
|
( crontab -l 2>/dev/null; printf '%s\n' "$cron_line" ) | crontab -
|
|
printf 'Installed hourly cron entry:\n%s\n' "$cron_line"
|
|
fi
|
|
}
|
|
|
|
uninstall_cron() {
|
|
if crontab -l 2>/dev/null | grep -Fq "$DIR/scripts/blocklists-sync.sh"; then
|
|
crontab -l 2>/dev/null | grep -Fv "$DIR/scripts/blocklists-sync.sh" | crontab -
|
|
printf 'Removed cron entry matching %s.\n' "$DIR/scripts/blocklists-sync.sh"
|
|
else
|
|
printf 'No cron entry to remove.\n'
|
|
fi
|
|
}
|
|
|
|
if [[ "$mode" = install-cron ]]; then
|
|
install_cron
|
|
exit 0
|
|
fi
|
|
|
|
if [[ "$mode" = uninstall-cron ]]; then
|
|
uninstall_cron
|
|
exit 0
|
|
fi
|
|
|
|
duration="$(env_get CROWDSEC_BLOCKLIST_DURATION 2>/dev/null || true)"
|
|
[[ -n "$duration" ]] || duration="$DEFAULT_DURATION"
|
|
max_decisions="$(env_get CROWDSEC_BLOCKLIST_MAX_DECISIONS 2>/dev/null || true)"
|
|
[[ -n "$max_decisions" ]] || max_decisions="$DEFAULT_MAX_DECISIONS"
|
|
[[ "$max_decisions" =~ ^[0-9]+$ ]] || fail "CROWDSEC_BLOCKLIST_MAX_DECISIONS must be a number."
|
|
allowlist="${CROWDSEC_BLOCKLIST_ALLOW:-$(env_get CROWDSEC_BLOCKLIST_ALLOW 2>/dev/null || true)}"
|
|
|
|
work="$(mktemp -d "$DIR/.blocklists.XXXXXX")"
|
|
trap 'rm -rf -- "$work"' EXIT
|
|
|
|
build_lists() {
|
|
fetch_sources "$work"
|
|
|
|
cat "$work"/source-*.txt 2>/dev/null | awk '{print $1}' \
|
|
| grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){2,}[0-9a-fA-F:]+(/[0-9]+)?' \
|
|
| awk '
|
|
{
|
|
if (index($0, "/") > 0) {
|
|
n = split($0, seg, "/")
|
|
if (n != 2 || seg[2] !~ /^[0-9]+$/) next
|
|
if (index(seg[1], ":") > 0) { if (seg[2] + 0 <= 128) print; next }
|
|
if (seg[2] + 0 <= 32) print
|
|
next
|
|
}
|
|
n = split($0, part, ".")
|
|
if (n != 4) next
|
|
ok = 1
|
|
for (i = 1; i <= 4; i++) {
|
|
if (part[i] !~ /^[0-9]+$/ || part[i] + 0 > 255) { ok = 0; break }
|
|
if (length(part[i]) > 1 && part[i] ~ /^0/) { ok = 0; break }
|
|
}
|
|
if (ok) print
|
|
}' \
|
|
| sort -u > "$work/candidates.txt"
|
|
|
|
if [[ -n "$allowlist" ]]; then
|
|
printf '%s\n' "$allowlist" | tr ',' '\n' | while IFS= read -r line; do printf '%s\n' "$line"; done | sort -u > "$work/allow.txt"
|
|
comm -23 "$work/candidates.txt" "$work/allow.txt" > "$work/final.txt"
|
|
else
|
|
cp "$work/candidates.txt" "$work/final.txt"
|
|
fi
|
|
|
|
if [[ "$(wc -l < "$work/final.txt" | tr -d ' ')" -gt "$max_decisions" ]]; then
|
|
sort -u "$work/final.txt" | head -n "$max_decisions" > "$work/final.limited.txt" || true
|
|
mv "$work/final.limited.txt" "$work/final.txt"
|
|
printf 'Note: capped the combined list at %s decisions (CROWDSEC_BLOCKLIST_MAX_DECISIONS).\n' "$max_decisions"
|
|
fi
|
|
|
|
count_total=$(wc -l < "$work/final.txt" | tr -d ' ')
|
|
count_ip=$(grep -cv '/' "$work/final.txt" || true)
|
|
count_range=$(grep -c '/' "$work/final.txt" || true)
|
|
if [[ "$count_total" -lt 1 ]]; then
|
|
fail "No valid addresses could be parsed from the configured sources. Configure CROWDSEC_BLOCKLIST_SOURCES."
|
|
fi
|
|
}
|
|
|
|
build_lists
|
|
|
|
printf 'Parsed %s targets (%s IPs, %s ranges).\n' "$count_total" "$count_ip" "$count_range"
|
|
|
|
if $dry_run; then
|
|
printf 'Dry run: would replace the cscli-import decisions with these %s targets.\n' "$count_total"
|
|
exit 0
|
|
fi
|
|
|
|
container_running || fail "The CrowdSec engine is not running. Start it first with: bash cms security"
|
|
|
|
printf 'Removing previous cscli-import decisions...\n'
|
|
cscli_exec decisions delete --origin cscli-import >/dev/null 2>&1 || true
|
|
|
|
{
|
|
printf 'duration,scope,value\n'
|
|
awk -v d="$duration" '{ if (index($0, "/") > 0) printf "%s,range,%s\n", d, $0; else printf "%s,ip,%s\n", d, $0 }' "$work/final.txt"
|
|
} > "$work/import.csv"
|
|
|
|
printf 'Importing %s decisions into the local LAPI (duration %s)...\n' "$count_total" "$duration"
|
|
cscli_exec decisions import -i "$work/import.csv" --format csv --batch 1000
|
|
|
|
printf 'Done. The app bouncer picks these up within a few seconds.\n' |