fix(security): drop URLhaus feed, validate CIDR ranges, pass unknown client IPs

scripts/blocklists-sync.sh

@@ -26,7 +26,6 @@ DEFAULT_SOURCES=(
   # Malware C2 / botnets
   "https://feodotracker.abuse.ch/downloads/ipblocklist.txt"
   "https://sslbl.abuse.ch/blacklist/sslipblacklist.txt"
-  "https://urlhaus.abuse.ch/downloads/text_online/"
   "https://www.botvrij.eu/data/ioclist.ip-dst.raw"
   # Aggregated threat intel
   "https://raw.githubusercontent.com/stamparm/ipsum/master/levels/3.txt"
@@ -143,14 +142,59 @@ build_lists() {
   fetch_sources "$work"
 
   cat "$work"/source-*.txt 2>/dev/null | awk '{print $1}' \
-    | grep -Eo '([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]+)?|([0-9a-fA-F]{1,4}:){2,}[0-9a-fA-F:]+(/[0-9]+)?' \
+    | grep -E '^([0-9]{1,3}\.){3}[0-9]{1,3}(/[0-9]{1,2})?$|^([0-9a-fA-F]{1,4}:){2,}[0-9a-fA-F:]*[0-9a-fA-F](/[0-9]{1,3})?$' \
     | awk '
+        # Only globally routable attacker space may become a decision. Reserved,
+        # private, loopback, link-local, CGNAT, test and multicast ranges never
+        # represent an external attacker and must not be imported (they could
+        # otherwise block the origin itself or internal traffic).
+        function isReserved4(prefix, a, b, c) {
+          if (a == 0 || a == 127 || a >= 224) return 1
+          if (a == 10) return 1
+          if (a == 100 && (prefix < 10 || (prefix >= 10 && b >= 64 && b <= 127))) return 1
+          if (a == 169 && (prefix < 16 || (prefix >= 16 && b == 254))) return 1
+          if (a == 172 && (prefix < 12 || (prefix >= 12 && b >= 16 && b <= 31))) return 1
+          if (a == 192 && b == 168) return 1
+          if (a == 192 && b == 0) return 1
+          if ((a == 198 && (b == 18 || b == 19)) || (a == 198 && b == 51 && c == 100)) return 1
+          if (a == 203 && b == 0 && c == 113) return 1
+          return 0
+        }
+        function isReserved6(line, prefix,   first, h) {
+          if (prefix < 32) return 1
+          if (line ~ /^::/) return 1
+          first = tolower(line); sub(/^::?/, "", first); sub(/:.*/, "", first)
+          h = "0x" substr(first, 1, 2)
+          if (h >= 252) return 1    # ULA fc00::/7, link-local fe80::/10, multicast ff00::/8
+          return 0
+        }
         {
           if (index($0, "/") > 0) {
             n = split($0, seg, "/")
             if (n != 2 || seg[2] !~ /^[0-9]+$/) next
-            if (index(seg[1], ":") > 0) { if (seg[2] + 0 <= 128) print; next }
-            if (seg[2] + 0 <= 32) print
+            if (index(seg[1], ":") > 0) {
+              pref = seg[2] + 0
+              if (pref < 32 || pref > 128) next
+              if (isReserved6(seg[1], pref)) next
+              print
+              next
+            }
+            pref = seg[2] + 0
+            if (pref < 8 || pref > 32) next
+            split(seg[1], oct, ".")
+            ok = 1
+            for (i = 1; i <= 4; i++) {
+              if (oct[i] !~ /^[0-9]+$/ || oct[i] + 0 > 255) { ok = 0; break }
+              if (length(oct[i]) > 1 && oct[i] ~ /^0/) { ok = 0; break }
+            }
+            if (!ok) next
+            if (isReserved4(pref, oct[1] + 0, oct[2] + 0, oct[3] + 0)) next
+            print
+            next
+          }
+          if (index($0, ":") > 0) {
+            if (isReserved6($0, 128)) next
+            print
             next
           }
           n = split($0, part, ".")
@@ -160,7 +204,9 @@ build_lists() {
             if (part[i] !~ /^[0-9]+$/ || part[i] + 0 > 255) { ok = 0; break }
             if (length(part[i]) > 1 && part[i] ~ /^0/) { ok = 0; break }
           }
-          if (ok) print
+          if (!ok) next
+          if (isReserved4(32, part[1] + 0, part[2] + 0, part[3] + 0)) next
+          print
         }' \
     | sort -u > "$work/candidates.txt"