From 90b65c92a22180eb49f71455900d9a59456e9fb3 Mon Sep 17 00:00:00 2001 From: openhands Date: Mon, 28 Sep 2026 23:35:00 +0200 Subject: [PATCH] feat(proxy): sync Cloudflare ranges at nginx+Traefik, block IP spoofing - cloudflare-ips.conf (new): geo $cms_trusted_edge + set_real_ip_from from live CF IPv4/IPv6 ranges plus Traefik bridge and loopback - nginx-cms.conf: forward real client IP only from trusted peers, strip incoming CF-Connecting-IP, 403 any other peer that presents one (spoof gate); direct game clients on :9443 stay unaffected - cf-ips-sync.sh (new): fetch cloudflare.com/ips-v4/-v6, regenerate the nginx snippet and Traefik websecure.forwardedHeaders.trustedIPs - nginx-sync.sh: install the cloudflare-ips.conf snippet - cms_upstream_servers.conf: point default at the live green slot 3003 --- deployment/proxy/cloudflare-ips.conf | 81 +++++++++ deployment/proxy/cms_upstream_servers.conf | 2 +- deployment/proxy/nginx-cms.conf | 63 +++++-- deployment/proxy/nginx.conf | 8 +- scripts/cf-ips-sync.sh | 186 +++++++++++++++++++++ scripts/nginx-sync.sh | 2 + 6 files changed, 322 insertions(+), 20 deletions(-) create mode 100644 deployment/proxy/cloudflare-ips.conf create mode 100755 scripts/cf-ips-sync.sh diff --git a/deployment/proxy/cloudflare-ips.conf b/deployment/proxy/cloudflare-ips.conf new file mode 100644 index 00000000..c734a136 --- /dev/null +++ b/deployment/proxy/cloudflare-ips.conf @@ -0,0 +1,81 @@ +# Trusted edge networks + live Cloudflare CDN ranges. +# Managed/regenerated by scripts/cf-ips-sync.sh - do not hand-edit the ranges. + +# Topology: Cloudflare -> Traefik (:443, docker bridge proxy_traefik-proxy) -> +# nginx (:9443) -> CMS. nginx ALSO receives direct connections on :9443 from +# Cloudflare edges and from the game client (ws.epicnabbo.nl is not proxied). + +# nginx only trusts the peers listed here as a source of $remote_addr +# (via CF-Connecting-IP). Anyone else presenting a CF-Connecting-IP or +# CF-ray header is spoofing and is rejected in nginx-cms.conf. + +# 1 = peer is a trusted edge or internal network (keyed on the raw peer, +# unaffected by real_ip rewrites). +geo $realip_remote_addr $cms_trusted_edge { + default 0; + 127.0.0.0/8 1; # localhost (health checks, admin) + ::1 1; # localhost v6 + 172.22.0.0/16 1; # Traefik (proxyserver_traefik-proxy) + # --- Cloudflare IPv4 ranges (live from cloudflare.com/ips-v4) --- + 173.245.48.0/20 1; + 103.21.244.0/22 1; + 103.22.200.0/22 1; + 103.31.4.0/22 1; + 141.101.64.0/18 1; + 108.162.192.0/18 1; + 190.93.240.0/20 1; + 188.114.96.0/20 1; + 197.234.240.0/22 1; + 198.41.128.0/17 1; + 162.158.0.0/15 1; + 104.16.0.0/13 1; + 104.24.0.0/14 1; + 172.64.0.0/13 1; + 131.0.72.0/22 1; + # --- Cloudflare IPv6 ranges (live from cloudflare.com/ips-v6) --- + 2400:cb00::/32 1; + 2606:4700::/32 1; + 2803:f800::/32 1; + 2405:b500::/32 1; + 2405:8100::/32 1; + 2a06:98c0::/29 1; + 2c0f:f248::/32 1; +} + +# 1 when an UNTRUSTED peer still presents a CF-Connecting-IP header: that is a +# spoof attempt (only real Cloudflare edges or Traefik may do that lawfully). +map "$cms_trusted_edge:$http_cf_connecting_ip" $cms_disallow_forwarding { + default 0; + "~^0:.+" 1; +} + +# Rewrite $remote_addr from CF-Connecting-IP but ONLY for the trusted peers +# above. Direct game clients (untrusted) keep their real peer address. +set_real_ip_from 127.0.0.0/8; +set_real_ip_from ::1; +set_real_ip_from 172.22.0.0/16; +set_real_ip_from 173.245.48.0/20; +set_real_ip_from 103.21.244.0/22; +set_real_ip_from 103.22.200.0/22; +set_real_ip_from 103.31.4.0/22; +set_real_ip_from 141.101.64.0/18; +set_real_ip_from 108.162.192.0/18; +set_real_ip_from 190.93.240.0/20; +set_real_ip_from 188.114.96.0/20; +set_real_ip_from 197.234.240.0/22; +set_real_ip_from 198.41.128.0/17; +set_real_ip_from 162.158.0.0/15; +set_real_ip_from 104.16.0.0/13; +set_real_ip_from 104.24.0.0/14; +set_real_ip_from 172.64.0.0/13; +set_real_ip_from 131.0.72.0/22; +set_real_ip_from 2400:cb00::/32; +set_real_ip_from 2606:4700::/32; +set_real_ip_from 2803:f800::/32; +set_real_ip_from 2405:b500::/32; +set_real_ip_from 2405:8100::/32; +set_real_ip_from 2a06:98c0::/29; +set_real_ip_from 2c0f:f248::/32; + +real_ip_header CF-Connecting-IP; +real_ip_recursive off; diff --git a/deployment/proxy/cms_upstream_servers.conf b/deployment/proxy/cms_upstream_servers.conf index 786b1856..ec033ec2 100644 --- a/deployment/proxy/cms_upstream_servers.conf +++ b/deployment/proxy/cms_upstream_servers.conf @@ -1,2 +1,2 @@ # Default; ci-deploy.sh (blue/green) herschrijft dit bestand bij elke switch. -server 127.0.0.1:3002; \ No newline at end of file +server 127.0.0.1:3003; \ No newline at end of file diff --git a/deployment/proxy/nginx-cms.conf b/deployment/proxy/nginx-cms.conf index 638e2596..6446a38d 100644 --- a/deployment/proxy/nginx-cms.conf +++ b/deployment/proxy/nginx-cms.conf @@ -109,6 +109,16 @@ server { ssl_protocols TLSv1.2 TLSv1.3; ssl_prefer_server_ciphers off; + # ─── Trusted Edge Gate ─── + # Real Cloudflare edges and Traefik are the only peers trusted to supply a + # CF-Connecting-IP (see cloudflare-ips.conf). Any other peer that does is + # spoofing and is rejected before it reaches the CMS. Legitimate direct + # visitors (game client, :9443) never carry that header and pass through + # with their real peer address. + if ($cms_disallow_forwarding) { + return 403; + } + location /health { access_log off; return 200 "OK"; @@ -122,10 +132,10 @@ server { proxy_set_header Connection "upgrade"; proxy_set_header Host $host; - # Stuur het échte client IP direct door naar Polaris - proxy_set_header CF-Connecting-IP $http_cf_connecting_ip; - proxy_set_header X-Real-IP $http_cf_connecting_ip; - proxy_set_header X-Forwarded-For $http_cf_connecting_ip; +# Echt client IP (trusted peers via real_ip, directe clients = eigen peer) + proxy_set_header CF-Connecting-IP ""; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; proxy_read_timeout 86400s; @@ -162,6 +172,16 @@ server { add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; add_header Strict-Transport-Security "max-age=63072000; includeSubDomains; preload" always; + # ─── Trusted Edge Gate ─── + # Real Cloudflare edges / Traefik are the only peers allowed to supply a + # CF-Connecting-IP (see cloudflare-ips.conf). Any other peer presenting one + # is spoofing (direct :9443 traffic), and is rejected before it reaches the + # CMS. Legitimate direct visitors never carry that header and pass through + # with their real peer address. + if ($cms_disallow_forwarding) { + return 403; + } + # ─── Client Limits & Timeouts ─── client_max_body_size 20m; client_body_buffer_size 16k; @@ -262,9 +282,10 @@ server { proxy_pass http://cms_app; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header X-Real-IP $http_cf_connecting_ip; - proxy_set_header X-Forwarded-For $http_cf_connecting_ip; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header CF-Connecting-IP ""; proxy_set_header Connection ""; # Auth is per sessie: nooit cachen, en de app-header onderdrukken zodat # er precies één Cache-Control overblijft. @@ -284,9 +305,10 @@ server { proxy_pass http://cms_app; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header X-Real-IP $http_cf_connecting_ip; - proxy_set_header X-Forwarded-For $http_cf_connecting_ip; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header CF-Connecting-IP ""; proxy_set_header Connection ""; proxy_hide_header Cache-Control; @@ -311,9 +333,10 @@ server { proxy_pass http://cms_app; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header X-Real-IP $http_cf_connecting_ip; - proxy_set_header X-Forwarded-For $http_cf_connecting_ip; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header CF-Connecting-IP ""; proxy_set_header Connection ""; proxy_buffering off; @@ -334,9 +357,10 @@ server { proxy_pass http://127.0.0.1:2096; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header X-Real-IP $http_cf_connecting_ip; - proxy_set_header X-Forwarded-For $http_cf_connecting_ip; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header CF-Connecting-IP ""; proxy_set_header Connection ""; # De emulator levert zelf geen Cache-Control; zonder proxy_hide_header # zou de app-header hier een tweede keer worden toegevoegd. @@ -351,9 +375,10 @@ server { proxy_pass http://cms_app; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header X-Real-IP $http_cf_connecting_ip; - proxy_set_header X-Forwarded-For $http_cf_connecting_ip; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header CF-Connecting-IP ""; proxy_set_header Connection ""; proxy_read_timeout 30s; } @@ -362,9 +387,10 @@ server { proxy_pass http://cms_app; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header X-Real-IP $http_cf_connecting_ip; - proxy_set_header X-Forwarded-For $http_cf_connecting_ip; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header CF-Connecting-IP ""; proxy_set_header Connection ""; } @@ -373,9 +399,10 @@ server { proxy_pass http://cms_app; proxy_http_version 1.1; proxy_set_header Host $host; - proxy_set_header X-Real-IP $http_cf_connecting_ip; - proxy_set_header X-Forwarded-For $http_cf_connecting_ip; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header CF-Connecting-IP ""; proxy_set_header Connection ""; # De HTML is per sessie: `auth()` in de homepage-layout stuurt # ingelogde bezoekers door naar /me, en de CSP-nonce is per request. diff --git a/deployment/proxy/nginx.conf b/deployment/proxy/nginx.conf index cea267e4..9a4094d2 100644 --- a/deployment/proxy/nginx.conf +++ b/deployment/proxy/nginx.conf @@ -3,7 +3,9 @@ # and this file. Installed/synced by scripts/nginx-sync.sh so it cannot be # lost again while nginx keeps running on an in-memory copy. # -# Traffic path: Cloudflare -> Traefik (:443) -> nginx (:9443) -> CMS (:3002). +# Traffic path: Cloudflare -> Traefik (:443) -> nginx (:9443) -> CMS (:3002), +# with direct Cloudflare-origin and game-client (ws.epicnabbo.nl) connections +# also terminating on :9443. # nginx is the last layer that can still rewrite Cache-Control, so it owns the # headers it adds explicitly; everything proxied to the CMS is passed through # untouched unless this file says otherwise. @@ -46,4 +48,8 @@ http { # Cache policy maps and server blocks live in the site file so they are # synced together and can never drift apart. include /etc/nginx/sites-enabled/*.conf; + + # Trusted edge / real-IP handling (regenerated by scripts/cf-ips-sync.sh + # from the live Cloudflare ranges; installed via scripts/nginx-sync.sh). + include /etc/nginx/conf.d/cloudflare-ips.conf; } \ No newline at end of file diff --git a/scripts/cf-ips-sync.sh b/scripts/cf-ips-sync.sh new file mode 100755 index 00000000..401926c4 --- /dev/null +++ b/scripts/cf-ips-sync.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +# Keep the Cloudflare IP ranges in sync for BOTH proxy layers: +# 1) deployment/proxy/cloudflare-ips.conf (nginx: geo + set_real_ip_from) +# 2) entryPoints.websecure.forwardedHeaders.trustedIPs in /docker/proxyserver/traefik.yml +# +# The repo file is the source of truth for nginx (installed by nginx-sync.sh); +# Traefik's static config lives outside the repo and is regenerated in place. +# Traefik only picks it up after a container restart (static config), which +# --install performs automatically when the list actually changed. +# +# Usage: +# scripts/cf-ips-sync.sh # regen repo + traefik files if ranges changed +# scripts/cf-ips-sync.sh --check # report what would change (exit 1 if any) +# sudo scripts/cf-ips-sync.sh --install # + run nginx-sync.sh and restart Traefik +# +# The Traefik bridge subnet is auto-detected (env TRUSTED_SUBNET overrides), +# because nginx trusts it as a TLS-terminating peer. +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PROXY_DIR="$SCRIPT_DIR/../deployment/proxy" +TARGET="$PROXY_DIR/cloudflare-ips.conf" +TRAEFIK_CONFIG="${TRAEFIK_CONFIG:-/docker/proxyserver/traefik.yml}" +IPV4_URL="https://www.cloudflare.com/ips-v4" +IPV6_URL="https://www.cloudflare.com/ips-v6" +MODE="status" + +for arg in "$@"; do + case "$arg" in + --check) MODE="check" ;; + --install) MODE="install" ;; + --no-traefik) TRAEFIK_CONFIG="" ;; + esac +done + +TRUSTED_SUBNET="${TRUSTED_SUBNET:-}" +if [[ -z "$TRUSTED_SUBNET" ]]; then + TRUSTED_SUBNET="$(docker network inspect proxyserver_traefik-proxy --format '{{range .IPAM.Config}}{{.Subnet}}{{end}}' 2>/dev/null || true)" +fi +if [[ -z "$TRUSTED_SUBNET" ]]; then + TRUSTED_SUBNET="172.22.0.0/16" + echo "warning: could not auto-detect Traefik bridge subnet, using $TRUSTED_SUBNET" >&2 +fi + +ipv4="$(mktemp)" +ipv6="$(mktemp)" +trap 'rm -f "$ipv4" "$ipv6"' EXIT + +if ! curl -sf "$IPV4_URL" -o "$ipv4" || ! curl -sf "$IPV6_URL" -o "$ipv6"; then + echo "error: could not fetch Cloudflare ranges" >&2 + if [[ -f "$TARGET" ]]; then + echo "keeping existing $TARGET (ranges not refreshed)" >&2 + exit 0 + fi + exit 1 +fi + +gen_nginx_conf() { + { + printf '%s\n' "# Trusted edge networks + live Cloudflare CDN ranges." + printf '%s\n' "# Managed/regenerated by scripts/cf-ips-sync.sh - do not hand-edit the ranges." + printf '%s\n' "" + printf '%s\n' "# Topology: Cloudflare -> Traefik (:443, docker bridge proxy_traefik-proxy) ->" + printf '%s\n' "# nginx (:9443) -> CMS. nginx ALSO receives direct connections on :9443 from" + printf '%s\n' "# Cloudflare edges and from the game client (ws.epicnabbo.nl is not proxied)." + printf '%s\n' "" + printf '%s\n' "# nginx only trusts the peers listed here as a source of \$remote_addr" + printf '%s\n' "# (via CF-Connecting-IP). Anyone else presenting a CF-Connecting-IP or" + printf '%s\n' "# CF-ray header is spoofing and is rejected in nginx-cms.conf." + printf '%s\n' "" + printf '%s\n' "# 1 = peer is a trusted edge or internal network (keyed on the raw peer," + printf '%s\n' "# unaffected by real_ip rewrites)." + printf '%s\n' "geo \$realip_remote_addr \$cms_trusted_edge {" + printf '%s\n' " default 0;" + printf '%s\n' " 127.0.0.0/8 1; # localhost (health checks, admin)" + printf '%s\n' " ::1 1; # localhost v6" + printf '%s\n' " $TRUSTED_SUBNET 1; # Traefik (proxyserver_traefik-proxy)" + printf '%s\n' " # --- Cloudflare IPv4 ranges (live from cloudflare.com/ips-v4) ---" + awk '{print " "$0" 1;"}' "$ipv4" + printf '%s\n' " # --- Cloudflare IPv6 ranges (live from cloudflare.com/ips-v6) ---" + awk '{print " "$0" 1;"}' "$ipv6" + printf '%s\n' "}" + printf '%s\n' "" + printf '%s\n' "# 1 when an UNTRUSTED peer still presents a CF-Connecting-IP header: that is a" + printf '%s\n' "# spoof attempt (only real Cloudflare edges or Traefik may do that lawfully)." + printf '%s\n' "map \"\$cms_trusted_edge:\$http_cf_connecting_ip\" \$cms_disallow_forwarding {" + printf '%s\n' " default 0;" + printf '%s\n' " \"~^0:.+\" 1;" + printf '%s\n' "}" + printf '%s\n' "" + printf '%s\n' "# Rewrite \$remote_addr from CF-Connecting-IP but ONLY for the trusted peers" + printf '%s\n' "# above. Direct game clients (untrusted) keep their real peer address." + printf '%s\n' "set_real_ip_from 127.0.0.0/8;" + printf '%s\n' "set_real_ip_from ::1;" + printf '%s\n' "set_real_ip_from $TRUSTED_SUBNET;" + awk '{print "set_real_ip_from "$0";"}' "$ipv4" + awk '{print "set_real_ip_from "$0";"}' "$ipv6" + printf '%s\n' "" + printf '%s\n' "real_ip_header CF-Connecting-IP;" + printf '%s\n' "real_ip_recursive off;" + } > "$TARGET.tmp" +} + +gen_nginx_conf +changed=0 +if cmp -s "$TARGET" "$TARGET.tmp"; then + rm -f "$TARGET.tmp" + echo "= $TARGET up to date (Cloudflare ranges unchanged)" +else + changed=1 + if [[ "$MODE" == "check" ]]; then + rm -f "$TARGET.tmp" + echo "- Cloudflare ranges DIFFER; $TARGET would be regenerated" + else + mv "$TARGET.tmp" "$TARGET" + echo "+ regenerated $TARGET" + fi +fi + +traefik_changed=0 +if [[ -n "$TRAEFIK_CONFIG" ]]; then + if [[ ! -f "$TRAEFIK_CONFIG" ]]; then + echo "warning: $TRAEFIK_CONFIG not found, skipping Traefik sync" >&2 + else + new_traefik="$(CF_V4="$ipv4" CF_V6="$ipv6" python3 - "$TRAEFIK_CONFIG" <<'PY' +import os, sys +path = sys.argv[1] +v4 = sorted(x.rstrip("\n") for x in open(os.environ["CF_V4"]) if x.strip()) +v6 = sorted(x.rstrip("\n") for x in open(os.environ["CF_V6"]) if x.strip()) +lines = open(path).read().split("\n") +out, i, n = [], 0, len(lines) +while i < n: + line = lines[i] + if line.startswith(" trustedIPs:"): + out.append(line) + i += 1 + while i < n: + s = lines[i] + if not s.strip() or s.startswith(" - ") or s.startswith(" #"): + i += 1 + else: + break + out.append(" # Cloudflare IPv4 reeksen (gesynct door cf-ips-sync.sh)") + out += [" - " + c for c in v4] + out.append(" # Cloudflare IPv6 reeksen") + out += [" - " + c for c in v6] + continue + out.append(line) + i += 1 +sys.stdout.write("\n".join(out)) +PY +)" + if grep -q 'trustedIPs:' <<< "$new_traefik" \ + && grep -cq '^ - ' <<< "$new_traefik"; then + if [[ "$new_traefik" == "$(cat "$TRAEFIK_CONFIG")" ]]; then + echo "= $TRAEFIK_CONFIG up to date (Cloudflare ranges unchanged)" + else + traefik_changed=1 + if [[ "$MODE" == "check" ]]; then + echo "- $TRAEFIK_CONFIG differs from live Cloudflare ranges" + else + cp -a "$TRAEFIK_CONFIG" "$TRAEFIK_CONFIG.bak-$(date +%Y%m%d-%H%M%S)" + printf '%s\n' "$new_traefik" > "$TRAEFIK_CONFIG" + echo "+ updated $TRAEFIK_CONFIG" + fi + fi + else + echo "error: generated Traefik config is missing its trustedIPs block; NOT writing" >&2 + exit 1 + fi + fi +fi + +[[ "$MODE" == "check" ]] && exit $((changed || traefik_changed)) + +if [[ "$MODE" == "install" ]]; then + "$SCRIPT_DIR/nginx-sync.sh" + if [[ "$traefik_changed" -eq 1 ]]; then + if docker inspect traefik >/dev/null 2>&1; then + echo "--- restarting traefik (static config changed) ---" + docker restart traefik + else + echo "warning: traefik container not found; restart it manually" >&2 + fi + fi +fi \ No newline at end of file diff --git a/scripts/nginx-sync.sh b/scripts/nginx-sync.sh index 16e68ce2..39a6df8c 100755 --- a/scripts/nginx-sync.sh +++ b/scripts/nginx-sync.sh @@ -16,6 +16,7 @@ # nginx.conf -> /etc/nginx/nginx.conf # nginx-mime.types -> /etc/nginx/mime.types # nginx-cms.conf -> /etc/nginx/sites-available/cms.conf +# cloudflare-ips.conf -> /etc/nginx/conf.d/cloudflare-ips.conf # cms_upstream_servers.conf -> /etc/nginx/snippets/cms_upstream_servers.conf # symlink sites-enabled/cms.conf -> ../sites-available/cms.conf set -euo pipefail @@ -66,6 +67,7 @@ changed=0 if install_file "$PROXY_DIR/nginx.conf" "$NGINX_DIR/nginx.conf"; then changed=1; fi if install_file "$PROXY_DIR/nginx-mime.types" "$NGINX_DIR/mime.types"; then changed=1; fi if install_file "$PROXY_DIR/nginx-cms.conf" "$NGINX_DIR/sites-available/cms.conf"; then changed=1; fi +if install_file "$PROXY_DIR/cloudflare-ips.conf" "$NGINX_DIR/conf.d/cloudflare-ips.conf"; then changed=1; fi if install_file "$PROXY_DIR/cms_upstream_servers.conf" "$NGINX_DIR/snippets/cms_upstream_servers.conf"; then changed=1; fi if [[ ! -f "$NGINX_DIR/sites-enabled/cms.conf" ]]; then