EDIT: fix now, there we some config issues that I have since been able to find
(Repost with some redactions, sorry for the confusion)
Just switch to Traefik as my main proxy to better take control of my ingress, specifically with crowdsec in mind. So far I’ve been able to set it up correctly for internal use. Sadly for external traffic all connections to services tagged with the crowdsec-bouncer middleware get blocked and I have not been able to figure out why exactly. All services are behind proxied Cloudflare DNS, which is why all cloudflare IPs are listed. Traefik logs do show original IPs. 2 Examples will be list, one working, the other one recieving the 403 code. It is most likely the TrustedIPs being set up incorrectly, as I can get the same effect for my internal network simply by removing 192.168.0.0/16 for example. Sadly I can’t just add all of my ISPs IP addresses and I don’t think that is how it is supposed to work. I simply just configured it incorrectly.
The middleware is added to a test-service via traefik.http.routers.app.middlewares=crowdsec@docker
Some help will be greatly appreciated. Please consider me to be a novice in both Traefik and Crowdsec.
working
{"ClientAddr":"162.158.102.102:11304","ClientHost":"2a00:20:6340:7a49:e530:366e:4be0:45d","ClientPort":"11304","ClientUsername":"-","DownstreamContentSize":118764,"DownstreamStatus":200,"Duration":33290841,"OriginContentSize":118764,"OriginDuration":32893252,"OriginStatus":200,"Overhead":397589,"RequestAddr":"redacted","RequestContentSize":0,"RequestCount":526,"RequestHost":"redacted","RequestMethod":"GET","RequestPath":"/Items/ea086d60fb06de5dab1059b740c8544b/Images/Thumb?fillHeight=478\u0026fillWidth=849\u0026quality=96\u0026tag=bd9d0ddf58af1f4260bdf0cd3d478f25","RequestPort":"-","RequestProtocol":"HTTP/2.0","RequestScheme":"https","RetryAttempts":0,"RouterName":"redacted@docker","ServiceAddr":"172.16.13.2:8096","ServiceName":"jellyfin-ix-jellyfin@docker","ServiceURL":"http://172.16.13.2:8096","StartLocal":"2025-06-15T17:46:58.163841387Z","StartUTC":"2025-06-15T17:46:58.163841387Z","TLSCipher":"TLS_AES_128_GCM_SHA256","TLSVersion":"1.3","entryPointName":"websecure","level":"info","msg":"","request_User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/139.1 Mobile/15E148 Safari/605.1.15","time":"2025-06-15T17:46:58Z"}
blocked
{"ClientAddr":"162.158.102.103:60380","ClientHost":"2a00:20:6340:7a49:e530:366e:4be0:45d","ClientPort":"60380","ClientUsername":"-","DownstreamContentSize":0,"DownstreamStatus":403,"Duration":4378884,"OriginContentSize":0,"OriginDuration":0,"OriginStatus":0,"Overhead":4378884,"RequestAddr":"redacted","RequestContentSize":0,"RequestCount":533,"RequestHost":"redacted","RequestMethod":"GET","RequestPath":"/web/","RequestPort":"-","RequestProtocol":"HTTP/2.0","RequestScheme":"https","RetryAttempts":0,"RouterName":"redacted@docker","StartLocal":"2025-06-15T17:50:08.239486754Z","StartUTC":"2025-06-15T17:50:08.239486754Z","TLSCipher":"TLS_AES_128_GCM_SHA256","TLSVersion":"1.3","entryPointName":"websecure","level":"info","msg":"","request_User-Agent":"Mozilla/5.0 (iPhone; CPU iPhone OS 18_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) FxiOS/139.1 Mobile/15E148 Safari/605.1.15","time":"2025-06-15T17:50:08Z"}
I want to learn how to debug this properly and figure out where exactly my configuration mistake lies.
I followed this guide for the initial setup.
Platform: TrueNas 25.04.1 custom app yaml for both traefik and crowdsec docker compose
Currently I heavily rely on cli syntax.
Crowdsec yaml
services:
crowdsec:
container_name: crowdsec
environment:
- GID=1000
- >-
COLLECTIONS=crowdsecurity/traefik crowdsecurity/http-cve
crowdsecurity/base-http-scenarios crowdsecurity/sshd crowdsecurity/linux
crowdsecurity/appsec-generic-rules crowdsecurity/appsec-virtual-patching
crowdsecurity/appsec-crs
expose:
- 8081
- 6060
- 7422
image: crowdsecurity/crowdsec:v1.6.8
labels:
- traefik.enable=false
ports:
- 127.0.0.1:9876:8081
restart: unless-stopped
volumes:
- /mnt/vdev1/docker/crowdsec/data:/var/lib/crowdsec/data
- /mnt/vdev1/docker/crowdsec/etc:/etc/crowdsec
- /var/log/auth.log:/var/log/auth.log:ro
- /var/log/syslog:/var/log/syslog:ro
- /mnt/vdev1/docker/traefik/logs:/var/log/traefik:ro
Crowdsec acquis.yaml
filenames:
- /var/log/nginx/*.log
- ./tests/nginx/nginx.log
#this is not a syslog log, indicate which kind of logs it is
labels:
type: nginx
---
filenames:
- /var/log/auth.log
- /var/log/syslog
labels:
type: syslog
---
filename: /var/log/apache2/*.log
labels:
type: apache2
---
poll_without_inotify: false
filenames:
- /var/log/traefik/traefik.log
labels:
type: traefik
---
listen_addr: 0.0.0.0:7422
appsec_config: crowdsecurity/appsec-default
name: myAppSecComponent
source: appsec
labels:
type: appsec
Traefik yaml
networks:
ix-adguard-home_default:
external: True
ix-collabora_default:
external: True
ix-crowdsec_default:
external: True
ix-home-assistant_default:
external: True
ix-immich_default:
external: True
ix-jellyfin_default:
external: True
ix-metube_default:
external: True
ix-nextcloud_default:
external: True
ix-nginx-proxy-manager_default:
external: True
ix-stirling-pdf_default:
external: True
services:
traefik:
command:
- '--accesslog=true'
- '--accesslog.filePath=/logs/traefik.log'
- '--accesslog.format=json'
- '--accesslog.filters.statuscodes="200-299","400-599"'
- '--accesslog.bufferingSize=0'
- '--accesslog.fields.headers.defaultMode=drop'
- '--accesslog.fields.headers.names.User-Agent=keep'
- '--metrics.prometheus=false'
- '--tracing=false'
- '--log.level=DEBUG'
- '--api.insecure=false'
- '--providers.docker=true'
- '--providers.docker.exposedbydefault=false'
- '--entryPoints.web.address=:80'
- >-
--entryPoints.web.forwardedHeaders.trustedIPs=127.0.0.1/32,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,103.21.244.0/22,
103.22.200.0/22, 103.31.4.0/22, 104.16.0.0/13, 104.24.0.0/14,
108.162.192.0/18, 131.0.72.0/22, 141.101.64.0/18, 162.158.0.0/15,
172.64.0.0/13, 173.245.48.0/20, 188.114.96.0/20, 190.93.240.0/20,
197.234.240.0/22, 198.41.128.0/17, 172.16.0.0/12, 192.168.2.0/24,
2400:cb00::/32, 2606:4700::/32, 2803:f800::/32, 2405:b500::/32,
2405:8100::/32, 2a06:98c0::/29, 2c0f:f248::/32
- '--entrypoints.web.http.redirections.entryPoint.to=websecure'
- '--entrypoints.web.http.redirections.entryPoint.scheme=https'
- '--entrypoints.web.http.redirections.entrypoint.permanent=true'
- '--entryPoints.websecure.address=:443'
- >-
--entryPoints.websecure.forwardedHeaders.trustedIPs=127.0.0.1/32,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,103.21.244.0/22,
103.22.200.0/22, 103.31.4.0/22, 104.16.0.0/13, 104.24.0.0/14,
108.162.192.0/18, 131.0.72.0/22, 141.101.64.0/18, 162.158.0.0/15,
172.64.0.0/13, 173.245.48.0/20, 188.114.96.0/20, 190.93.240.0/20,
197.234.240.0/22, 198.41.128.0/17, 172.16.0.0/12, 192.168.2.0/24,
2400:cb00::/32, 2606:4700::/32, 2803:f800::/32, 2405:b500::/32,
2405:8100::/32, 2a06:98c0::/29, 2c0f:f248::/32
- >-
--entryPoints.websecure.proxyProtocol.trustedIPs=127.0.0.1/32,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,103.21.244.0/22,
103.22.200.0/22, 103.31.4.0/22, 104.16.0.0/13, 104.24.0.0/14,
108.162.192.0/18, 131.0.72.0/22, 141.101.64.0/18, 162.158.0.0/15,
172.64.0.0/13, 173.245.48.0/20, 188.114.96.0/20, 190.93.240.0/20,
197.234.240.0/22, 198.41.128.0/17, 172.16.0.0/12, 192.168.2.0/24,
2400:cb00::/32, 2606:4700::/32, 2803:f800::/32, 2405:b500::/32,
2405:8100::/32, 2a06:98c0::/29, 2c0f:f248::/32
- '--certificatesresolvers.cloudflare.acme.dnschallenge=true'
- '--certificatesresolvers.cloudflare.acme.keyType=EC256'
- '--certificatesresolvers.cloudflare.acme.email=redacted'
- >-
--certificatesresolvers.cloudflare.acme.storage=/traefik/sslcerts/cloudflare-acme.json
- '--certificatesresolvers.cloudflare.acme.dnschallenge.provider=cloudflare'
- >-
--certificatesResolvers.cloudflare.acme.dnschallenge.resolvers=1.1.1.1:53,8.8.8.8:53
- >-
--certificatesresolvers.cloudflare.acme.caServer=https://acme-v02.api.letsencrypt.org/directory
- >-
--experimental.plugins.crowdsec-bouncer.modulename=github.com/maxlerebourg/crowdsec-bouncer-traefik-plugin
- '--experimental.plugins.crowdsec-bouncer.version=v1.4.2'
container_name: traefik
environment:
- CF_DNS_API_TOKEN=redacted
image: traefik:v3.4
labels:
- traefik.enable=true
- traefik.http.routers.dashboard.rule=Host(`traefik.redacted`)
- traefik.http.routers.dashboard.service=api@internal
- traefik.http.routers.dashboard.tls=true
- traefik.http.routers.dashboard.tls.certresolver=cloudflare
- traefik.http.routers.dashboard.entrypoints=websecure
- traefik.http.routers.dashboard.rule=Host(`traefik.redacted`)
- traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.enabled=true
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.defaultDecisionSeconds=60
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecMode=stream
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecAppsecEnabled=false
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecAppsecHost=crowdsec:7422
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecAppsecFailureBlock=true
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecAppsecUnreachableBlock=true
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecLapiKey=redacted
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecLapiHost=crowdsec:8081
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecLapiScheme=http
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.crowdsecLapiTLSInsecureVerify=false
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.forwardedHeadersTrustedIPs=127.0.0.1/32,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16,103.21.244.0/22,
103.22.200.0/22, 103.31.4.0/22, 104.16.0.0/13, 104.24.0.0/14,
108.162.192.0/18, 131.0.72.0/22, 141.101.64.0/18, 162.158.0.0/15,
172.64.0.0/13, 173.245.48.0/20, 188.114.96.0/20, 190.93.240.0/20,
197.234.240.0/22, 198.41.128.0/17, 172.16.0.0/12, 192.168.2.0/24,
2400:cb00::/32, 2606:4700::/32, 2803:f800::/32, 2405:b500::/32,
2405:8100::/32, 2a06:98c0::/29, 2c0f:f248::/32
- >-
traefik.http.middlewares.crowdsec.plugin.crowdsec-bouncer.clientTrustedIPs=127.0.0.1/32,10.0.0.0/8,172.16.0.0/12,192.168.0.0/16
networks:
- ix-jellyfin_default
- ix-adguard-home_default
- ix-home-assistant_default
- ix-immich_default
- ix-nextcloud_default
- ix-collabora_default
- ix-metube_default
- ix-stirling-pdf_default
- ix-nginx-proxy-manager_default
- ix-crowdsec_default
ports:
- '80:80'
- '443:443'
- '8080:8080'
volumes:
- /var/run/docker.sock:/var/run/docker.sock:ro
- /mnt/vdev1/docker/traefik/sslcerts:/traefik/sslcerts/
- /mnt/vdev1/docker/traefik/:/traefik/
- /mnt/vdev1/docker/traefik/logs:/logs