Expected Behavior
Nginx reverse proxy sends log data to Graylog and Graylog imports it.
Actual Behavior
Log data makes it to the Graylog container but Graylog doesn’t import it.
If proxy_bind $remote_addr transparent; is not included in nginx config, log data does import correctly, although with the source IP of the reverse proxy.
Additional Information
I verified this with tcpdump in the Graylog container. Logs are being shipped and reaching the container but for some reason Graylog doesn’t import them. It doesn’t appear to have any logs indicating issues either.
Configurations
Nginx docker startup script:
docker service create \
--mode global \
--name nginx-service \
--network lb-net \
--publish published=2514,target=2514,protocol=udp,mode=host \
--endpoint-mode dnsrr \
--mount type=bind,source=/mnt/shared/etc/nginx/rsyslog.conf,destination=/etc/nginx/nginx.conf \
--constraint node.labels.LB-NODE==yes \
nginx:alpine
/mnt/shared/etc/nginx/rsyslog.conf aka /etc/nginx/nginx.conf:
user root;
worker_processes 1;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events { worker_connections 1024; }
stream {
resolver 127.0.0.11 valid=5s;
upstream graylog_udp {
hash $remote_addr;
server graylog:2514;
}
server {
listen 2514 udp;
proxy_timeout 10s;
#proxy_bind $remote_addr transparent; #if this line isn't commented then logs don't make it into graylog.
proxy_pass graylog_udp;
proxy_responses 0;
#return $remote_addr;
}
}
docker-compose.yml for stack (simplified):
version: '3.2'
services:
mongo:
image: mongo:3
volumes:
- gl_mongodb:/data/db
networks:
lb-net: {}
deploy:
placement:
constraints: [node.role == worker]
mode: replicated
replicas: 1
endpoint_mode: dnsrr
restart_policy:
condition: on-failure
elasticsearch:
image: 'bitnami/elasticsearch:6.8.0-debian-9-r2'
volumes:
- 'gl_elastic:/bitnami/elasticsearch/data'
environment:
- ELASTICSEARCH_CLUSTER_NAME=graylog
networks:
lb-net: {}
deploy:
placement:
constraints: [node.role == worker]
mode: replicated
replicas: 1
endpoint_mode: dnsrr
restart_policy:
condition: on-failure
# Graylog: https://hub.docker.com/r/graylog/graylog/
graylog:
image: registry.redacted.com/graylog:latest
hostname: gl_master
volumes:
- "/mnt/shared/services/graylog/config/node-id-master:/usr/share/graylog/data/config/node-id"
- gl_journal:/graylog
- "/mnt/shared/services/graylog/shadowCA/cert/CA/shadowCA.der:/usr/share/graylog/cert/CA/shadowCA.der"
- "/mnt/shared/services/graylog/shadowCA/cert/graylog.redacted.com/graylog.redacted.com.crt:/usr/share/graylog/cert/graylog.crt"
- "/mnt/shared/services/graylog/shadowCA/cert/graylog.redacted.com/graylog.redacted.com.key:/usr/share/graylog/cert/graylog.key"
- "/mnt/shared/services/graylog/certs-init.sh:/usr/local/bin/graylog-custom.sh"
- "/mnt/shared/services/graylog/shadowCA/cert/trusted_clients:/usr/share/graylog/cert/trusted_clients"
environment:
- GRAYLOG_IS_MASTER=true
- GRAYLOG_PASSWORD_SECRET=redacted
- GRAYLOG_ROOT_PASSWORD_SHA2=redacted
- GRAYLOG_HTTP_EXTERNAL_URI=https://grayl.redacted.com/
- GRAYLOG_TRUSTED_PROXIES=192.168.5.4/32,10.0.2.0/24,184.69.119.142/32,184.71.21.162/32,184.69.23.106/32,172.18.0.0/24
- GRAYLOG_ELASTICSEARCH_HOSTS=http://elasticsearch:9200
- GRAYLOG_JAVA_OPTS="-Djavax.net.ssl.trustStore=/usr/share/graylog/cacerts.jks"
- GRAYLOG_ROOT_TIMEZONE=Canada/Pacific
networks:
lb-net: {}
deploy:
placement:
constraints: [node.role == worker]
mode: replicated
replicas: 1
endpoint_mode: dnsrr
restart_policy:
condition: on-failure
depends_on:
- mongodb
- elasticsearch
command:
- graylog-custom.sh
volumes:
gl_elastic:
external: true
gl_mongodb:
external: true
gl_journal:
external: true
networks:
lb-net:
external: true
Sorry if this isn’t enough information - if more is needed I’ll provide it.