Hello Guys,
In our infrastructure we are using the collectors output on TCP/TLS, it was working ok when it was a single node, but now in a cluster environment we been facing a lot of challenges, one of them is the Load balancing, so we were giving a try using HAProxy, does someone have any experience with this, currently we have a running configuration but I feel needs a lot more of work, bellow you will find the configuration, right now it is passing traffic, but the collectors are printing this messages:
2017-11-06 05:47:30 INFO connecting to phxiograylogp01.internal.mcmcg.com:5044
2017-11-06 05:47:30 INFO successfully connected to phxiograylogp01.internal.mcmcg.com:5044
2017-11-06 05:47:30 INFO connecting to phxiograylogp01.internal.mcmcg.com:5044
2017-11-06 05:47:30 INFO remote socket was closed during SSL handshake
2017-11-06 05:47:30 INFO reconnecting in 1 seconds
2017-11-06 05:47:30 INFO successfully connected to phxiograylogp01.internal.mcmcg.com:5044
2017-11-06 05:47:30 INFO remote socket was closed during SSL handshake
2017-11-06 05:47:30 INFO reconnecting in 1 seconds
2017-11-06 05:47:31 INFO connecting to phxiograylogp01.internal.mcmcg.com:5047
2017-11-06 05:47:31 INFO successfully connected to phxiograylogp01.internal.mcmcg.com:5047
2017-11-06 05:47:31 INFO remote socket was closed during SSL handshake
2017-11-06 05:47:31 INFO reconnecting in 1 seconds
2017-11-06 05:47:31 INFO connecting to phxiograylogp01.internal.mcmcg.com:5044
2017-11-06 05:47:31 INFO successfully connected to phxiograylogp01.internal.mcmcg.com:5044
#---------------------------------------------------------------------
# Example configuration for a possible web application. See the
# full configuration options online.
#
# http://haproxy.1wt.eu/download/1.4/doc/configuration.txt
#
#---------------------------------------------------------------------
#---------------------------------------------------------------------
# Global settings
#---------------------------------------------------------------------
global
# to have these messages end up in /var/log/haproxy.log you will
# need to:
#
# 1) configure syslog to accept network log events. This is done
# by adding the '-r' option to the SYSLOGD_OPTIONS in
# /etc/sysconfig/syslog
#
# 2) configure local2 events to go to the /var/log/haproxy.log
# file. A line like the following can be added to
# /etc/sysconfig/syslog
#
# local2.* /var/log/haproxy.log
#
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 4000
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/stats
#---------------------------------------------------------------------
# common defaults that all the 'listen' and 'backend' sections will
# use if not designated in their block
#---------------------------------------------------------------------
defaults
mode http
log global
option httplog
option dontlognull
option http-server-close
option forwardfor except 127.0.0.0/8
option redispatch
retries 3
timeout http-request 3s
timeout queue 1m
timeout connect 3s
timeout client 1m
timeout server 1m
timeout http-keep-alive 3s
timeout check port 9000 3s
maxconn 3000
#---------------------------------------------------------------------
# main frontend which proxys to the backends
#---------------------------------------------------------------------
#frontend main *:5000
# acl url_static path_beg -i /static /images /javascript /stylesheets
# acl url_static path_end -i .jpg .gif .png .css .js
# use_backend static if url_static
# default_backend app
#---------------------------------------------------------------------
# static backend for serving up images, stylesheets and such
#---------------------------------------------------------------------
#backend static
# balance roundrobin
# server static 127.0.0.1:4331 check port 9000
#---------------------------------------------------------------------
# round robin balancing between the various backends
#---------------------------------------------------------------------
#backend app
# balance roundrobin
# server app1 127.0.0.1:5001 check port 9000
# server app2 127.0.0.1:5002 check port 9000
# server app3 127.0.0.1:5003 check port 9000
# server app4 127.0.0.1:5004 check port 9000
listen graylog_5047 :5047
mode tcp
option tcplog
balance roundrobin
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:5047 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:5047 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:5047 check port 9000
listen graylog_5044 :5044
mode tcp
option tcplog
balance roundrobin /api/system/lbstatus
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:5044 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:5044 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:5047 check port 9000
listen graylog_5050 :5050
mode tcp
option tcplog
balance roundrobin
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:5050 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:5050 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:5050 check port 9000
listen graylog_12202 :12202
mode tcp
option tcplog
balance roundrobin
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:12202 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:12202 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:12202 check port 9000
listen graylog_12201 :12201
mode tcp
option tcplog
balance roundrobin
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:12201 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:12201 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:12201 check port 9000
listen graylog_12203 :12203
mode tcp
option tcplog
balance roundrobin
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:12203 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:12203 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:12203 check port 9000
listen graylog_12020 :12020
mode tcp
option tcplog
balance roundrobin
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:12020 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:12020 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:12020 check port 9000
listen graylog_5045 :5045
mode tcp
option tcplog
balance roundrobin
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:5045 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:5045 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:5045 check port 9000
listen graylog_12189 :12189
mode tcp
option tcplog
balance roundrobin
option httpchk HEAD /api/system/lbstatus
server graylog1 phxiograylogp02.internal.mcmcg.com:12189 check port 9000
server graylog2 phxiograylogp03.internal.mcmcg.com:12189 check port 9000
server graylog3 phxiograylogp04.internal.mcmcg.com:12189 check port 9000