Hi
I have a 2 cluster node, it is working fine, but if I restart graylog-server service, it can’t reconnect to the elasticsearch.
If I restart the elasticsearch service (the master one), the connection recover. All two server do the same.
I tried to search, I find a lot of information, but I can’t find the solution.
I double checked the config, it is the same on the other side (except the IP, and host name, is_master)
I marked 3 lines with ## I tried to play with it, delete it, try ip, hostname, one host, all hosts, etc…
I also tried to check the network traffic, and I saw no traffix on the 9200,9300,9350 ports to not the other host (the other host worked well, so it connected to the node1 elasticsearch node)
I replaced the htt protocol with AA (as new user I can instert only 2 links.)
Hostnames are in the local hosts file, and in DNS also correct.
I tried to copy all information, what I have, please let us know if you need anything else, or have any idea.
[root@graylog-t-node-01 ~]# yum list installed | grep -e 'elasticsearch\|graylog'
elasticsearch.noarch 2.3.5-1 @elasticsearch-2.x
graylog-2.1-repository.noarch 1-3 installed
graylog-server.noarch 2.1.3-1 @graylog
[root@graylog-t-node-01 ~]# cat /etc/hosts
10.14.0.91 graylog-t-node-01 graylog-t-node-01.mgmt.internal
10.14.0.92 graylog-t-node-02 graylog-t-node-02.mgmt.internal
[root@graylog-t-node-01 ~]# curl -XGET "10.14.0.91:9200/_cluster/state?pretty" 2>/dev/null | grep "transport_address" | sort -n
"transport_address" : "10.14.0.91:9300",
"transport_address" : "10.14.0.92:9300",
"transport_address" : "10.14.0.92:9350",
[root@graylog-t-node-01 ~]# systemctl restart elasticsearch
[root@graylog-t-node-01 ~]# curl -XGET "10.14.0.91:9200/_cluster/state?pretty" 2>/dev/null | grep "transport_address" | sort -n
"transport_address" : "10.14.0.91:9300",
**"transport_address" : "10.14.0.91:9350",**
"transport_address" : "10.14.0.92:9300",
"transport_address" : "10.14.0.92:9350",
[root@graylog-t-node-01 ~]# grep -v "#" /etc/graylog/server/server.conf /etc/elasticsearch/elasticsearch.yml | grep -v ":$"
/etc/graylog/server/server.conf:is_master = true
/etc/graylog/server/server.conf:node_id_file = /etc/graylog/server/node-id
/etc/graylog/server/server.conf:password_secret = XXX
/etc/graylog/server/server.conf:root_password_sha2 = XXX
/etc/graylog/server/server.conf:plugin_dir = /usr/share/graylog-server/plugin
/etc/graylog/server/server.conf:rest_listen_uri = AA10.14.0.91:9000/api/
/etc/graylog/server/server.conf:rest_transport_uri = AA10.14.0.91:9000/api/
/etc/graylog/server/server.conf:web_listen_uri = AA10.14.0.91:9000/
/etc/graylog/server/server.conf:elasticsearch_config_file = /etc/elasticsearch/elasticsearch.yml
/etc/graylog/server/server.conf:rotation_strategy = count
/etc/graylog/server/server.conf:elasticsearch_max_docs_per_index = 20000000
/etc/graylog/server/server.conf:rotation_strategy = count
/etc/graylog/server/server.conf:elasticsearch_max_docs_per_index = 20000000
/etc/graylog/server/server.conf:elasticsearch_max_number_of_indices = 20
/etc/graylog/server/server.conf:retention_strategy = delete
/etc/graylog/server/server.conf:elasticsearch_max_number_of_indices = 20
/etc/graylog/server/server.conf:retention_strategy = delete
/etc/graylog/server/server.conf:elasticsearch_shards = 1
/etc/graylog/server/server.conf:elasticsearch_replicas = 1
/etc/graylog/server/server.conf:elasticsearch_index_prefix = graylog
/etc/graylog/server/server.conf:allow_leading_wildcard_searches = false
/etc/graylog/server/server.conf:allow_highlighting = true
/etc/graylog/server/server.conf:elasticsearch_node_name_prefix = graylog-
**/etc/graylog/server/server.conf:##elasticsearch_discovery_zen_ping_unicast_hosts = 10.14.0.91:9300, 10.14.0.92:9300**
**/etc/graylog/server/server.conf:##elasticsearch_discovery_zen_ping_multicast_enabled = false**
**/etc/graylog/server/server.conf:##elasticsearch_network_host = 10.14.0.91, 10.14.0.92**
/etc/graylog/server/server.conf:elasticsearch_analyzer = standard
/etc/graylog/server/server.conf:output_batch_size = 500
/etc/graylog/server/server.conf:output_flush_interval = 1
/etc/graylog/server/server.conf:output_fault_count_threshold = 5
/etc/graylog/server/server.conf:output_fault_penalty_seconds = 30
/etc/graylog/server/server.conf:processbuffer_processors = 5
/etc/graylog/server/server.conf:outputbuffer_processors = 3
/etc/graylog/server/server.conf:processor_wait_strategy = blocking
/etc/graylog/server/server.conf:ring_size = 65536
/etc/graylog/server/server.conf:inputbuffer_ring_size = 65536
/etc/graylog/server/server.conf:inputbuffer_processors = 2
/etc/graylog/server/server.conf:inputbuffer_wait_strategy = blocking
/etc/graylog/server/server.conf:message_journal_enabled = true
/etc/graylog/server/server.conf:message_journal_dir = /var/lib/graylog-server/journal
/etc/graylog/server/server.conf:lb_recognition_period_seconds = 3
/etc/graylog/server/server.conf:stale_master_timeout = 10000
/etc/graylog/server/server.conf:mongodb_uri = mongodb://graylog_user:XXX@graylog-t-node-01:27017,graylog-t-node-02:27017/graylog
/etc/graylog/server/server.conf:mongodb_max_connections = 1000
/etc/graylog/server/server.conf:mongodb_threads_allowed_to_block_multiplier = 5
/etc/graylog/server/server.conf:content_packs_dir = /usr/share/graylog-server/contentpacks
/etc/graylog/server/server.conf:content_packs_auto_load = grok-patterns.json
/etc/graylog/server/server.conf:proxied_requests_thread_pool_size = 32
/etc/graylog/server/server.conf:usage_statistics_enabled = false
/etc/elasticsearch/elasticsearch.yml: cluster.name: graylog-elasticsearch2
/etc/elasticsearch/elasticsearch.yml: node.name: graylog-t-node-01
/etc/elasticsearch/elasticsearch.yml: path.data: /var/lib/elasticsearch
/etc/elasticsearch/elasticsearch.yml: path.logs: /var/log/elasticsearch
/etc/elasticsearch/elasticsearch.yml: network.host: 10.14.0.91
/etc/elasticsearch/elasticsearch.yml: discovery.zen.ping.unicast.hosts: ["graylog-t-node-01", "graylog-t-node-02", "graylog-t-node-01:9350", "graylog-t-node-02:9350"]
/etc/elasticsearch/elasticsearch.yml: discovery.zen.ping.multicast.enabled: false
/etc/elasticsearch/elasticsearch.yml: http.cors.allow-origin: "*"
/etc/elasticsearch/elasticsearch.yml: http.cors.enabled: true
/etc/elasticsearch/elasticsearch.yml: http.cors.allow-methods: ["HEAD", "GET"]
[root@graylog-t-node-01 ~]# netstat -ntlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 10.14.0.91:9200 0.0.0.0:* LISTEN 5966/java
tcp 0 0 10.14.0.91:9300 0.0.0.0:* LISTEN 5966/java
tcp 0 0 10.14.0.91:9350 0.0.0.0:* LISTEN 6179/java
tcp 0 0 10.14.0.91:9000 0.0.0.0:* LISTEN 6179/java
start log:
2017-04-24T16:25:41.771+02:00 INFO [CmdLineTool] Loaded plugin: Elastic Beats Input 1.1.5 [org.graylog.plugins.beats.BeatsInputPlugin]
2017-04-24T16:25:41.773+02:00 INFO [CmdLineTool] Loaded plugin: Collector 1.1.3 [org.graylog.plugins.collector.CollectorPlugin]
2017-04-24T16:25:41.773+02:00 INFO [CmdLineTool] Loaded plugin: Enterprise Integration Plugin 1.1.3 [org.graylog.plugins.enterprise_integration.EnterpriseIntegrationPlugin]
2017-04-24T16:25:41.774+02:00 INFO [CmdLineTool] Loaded plugin: MapWidgetPlugin 1.1.3 [org.graylog.plugins.map.MapWidgetPlugin]
2017-04-24T16:25:41.774+02:00 INFO [CmdLineTool] Loaded plugin: Pipeline Processor Plugin 1.1.3 [org.graylog.plugins.pipelineprocessor.ProcessorPlugin]
2017-04-24T16:25:41.774+02:00 INFO [CmdLineTool] Loaded plugin: Anonymous Usage Statistics 2.1.3 [org.graylog.plugins.usagestatistics.UsageStatsPlugin]
2017-04-24T16:25:42.003+02:00 INFO [CmdLineTool] Running with JVM arguments: -Xms1g -Xmx1g -XX:NewRatio=1 -XX:+ResizeTLAB -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseParNewGC -XX:-OmitStackTraceInFastThrow -Dlog4j.configurationFile=file:///etc/graylog/server/log4j2.xml -Djava.library.path=/usr/share/graylog-server/lib/sigar -Dgraylog2.installation_source=rpm
2017-04-24T16:25:42.417+02:00 INFO [Version] HV000001: Hibernate Validator 5.2.4.Final
2017-04-24T16:25:47.166+02:00 INFO [InputBufferImpl] Message journal is enabled.
2017-04-24T16:25:47.218+02:00 INFO [NodeId] Node ID: 11111111-aed8-4e8c-9481-111111111111
2017-04-24T16:25:47.541+02:00 INFO [LogManager] Loading logs.
2017-04-24T16:25:47.661+02:00 INFO [LogManager] Logs loading complete.
2017-04-24T16:25:47.662+02:00 INFO [KafkaJournal] Initialized Kafka based journal at /var/lib/graylog-server/journal
2017-04-24T16:25:47.717+02:00 INFO [InputBufferImpl] Initialized InputBufferImpl with ring size <65536> and wait strategy <BlockingWaitStrategy>, running 2 parallel message handlers.
2017-04-24T16:25:47.772+02:00 INFO [cluster] Cluster created with settings {hosts=[graylog-t-node-01:27017, graylog-t-node-02:27017], mode=MULTIPLE, requiredClusterType=UNKNOWN, serverSelectionTimeout='30000 ms', maxWaitQueueSize=5000}
2017-04-24T16:25:47.773+02:00 INFO [cluster] Adding discovered server graylog-t-node-01:27017 to client view of cluster
2017-04-24T16:25:47.812+02:00 INFO [cluster] Adding discovered server graylog-t-node-02:27017 to client view of cluster
2017-04-24T16:25:47.904+02:00 INFO [cluster] No server chosen by ReadPreferenceServerSelector{readPreference=primary} from cluster description ClusterDescription{type=UNKNOWN, connectionMode=MULTIPLE, serverDescriptions=[ServerDescription{address=graylog-t-node-02:27017, type=UNKNOWN, state=CONNECTING}, ServerDescription{address=graylog-t-node-01:27017, type=UNKNOWN, state=CONNECTING}]}. Waiting for 30000 ms before timing out
2017-04-24T16:25:48.364+02:00 INFO [connection] Opened connection [connectionId{localValue:1, serverValue:587}] to graylog-t-node-01:27017
2017-04-24T16:25:48.371+02:00 INFO [cluster] Monitor thread successfully connected to server with description ServerDescription{address=graylog-t-node-01:27017, type=REPLICA_SET_PRIMARY, state=CONNECTED, ok=true, version=ServerVersion{versionList=[3, 4, 2]}, minWireVersion=0, maxWireVersion=5, maxDocumentSize=16777216, roundTripTimeNanos=1293797, setName='uvp_graylog_test_repl', canonicalAddress=graylog-t-node-01.mgmt.internal:27017, hosts=[graylog-t-node-01.mgmt.internal:27017, graylog-t-node-02.mgmt.internal:27017], passives=[], arbiters=[], primary='graylog-t-node-01.mgmt.internal:27017', tagSet=TagSet{[]}, electionId=7fffffff0000000000000005, setVersion=2}
2017-04-24T16:25:48.375+02:00 INFO [connection] Opened connection [connectionId{localValue:2, serverValue:90}] to graylog-t-node-02:27017
2017-04-24T16:25:48.377+02:00 INFO [cluster] Monitor thread successfully connected to server with description ServerDescription{address=graylog-t-node-02:27017, type=REPLICA_SET_SECONDARY, state=CONNECTED, ok=true, version=ServerVersion{versionList=[3, 4, 2]}, minWireVersion=0, maxWireVersion=5, maxDocumentSize=16777216, roundTripTimeNanos=1558411, setName='uvp_graylog_test_repl', canonicalAddress=graylog-t-node-02.mgmt.internal:27017, hosts=[graylog-t-node-01.mgmt.internal:27017, graylog-t-node-02.mgmt.internal:27017], passives=[], arbiters=[], primary='graylog-t-node-01.mgmt.internal:27017', tagSet=TagSet{[]}, electionId=null, setVersion=2}
2017-04-24T16:25:48.377+02:00 INFO [cluster] Discovered cluster type of REPLICA_SET
2017-04-24T16:25:48.378+02:00 INFO [cluster] Adding discovered server graylog-t-node-01.mgmt.internal:27017 to client view of cluster
2017-04-24T16:25:48.391+02:00 INFO [cluster] Adding discovered server graylog-t-node-02.mgmt.internal:27017 to client view of cluster
2017-04-24T16:25:48.404+02:00 INFO [cluster] Canonical address graylog-t-node-02.mgmt.internal:27017 does not match server address. Removing graylog-t-node-02:27017 from client view of cluster
2017-04-24T16:25:48.407+02:00 INFO [cluster] Server graylog-t-node-01:27017 is no longer a member of the replica set. Removing from client view of cluster.
2017-04-24T16:25:48.407+02:00 INFO [cluster] Canonical address graylog-t-node-01.mgmt.internal:27017 does not match server address. Removing graylog-t-node-01:27017 from client view of cluster
2017-04-24T16:25:48.584+02:00 INFO [connection] Opened connection [connectionId{localValue:3, serverValue:588}] to graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:48.589+02:00 INFO [cluster] Monitor thread successfully connected to server with description ServerDescription{address=graylog-t-node-01.mgmt.internal:27017, type=REPLICA_SET_PRIMARY, state=CONNECTED, ok=true, version=ServerVersion{versionList=[3, 4, 2]}, minWireVersion=0, maxWireVersion=5, maxDocumentSize=16777216, roundTripTimeNanos=3765965, setName='uvp_graylog_test_repl', canonicalAddress=graylog-t-node-01.mgmt.internal:27017, hosts=[graylog-t-node-01.mgmt.internal:27017, graylog-t-node-02.mgmt.internal:27017], passives=[], arbiters=[], primary='graylog-t-node-01.mgmt.internal:27017', tagSet=TagSet{[]}, electionId=7fffffff0000000000000005, setVersion=2}
2017-04-24T16:25:48.589+02:00 INFO [cluster] Setting max election id to 7fffffff0000000000000005 from replica set primary graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:48.589+02:00 INFO [cluster] Setting max set version to 2 from replica set primary graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:48.589+02:00 INFO [cluster] Discovered replica set primary graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:48.594+02:00 INFO [connection] Opened connection [connectionId{localValue:4, serverValue:91}] to graylog-t-node-02.mgmt.internal:27017
2017-04-24T16:25:48.596+02:00 INFO [cluster] Monitor thread successfully connected to server with description ServerDescription{address=graylog-t-node-02.mgmt.internal:27017, type=REPLICA_SET_SECONDARY, state=CONNECTED, ok=true, version=ServerVersion{versionList=[3, 4, 2]}, minWireVersion=0, maxWireVersion=5, maxDocumentSize=16777216, roundTripTimeNanos=1910637, setName='uvp_graylog_test_repl', canonicalAddress=graylog-t-node-02.mgmt.internal:27017, hosts=[graylog-t-node-01.mgmt.internal:27017, graylog-t-node-02.mgmt.internal:27017], passives=[], arbiters=[], primary='graylog-t-node-01.mgmt.internal:27017', tagSet=TagSet{[]}, electionId=null, setVersion=2}
2017-04-24T16:25:48.709+02:00 INFO [connection] Opened connection [connectionId{localValue:5, serverValue:589}] to graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:49.456+02:00 INFO [node] [graylog-t-node-01] version[2.3.5], pid[6179], build[90f439f/2016-07-27T10:36:52Z]
2017-04-24T16:25:49.456+02:00 INFO [node] [graylog-t-node-01] initializing ...
2017-04-24T16:25:49.474+02:00 INFO [plugins] [graylog-t-node-01] modules [], plugins [graylog-monitor], sites []
2017-04-24T16:25:53.044+02:00 INFO [node] [graylog-t-node-01] initialized
2017-04-24T16:25:53.310+02:00 INFO [ProcessBuffer] Initialized ProcessBuffer with ring size <65536> and wait strategy <BlockingWaitStrategy>.
2017-04-24T16:25:56.107+02:00 INFO [RulesEngineProvider] No static rules file loaded.
2017-04-24T16:25:56.456+02:00 INFO [connection] Opened connection [connectionId{localValue:6, serverValue:590}] to graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:56.578+02:00 WARN [GeoIpResolverEngine] GeoIP database file does not exist: /tmp/GeoLite2-City.mmdb
2017-04-24T16:25:56.586+02:00 INFO [OutputBuffer] Initialized OutputBuffer with ring size <65536> and wait strategy <BlockingWaitStrategy>.
2017-04-24T16:25:56.753+02:00 WARN [GeoIpResolverEngine] GeoIP database file does not exist: /tmp/GeoLite2-City.mmdb
2017-04-24T16:25:56.878+02:00 WARN [GeoIpResolverEngine] GeoIP database file does not exist: /tmp/GeoLite2-City.mmdb
2017-04-24T16:25:57.014+02:00 WARN [GeoIpResolverEngine] GeoIP database file does not exist: /tmp/GeoLite2-City.mmdb
2017-04-24T16:25:57.133+02:00 WARN [GeoIpResolverEngine] GeoIP database file does not exist: /tmp/GeoLite2-City.mmdb
2017-04-24T16:25:58.857+02:00 INFO [ServerBootstrap] Graylog server 2.1.3+040d371 starting up
2017-04-24T16:25:58.857+02:00 INFO [ServerBootstrap] JRE: Oracle Corporation 1.8.0_121 on Linux 3.10.0-514.el7.x86_64
2017-04-24T16:25:58.858+02:00 INFO [ServerBootstrap] Deployment: rpm
2017-04-24T16:25:58.858+02:00 INFO [ServerBootstrap] OS: CentOS Linux 7 (Core) (centos)
2017-04-24T16:25:58.858+02:00 INFO [ServerBootstrap] Arch: amd64
2017-04-24T16:25:58.884+02:00 WARN [DeadEventLoggingListener] Received unhandled event of type <org.graylog2.plugin.lifecycles.Lifecycle> from event bus <AsyncEventBus{graylog-eventbus}>
2017-04-24T16:25:58.953+02:00 INFO [PeriodicalsService] Starting 25 periodicals ...
2017-04-24T16:25:58.970+02:00 INFO [node] [graylog-t-node-01] starting ...
2017-04-24T16:25:58.953+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.ThroughputCalculator] periodical in [0s], polling every [1s].
2017-04-24T16:25:59.019+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.AlertScannerThread] periodical in [10s], polling every [60s].
2017-04-24T16:25:59.020+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.BatchedElasticSearchOutputFlushThread] periodical in [0s], polling every [1s].
2017-04-24T16:25:59.022+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.ClusterHealthCheckThread] periodical in [120s], polling every [20s].
2017-04-24T16:25:59.025+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.ContentPackLoaderPeriodical] periodical, running forever.
2017-04-24T16:25:59.025+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.GarbageCollectionWarningThread] periodical, running forever.
2017-04-24T16:25:59.028+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.IndexerClusterCheckerThread] periodical in [0s], polling every [30s].
2017-04-24T16:25:59.036+02:00 INFO [IndexerClusterCheckerThread] Indexer not fully initialized yet. Skipping periodic cluster check.
2017-04-24T16:25:59.037+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.IndexRetentionThread] periodical in [0s], polling every [300s].
**2017-04-24T16:25:59.038+02:00 INFO [IndexRetentionThread] Elasticsearch cluster not available, skipping index retention checks.**
2017-04-24T16:25:59.038+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.IndexRotationThread] periodical in [0s], polling every [10s].
2017-04-24T16:25:59.039+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.NodePingThread] periodical in [0s], polling every [1s].
2017-04-24T16:25:59.040+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.VersionCheckThread] periodical in [300s], polling every [1800s].
2017-04-24T16:25:59.040+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.ThrottleStateUpdaterThread] periodical in [1s], polling every [1s].
2017-04-24T16:25:59.041+02:00 INFO [Periodicals] Starting [org.graylog2.events.ClusterEventPeriodical] periodical in [0s], polling every [1s].
2017-04-24T16:25:59.045+02:00 INFO [Periodicals] Starting [org.graylog2.events.ClusterEventCleanupPeriodical] periodical in [0s], polling every [86400s].
2017-04-24T16:25:59.047+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.ClusterIdGeneratorPeriodical] periodical, running forever.
2017-04-24T16:25:59.047+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.IndexRangesMigrationPeriodical] periodical, running forever.
2017-04-24T16:25:59.048+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.IndexRangesCleanupPeriodical] periodical in [15s], polling every [3600s].
2017-04-24T16:25:59.240+02:00 INFO [connection] Opened connection [connectionId{localValue:7, serverValue:591}] to graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:59.241+02:00 INFO [connection] Opened connection [connectionId{localValue:10, serverValue:594}] to graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:59.242+02:00 INFO [connection] Opened connection [connectionId{localValue:9, serverValue:593}] to graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:59.244+02:00 INFO [connection] Opened connection [connectionId{localValue:8, serverValue:592}] to graylog-t-node-01.mgmt.internal:27017
2017-04-24T16:25:59.333+02:00 INFO [PeriodicalsService] Not starting [org.graylog2.periodical.UserPermissionMigrationPeriodical] periodical. Not configured to run on this node.
2017-04-24T16:25:59.334+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.AlarmCallbacksMigrationPeriodical] periodical, running forever.
2017-04-24T16:25:59.388+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.ConfigurationManagementPeriodical] periodical, running forever.
2017-04-24T16:25:59.543+02:00 INFO [PeriodicalsService] Not starting [org.graylog2.periodical.LdapGroupMappingMigration] periodical. Not configured to run on this node.
2017-04-24T16:25:59.544+02:00 INFO [Periodicals] Starting [org.graylog2.periodical.IndexFailuresPeriodical] periodical, running forever.
2017-04-24T16:25:59.562+02:00 INFO [PeriodicalsService] Not starting [org.graylog.plugins.usagestatistics.UsageStatsNodePeriodical] periodical. Not configured to run on this node.
2017-04-24T16:25:59.562+02:00 INFO [PeriodicalsService] Not starting [org.graylog.plugins.usagestatistics.UsageStatsClusterPeriodical] periodical. Not configured to run on this node.
2017-04-24T16:25:59.562+02:00 INFO [Periodicals] Starting [org.graylog.plugins.collector.periodical.PurgeExpiredCollectorsThread] periodical in [0s], polling every [3600s].
2017-04-24T16:26:00.013+02:00 INFO [transport] [graylog-t-node-01] publish_address {10.14.0.91:9350}, bound_addresses {10.14.0.91:9350}
2017-04-24T16:26:00.019+02:00 INFO [discovery] [graylog-t-node-01] graylog-elasticsearch2/FgNcFLmASuKBmRNJqaCKLw
2017-04-24T16:26:00.630+02:00 INFO [JerseyService] Enabling CORS for HTTP endpoint
2017-04-24T16:26:03.069+02:00 WARN [discovery] [graylog-t-node-01] waited for 3s and no initial state was set by the discovery
2017-04-24T16:26:03.069+02:00 INFO [node] [graylog-t-node-01] started
**2017-04-24T16:26:08.078+02:00 WARN [IndexerSetupService] Could not connect to Elasticsearch**
**2017-04-24T16:26:08.078+02:00 INFO [IndexerSetupService] If you're using multicast, check that it is working in your network and that Elasticsearch is accessible. Also check that the cluster name setting is correct.**
**2017-04-24T16:26:08.079+02:00 INFO [IndexerSetupService] See AAdocs.graylog.org/en/2.1/pages/configuration/elasticsearch.html for details.**
2017-04-24T16:26:14.048+02:00 INFO [IndexRangesCleanupPeriodical] Skipping index range cleanup because the Elasticsearch cluster is unreachable or unhealthy
2017-04-24T16:26:15.630+02:00 INFO [NetworkListener] Started listener bound to [10.14.0.91:9000]
2017-04-24T16:26:15.632+02:00 INFO [HttpServer] [HttpServer] Started.
2017-04-24T16:26:15.632+02:00 INFO [JerseyService] Started REST API at <AA10.14.0.91:9000/api/>
2017-04-24T16:26:15.633+02:00 INFO [JerseyService] Started Web Interface at <AA10.14.0.91:9000/>
2017-04-24T16:26:15.636+02:00 INFO [ServiceManagerListener] Services are healthy
2017-04-24T16:26:15.638+02:00 INFO [ServerBootstrap] Services started, startup times in ms: {OutputSetupService [RUNNING]=13, BufferSynchronizerService [RUNNING]=14, KafkaJournal [RUNNING]=122, InputSetupService [RUNNING]=314, PeriodicalsService [RUNNING]=700, JournalReader [RUNNING]=882, IndexerSetupService [RUNNING]=9139, JerseyService [RUNNING]=16696}
2017-04-24T16:26:15.653+02:00 INFO [ServerBootstrap] Graylog server up and running.
2017-04-24T16:26:15.653+02:00 INFO [InputSetupService] Triggering launching persisted inputs, node transitioned from Uninitialized [LB:DEAD] to Running [LB:ALIVE]
2017-04-24T16:26:15.729+02:00 INFO [InputStateListener] Input [GELF TCP/589b2e028562a80fdf1a4032] is now STARTING
2017-04-24T16:26:15.745+02:00 INFO [InputStateListener] Input [Syslog TCP/589b2e548562a80fdf1a4088] is now STARTING
2017-04-24T16:26:15.775+02:00 WARN [NettyTransport] receiveBufferSize (SO_RCVBUF) for input SyslogTCPInput{title=Syslog TCP, type=org.graylog2.inputs.syslog.tcp.SyslogTCPInput, nodeId=null} should be 1048576 but is 212992.
2017-04-24T16:26:15.779+02:00 INFO [InputStateListener] Input [Syslog TCP/589b2e548562a80fdf1a4088] is now RUNNING
2017-04-24T16:26:15.790+02:00 WARN [NettyTransport] receiveBufferSize (SO_RCVBUF) for input GELFTCPInput{title=GELF TCP, type=org.graylog2.inputs.gelf.tcp.GELFTCPInput, nodeId=null} should be 1048576 but is 212992.
2017-04-24T16:26:15.795+02:00 INFO [InputStateListener] Input [GELF TCP/589b2e028562a80fdf1a4032] is now RUNNING
[root@graylog-t-node-01 ~]# tcpdump -nn -i any \( port 9350 or port 9200 or port 9300 \) and not host 10.14.0.92
tcpdump: verbose output suppressed, use -v or -vv for full protocol decode
listening on any, link-type LINUX_SLL (Linux cooked), capture size 65535 bytes
0 packets captured
0 packets received by filter
0 packets dropped by kernel
[root@graylog-t-node-01 ~]# getenforce
Permissive