Show received messages spinning with no data

Graylog Central (peer support)
1

I am unable to display received messages . My 3 nodes show green in ES. Could someone help with this issue ?
server.conf

s_master = true
node_id_file = /etc/graylog/server/node-id
password_secret =
root_username = sdpadmin
root_password_sha2 =
root_email = ""
root_timezone = America/New_York
plugin_dir = /usr/share/graylog-server/plugin
rest_listen_uri = https://gray1.philasd.net:9000/api/
rest_transport_uri = https://gray1.philasd.net:9000/api/
rest_enable_tls = true
rest_tls_cert_file = /usr/ssl/wildcard-all.crt
rest_tls_key_file = /usr/ssl/graylog-key-pkcs8.pem
rest_tls_key_password = xxxxxx
rest_thread_pool_size = 16
web_enable = true
web_listen_uri = https://gray1.philasd.net:9000/
web_enable_tls = true
web_tls_cert_file = /usr/ssl/wildcard-all.crt
web_tls_key_file = /usr/ssl/wildcard-key-pkcs8.pem
web_tls_key_password = 2ltmbsd3
web_thread_pool_size = 16
elasticsearch_config_file = /etc/elasticsearch/elasticsearch.yml
elasticsearch_hosts = http://198.16.5.11:9200, http://198.16.5.13:9200, http://198.16.5.14:9200
elasticsearch_discovery_enabled =  true
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
elasticsearch_max_time_per_index = 1d
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
elasticsearch_max_size_per_index = 1073741824
elasticsearch_max_time_per_index = 1d
elasticsearch_disable_version_check = true
elasticsearch_max_number_of_indices = 20
elasticsearch_max_number_of_indices = 20
retention_strategy = delete
elasticsearch_shards = 5
elasticsearch_replicas = 2
elasticsearch_index_prefix = graylog
elasticsearch_template_name = graylog-internal
allow_leading_wildcard_searches = false
allow_highlighting = false
elasticsearch_cluster_name = graylog
elasticsearch_node_name_prefix = graylog-
elasticsearch_node_name = graylog1-mgmt.philasd.net
elasticsearch_discovery_zen_ping_multicast_enabled = false
discovery.zen.ping.unicast.hosts: ["graylog1-mgmt.philasd.net:9300", "graylog2-mgmt.philasd.net:9300", "graylog3-mgmt.philasd.net:9300" ]
elasticsearch_cluster_discovery_timeout = 30000
elasticsearch_network_host = 198.16.5.11
elasticsearch_network_bind_host = 198.16.5.11
elasticsearch_network_publish_host = 198.16.5.11
elasticsearch_discovery_initial_state_timeout = 30s
elasticsearch_analyzer = standard
output_batch_size = 500
output_flush_interval = 1
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 5
outputbuffer_processors = 3
outputbuffer_processor_threads_core_pool_size = 3
outputbuffer_processor_threads_max_pool_size = 30
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
message_journal_enabled = true
message_journal_dir = /var/lib/graylog-server/journal
async_eventbus_processors = 3
lb_recognition_period_seconds = 3
mongodb_uri = mongodb://admin:xxxxxxxxx@graylog1-mgmt.philasd.net:27017,graylog2-mgmt.philasd.net:27017,graylog3-mgmt.philasd.net:27017/graylog
mongodb_max_connections = 1000
mongodb_threads_allowed_to_block_multiplier = 5
rules_file = /etc/graylog/server/rules.drl
transport_email_enabled = true
transport_email_hostname = mta04.philasd.org
transport_email_port = 25
transport_email_use_tls = false
transport_email_use_ssl = false
transport_email_subject_prefix = [graylog]
transport_email_from_email = graylog@philasd.org
content_packs_dir = /usr/share/graylog-server/contentpacks
content_packs_auto_load = grok-patterns.json
proxied_requests_thread_pool_size = 32

elasticsearch.yml

cluster.name: graylog
node.master: true
node.data: true
node.name: "graylog1-mgmt.philasd.net"
discovery.zen.minimum_master_nodes: 1
http.bind_host: graylog1-mgmt.philasd.net
network.host: 172.16.5.11
bootstrap.system_call_filter: false
discovery.zen.ping.unicast.hosts: ["graylog1-mgmt.philasd.net", "graylog2-mgmt.philasd.net", "graylog3-mgmt.philasd.net"]

[root@graylog1 server]# curl -XGET 'http://172.16.5.11:9200/_cluster/health?pretty'
{
  "cluster_name" : "graylog",
  "status" : "green",
  "timed_out" : false,
  "number_of_nodes" : 3,
  "number_of_data_nodes" : 3,
  "active_primary_shards" : 9,
  "active_shards" : 14,
  "relocating_shards" : 0,
  "initializing_shards" : 0,
  "unassigned_shards" : 0,
  "delayed_unassigned_shards" : 0,
  "number_of_pending_tasks" : 0,
  "number_of_in_flight_fetch" : 0,
  "task_max_waiting_in_queue_millis" : 0,
  "active_shards_percent_as_number" : 100.0
}

all three nodes are green

2

I guess these should be identical.

3

yes I just tried to mask the ips

4

Well, if you decide to make things up, then at least be consistent.

What’s in the logs of your Graylog and Elasticsearch nodes?

5

root@graylog1 ~]# tail -f /var/log/elasticsearch/graylog.log
[2018-03-19T16:36:42,947][INFO ][o.e.n.Node ] [graylog1-mgmt.philasd.net] starting …
[2018-03-19T16:36:43,099][INFO ][o.e.t.TransportService ] [graylog1-mgmt.philasd.net] publish_address {172.16.5.11:9300}, bound_addresses {172.16.5.11:9300}
[2018-03-19T16:36:43,108][INFO ][o.e.b.BootstrapChecks ] [graylog1-mgmt.philasd.net] bound or publishing to a non-loopback address, enforcing bootstrap checks
[2018-03-19T16:36:46,171][INFO ][o.e.c.s.ClusterService ] [graylog1-mgmt.philasd.net] new_master {graylog1-mgmt.philasd.net}{gOhOsdXwR4WCr1Lg2Rq03A}{yublNotDTkKgD2LExogV7w}{172.16.5.11}{172.16.5.11:9300}, added {{graylog3-mgmt.philasd.net}{Ba3FETmwTmuQrW-NK9cq7Q}{QL7oPJAYTZKgewbX7b4Npg}{172.16.5.14}{172.16.5.14:9300},{graylog2-mgmt.philasd.net}{8IEUMkwGQGKB_kuIx8vFxw}{O-qRb7SMSIu13MJR28U09A}{172.16.5.13}{172.16.5.13:9300},}, reason: zen-disco-elected-as-master ([2] nodes joined)[{graylog3-mgmt.philasd.net}{Ba3FETmwTmuQrW-NK9cq7Q}{QL7oPJAYTZKgewbX7b4Npg}{172.16.5.14}{172.16.5.14:9300}, {graylog2-mgmt.philasd.net}{8IEUMkwGQGKB_kuIx8vFxw}{O-qRb7SMSIu13MJR28U09A}{172.16.5.13}{172.16.5.13:9300}]
[2018-03-19T16:36:46,410][DEBUG][o.e.a.a.i.g.TransportGetIndexAction] [graylog1-mgmt.philasd.net] no known master node, scheduling a retry
[2018-03-19T16:36:46,474][INFO ][o.e.h.n.Netty4HttpServerTransport] [graylog1-mgmt.philasd.net] publish_address {172.16.5.11:9200}, bound_addresses {172.16.5.11:9200}
[2018-03-19T16:36:46,474][INFO ][o.e.n.Node ] [graylog1-mgmt.philasd.net] started
[2018-03-19T16:36:46,688][INFO ][o.e.g.GatewayService ] [graylog1-mgmt.philasd.net] recovered [2] indices into cluster_state
[2018-03-19T16:36:47,304][INFO ][o.e.c.r.a.AllocationService] [graylog1-mgmt.philasd.net] Cluster health status changed from [RED] to [YELLOW] (reason: [shards started [[graylog_0][1]] …]).
[2018-03-19T16:36:48,388][INFO ][o.e.c.r.a.AllocationService] [graylog1-mgmt.philasd.net] Cluster health status changed from [YELLOW] to [GREEN] (reason: [shards started [[api][4]] …]).
^C
[root@graylog1 ~]#

6

server.log log file

2018-03-16T15:28:31.053-04:00 INFO [ServerBootstrap] Services started, startup times in ms: {InputSetupService [RUNNING]=2, JournalReader [RUNNING]=15, ConfigurationEtagService [RUNNING]=27, OutputSetupService [RUNNING]=39, BufferSynchronizerService [RUNNING]=98, StreamCacheService [RUNNING]=105, KafkaJournal [RUNNING]=120, PeriodicalsService [RUNNING]=554, LookupTableService [RUNNING]=784, JerseyService [RUNNING]=13443}
2018-03-16T15:28:31.056-04:00 INFO [ServerBootstrap] Graylog server up and running.
2018-03-16T15:28:31.073-04:00 INFO [InputStateListener] Input [Syslog TCP/5aaaa4e9c510b72b4d30eef4] is now STARTING
2018-03-16T15:28:31.098-04:00 WARN [NettyTransport] receiveBufferSize (SO_RCVBUF) for input SyslogTCPInput{title=DEV Systems TCP Gray1, type=org.graylog2.inputs.syslog.tcp.SyslogTCPInput, nodeId=d35d7005-e335-4858-bd5a-7f69a83f3d76} should be 1048576 but is 212992.
2018-03-16T15:28:31.100-04:00 INFO [InputStateListener] Input [Syslog TCP/5aaaa4e9c510b72b4d30eef4] is now RUNNING
2018-03-17T09:30:31.146-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 1.3.6.1.4.1.18060.0.0.1
2018-03-17T09:30:31.149-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.7
2018-03-17T09:30:31.149-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.2
2018-03-17T09:30:31.150-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.18
2018-03-17T09:30:31.150-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 1.2.840.113556.1.4.319
2018-03-17T09:30:31.150-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.3
2018-03-17T09:30:31.151-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 1.3.6.1.4.1.4203.1.10.1
2018-03-17T09:30:31.151-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 1.2.840.113556.1.4.473
2018-03-17T09:30:31.151-04:00 INFO [DefaultLdapCodecService] Registered pre-bundled control factory: 1.2.840.113556.1.4.474
2018-03-17T09:30:31.152-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.3.6.1.4.1.18060.0.0.1
2018-03-17T09:30:31.152-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.7
2018-03-17T09:30:31.152-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.2
2018-03-17T09:30:31.152-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.18
2018-03-17T09:30:31.152-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.2.840.113556.1.4.319
2018-03-17T09:30:31.152-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.3
2018-03-17T09:30:31.152-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.3.6.1.4.1.4203.1.10.1
2018-03-17T09:30:31.152-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.3.6.1.4.1.42.2.27.8.5.1
2018-03-17T09:30:31.153-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.9
2018-03-17T09:30:31.153-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 2.16.840.1.113730.3.4.10
2018-03-17T09:30:31.153-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.3.6.1.4.1.4203.1.9.1.3
2018-03-17T09:30:31.153-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.3.6.1.4.1.4203.1.9.1.4
2018-03-17T09:30:31.154-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.3.6.1.4.1.4203.1.9.1.1
2018-03-17T09:30:31.154-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.3.6.1.4.1.4203.1.9.1.2
2018-03-17T09:30:31.154-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.2.840.113556.1.4.473
2018-03-17T09:30:31.154-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.2.840.113556.1.4.474
2018-03-17T09:30:31.154-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.2.840.113556.1.4.841
2018-03-17T09:30:31.155-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.2.840.113556.1.4.417
2018-03-17T09:30:31.155-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.2.840.113556.1.4.1413
2018-03-17T09:30:31.155-04:00 INFO [CodecFactoryUtil] Registered pre-bundled control factory: 1.2.840.113556.1.4.528
2018-03-17T09:30:31.156-04:00 INFO [CodecFactoryUtil] Registered pre-bundled extended operation factory: 1.3.6.1.1.8
2018-03-17T09:30:31.157-04:00 INFO [CodecFactoryUtil] Registered pre-bundled extended operation factory: 1.3.6.1.4.1.18060.0.1.8
2018-03-17T09:30:31.157-04:00 INFO [CodecFactoryUtil] Registered pre-bundled extended operation factory: 1.3.6.1.4.1.18060.0.1.3
2018-03-17T09:30:31.158-04:00 INFO [CodecFactoryUtil] Registered pre-bundled extended operation factory: 1.3.6.1.4.1.18060.0.1.6
2018-03-17T09:30:31.158-04:00 INFO [CodecFactoryUtil] Registered pre-bundled extended operation factory: 1.3.6.1.4.1.18060.0.1.5
2018-03-17T09:30:31.159-04:00 INFO [CodecFactoryUtil] Registered pre-bundled extended operation factory: 1.3.6.1.4.1.4203.1.11.1
2018-03-17T09:30:31.160-04:00 INFO [CodecFactoryUtil] Registered pre-bundled extended operation factory: 1.3.6.1.4.1.4203.1.11.3
2018-03-17T09:30:31.160-04:00 INFO [CodecFactoryUtil] Registered pre-bundled extended operation factory: 1.3.6.1.4.1.1466.20037
2018-03-19T14:51:15.724-04:00 INFO [connection] Opened connection [connectionId{localValue:13, serverValue:151}] to graylog3-mgmt.philasd.net:27017
2018-03-19T16:36:46.452-04:00 INFO [MongoIndexSet] Did not find a deflector alias. Setting one up now.
2018-03-19T16:36:46.694-04:00 INFO [MongoIndexSet] Pointing to already existing index target <graylog_0>

7

Please post the complete log files and not just some excerpts.

If they’re too big to post here, use a pastebin service such as https://gist.github.com/ or https://0bin.net/.

1 Like
8

I submitted the logs. I hope this helps.

9

should I resubmit the server.conf file ?

10

is_master = true
node_id_file = /etc/graylog/server/node-id
password_secret = xxxxxxxxxxxxxxx
root_username = sdpadmin
root_password_sha2 = xxxxxxxxxxxxxxx
root_email = "rgizynski@philasd.org"
root_timezone = America/New_York
plugin_dir = /usr/share/graylog-server/plugin
rest_listen_uri = https://graylog1.philasd.net:9000/api/
rest_transport_uri = https://graylog1.philasd.net:9000/api/
rest_enable_tls = true
rest_tls_cert_file = /usr/ssl/wildcard-all.crt
rest_tls_key_file = /usr/ssl/graylog-key-pkcs8.pem
rest_tls_key_password = xxxxxxxxxxxxxxx
rest_thread_pool_size = 16
web_enable = true
web_listen_uri = https://graylog1.philasd.net:9000/
web_enable_tls = true
web_tls_cert_file = /usr/ssl/wildcard-all.crt
web_tls_key_file = /usr/ssl/wildcard-key-pkcs8.pem
web_tls_key_password = xxxxxxxxxxxxxxx
web_thread_pool_size = 16
elasticsearch_config_file = /etc/elasticsearch/elasticsearch.yml
elasticsearch_hosts = http://172.16.5.11:9200, http://172.16.5.13:9200, http://172.16.5.14:9200
elasticsearch_discovery_enabled = true
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
elasticsearch_max_time_per_index = 1d
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
elasticsearch_max_size_per_index = 1073741824
elasticsearch_max_time_per_index = 1d
elasticsearch_disable_version_check = true
elasticsearch_max_number_of_indices = 20
elasticsearch_max_number_of_indices = 20
retention_strategy = delete
elasticsearch_shards = 5
elasticsearch_replicas = 2
elasticsearch_index_prefix = graylog
elasticsearch_template_name = graylog-internal
allow_leading_wildcard_searches = false
allow_highlighting = false
elasticsearch_cluster_name = graylog
elasticsearch_node_name_prefix = graylog-
elasticsearch_node_name = graylog1-mgmt.philasd.net
elasticsearch_discovery_zen_ping_multicast_enabled = false
discovery.zen.ping.unicast.hosts: [“graylog1-mgmt.philasd.net:9300”, “graylog2-mgmt.philasd.net:9300”, “graylog3-mgmt.philasd.net:9300” ]
elasticsearch_cluster_discovery_timeout = 30000
elasticsearch_network_host = 172.16.5.11
elasticsearch_network_bind_host = 172.16.5.11
elasticsearch_network_publish_host = 172.16.5.11
elasticsearch_discovery_initial_state_timeout = 30s
elasticsearch_analyzer = standard
output_batch_size = 500
output_flush_interval = 1
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 5
outputbuffer_processors = 3
outputbuffer_processor_threads_core_pool_size = 3
outputbuffer_processor_threads_max_pool_size = 30
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
message_journal_enabled = true
message_journal_dir = /var/lib/graylog-server/journal
async_eventbus_processors = 3
lb_recognition_period_seconds = 3
mongodb_uri = mongodb://admin:xxxxxxxxxxxxxxx@graylog1-mgmt.philasd.net:27017,graylog2-mgmt.philasd.net:27017,graylog3-mgmt.philasd.net:27017/graylog
mongodb_max_connections = 1000
mongodb_threads_allowed_to_block_multiplier = 5
rules_file = /etc/graylog/server/rules.drl
transport_email_enabled = true
transport_email_hostname = mta04.philasd.org
transport_email_port = 25
transport_email_use_tls = false
transport_email_use_ssl = false
transport_email_subject_prefix = [graylog]
transport_email_from_email = graylog@philasd.org
content_packs_dir = /usr/share/graylog-server/contentpacks
content_packs_auto_load = grok-patterns.json
proxied_requests_thread_pool_size = 32

11

[root@graylog1 server]# netstat -tulpn | egrep ‘9000|9200|9300|27017’
tcp 0 0 172.16.5.11:9200 0.0.0.0:* LISTEN 13947/java
tcp 0 0 172.16.5.11:9300 0.0.0.0:* LISTEN 13947/java
tcp 0 0 10.0.103.11:9000 0.0.0.0:* LISTEN 1441/java
tcp 0 0 0.0.0.0:27017 0.0.0.0:* LISTEN 1722/mongod
[root@graylog1 server]#

12

https://0bin.net/paste/8gYBTWB8QrZ6BRB6#ZSASW0g9n0+FrhRfwjnSuS3E3zPF1Npx6JpBU06W+5R

13

Can someone please help with this issue ? I’m not quite sure what is wrong. The shards start and all three server are green. I included the logs for better clarification.

Thanks !
Bob

14

Hi,

Thanks for getting back to me. Is there anything else I need to do ?

Bob

15

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.