Hi,
I have standalone instance of Graylog 3.0, ES and mongo running on same server. Everything was running fine but we were facing storage issue quiet frequently so we increased the storage of VM. But after that I am not able to get the list of Indices at system/indices webpage. I am getting 500 status_code in “/api/system/indices/index_sets?skip=0&limit=10&stats=true” and “api/system/indices/index_sets/stats” in webpage.
There are is no error in graylog/server/log or in elasticsearch. I tried deleting the indices in elasticsearch and restarting graylog. It does create new index in elasticsearch but nothing in graylog.
$ curl http://127.0.0.1:9200/_cat/indices
green open default_0 XT3Uvv1TRKKKceZ2VuqRiw 4 0 0 0 1kb 1kb
green open nginx__0 T1xbMBKjQSK8zfJlrBa_7g 4 0 0 0 1kb 1kb
green open graylog_0 pUDy3Xl_SbClrleYxhRa8A 1 0 0 0 261b 261b
$ curl http://127.0.0.1:9200/_cat/shards
default_0 1 p STARTED 0 261b 127.0.0.1 BAxxy2s
default_0 3 p STARTED 0 261b 127.0.0.1 BAxxy2s
default_0 2 p STARTED 0 261b 127.0.0.1 BAxxy2s
default_0 0 p STARTED 0 261b 127.0.0.1 BAxxy2s
graylog_0 0 p STARTED 0 261b 127.0.0.1 BAxxy2s
nginx__0 1 p STARTED 0 261b 127.0.0.1 BAxxy2s
nginx__0 3 p STARTED 0 261b 127.0.0.1 BAxxy2s
nginx__0 2 p STARTED 0 261b 127.0.0.1 BAxxy2s
nginx__0 0 p STARTED 0 261b 127.0.0.1 BAxxy2s
Graylog server.conf file
$ sudo cat /etc/graylog/server/server.conf
is_master = true
node_id_file = /etc/graylog/server/node-id
password_secret = qZx4gYX1Ofc05MMPw7DGJjoS5ShRkanZXCeaglTotqJiO8jf0mPzodsYm103cvavf7hjB1nyL7WJfHK4ycrHo1i3DhyGeh0s
root_password_sha2 = c70f30de786b86042a8081dfebce10274ea847b85aabcd29c595523551712e21
bin_dir = /usr/share/graylog-server/bin
data_dir = /var/lib/graylog-server
plugin_dir = /usr/share/graylog-server/plugin
http_bind_address = 0.0.0.0:9000
http_external_uri = http://127.0.0.1:9000/
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
elasticsearch_max_number_of_indices = 20
retention_strategy = delete
elasticsearch_shards = 4
elasticsearch_replicas = 0
elasticsearch_index_prefix = graylog
allow_leading_wildcard_searches = false
allow_highlighting = false
elasticsearch_analyzer = standard
output_batch_size = 500
output_flush_interval = 1
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 5
outputbuffer_processors = 3
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
message_journal_enabled = true
message_journal_dir = /var/lib/graylog-server/journal
lb_recognition_period_seconds = 3
mongodb_uri = mongodb://graylog:PASSWORD@localhost:27017/graylog
mongodb_max_connections = 1000
mongodb_threads_allowed_to_block_multiplier = 5
proxied_requests_thread_pool_size = 32
elasticsearch_http_enabled = false
elasticsearch_discovery_zen_ping_unicast_hosts = 127.0.0.1:9200
/etc/elasticsearch/elasticsearch.yml
path.data: /var/lib/elasticsearch
path.logs: /var/log/elasticsearch
cluster.name: graylog
bootstrap.system_call_filter: false