upgrding a graylog server from 2.5 to 3.0
Expect:
Graylog has to be running.
Reality:
getting 503 Service Unavailable
The server is temporarily unable to service your request due to maintenance downtime or capacity problems. Please try again later.
OS and application
CentOS Linux release 7.7.1908 (Core)
[root@graylog elasticsearch]# rpm -qa | grep -i mongo
mongodb-org-tools-4.0.14-1.el7.x86_64
mongodb-org-server-4.0.14-1.el7.x86_64
mongodb-org-4.0.14-1.el7.x86_64
mongodb-org-mongos-4.0.14-1.el7.x86_64
mongodb-org-shell-4.0.14-1.el7.x86_64
[root@graylog elasticsearch]# rpm -qa | grep -i elastic
elasticsearch-6.8.6-1.noarch
[root@graylog elasticsearch]# rpm -qa | grep -i graylog
graylog-3.1-repository-1-1.noarch
graylog-server-3.0.2-1.noarch
[root@graylog elasticsearch]# java -version
openjdk version "1.8.0_232"
OpenJDK Runtime Environment (build 1.8.0_232-b09)
OpenJDK 64-Bit Server VM (build 25.232-b09, mixed mode)
Services
[root@graylog elasticsearch]# systemctl status mongod.service
● mongod.service - MongoDB Database Server
Loaded: loaded (/usr/lib/systemd/system/mongod.service; enabled; vendor preset: disabled)
Active: active (running) since Fri 2020-01-03 10:30:44 IST; 4 days ago
Docs: https://docs.mongodb.org/manual
Main PID: 3561 (mongod)
Tasks: 37
CGroup: /system.slice/mongod.service
└─3561 /usr/bin/mongod -f /etc/mongod.conf
[root@graylog elasticsearch]# systemctl status elasticsearch.service
● elasticsearch.service - Elasticsearch
Loaded: loaded (/etc/systemd/system/elasticsearch.service; enabled; vendor preset: disabled)
Active: active (running) since Tue 2020-01-07 11:09:28 IST; 1h 38min ago
Docs: http://www.elastic.co
Main PID: 9210 (java)
Tasks: 104
CGroup: /system.slice/elasticsearch.service
├─9210 /bin/java -Xms4g -Xmx4g -XX:+UseConcMarkSweepGC -XX:CMSInitiatingOccupancyFraction=75 -XX:+UseCMSInitiatingOccupancyOnly -De...
└─9364 /usr/share/elasticsearch/modules/x-pack-ml/platform/linux-x86_64/bin/controller
[root@graylog elasticsearch]# curl -XGET 'http://localhost:9200/_cluster/health?pretty=true'
{
"cluster_name" : "graylog",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"active_primary_shards" : 4,
"active_shards" : 4,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
[root@graylog elasticsearch]# curl -XGET http://localhost:9200
{
"name" : "9Z8I0X5",
"cluster_name" : "graylog",
"cluster_uuid" : "bcbt97xeTHCLHcq5yr8rtw",
"version" : {
"number" : "6.8.6",
"build_flavor" : "default",
"build_type" : "rpm",
"build_hash" : "3d9f765",
"build_date" : "2019-12-13T17:11:52.013738Z",
"build_snapshot" : false,
"lucene_version" : "7.7.2",
"minimum_wire_compatibility_version" : "5.6.0",
"minimum_index_compatibility_version" : "5.0.0"
},
"tagline" : "You Know, for Search"
}
[root@graylog elasticsearch]# systemctl status graylog-server.service
● graylog-server.service - Graylog server
Loaded: loaded (/usr/lib/systemd/system/graylog-server.service; enabled; vendor preset: disabled)
Active: active (running) since Tue 2020-01-07 10:53:55 IST; 1h 54min ago
Docs: http://docs.graylog.org/
Main PID: 7131 (graylog-server)
Tasks: 128
CGroup: /system.slice/graylog-server.service
├─7131 /bin/sh /usr/share/graylog-server/bin/graylog-server
└─7132 /usr/bin/java -Xms20g -Xmx20g -XX:NewRatio=1 -server -XX:+ResizeTLAB -XX:+UseConcMarkSweepGC -XX:+CMSConcurrentMTEnabled -XX...
elasticsearch.yml file
[root@graylog elasticsearch]# cat elasticsearch.yml
# ======================== Elasticsearch Configuration =========================
#
# Please consult the documentation for further information on configuration options:
# https://www.elastic.co/guide/en/elasticsearch/reference/index.html
#
# ---------------------------------- Cluster -----------------------------------
# Use a descriptive name for your cluster:
#cluster.name: my-application
cluster.name: graylog
# ------------------------------------ Node ------------------------------------
#
# Use a descriptive name for the node:
#node.name: node-1
# Add custom attributes to the node:
#node.attr.rack: r1
# ----------------------------------- Paths ------------------------------------
# Path to directory where to store the data (separate multiple locations by comma):
path.data: /var/lib/elasticsearch
# Path to log files:
path.logs: /var/log/elasticsearch
# ----------------------------------- Memory -----------------------------------
# Lock the memory on startup:
#bootstrap.memory_lock: true
# Make sure that the heap size is set to about half the memory available
# on the system and that the owner of the process is allowed to use this
# limit.
# Elasticsearch performs poorly when the system is swapping the memory.
# ---------------------------------- Network -----------------------------------
# Set the bind address to a specific IP (IPv4 or IPv6):
#network.host: 192.168.0.1
network.host: 0.0.0.0
# Set a custom port for HTTP:
http.port: 9200
# For more information, consult the network module documentation.
# --------------------------------- Discovery ----------------------------------
# Pass an initial list of hosts to perform discovery when new node is started:
# The default list of hosts is ["127.0.0.1", "[::1]"]
#discovery.zen.ping.unicast.hosts: ["host1", "host2"]
# Prevent the "split brain" by configuring the majority of nodes (total number of master-eligible nodes / 2 + 1):
#discovery.zen.minimum_master_nodes:
# For more information, consult the zen discovery module documentation.
# ---------------------------------- Gateway -----------------------------------
# Block initial recovery after a full cluster restart until N nodes are started:
#gateway.recover_after_nodes: 3
# For more information, consult the gateway module documentation.
# ---------------------------------- Various -----------------------------------
# Require explicit names when deleting indices:
#action.destructive_requires_name: true
jvm configuration
[root@graylog elasticsearch]# cat jvm.options
## JVM configuration
################################################################
## IMPORTANT: JVM heap size
################################################################
##
-Xms4g
-Xmx4g
## GC configuration
8-13:-XX:+UseConcMarkSweepGC
8-13:-XX:CMSInitiatingOccupancyFraction=75
8-13:-XX:+UseCMSInitiatingOccupancyOnly
## G1GC Configuration
# NOTE: G1 GC is only supported on JDK version 10 or later
# to use G1GC, uncomment the next two lines and update the version on the
# following three lines to your version of the JDK
# 10-13:-XX:-UseConcMarkSweepGC
# 10-13:-XX:-UseCMSInitiatingOccupancyOnly
14-:-XX:+UseG1GC
14-:-XX:G1ReservePercent=25
14-:-XX:InitiatingHeapOccupancyPercent=30
## DNS cache policy
# cache ttl in seconds for positive DNS lookups noting that this overrides the
# JDK security property networkaddress.cache.ttl; set to -1 to cache forever
-Des.networkaddress.cache.ttl=60
# cache ttl in seconds for negative DNS lookups noting that this overrides the
# JDK security property networkaddress.cache.negative ttl; set to -1 to cache
# forever
-Des.networkaddress.cache.negative.ttl=10
## optimizations
# pre-touch memory pages used by the JVM during initialization
-XX:+AlwaysPreTouch
## basic
# explicitly set the stack size
-Xss1m
# set to headless, just in case
-Djava.awt.headless=true
# ensure UTF-8 encoding by default (e.g. filenames)
-Dfile.encoding=UTF-8
# use our provided JNA always versus the system one
-Djna.nosys=true
# turn off a JDK optimization that throws away stack traces for common
# exceptions because stack traces are important for debugging
-XX:-OmitStackTraceInFastThrow
# flags to configure Netty
-Dio.netty.noUnsafe=true
-Dio.netty.noKeySetOptimization=true
-Dio.netty.recycler.maxCapacityPerThread=0
# log4j 2
-Dlog4j.shutdownHookEnabled=false
-Dlog4j2.disable.jmx=true
-Djava.io.tmpdir=${ES_TMPDIR}
## heap dumps
# generate a heap dump when an allocation from the Java heap fails
# heap dumps are created in the working directory of the JVM
-XX:+HeapDumpOnOutOfMemoryError
# specify an alternative path for heap dumps; ensure the directory exists and
# has sufficient space
-XX:HeapDumpPath=/var/lib/elasticsearch
# specify an alternative path for JVM fatal error logs
-XX:ErrorFile=/var/log/elasticsearch/hs_err_pid%p.log
## JDK 8 GC logging
8:-XX:+PrintGCDetails
8:-XX:+PrintGCDateStamps
8:-XX:+PrintTenuringDistribution
8:-XX:+PrintGCApplicationStoppedTime
8:-Xloggc:/var/log/elasticsearch/gc.log
8:-XX:+UseGCLogFileRotation
8:-XX:NumberOfGCLogFiles=32
8:-XX:GCLogFileSize=64m
# JDK 9+ GC logging
9-:-Xlog:gc*,gc+age=trace,safepoint:file=/var/log/elasticsearch/gc.log:utctime,pid,tags:filecount=32,filesize=64m
# due to internationalization enhancements in JDK 9 Elasticsearch need to set the provider to COMPAT otherwise
# time/date parsing will break in an incompatible way for some date patterns and locals
9-:-Djava.locale.providers=COMPAT
# temporary workaround for C2 bug with JDK 10 on hardware with AVX-512
10-:-XX:UseAVX=2
[root@graylog elasticsearch]#
graylog configuration
############################
# GRAYLOG CONFIGURATION FILE
############################
#
# * Entries are generally expected to be a single line of the form, one of the following:
#
# propertyName=propertyValue
# propertyName:propertyValue
#
# * White space that appears between the property name and property value is ignored,
# so the following are equivalent:
#
# name=Stephen
# name = Stephen
#
# targetCities=\
# Detroit,\
# Chicago,\
# Los Angeles
# path=c:\\docs\\doc1
# instances as master. The master will perform some periodical tasks that non-masters won't perform.
is_master = true
# The auto-generated node ID will be stored in this file and read after restarts. It is a good idea
# to use an absolute file path here if you are starting Graylog server from init scripts or similar.
node_id_file = /etc/graylog/server/node-id
# You MUST set a secret to secure/pepper the stored user passwords here. Use at least 64 characters.
# Generate one by using for example: pwgen -N 1 -s 96
password_secret = Iei1zp1fQyXWRGL8gyaAuuFaa4ZrNr0DCWhvO1jhbLZJaIyzJykL3APi3dHYXLkD7ksvCcplfAiRRBvYvkpiRdl6dPM1ee1s
# The default root user is named 'admin'
#root_username = admin
# and put the resulting hash value into the following line
root_password_sha2 = 4813494d137e1631bba301d5acab6e7bb7aa74ce1185d456565ef51d737677b2
# The email address of the root user.
# Default is empty
#root_email = ""
# The time zone setting of the root user. See http://www.joda.org/joda-time/timezones_masters.html for a list of valid time zones.
# Default is UTC
#root_timezone = UTC
# Set the bin directory here (relative or absolute)
# This directory contains binaries that are used by the Graylog server.
# Default: bin
bin_dir = /usr/share/graylog-server/bin
# Set the data directory here (relative or absolute)
# This directory is used to store Graylog server state.
# Default: data
data_dir = /var/lib/graylog-server
# Set plugin directory here (relative or absolute)
plugin_dir = /usr/share/graylog-server/plugin
###############
# HTTP settings
###############
#### HTTP bind address
#
# The network interface used by the Graylog HTTP interface.
#
# This network interface must be accessible by all Graylog nodes in the cluster and by all clients
# using the Graylog web interface.
#
# If the port is omitted, Graylog will use port 9000 by default.
#
# Default: 127.0.0.1:9000
#http_bind_address = 3.136.154.11:9000
http_bind_address = 172.31.46.79:9000
#http_bind_address = [2001:db8::1]:9000
#### HTTP publish URI
# Default: http://$http_bind_address/
#http_publish_uri = http://192.168.1.1:9000/
http_publish_uri = http://$http_bind_address/
#### External Graylog URI
#
# Default: $http_publish_uri
#http_external_uri =
http_external_uri = $http_publish_uri/
#### Enable CORS headers for HTTP interface
#
# This is necessary for JS-clients accessing the server directly.
# If these are disabled, modern browsers will not be able to retrieve resources from the server.
# This is enabled by default. Uncomment the next line to disable it.
http_enable_cors = true
#### Enable GZIP support for HTTP interface
#
# This compresses API responses and therefore helps to reduce
# overall round trip times. This is enabled by default. Uncomment the next line to disable it.
http_enable_gzip = true
# The maximum size of the HTTP request headers in bytes.
http_max_header_size = 8192
# The size of the thread pool used exclusively for serving the HTTP interface.
http_thread_pool_size = 16
################
# HTTPS settings
################
#### Enable HTTPS support for the HTTP interface
#
# This secures the communication with the HTTP interface with TLS to prevent request forgery and eavesdropping.
#
# Default: false
#http_enable_tls = true
# The X.509 certificate chain file in PEM format to use for securing the HTTP interface.
#http_tls_cert_file = /path/to/graylog.crt
# The PKCS#8 private key file in PEM format to use for securing the HTTP interface.
#http_tls_key_file = /path/to/graylog.key
# The password to unlock the private key used for securing the HTTP interface.
#http_tls_key_password = secret
# Comma separated list of trusted proxies that are allowed to set the client address with X-Forwarded-For
# header. May be subnets, or hosts.
#trusted_proxies = 127.0.0.1/32, 0:0:0:0:0:0:0:1/128
# Default: http://127.0.0.1:9200
#elasticsearch_hosts = http://node1:9200,http://user:password@node2:19200
# Maximum amount of time to wait for successfull connection to Elasticsearch HTTP port.
#
# Default: 10 Seconds
#elasticsearch_connect_timeout = 10s
# Maximum amount of time to wait for reading back a response from an Elasticsearch server.
#
# Default: 60 seconds
#elasticsearch_socket_timeout = 60s
# Maximum idle time for an Elasticsearch connection. If this is exceeded, this connection will
# be tore down.
#
# Default: inf
#elasticsearch_idle_timeout = -1s
# Maximum number of total connections to Elasticsearch.
#
# Default: 20
#elasticsearch_max_total_connections = 20
# Maximum number of total connections per Elasticsearch route (normally this means per
# elasticsearch server).
#
# Default: 2
#elasticsearch_max_total_connections_per_route = 2
# Maximum number of times Graylog will retry failed requests to Elasticsearch.
#
# Default: 2
#elasticsearch_max_retries = 2
# Enable automatic Elasticsearch node discovery through Nodes Info,
# see https://www.elastic.co/guide/en/elasticsearch/reference/5.4/cluster-nodes-info.html
#
# WARNING: Automatic node discovery does not work if Elasticsearch requires authentication, e. g. with Shield.
#
# Default: false
#elasticsearch_discovery_enabled = true
# Filter for including/excluding Elasticsearch nodes in discovery according to their custom attributes,
# see https://www.elastic.co/guide/en/elasticsearch/reference/5.4/cluster.html#cluster-nodes
#
# Default: empty
#elasticsearch_discovery_filter = rack:42
# Frequency of the Elasticsearch node discovery.
#
# Default: 30s
# elasticsearch_discovery_frequency = 30s
# Enable payload compression for Elasticsearch requests.
#
# Default: false
#elasticsearch_compression_enabled = true
rotation_strategy = count
elasticsearch_max_docs_per_index = 20000000
#elasticsearch_max_size_per_index = 1073741824
#elasticsearch_max_time_per_index = 1d
#elasticsearch_disable_version_check = true
# Disable message retention on this node, i. e. disable Elasticsearch index rotation.
#no_retention = false
# How many indices do you want to keep?
#
# Also see http://docs.graylog.org/en/2.3/pages/configuration/index_model.html#index-set-configuration.
elasticsearch_max_number_of_indices = 20
retention_strategy = delete
elasticsearch_shards = 4
elasticsearch_replicas = 0
elasticsearch_index_prefix = graylog
#elasticsearch_template_name = graylog-internal
# Do you want to allow searches with leading wildcards? This can be extremely resource hungry and should only
# be enabled with care. See also: http://docs.graylog.org/en/2.1/pages/queries.html
allow_leading_wildcard_searches = false
# Do you want to allow searches to be highlighted? Depending on the size of your messages this can be memory hungry and
# should only be enabled after making sure your Elasticsearch cluster has enough memory.
allow_highlighting = false
elasticsearch_analyzer = standard
# Global request timeout for Elasticsearch requests (e. g. during search, index creation, or index time-range
# calculations) based on a best-effort to restrict the runtime of Elasticsearch operations.
# Default: 1m
#elasticsearch_request_timeout = 1m
# Global timeout for index optimization (force merge) requests.
# Default: 1h
#elasticsearch_index_optimization_timeout = 1h
# Maximum number of concurrently running index optimization (force merge) jobs.
# If you are using lots of different index sets, you might want to increase that number.
# Default: 20
#elasticsearch_index_optimization_jobs = 20
# Time interval for index range information cleanups. This setting defines how often stale index range information
# is being purged from the database.
# Default: 1h
#index_ranges_cleanup_interval = 1h
# Time interval for the job that runs index field type maintenance tasks like cleaning up stale entries. This doesn't
# need to run very often.
# Default: 1h
#index_field_type_periodical_interval = 1h
# ("outputbuffer_processors" variable)
output_batch_size = 500
# for this time period is less than output_batch_size * outputbuffer_processors.
output_flush_interval = 1
# not be tried again for an also configurable amount of seconds.
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 5
outputbuffer_processors = 3
#outputbuffer_processor_keep_alive_time = 5000
# The number of threads to keep in the pool, even if they are idle, unless allowCoreThreadTimeOut is set
# Default: 3
#outputbuffer_processor_threads_core_pool_size = 3
# The maximum number of threads to allow in the pool
# Default: 30
#outputbuffer_processor_threads_max_pool_size = 30
# UDP receive buffer size for all message inputs (e. g. SyslogUDPInput).
#udp_recvbuffer_sizes = 1048576
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
# Enable the disk based message journal.
message_journal_enabled = true
message_journal_dir = /var/lib/graylog-server/journal
# Journal hold messages before they could be written to Elasticsearch.
# For a maximum of 12 hours or 5 GB whichever happens first.
# During normal operation the journal will be smaller.
#message_journal_max_age = 12h
#message_journal_max_size = 5gb
#message_journal_flush_age = 1m
#message_journal_flush_interval = 1000000
#message_journal_segment_age = 1h
#message_journal_segment_size = 100mb
# Number of threads used exclusively for dispatching internal events. Default is 2.
#async_eventbus_processors = 2
# How many seconds to wait between marking node as DEAD for possible load balancers and starting the actual
# shutdown process. Set to 0 if you have no status checking load balancers in front.
lb_recognition_period_seconds = 3
# Journal usage percentage that triggers requesting throttling for this server node from load balancers. The feature is
# disabled if not set.
#lb_throttle_threshold_percentage = 95
#stream_processing_timeout = 2000
#stream_processing_max_faults = 3
# Length of the interval in seconds in which the alert conditions for all streams should be checked
# and alarms are being sent.
#alert_check_interval = 60
# Since 0.21 the Graylog server supports pluggable output modules. This means a single message can be written to multiple
# outputs. The next setting defines the timeout for a single output module, including the default output module where all
# messages end up.
#
# Time in milliseconds to wait for all message outputs to finish writing a single message.
#output_module_timeout = 10000
# Time in milliseconds after which a detected stale master node is being rechecked on startup.
#stale_master_timeout = 2000
# Time in milliseconds which Graylog is waiting for all threads to stop on shutdown.
#shutdown_timeout = 30000
# MongoDB connection string
# See https://docs.mongodb.com/manual/reference/connection-string/ for details
mongodb_uri = mongodb://localhost/graylog
# Authenticate against the MongoDB server
# '+'-signs in the username or password need to be replaced by '%2B'
#mongodb_uri = mongodb://grayloguser:secret@localhost:27017/graylog
# Use a replica set instead of a single host
#mongodb_uri = mongodb://grayloguser:secret@localhost:27017,localhost:27018,localhost:27019/graylog
# Increase this value according to the maximum connections your MongoDB server can handle from a single client
# if you encounter MongoDB connection problems.
mongodb_max_connections = 1000
# then 500 threads can block. More than that and an exception will be thrown.
# http://api.mongodb.com/java/current/com/mongodb/MongoOptions.html#threadsAllowedToBlockForConnectionMultiplier
mongodb_threads_allowed_to_block_multiplier = 5
# Email transport
#transport_email_enabled = false
#transport_email_hostname = mail.example.com
#transport_email_port = 587
#transport_email_use_auth = true
#transport_email_auth_username = you@example.com
#transport_email_auth_password = secret
#transport_email_subject_prefix = [graylog]
#transport_email_from_email = graylog@example.com
# Encryption settings
#
# ATTENTION:
# Using SMTP with STARTTLS *and* SMTPS at the same time is *not* possible.
# Use SMTP with STARTTLS, see https://en.wikipedia.org/wiki/Opportunistic_TLS
#transport_email_use_tls = true
# Use SMTP over SSL (SMTPS), see https://en.wikipedia.org/wiki/SMTPS
# This is deprecated on most SMTP services!
#transport_email_use_ssl = true
#transport_email_web_interface_url = https://graylog.example.com
# The default connect timeout for outgoing HTTP connections.
# Values must be a positive duration (and between 1 and 2147483647 when converted to milliseconds).
# Default: 5s
#http_connect_timeout = 5s
# The default read timeout for outgoing HTTP connections.
# Values must be a positive duration (and between 1 and 2147483647 when converted to milliseconds).
# Default: 10s
#http_read_timeout = 10s
# The default write timeout for outgoing HTTP connections.
# Values must be a positive duration (and between 1 and 2147483647 when converted to milliseconds).
# Default: 10s
#http_write_timeout = 10s
# HTTP proxy for outgoing HTTP connections
#http_proxy_uri =
#http_non_proxy_hosts =
#disable_index_optimization = true
# Optimize the index down to <= index_optimization_max_num_segments. A higher number may take some load from Elasticsearch
# on heavily used systems with large indices, but it will decrease search performance. The default is 1.
#
#index_optimization_max_num_segments = 1
# The threshold of the garbage collection runs. If GC runs take longer than this threshold, a system notification
# will be generated to warn the administrator about possible problems with the system. Default is 1 second.
#gc_warning_threshold = 1s
# Connection timeout for a configured LDAP server (e. g. ActiveDirectory) in milliseconds.
#ldap_connection_timeout = 2000
# Disable the use of SIGAR for collecting system stats
#disable_sigar = false
# The default cache time for dashboard widgets. (Default: 10 seconds, minimum: 1 second)
#dashboard_widget_default_cache_time = 10s
# For some cluster-related REST requests, the node must query all other nodes in the cluster. This is the maximum number
# of threads available for this. Increase it, if '/cluster/*' requests take long to complete.
# Should be http_thread_pool_size * average_cluster_size if you have a high number of concurrent users.
proxied_requests_thread_pool_size = 32
Logs for elasticsearch
[root@graylog elasticsearch]# tail graylog.log
[2020-01-07T10:34:52,746][INFO ][o.e.t.TransportService ] [9Z8I0X5] publish_address {172.29.55.217:9300}, bound_addresses {[::]:9300}
[2020-01-07T10:34:52,772][INFO ][o.e.b.BootstrapChecks ] [9Z8I0X5] bound or publishing to a non-loopback address, enforcing bootstrap checks
[2020-01-07T10:34:55,897][INFO ][o.e.c.s.MasterService ] [9Z8I0X5] zen-disco-elected-as-master ([0] nodes joined), reason: new_master {9Z8I0X5}{9Z8I0X5ATS2Zk-WX8WMOrw}{HQo_ib4aRf28Y-gkUR7ifw}{172.29.55.217}{172.29.55.217:9300}{ml.machine_memory=50462601216, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true}
[2020-01-07T10:34:55,905][INFO ][o.e.c.s.ClusterApplierService] [9Z8I0X5] new_master {9Z8I0X5}{9Z8I0X5ATS2Zk-WX8WMOrw}{HQo_ib4aRf28Y-gkUR7ifw}{172.29.55.217}{172.29.55.217:9300}{ml.machine_memory=50462601216, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true}, reason: apply cluster state (from master [master {9Z8I0X5}{9Z8I0X5ATS2Zk-WX8WMOrw}{HQo_ib4aRf28Y-gkUR7ifw}{172.29.55.217}{172.29.55.217:9300}{ml.machine_memory=50462601216, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true} committed version [1] source [zen-disco-elected-as-master ([0] nodes joined)]])
[2020-01-07T10:34:55,970][INFO ][o.e.h.n.Netty4HttpServerTransport] [9Z8I0X5] publish_address {172.29.55.217:9200}, bound_addresses {[::]:9200}
[2020-01-07T10:34:55,971][INFO ][o.e.n.Node ] [9Z8I0X5] started
[2020-01-07T10:34:56,675][WARN ][o.e.x.s.a.s.m.NativeRoleMappingStore] [9Z8I0X5] Failed to clear cache for realms [[]]
[2020-01-07T10:34:56,778][INFO ][o.e.l.LicenseService ] [9Z8I0X5] license [27c3d6e4-f3dd-4598-a2d6-9ae6c43ab6ea] mode [basic] - valid
[2020-01-07T10:34:56,794][INFO ][o.e.g.GatewayService ] [9Z8I0X5] recovered [2] indices into cluster_state
[2020-01-07T10:34:57,495][INFO ][o.e.c.r.a.AllocationService] [9Z8I0X5] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[graylog_0][1], [graylog_0][2], [graylog_0][3], [graylog_0][0]] ...]).
[root@graylog elasticsearch]# tail graylog_deprecation.log
[2020-01-07T05:30:07,371][WARN ][o.e.d.a.a.i.t.p.PutIndexTemplateRequest] [9Z8I0X5] Deprecated field [template] used, replaced by [index_patterns]
[2020-01-07T05:30:38,544][WARN ][o.e.d.s.a.I.Parser ] [9Z8I0X5] Deprecated aggregation order key [_term] used, replaced by [_key]
[2020-01-07T10:34:56,137][WARN ][o.e.d.c.j.Joda ] [9Z8I0X5] 'y' year should be replaced with 'u'. Use 'y' for year-of-era. Prefix your date format with '8' to use the new specifier.
[2020-01-07T10:34:58,405][WARN ][o.e.d.r.a.a.i.RestGetMappingAction] [9Z8I0X5] [types removal] The parameter include_type_name should be explicitly specified in get mapping requests to prepare for 7.0. In 7.0 include_type_name will default to 'false', which means responses will omit the type name in mapping definitions.
[root@graylog elasticsearch]#
[root@graylog elasticsearch]# tail gc.log.0.current
2020-01-07T10:34:41.967+0530: 5.033: [GC (CMS Initial Mark) [1 CMS-initial-mark: 0K(3086784K)] 727090K(4083584K), 0.1205775 secs] [Times: user=0.58 sys=0.01, real=0.12 secs]
2020-01-07T10:34:42.088+0530: 5.154: Total time for which application threads were stopped: 0.1207925 seconds, Stopping threads took: 0.0000441 seconds
2020-01-07T10:34:42.088+0530: 5.154: [CMS-concurrent-mark-start]
2020-01-07T10:34:42.091+0530: 5.157: [CMS-concurrent-mark: 0.003/0.003 secs] [Times: user=0.01 sys=0.01, real=0.01 secs]
2020-01-07T10:34:42.091+0530: 5.157: [CMS-concurrent-preclean-start]
2020-01-07T10:34:42.120+0530: 5.187: [CMS-concurrent-preclean: 0.029/0.029 secs] [Times: user=0.07 sys=0.02, real=0.03 secs]
2020-01-07T10:34:42.121+0530: 5.187: [CMS-concurrent-abortable-preclean-start]
2020-01-07T10:34:42.319+0530: 5.385: Total time for which application threads were stopped: 0.0003473 seconds, Stopping threads took: 0.0000414 seconds
2020-01-07T10:34:42.372+0530: 5.438: Total time for which application threads were stopped: 0.0002243 seconds, Stopping threads took: 0.0000475 seconds
2020-01-07T10:34:42.728+0530: 5.794: [GC (Allocation Failure) 2020-01-07T10:34:42.728+0530: 5.794: [ParNew2020-01-07T10:34:42.814+0530: 5.880: [CMS-concurrent-abortable-preclean: 0.072/0.694 secs] [Times: user=2.13 sys=0.19, real=0.69 secs]
Desired survivor size 56688640 bytes, new threshold 6 (max 6)
- age 1: 40151056 bytes, 40151056 total
: 886080K->39298K(996800K), 0.0881263 secs] 886080K->39298K(4083584K), 0.0883108 secs] [Times: user=0.50 sys=0.03, real=0.09 secs]
2020-01-07T10:34:42.816+0530: 5.882: Total time for which application threads were stopped: 0.0886332 seconds, Stopping threads took: 0.0000530 seconds
2020-01-07T10:34:42.817+0530: 5.883: [GC (CMS Final Remark) [YG occupancy: 56131 K (996800 K)]2020-01-07T10:34:42.817+0530: 5.883: [Rescan (parallel) , 0.0274550 secs]2020-01-07T10:34:42.844+0530: 5.910: [weak refs processing, 0.0000610 secs]2020-01-07T10:34:42.844+0530: 5.910: [class unloading, 0.0191991 secs]2020-01-07T10:34:42.863+0530: 5.930: [scrub symbol table, 0.0062919 secs]2020-01-07T10:34:42.870+0530: 5.936: [scrub string table, 0.0006374 secs][1 CMS-remark: 0K(3086784K)] 56131K(4083584K), 0.0553239 secs] [Times: user=0.40 sys=0.03, real=0.06 secs]
2020-01-07T10:34:42.872+0530: 5.938: Total time for which application threads were stopped: 0.0554856 seconds, Stopping threads took: 0.0000464 seconds
2020-01-07T10:34:42.872+0530: 5.938: [CMS-concurrent-sweep-start]
2020-01-07T10:34:42.872+0530: 5.938: [CMS-concurrent-sweep: 0.000/0.000 secs] [Times: user=0.00 sys=0.00, real=0.00 secs]
2020-01-07T10:34:42.872+0530: 5.938: [CMS-concurrent-reset-start]
2020-01-07T10:39:57.901+0530: 320.968: Total time for which application threads were stopped: 0.0002806 seconds, Stopping threads took: 0.0001159 seconds
2020-01-07T10:39:57.909+0530: 320.975: Total time for which application threads were stopped: 0.0002280 seconds, St[root@graylog elasticsearch]#
logs for graylog
tail -f /var/log/graylog-server/server.log
[0s], polling every [3600s].
2020-01-07T10:43:39.422+05:30 INFO [JerseyService] Enabling CORS for HTTP endpoint
2020-01-07T10:44:03.775+05:30 INFO [NetworkListener] Started listener bound to [127.0.0.1:9000]
2020-01-07T10:44:03.777+05:30 INFO [HttpServer] [HttpServer] Started.
2020-01-07T10:44:03.777+05:30 INFO [JerseyService] Started REST API at <127.0.0.1:9000>
2020-01-07T10:44:03.778+05:30 INFO [ServiceManagerListener] Services are healthy
2020-01-07T10:44:03.778+05:30 INFO [InputSetupService] Triggering launching persisted inputs, node transitioned from Uninitialized [LB:DEAD] to Running [LB:ALIVE]
2020-01-07T10:44:03.779+05:30 INFO [ServerBootstrap] Services started, startup times in ms: {OutputSetupService [RUNNING]=57, BufferSynchronizerService [RUNNING]=60, KafkaJournal [RUNNING]=62, GracefulShutdownService [RUNNING]=63, InputSetupService [RUNNING]=150, ConfigurationEtagService [RUNNING]=221, EtagService [RUNNING]=222, JournalReader [RUNNING]=225, LookupTableService [RUNNING]=226, StreamCacheService [RUNNING]=364, PeriodicalsService [RUNNING]=371, JerseyService [RUNNING]=25270}
2020-01-07T10:44:03.784+05:30 INFO [KafkaJournal] Read offset 59654652778 before start of log at 59665435301, starting to read from the beginning of the journal.
2020-01-07T10:44:03.786+05:30 INFO [ServerBootstrap] Graylog server up and running.
Please help me to resolve this issue, Graylog server is down.
Thanks,
Prakash A