Hello,
To me, Im new here and this is my first post.
I have a performance problem with Graylog Server running on Ubuntu 18.04 . As soon as I create grok extractors, process buffer will be full in a couple of minutes. Its a vm in azure cloud. ES and Graylog are running on the same vm with 1 node. The log rate is on average 50 logs/s.
Graylog Version: v2.5.1+34194da
Ressources: 4 VCPUs, 16GB RAM
Reserved Rams for Graylog: 2 GB
Reserved Rams for Elasticsearch: 8 GB
I am grateful for any help!
server.conf:
output_batch_size = 10000
output_flush_interval = 1
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 5
outputbuffer_processors = 2
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
Grok pattern:
{
"extractors": [
{
"title": "SYSTEM_LOGS",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "message",
"extractor_config": {
"grok_pattern": "%{BASE10NUM:UNWANTED},%{DATA:time_event;date;yyyy/MM/dd HH:mm:ss},%{BASE10NUM:serial},%{WORD:type},%{WORD:type_sub},%{BASE10NUM:UNWANTED},%{DATA:time_generated;date;yyyy/MM/dd HH:mm:ss},%{DATA:UNWANTED},%{DATA:id_event},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:severity},%{DATA:description},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{HOSTNAME:device_name}"
},
"condition_type": "regex",
"condition_value": "^(.*,SYSTEM,.*)"
},
{
"title": "THREAT_LOGS",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "message",
"extractor_config": {
"grok_pattern": "%{BASE10NUM:UNWANTED},%{DATA:time_event;date;yyyy/MM/dd HH:mm:ss},%{BASE10NUM:serial},%{WORD:type},%{WORD:type_sub},%{BASE10NUM:UNWANTED},%{DATA:time_generated;date;yyyy/MM/dd HH:mm:ss},%{IPV4:src},%{IPV4:dst},%{IPV4:src_nat},%{IPV4:dst_nat},%{DATA:rule},%{DATA:src_user},%{DATA:dst_user},%{DATA:app},%{DATA:UNWANTED},%{WORD:src_zone},%{WORD:dst_zone},%{DATA:interface_in},%{DATA:interface_out},%{DATA:action_log},%{DATA:UNWANTED},%{DATA:id_session},%{BASE10NUM:count},%{BASE10NUM:src_port},%{BASE10NUM:dst_port},%{BASE10NUM:src_nat_port},%{BASE10NUM:dst_nat_port},%{DATA:UNWANTED},%{DATA:protocol},%{DATA:action},%{QUOTEDSTRING:url},%{DATA:id_threat},%{DATA:category},%{DATA:severity},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:src_location},%{DATA:dst_location},%{DATA:UNWANTED},%{DATA:type_content},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{HOSTNAME:device_name},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:http_method},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED}",
"named_captures_only": false
},
"condition_type": "regex",
"condition_value": "^(.*,THREAT,.*)"
},
{
"title": "TRAFFIC_LOGS",
"extractor_type": "grok",
"converters": [],
"order": 0,
"cursor_strategy": "copy",
"source_field": "message",
"target_field": "message",
"extractor_config": {
"grok_pattern": "%{BASE10NUM:UNWANTED},%{DATA:time_event;date;yyyy/MM/dd HH:mm:ss},%{BASE10NUM:serial},%{WORD:type},%{WORD:type_sub},%{BASE10NUM:UNWANTED},%{DATA:time_generated;date;yyyy/MM/dd HH:mm:ss},%{IPV4:src},%{IPV4:dst},%{IPV4:src_nat},%{IPV4:dst_nat},%{DATA:rule},%{DATA:src_user},%{DATA:dst_user},%{DATA:app},%{DATA:UNWANTED},%{WORD:src_zone},%{WORD:dst_zone},%{DATA:interface_in},%{DATA:interface_out},%{DATA:action_log},%{DATA:UNWANTED},%{DATA:id_session},%{BASE10NUM:count},%{BASE10NUM:src_port},%{BASE10NUM:dst_port},%{BASE10NUM:src_nat_port},%{BASE10NUM:dst_nat_port},%{DATA:UNWANTED},%{DATA:protocol},%{DATA:action},%{NUMBER:bytes;long},%{NUMBER:bytes_sent;long},%{NUMBER:bytes_recieved;long},%{NUMBER:packets;long},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:category},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:src_location},%{DATA:dst_location},%{DATA:UNWANTED},%{NUMBER:packets_sent;long},%{NUMBER:packets_recieved;long},%{DATA:session_end_reason},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{HOSTNAME:device_name},%{DATA:src_action},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:type_tunnel},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED},%{DATA:UNWANTED}"
},
"condition_type": "regex",
"condition_value": "^(.*,TRAFFIC,.*)"
}
],
"version": "2.5.1"
}
server logs:
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317) [graylog.jar:?]
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305) [graylog.jar:?]
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154) [graylog.jar:?]
at org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer.service(GrizzlyHttpContainer.java:384) [graylog.jar:?]
at org.glassfish.grizzly.http.server.HttpHandler$1.run(HttpHandler.java:224) [graylog.jar:?]
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176) [graylog.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
2019-03-22T12:55:57.408Z WARN [ProxiedResource] Unable to call http://192.168.64.4:2095/api/system/inputstates on node <665dd38a-8e18-4e0b-8634-92cfb94af805>
java.net.SocketTimeoutException: timeout
at okio.Okio$4.newTimeoutException(Okio.java:230) ~[graylog.jar:?]
at okio.AsyncTimeout.exit(AsyncTimeout.java:285) ~[graylog.jar:?]
at okio.AsyncTimeout$2.read(AsyncTimeout.java:241) ~[graylog.jar:?]
at okio.RealBufferedSource.indexOf(RealBufferedSource.java:345) ~[graylog.jar:?]
at okio.RealBufferedSource.readUtf8LineStrict(RealBufferedSource.java:217) ~[graylog.jar:?]
at okio.RealBufferedSource.readUtf8LineStrict(RealBufferedSource.java:211) ~[graylog.jar:?]
at okhttp3.internal.http1.Http1Codec.readResponseHeaders(Http1Codec.java:187) ~[graylog.jar:?]
at okhttp3.internal.http.CallServerInterceptor.intercept(CallServerInterceptor.java:88) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[graylog.jar:?]
at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:45) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[graylog.jar:?]
at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[graylog.jar:?]
at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[graylog.jar:?]
at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:125) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[graylog.jar:?]
at org.graylog2.rest.RemoteInterfaceProvider.lambda$get$0(RemoteInterfaceProvider.java:61) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121) ~[graylog.jar:?]
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:200) ~[graylog.jar:?]
at okhttp3.RealCall.execute(RealCall.java:77) ~[graylog.jar:?]
at retrofit2.OkHttpCall.execute(OkHttpCall.java:180) ~[graylog.jar:?]
at org.graylog2.shared.rest.resources.ProxiedResource.lambda$getForAllNodes$0(ProxiedResource.java:76) ~[graylog.jar:?]
at java.util.concurrent.FutureTask.run(FutureTask.java:266) [?:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_191]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_191]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_191]
Caused by: java.net.SocketTimeoutException: Read timed out
at java.net.SocketInputStream.socketRead0(Native Method) ~[?:1.8.0_191]
at java.net.SocketInputStream.socketRead(SocketInputStream.java:116) ~[?:1.8.0_191]
at java.net.SocketInputStream.read(SocketInputStream.java:171) ~[?:1.8.0_191]
at java.net.SocketInputStream.read(SocketInputStream.java:141) ~[?:1.8.0_191]
at okio.Okio$2.read(Okio.java:139) ~[graylog.jar:?]
at okio.AsyncTimeout$2.read(AsyncTimeout.java:237) ~[graylog.jar:?]
... 28 more
2019-03-22T13:51:55.837Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now STOPPING
2019-03-22T13:51:55.838Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now STOPPED
2019-03-22T13:51:55.838Z INFO [connection] Opened connection [connectionId{localValue:10, serverValue:34}] to localhost:27017
2019-03-22T13:51:55.839Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now TERMINATED
2019-03-22T13:51:55.842Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now STARTING
2019-03-22T13:51:55.855Z WARN [NettyTransport] receiveBufferSize (SO_RCVBUF) for input SyslogUDPInput{title=Syslog test, type=org.graylog2.inputs.syslog.udp.SyslogUDPInput, nodeId=665dd38a-8e18-4e0b-8634-92cfb94af805} should be 262144 but is 212992.
2019-03-22T13:51:55.857Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now RUNNING
2019-03-22T13:52:51.556Z INFO [ExtractorsResource] Added extractor <c8e6e570-4ca9-11e9-9040-000d3a458424> of type [grok] to input <5c1fa7370ae6f404da5f4b04>.
2019-03-22T13:52:51.620Z INFO [ExtractorsResource] Added extractor <c8e8e140-4ca9-11e9-9040-000d3a458424> of type [grok] to input <5c1fa7370ae6f404da5f4b04>.
2019-03-22T13:52:51.625Z INFO [ExtractorsResource] Added extractor <c8e75aa0-4ca9-11e9-9040-000d3a458424> of type [grok] to input <5c1fa7370ae6f404da5f4b04>.
2019-03-22T13:52:51.855Z INFO [connection] Opened connection [connectionId{localValue:14, serverValue:38}] to localhost:27017
2019-03-22T13:52:51.866Z INFO [connection] Opened connection [connectionId{localValue:13, serverValue:37}] to localhost:27017
2019-03-22T13:52:51.875Z INFO [connection] Opened connection [connectionId{localValue:11, serverValue:35}] to localhost:27017
2019-03-22T13:52:51.921Z INFO [connection] Opened connection [connectionId{localValue:12, serverValue:36}] to localhost:27017
2019-03-22T13:56:42.833Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now STOPPING
2019-03-22T13:56:42.837Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now STOPPED
2019-03-22T13:56:42.837Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now TERMINATED
2019-03-22T13:56:42.878Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now STARTING
2019-03-22T13:56:43.056Z INFO [InputStateListener] Input [Syslog UDP/5c1fa7370ae6f404da5f4b04] is now RUNNING
Elasticsearch logs:
[2019-03-22T11:55:39,638][INFO ][o.e.p.PluginsService ] [node-1] loaded module [x-pack-sql]
[2019-03-22T11:55:39,638][INFO ][o.e.p.PluginsService ] [node-1] loaded module [x-pack-upgrade]
[2019-03-22T11:55:39,638][INFO ][o.e.p.PluginsService ] [node-1] loaded module [x-pack-watcher]
[2019-03-22T11:55:39,638][INFO ][o.e.p.PluginsService ] [node-1] no plugins loaded
[2019-03-22T11:55:44,481][INFO ][o.e.x.s.a.s.FileRolesStore] [node-1] parsed [0] roles from file [/etc/elasticsearch/roles.yml]
[2019-03-22T11:55:45,178][INFO ][o.e.x.m.j.p.l.CppLogMessageHandler] [node-1] [controller/2176] [Main.cc@109] controller (64 bit): Version 6.5.4 (Build b616085ef32393) Copyright (c) 2018 Elasticsearch BV
[2019-03-22T11:55:46,017][DEBUG][o.e.a.ActionModule ] [node-1] Using REST wrapper from plugin org.elasticsearch.xpack.security.Security
[2019-03-22T11:55:46,629][INFO ][o.e.d.DiscoveryModule ] [node-1] using discovery type [zen] and host providers [settings]
[2019-03-22T11:55:47,622][INFO ][o.e.n.Node ] [node-1] initialized
[2019-03-22T11:55:47,623][INFO ][o.e.n.Node ] [node-1] starting ...
[2019-03-22T11:55:47,793][INFO ][o.e.t.TransportService ] [node-1] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}
[2019-03-22T11:55:50,933][INFO ][o.e.c.s.MasterService ] [node-1] zen-disco-elected-as-master ([0] nodes joined), reason: new_master {node-1}{AYzavL2VS8-P6osaii9F9w}{M_ILoUg4RrSPsPuoQRNsgA}{127.0.0.1}{127.0.0.1:9300}{ml.machine_memory=16818376704, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true}
[2019-03-22T11:55:50,940][INFO ][o.e.c.s.ClusterApplierService] [node-1] new_master {node-1}{AYzavL2VS8-P6osaii9F9w}{M_ILoUg4RrSPsPuoQRNsgA}{127.0.0.1}{127.0.0.1:9300}{ml.machine_memory=16818376704, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true}, reason: apply cluster state (from master [master {node-1}{AYzavL2VS8-P6osaii9F9w}{M_ILoUg4RrSPsPuoQRNsgA}{127.0.0.1}{127.0.0.1:9300}{ml.machine_memory=16818376704, xpack.installed=true, ml.max_open_jobs=20, ml.enabled=true} committed version [1] source [zen-disco-elected-as-master ([0] nodes joined)]])
[2019-03-22T11:55:51,054][INFO ][o.e.x.s.t.n.SecurityNetty4HttpServerTransport] [node-1] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
[2019-03-22T11:55:51,055][INFO ][o.e.n.Node ] [node-1] started
[2019-03-22T11:55:52,915][WARN ][o.e.x.s.a.s.m.NativeRoleMappingStore] [node-1] Failed to clear cache for realms [[]]
[2019-03-22T11:55:53,104][INFO ][o.e.l.LicenseService ] [node-1] license [e19dd89c-17fe-4c05-89ed-ea18ea04291d] mode [basic] - valid
[2019-03-22T11:55:53,160][INFO ][o.e.g.GatewayService ] [node-1] recovered [12] indices into cluster_state
[2019-03-22T11:55:53,626][DEBUG][o.e.a.s.TransportSearchAction] [node-1] All shards failed for phase: [query]
[2019-03-22T11:56:53,739][INFO ][o.e.c.r.a.AllocationService] [node-1] Cluster health status changed from [RED] to [GREEN] (reason: [shards started [[graylog_0][1], [graylog_0][0]] ...]).
[2019-03-22T12:06:58,334][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][668] overhead, spent [382ms] collecting in the last [1s]
[2019-03-22T12:07:02,357][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][672] overhead, spent [464ms] collecting in the last [1s]
[2019-03-22T12:07:06,370][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][676] overhead, spent [362ms] collecting in the last [1s]
[2019-03-22T12:07:11,380][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][681] overhead, spent [277ms] collecting in the last [1s]
[2019-03-22T12:07:21,585][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][691] overhead, spent [282ms] collecting in the last [1s]
[2019-03-22T12:07:35,598][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][705] overhead, spent [342ms] collecting in the last [1s]
[2019-03-22T12:07:46,023][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][715] overhead, spent [334ms] collecting in the last [1.2s]
[2019-03-22T12:08:04,037][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][733] overhead, spent [325ms] collecting in the last [1s]
[2019-03-22T12:08:49,272][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][778] overhead, spent [268ms] collecting in the last [1s]
[2019-03-22T12:09:19,868][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][808] overhead, spent [303ms] collecting in the last [1.2s]
[2019-03-22T12:09:50,067][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][838] overhead, spent [307ms] collecting in the last [1s]
[2019-03-22T12:10:03,586][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][851] overhead, spent [254ms] collecting in the last [1s]
[2019-03-22T12:10:31,610][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][879] overhead, spent [253ms] collecting in the last [1s]
[2019-03-22T12:11:00,616][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][908] overhead, spent [313ms] collecting in the last [1s]
[2019-03-22T12:11:07,623][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][915] overhead, spent [370ms] collecting in the last [1s]
[2019-03-22T12:11:19,724][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][927] overhead, spent [267ms] collecting in the last [1s]
[2019-03-22T12:11:24,737][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][932] overhead, spent [253ms] collecting in the last [1s]
[2019-03-22T12:11:39,746][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][947] overhead, spent [266ms] collecting in the last [1s]
[2019-03-22T12:11:54,754][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][962] overhead, spent [255ms] collecting in the last [1s]
[2019-03-22T12:12:23,765][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][991] overhead, spent [284ms] collecting in the last [1s]
[2019-03-22T12:12:31,767][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][999] overhead, spent [352ms] collecting in the last [1s]
[2019-03-22T12:12:38,769][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1006] overhead, spent [282ms] collecting in the last [1s]
[2019-03-22T12:13:06,950][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1034] overhead, spent [258ms] collecting in the last [1s]
[2019-03-22T12:13:22,964][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1050] overhead, spent [328ms] collecting in the last [1s]
[2019-03-22T12:13:33,003][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1060] overhead, spent [346ms] collecting in the last [1s]
[2019-03-22T12:13:40,015][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1067] overhead, spent [267ms] collecting in the last [1s]
[2019-03-22T12:13:47,019][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1074] overhead, spent [317ms] collecting in the last [1s]
[2019-03-22T12:14:33,274][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1120] overhead, spent [340ms] collecting in the last [1s]
[2019-03-22T12:14:54,282][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1141] overhead, spent [280ms] collecting in the last [1s]
[2019-03-22T12:14:59,290][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1146] overhead, spent [262ms] collecting in the last [1s]
[2019-03-22T12:15:25,307][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1172] overhead, spent [292ms] collecting in the last [1s]
[2019-03-22T12:15:45,555][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1192] overhead, spent [305ms] collecting in the last [1s]
[2019-03-22T12:16:12,578][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1219] overhead, spent [285ms] collecting in the last [1s]
[2019-03-22T12:16:26,763][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1233] overhead, spent [291ms] collecting in the last [1s]
[2019-03-22T12:16:30,764][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1237] overhead, spent [274ms] collecting in the last [1s]
[2019-03-22T12:16:42,770][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1249] overhead, spent [272ms] collecting in the last [1s]
[2019-03-22T12:16:56,926][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][1263] overhead, spent [284ms] collecting in the last [1s]
[2019-03-22T12:37:13,261][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][2479] overhead, spent [481ms] collecting in the last [1s]
[2019-03-22T12:37:17,280][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][2483] overhead, spent [345ms] collecting in the last [1s]
[2019-03-22T12:37:25,300][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][2491] overhead, spent [340ms] collecting in the last [1s]
[2019-03-22T12:37:33,366][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][2499] overhead, spent [312ms] collecting in the last [1s]
[2019-03-22T12:37:41,657][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][2507] overhead, spent [408ms] collecting in the last [1.2s]
[2019-03-22T12:53:25,342][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][3450] overhead, spent [262ms] collecting in the last [1s]
[2019-03-22T12:53:34,344][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][3459] overhead, spent [281ms] collecting in the last [1s]
[2019-03-22T12:55:32,940][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][3577] overhead, spent [425ms] collecting in the last [1.4s]
[2019-03-22T12:55:37,956][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][3582] overhead, spent [314ms] collecting in the last [1s]
[2019-03-22T12:55:51,161][INFO ][o.e.m.j.JvmGcMonitorService] [node-1] [gc][3595] overhead, spent [351ms] collecting in the last [1.1s]