Server currently unavailable - new cluster setup

Web Page displays login, briefly. Before stating that the server is unavailable

/var/log/graylog-server/server.log
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154)
at org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer.service(GrizzlyHttpContainer.java:384)
at org.glassfish.grizzly.http.server.HttpHandler$1.run(HttpHandler.java:224)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

2018-08-02T15:38:32.616+01:00 WARN [Errors] The following warnings have been detected: WARNING: Unknown HK2 failure detected:
MultiException stack 1 of 2
java.lang.RuntimeException: IOException encountered while reading from a byte array input stream
at org.mongojack.internal.stream.JacksonDBDecoder.decode(JacksonDBDecoder.java:67)
at com.mongodb.DBDecoderAdapter.decode(DBDecoderAdapter.java:49)
at com.mongodb.DBDecoderAdapter.decode(DBDecoderAdapter.java:29)
at com.mongodb.operation.CommandResultArrayCodec.decode(CommandResultArrayCodec.java:48)
at com.mongodb.operation.CommandResultDocumentCodec.readValue(CommandResultDocumentCodec.java:53)
at org.bson.codecs.BsonDocumentCodec.decode(BsonDocumentCodec.java:81)
at org.bson.codecs.BsonDocumentCodec.decode(BsonDocumentCodec.java:40)
at org.bson.codecs.configuration.LazyCodec.decode(LazyCodec.java:47)
at org.bson.codecs.BsonDocumentCodec.readValue(BsonDocumentCodec.java:98)
at com.mongodb.operation.CommandResultDocumentCodec.readValue(CommandResultDocumentCodec.java:56)
at org.bson.codecs.BsonDocumentCodec.decode(BsonDocumentCodec.java:81)
at org.bson.codecs.BsonDocumentCodec.decode(BsonDocumentCodec.java:40)
at com.mongodb.connection.ReplyMessage.(ReplyMessage.java:57)
at com.mongodb.connection.CommandProtocol.getResponseDocument(CommandProtocol.java:149)
at com.mongodb.connection.CommandProtocol.execute(CommandProtocol.java:118)
at com.mongodb.connection.DefaultServer$DefaultServerProtocolExecutor.execute(DefaultServer.java:159)
at com.mongodb.connection.DefaultServerConnection.executeProtocol(DefaultServerConnection.java:286)
at com.mongodb.connection.DefaultServerConnection.command(DefaultServerConnection.java:173)
at com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocol(CommandOperationHelper.java:215)
at com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocol(CommandOperationHelper.java:206)
at com.mongodb.operation.CommandOperationHelper.executeWrappedCommandProtocol(CommandOperationHelper.java:112)
at com.mongodb.operation.FindOperation$1.call(FindOperation.java:487)
at com.mongodb.operation.FindOperation$1.call(FindOperation.java:482)
at com.mongodb.operation.OperationHelper.withConnectionSource(OperationHelper.java:239)
at com.mongodb.operation.OperationHelper.withConnection(OperationHelper.java:212)
at com.mongodb.operation.FindOperation.execute(FindOperation.java:482)
at com.mongodb.operation.FindOperation.execute(FindOperation.java:79)
at com.mongodb.Mongo.execute(Mongo.java:772)
at com.mongodb.Mongo$2.execute(Mongo.java:759)
at com.mongodb.DBCursor.initializeCursor(DBCursor.java:851)
at com.mongodb.DBCursor.hasNext(DBCursor.java:152)
at org.mongojack.DBCursor.hasNext(DBCursor.java:330)
at org.mongojack.JacksonDBCollection.findOne(JacksonDBCollection.java:1375)
at org.mongojack.JacksonDBCollection.findOne(JacksonDBCollection.java:1336)
at org.mongojack.JacksonDBCollection.findOne(JacksonDBCollection.java:1310)
at org.graylog2.cluster.ClusterConfigServiceImpl.get(ClusterConfigServiceImpl.java:103)
at org.graylog2.cluster.ClusterConfigServiceImpl.getOrDefault(ClusterConfigServiceImpl.java:120)
at org.graylog2.rest.resources.system.ClusterResource.(ClusterResource.java:63)
at sun.reflect.GeneratedConstructorAccessor84.newInstance(Unknown Source)
at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
at org.glassfish.hk2.utilities.reflection.ReflectionHelper.makeMe(ReflectionHelper.java:1350)
at org.jvnet.hk2.internal.ClazzCreator.createMe(ClazzCreator.java:271)
at org.jvnet.hk2.internal.ClazzCreator.create(ClazzCreator.java:365)
at org.jvnet.hk2.internal.SystemDescriptor.create(SystemDescriptor.java:471)
at org.glassfish.jersey.process.internal.RequestScope.findOrCreate(RequestScope.java:162)
at org.jvnet.hk2.internal.Utilities.createService(Utilities.java:2072)
at org.jvnet.hk2.internal.ServiceLocatorImpl.internalGetService(ServiceLocatorImpl.java:767)
at org.jvnet.hk2.internal.ServiceLocatorImpl.getService(ServiceLocatorImpl.java:706)
at org.glassfish.jersey.internal.inject.Injections.getOrCreate(Injections.java:172)
at org.glassfish.jersey.server.model.MethodHandler$ClassBasedMethodHandler.getInstance(MethodHandler.java:284)
at org.glassfish.jersey.server.internal.routing.PushMethodHandlerRouter.apply(PushMethodHandlerRouter.java:74)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:109)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:92)
at org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:61)
at org.glassfish.jersey.process.internal.Stages.process(Stages.java:197)
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:318)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154)
at org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer.service(GrizzlyHttpContainer.java:384)
at org.glassfish.grizzly.http.server.HttpHandler$1.run(HttpHandler.java:224)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: com.fasterxml.jackson.databind.JsonMappingException: Can not deserialize instance of org.joda.time.DateTime out of VALUE_EMBEDDED_OBJECT token
at [Source: de.undercouch.bson4jackson.io.LittleEndianInputStream@509809a0; pos: 146] (through reference chain: org.graylog2.cluster.ClusterConfig[“last_updated”])
at com.fasterxml.jackson.databind.JsonMappingException.from(JsonMappingException.java:148)
at com.fasterxml.jackson.databind.DeserializationContext.mappingException(DeserializationContext.java:854)
at com.fasterxml.jackson.databind.DeserializationContext.mappingException(DeserializationContext.java:850)
at com.fasterxml.jackson.datatype.joda.deser.DateTimeDeserializer.deserialize(DateTimeDeserializer.java:92)
at com.fasterxml.jackson.datatype.joda.deser.DateTimeDeserializer.deserialize(DateTimeDeserializer.java:22)
at com.fasterxml.jackson.databind.deser.SettableBeanProperty.deserialize(SettableBeanProperty.java:520)
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeWithErrorWrapping(BeanDeserializer.java:461)
at com.fasterxml.jackson.databind.deser.BeanDeserializer._deserializeUsingPropertyBased(BeanDeserializer.java:376)
at com.fasterxml.jackson.databind.deser.BeanDeserializerBase.deserializeFromObjectUsingNonDefault(BeanDeserializerBase.java:1099)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserializeFromObject(BeanDeserializer.java:294)
at com.fasterxml.jackson.databind.deser.BeanDeserializer.deserialize(BeanDeserializer.java:131)
at com.fasterxml.jackson.databind.ObjectMapper._readValue(ObjectMapper.java:3703)
at com.fasterxml.jackson.databind.ObjectMapper.readValue(ObjectMapper.java:2072)
at org.mongojack.internal.stream.JacksonDBDecoder.decode(JacksonDBDecoder.java:77)
at org.mongojack.internal.stream.JacksonDBDecoder.decode(JacksonDBDecoder.java:64)
… 74 more
MultiException stack 2 of 2
java.lang.IllegalStateException: Unable to perform operation: create on org.graylog2.rest.resources.system.ClusterResource
at org.jvnet.hk2.internal.ClazzCreator.create(ClazzCreator.java:392)
at org.jvnet.hk2.internal.SystemDescriptor.create(SystemDescriptor.java:471)
at org.glassfish.jersey.process.internal.RequestScope.findOrCreate(RequestScope.java:162)
at org.jvnet.hk2.internal.Utilities.createService(Utilities.java:2072)
at org.jvnet.hk2.internal.ServiceLocatorImpl.internalGetService(ServiceLocatorImpl.java:767)
at org.jvnet.hk2.internal.ServiceLocatorImpl.getService(ServiceLocatorImpl.java:706)
at org.glassfish.jersey.internal.inject.Injections.getOrCreate(Injections.java:172)
at org.glassfish.jersey.server.model.MethodHandler$ClassBasedMethodHandler.getInstance(MethodHandler.java:284)
at org.glassfish.jersey.server.internal.routing.PushMethodHandlerRouter.apply(PushMethodHandlerRouter.java:74)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:109)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage._apply(RoutingStage.java:112)
at org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:92)
at org.glassfish.jersey.server.internal.routing.RoutingStage.apply(RoutingStage.java:61)
at org.glassfish.jersey.process.internal.Stages.process(Stages.java:197)
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:318)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271)
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267)
at org.glassfish.jersey.internal.Errors.process(Errors.java:315)
at org.glassfish.jersey.internal.Errors.process(Errors.java:297)
at org.glassfish.jersey.internal.Errors.process(Errors.java:267)
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317)
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305)
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154)
at org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer.service(GrizzlyHttpContainer.java:384)
at org.glassfish.grizzly.http.server.HttpHandler$1.run(HttpHandler.java:224)
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:176)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)

graylog, elasticsearch and mongo all reside in the same box and this these are clustered over 4 servers. I have only installed graylog in one server at the moment to try and get it working.

When you format your post - https://community.graylog.org/faq#format-markdown - it is more likely that someone will help you as the message is more readable.

Did you checked the time of the servers (time jump/difference)?

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.