Hi all,
I’ve made a Graylog pod that contains Graylog 3.2.5.
I’ve created my own root CA, and used that to sign off on a certificate, and a plain pkcs8. These are attached to the container as in the documentation, and i’ve checked the java key store to make sure that:
A) both the certificate and root are in the java key store and attached
B) fingerprints between the jks and the certificates attached to graylog.
I then set my http_publish to MYDOMAIN:30003
Its on that port becuase of the kubernetes service.
Now, I can access the web portal just fine, log in, everything works, but whenver I try to get node information, the console tells me “No information available”
Upon checking the container logs, I get:
2020-07-02 14:51:34,659 ERROR: org.graylog2.shared.rest.exceptionmappers.AnyExceptionClassMapper - Unhandled exception in REST resource
java.net.SocketTimeoutException: connect timed out
at java.net.PlainSocketImpl.socketConnect(Native Method) ~[?:1.8.0_252]
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350) ~[?:1.8.0_252]
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206) ~[?:1.8.0_252]
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188) ~[?:1.8.0_252]
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392) ~[?:1.8.0_252]
at java.net.Socket.connect(Socket.java:607) ~[?:1.8.0_252]
at okhttp3.internal.platform.Platform.connectSocket(Platform.java:130) ~[graylog.jar:?]
at okhttp3.internal.connection.RealConnection.connectSocket(RealConnection.java:263) ~[graylog.jar:?]
at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:183) ~[graylog.jar:?]
at okhttp3.internal.connection.ExchangeFinder.findConnection(ExchangeFinder.java:224) ~[graylog.jar:?]
at okhttp3.internal.connection.ExchangeFinder.findHealthyConnection(ExchangeFinder.java:108) ~[graylog.jar:?]
at okhttp3.internal.connection.ExchangeFinder.find(ExchangeFinder.java:88) ~[graylog.jar:?]
at okhttp3.internal.connection.Transmitter.newExchange(Transmitter.java:169) ~[graylog.jar:?]
at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:41) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[graylog.jar:?]
at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:94) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[graylog.jar:?]
at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:88) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[graylog.jar:?]
at org.graylog2.rest.RemoteInterfaceProvider.lambda$get$0(RemoteInterfaceProvider.java:61) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[graylog.jar:?]
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:221) ~[graylog.jar:?]
at okhttp3.RealCall.execute(RealCall.java:81) ~[graylog.jar:?]
at retrofit2.OkHttpCall.execute(OkHttpCall.java:188) ~[graylog.jar:?]
at org.graylog2.rest.resources.cluster.ClusterSystemPluginResource.list(ClusterSystemPluginResource.java:76) ~[graylog.jar:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_252]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_252]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_252]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_252]
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$TypeOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:205) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102) ~[graylog.jar:?]
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:315) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:297) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:267) [graylog.jar:?]
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317) [graylog.jar:?]
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305) [graylog.jar:?]
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154) [graylog.jar:?]
at org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer.service(GrizzlyHttpContainer.java:384) [graylog.jar:?]
at org.glassfish.grizzly.http.server.HttpHandler$1.run(HttpHandler.java:224) [graylog.jar:?]
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:181) [graylog.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_252]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_252]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_252]
I’m really at a loss here. I don’t think I’ve misconfigured SSL, maybe its a kubernetes issue?
So I managed to find out that the SAN was being added to the cert. That seemed to help, i’m not getting this error:
2020-07-02 14:57:17,027 WARN : org.graylog2.shared.rest.resources.ProxiedResource - Unable to call https://cybermonitor:30003/api/system/inputstates on node <ba790603-7753-46e1-86ac-ca6af950bf70>: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: validity check failed
2020-07-02 14:57:21,028 ERROR: org.graylog2.shared.rest.exceptionmappers.AnyExceptionClassMapper - Unhandled exception in REST resource
javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: validity check failed
at sun.security.ssl.Alerts.getSSLException(Alerts.java:198) ~[?:1.8.0_252]
at sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1967) ~[?:1.8.0_252]
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:331) ~[?:1.8.0_252]
at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:325) ~[?:1.8.0_252]
at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1688) ~[?:1.8.0_252]
at sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:226) ~[?:1.8.0_252]
at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1082) ~[?:1.8.0_252]
at sun.security.ssl.Handshaker.process_record(Handshaker.java:1010) ~[?:1.8.0_252]
at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1079) ~[?:1.8.0_252]
at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1388) ~[?:1.8.0_252]
at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1416) ~[?:1.8.0_252]
at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1400) ~[?:1.8.0_252]
at okhttp3.internal.connection.RealConnection.connectTls(RealConnection.java:336) ~[graylog.jar:?]
at okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:300) ~[graylog.jar:?]
at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:185) ~[graylog.jar:?]
at okhttp3.internal.connection.ExchangeFinder.findConnection(ExchangeFinder.java:224) ~[graylog.jar:?]
at okhttp3.internal.connection.ExchangeFinder.findHealthyConnection(ExchangeFinder.java:108) ~[graylog.jar:?]
at okhttp3.internal.connection.ExchangeFinder.find(ExchangeFinder.java:88) ~[graylog.jar:?]
at okhttp3.internal.connection.Transmitter.newExchange(Transmitter.java:169) ~[graylog.jar:?]
at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:41) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[graylog.jar:?]
at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:94) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[graylog.jar:?]
at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:88) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[graylog.jar:?]
at org.graylog2.rest.RemoteInterfaceProvider.lambda$get$0(RemoteInterfaceProvider.java:61) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:142) ~[graylog.jar:?]
at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:117) ~[graylog.jar:?]
at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:221) ~[graylog.jar:?]
at okhttp3.RealCall.execute(RealCall.java:81) ~[graylog.jar:?]
at retrofit2.OkHttpCall.execute(OkHttpCall.java:188) ~[graylog.jar:?]
at org.graylog2.rest.resources.cluster.ClusterSystemPluginResource.list(ClusterSystemPluginResource.java:76) ~[graylog.jar:?]
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:1.8.0_252]
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) ~[?:1.8.0_252]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_252]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_252]
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory$1.invoke(ResourceMethodInvocationHandlerFactory.java:81) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:144) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:161) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$TypeOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:205) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:99) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:389) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:347) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:102) ~[graylog.jar:?]
at org.glassfish.jersey.server.ServerRuntime$2.run(ServerRuntime.java:326) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:271) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:267) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:315) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:297) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:267) [graylog.jar:?]
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:317) [graylog.jar:?]
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:305) [graylog.jar:?]
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:1154) [graylog.jar:?]
at org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer.service(GrizzlyHttpContainer.java:384) [graylog.jar:?]
at org.glassfish.grizzly.http.server.HttpHandler$1.run(HttpHandler.java:224) [graylog.jar:?]
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:181) [graylog.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_252]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_252]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_252]
Caused by: sun.security.validator.ValidatorException: PKIX path validation failed: java.security.cert.CertPathValidatorException: validity check failed
at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:380) ~[?:1.8.0_252]
at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:285) ~[?:1.8.0_252]
at sun.security.validator.Validator.validate(Validator.java:262) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:330) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:237) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:132) ~[?:1.8.0_252]
at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1670) ~[?:1.8.0_252]
... 59 more
Caused by: java.security.cert.CertPathValidatorException: validity check failed
at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:135) ~[?:1.8.0_252]
at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:233) ~[?:1.8.0_252]
at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:141) ~[?:1.8.0_252]
at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:80) ~[?:1.8.0_252]
at java.security.cert.CertPathValidator.validate(CertPathValidator.java:292) ~[?:1.8.0_252]
at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:375) ~[?:1.8.0_252]
at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:285) ~[?:1.8.0_252]
at sun.security.validator.Validator.validate(Validator.java:262) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:330) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:237) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:132) ~[?:1.8.0_252]
at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1670) ~[?:1.8.0_252]
... 59 more
Caused by: java.security.cert.CertificateNotYetValidException: NotBefore: Thu Jul 02 21:00:27 UTC 2020
at sun.security.x509.CertificateValidity.valid(CertificateValidity.java:270) ~[?:1.8.0_252]
at sun.security.x509.X509CertImpl.checkValidity(X509CertImpl.java:677) ~[?:1.8.0_252]
at sun.security.provider.certpath.BasicChecker.verifyValidity(BasicChecker.java:190) ~[?:1.8.0_252]
at sun.security.provider.certpath.BasicChecker.check(BasicChecker.java:144) ~[?:1.8.0_252]
at sun.security.provider.certpath.PKIXMasterCertPathValidator.validate(PKIXMasterCertPathValidator.java:125) ~[?:1.8.0_252]
at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:233) ~[?:1.8.0_252]
at sun.security.provider.certpath.PKIXCertPathValidator.validate(PKIXCertPathValidator.java:141) ~[?:1.8.0_252]
at sun.security.provider.certpath.PKIXCertPathValidator.engineValidate(PKIXCertPathValidator.java:80) ~[?:1.8.0_252]
at java.security.cert.CertPathValidator.validate(CertPathValidator.java:292) ~[?:1.8.0_252]
at sun.security.validator.PKIXValidator.doValidate(PKIXValidator.java:375) ~[?:1.8.0_252]
at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:285) ~[?:1.8.0_252]
at sun.security.validator.Validator.validate(Validator.java:262) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:330) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:237) ~[?:1.8.0_252]
at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:132) ~[?:1.8.0_252]
at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1670) ~[?:1.8.0_252]
can the pod reach itself with MYDOMAIN ? Because that is what the publish_uri is used for - that all Graylog nodes can speak to each others over that URI to get stats.
