I’m trying to correct the source fields on our graylog server, primarily from our ASAs’. We have multiple firewalls with multiple contexts and the source data shows up as the month. Now I know this is because the format of messages on the way the data is presented.
I tried extractors, although they worked, it made our CPU usage spike to 100%. Filled up our process buffer and our Journal. After removing the grok or regex extractors things returned to normal.
My next attempt was setting up our server as [Jan Doberstein] Working with Cisco ASA / Nexus on Graylog suggested. Unfortunately this causes issues as well. The Grok Pattern for CiscoTimeStamp wont be accepted. No issues with the Nexus Pattern. The error is below
2021-03-09T17:48:37.768-05:00 ERROR [AnyExceptionClassMapper] Unhandled exception in REST resource
java.lang.IllegalArgumentException: No definition for key ‘CISCOTIMESTAMP’ found, aborting
at io.krakens.grok.api.GrokCompiler.compile(GrokCompiler.java:177) ~[graylog.jar:?]
at io.krakens.grok.api.GrokCompiler.compile(GrokCompiler.java:177) ~[graylog.jar:?]
at io.krakens.grok.api.GrokCompiler.compile(GrokCompiler.java:126) ~[graylog.jar:?]
at io.krakens.grok.api.GrokCompiler.compile(GrokCompiler.java:122) ~[graylog.jar:?]
at org.graylog2.grok.MongoDbGrokPatternService.validate(MongoDbGrokPatternService.java:216) ~[graylog.jar:?]
at org.graylog2.grok.MongoDbGrokPatternService.save(MongoDbGrokPatternService.java:117) ~[graylog.jar:?]
at org.graylog2.rest.resources.system.GrokResource.createPattern(GrokResource.java:174) ~[graylog.jar:?]
at sun.reflect.GeneratedMethodAccessor952.invoke(Unknown Source) ~[?:?]
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:1.8.0_282]
at java.lang.reflect.Method.invoke(Method.java:498) ~[?:1.8.0_282]
at org.glassfish.jersey.server.model.internal.ResourceMethodInvocationHandlerFactory.lambda$static$0(ResourceMethodInvocationHandlerFactory.java:52) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher$1.run(AbstractJavaResourceMethodDispatcher.java:124) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.invoke(AbstractJavaResourceMethodDispatcher.java:167) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.JavaResourceMethodDispatcherProvider$ResponseOutInvoker.doDispatch(JavaResourceMethodDispatcherProvider.java:176) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.internal.AbstractJavaResourceMethodDispatcher.dispatch(AbstractJavaResourceMethodDispatcher.java:79) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.invoke(ResourceMethodInvoker.java:469) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:391) ~[graylog.jar:?]
at org.glassfish.jersey.server.model.ResourceMethodInvoker.apply(ResourceMethodInvoker.java:80) ~[graylog.jar:?]
at org.glassfish.jersey.server.ServerRuntime$1.run(ServerRuntime.java:253) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:248) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors$1.call(Errors.java:244) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:292) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:274) [graylog.jar:?]
at org.glassfish.jersey.internal.Errors.process(Errors.java:244) [graylog.jar:?]
at org.glassfish.jersey.process.internal.RequestScope.runInScope(RequestScope.java:265) [graylog.jar:?]
at org.glassfish.jersey.server.ServerRuntime.process(ServerRuntime.java:232) [graylog.jar:?]
at org.glassfish.jersey.server.ApplicationHandler.handle(ApplicationHandler.java:680) [graylog.jar:?]
at org.glassfish.jersey.grizzly2.httpserver.GrizzlyHttpContainer.service(GrizzlyHttpContainer.java:356) [graylog.jar:?]
at org.glassfish.grizzly.http.server.HttpHandler$1.run(HttpHandler.java:200) [graylog.jar:?]
at com.codahale.metrics.InstrumentedExecutorService$InstrumentedRunnable.run(InstrumentedExecutorService.java:180) [graylog.jar:?]
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149) [?:1.8.0_282]
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624) [?:1.8.0_282]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_282]
Fixed the Input ID, that was the issue there now I run into the below
2021-03-10T10:19:39.167-05:00 ERROR [GrokPatternRegistry] Unable to load grok pattern %{SYSLOG5424PRI}(%{NUMBER:log_sequence#})? %{NUMBER}:)? %{CISCOTIMESTAMPTZ:log_date}: %%{CISCO_REASON:facility}-%{INT:s$
java.lang.IllegalArgumentException: No definition for key ‘SYSLOG5424PRI’ found, aborting
at io.krakens.grok.api.GrokCompiler.compile(GrokCompiler.java:177) ~[graylog.jar:?]
at io.krakens.grok.api.GrokCompiler.compile(GrokCompiler.java:126) ~[graylog.jar:?]
at org.graylog2.grok.GrokPatternRegistry$GrokReloader.load(GrokPatternRegistry.java:132) ~[graylog.jar:?]
at org.graylog2.grok.GrokPatternRegistry$GrokReloader.load(GrokPatternRegistry.java:119) ~[graylog.jar:?]
at com.google.common.cache.CacheLoader$1.load(CacheLoader.java:188) ~[graylog.jar:?]
at com.google.common.cache.LocalCache$LoadingValueReference.loadFuture(LocalCache.java:3529) ~[graylog.jar:?]
at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2278) ~[graylog.jar:?]
at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2155) ~[graylog.jar:?]
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2045) ~[graylog.jar:?]
at com.google.common.cache.LocalCache.get(LocalCache.java:3951) ~[graylog.jar:?]
at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:3974) ~[graylog.jar:?]
at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4958) ~[graylog.jar:?]
at org.graylog2.grok.GrokPatternRegistry.cachedGrokForPattern(GrokPatternRegistry.java:97) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch.evaluate(GrokMatch.java:63) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.functions.strings.GrokMatch.evaluate(GrokMatch.java:34) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.ast.expressions.FunctionExpression.evaluateUnsafe(FunctionExpression.java:63) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.ast.expressions.Expression.evaluate(Expression.java:41) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement.evaluate(VarAssignStatement.java:33) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.ast.statements.VarAssignStatement.evaluate(VarAssignStatement.java:22) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter.evaluateStatement(PipelineInterpreter.java:385) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter.executeRuleActions(PipelineInterpreter.java:369) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter.evaluateStage(PipelineInterpreter.java:309) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter.processForResolvedPipelines(PipelineInterpreter.java:267) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter.process(PipelineInterpreter.java:147) [graylog.jar:?]
at org.graylog.plugins.pipelineprocessor.processors.PipelineInterpreter.process(PipelineInterpreter.java:103) [graylog.jar:?]
at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.handleMessage(ProcessBufferProcessor.java:136) [graylog.jar:?]
at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.dispatchMessage(ProcessBufferProcessor.java:121) [graylog.jar:?]
at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.onEvent(ProcessBufferProcessor.java:93) [graylog.jar:?]
at org.graylog2.shared.buffers.processors.ProcessBufferProcessor.onEvent(ProcessBufferProcessor.java:47) [graylog.jar:?]
at com.lmax.disruptor.WorkProcessor.run(WorkProcessor.java:143) [graylog.jar:?]
at com.codahale.metrics.InstrumentedThreadFactory$InstrumentedRunnable.run(InstrumentedThreadFactory.java:66) [graylog.jar:?]
at java.lang.Thread.run(Thread.java:748) [?:1.8.0_282]