ASA syslog plugin for syslog parsing

Development
1

Hi guys,

i’m facing a big problem i have a lot of ASA firewalls which are sending around 500 GB /day to the graylog box,
and because i have 14 grok extractors for that input sometimes the CPU goes through the roof, and sometimes i’m facing issues with loosing logs… which is not good at all.

now the question is can i write some kind of input plugin dedicated only for CISCO ASA firewalls to parse those messages without forcing the CPU to work hard?

if the answer is probably yes, can you offer me some guidance, considering the fact that i have 0 programming experience? i know something about python but not a lot…

Thanks,
Marius.

2

That’s a good amount of incoming log data.

Are you using pipelines at all, or just running the 14 extractors on every message? It might be useful to consider moving to pipelines, if only to reduce the amount of regex work needed to get those messages parsed. We still use extractors on our ASA logs, and haven’t moved those logs to pipelines yet. If you don’t get any immediate answers, I will take a look converting our ASA logs to pipelines and see it it makes a difference for cpu load.

As long as you have enough disk journal available and cpu to get through the day, you shouldn’t be loosing logs. If you are taking in more than you can process and get continually further and further behind, something will need to change.

It will take me a few days.

3

Hi Dustin,

many thanks for the help, the problem is that i have arround 20k msg/s and in the next months i’m expecting for something like 35k or 40k…
i’m not loosing too much but a considerable amount…

Thanks,
Marius.

4

tomorrow i’ll try to play with pipelines too see what i’ll get into

5

He

maybe this post can help you with that: https://jalogisch.de/2018/working-with-cisco-asa-nexus-on-graylog/

1 Like
6

Hi Jan,

thanks for the help with that one, i’ll start to write the lines and let you know what’s the outcome :slight_smile:

Marius.

7

hi Jan,

i did the rule(down bellow), however i get a an error at the line with let asa7, it says: “invalid expression line 14 column 110”
at column 110 i have “from” as a word i don’t think this should be any problem… right?

rule “ASA firewall grok”
when
has_field(“message”)
then

let message_field = to_string($message.message);

let asa1 = grok(pattern: "%{CISCO_APP:application} client %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})", value: message_field, only_named_captures: true);
let asa2 = grok(pattern: "%{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group %{DATA:access_group} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]", value: message_field, only_named_captures: true);	
let asa3 = grok(pattern: "%{CISCO_ACTION:action} local-host %{DATA:dst_interface}:%{IPV4:dst_ip}", value: message_field, only_named_captures: true);
let asa4 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} from %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} due to %{CISCO_REASON:reason}", value: message_field, only_named_captures: true);
let asa5 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} from %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} on interface %{DST_ACCT:dst_interface}", value: message_field, only_named_captures: true);
let asa6 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} from %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} on interface %{DST_ACCT:dst_interface}", value: message_field, only_named_captures: true);
let asa7 = grok(pattern: "%{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IPV4:src_xlated_ip}/%{INT:src_xlated_port}", value: message_field, only_named_captures: true);
let asa8 = grok(pattern: "%{CISCO_REASON:reason} for %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port}( \(%{IPV4:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port}( \(%{IPV4:dst_mapped_ip}/%{INT:dst_mapped_port}\))?", value: message_field, only_named_captures: true);
let asa9 = grok(pattern: "%{CISCO_REASON:reason} from %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port} with different initial sequence number", value: message_field, only_named_captures: true);
let asa10 = grok(pattern: "%{CISCO_IPS:Detected_by_IPS} requested to %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port}) to %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})", value: message_field, only_named_captures: true);
let asa11 = grok(pattern: "access-list %{DATA:acl_name} %{CISCO_ACTION:action} %{WORD:protocol} for user %{DATA:src_fwuser} %{DATA:src_interface}/%{IPV4:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IPV4:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]", value: message_field, only_named_captures: true);
let asa12 = grok(pattern: "%{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IPV4:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port}", value: message_field, only_named_captures: true);
let asa13 = grok(pattern: "%{CISCO_APP:application} client %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})", value: message_field, only_named_captures: true);

set_fields(asa1);
set_fields(asa2);
set_fields(asa3);
set_fields(asa4);
set_fields(asa5);
set_fields(asa6);
set_fields(asa7);
set_fields(asa8);
set_fields(asa9);
set_fields(asa10);
set_fields(asa11);
set_fields(asa12);
set_fields(asa13);

end

8

managed to fix it in this manner, now i get an error at line 18…

rule “ASA firewall grok”
when
has_field(“message”)
then

let message_field = to_string($message.message);

let asa1 = grok(pattern: "%{CISCO_APP:application} client %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})", value: message_field, only_named_captures: true);
let asa2 = grok(pattern: "%{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? dst %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})?(\(%{DATA:dst_fwuser}\))?( \(type %{INT:icmp_type}, code %{INT:icmp_code}\))? by access-group %{DATA:access_group} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]", value: message_field, only_named_captures: true);	
let asa3 = grok(pattern: "%{CISCO_ACTION:action} local-host %{DATA:dst_interface}:%{IPV4:dst_ip}", value: message_field, only_named_captures: true);
let asa4 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} %{DATA:UNWANTED} %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} due to %{CISCO_REASON:reason}", value: message_field, only_named_captures: true);
let asa5 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} %{DATA:UNWANTED} %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} on interface %{DST_ACCT:dst_interface}", value: message_field, only_named_captures: true);
let asa6 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} %{DATA:UNWANTED} %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} %{DATA:UNWANTED} %{DST_ACCT:dst_interface}", value: message_field, only_named_captures: true);
let asa7 = grok(pattern: "%{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} %{DATA:UNWANTED} %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})?(\(%{DATA:src_fwuser}\))? to %{DATA:src_xlated_interface}:%{IPV4:src_xlated_ip}/%{INT:src_xlated_port}", value: message_field, only_named_captures: true);
let asa8 = grok(pattern: "%{CISCO_REASON:reason} %{DATA:UNWANTED} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} %{DATA:UNWANTED} %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port}( \(%{IPV4:src_mapped_ip}/%{INT:src_mapped_port}\))?(\(%{DATA:src_fwuser}\))? to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port}( \(%{IPV4:dst_mapped_ip}/%{INT:dst_mapped_port}\))?", value: message_field, only_named_captures: true);
let asa9 = grok(pattern: "%{CISCO_REASON:reason} %{DATA:UNWANTED} %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port} %{DATA:UNWANTED} %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port} %{DATA:UNWANTED}", value: message_field, only_named_captures: true);
let asa10 = grok(pattern: "%{CISCO_IPS:Detected_by_IPS} %{DATA:UNWANTED} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:UNWANTED} %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port}) to %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})", value: message_field, only_named_captures: true);
let asa11 = grok(pattern: "access-list %{DATA:acl_name} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:UNWANTED} %{DATA:src_fwuser} %{DATA:src_interface}/%{IPV4:src_ip}\(%{INT:src_port}\)(\(%{DATA:src_fwuser}\))? -> %{DATA:dst_interface}/%{IPV4:dst_ip}\(%{INT:dst_port}\)(\(%{DATA:src_fwuser}\))? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\]", value: message_field, only_named_captures: true);
let asa12 = grok(pattern: "%{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IPV4:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port}", value: message_field, only_named_captures: true);
let asa13 = grok(pattern: "%{CISCO_APP:application} client %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})", value: message_field, only_named_captures: true);

set_fields(asa1);
set_fields(asa2);
set_fields(asa3);
set_fields(asa4);
set_fields(asa5);
set_fields(asa6);
set_fields(asa7);
set_fields(asa8);
set_fields(asa9);
set_fields(asa10);
set_fields(asa11);
set_fields(asa12);
set_fields(asa13);

end

9

He @mariusgeonea

I do not know what your line 18 is … in addition you should check the error message and then work on that.

10

line 18 is with let asa11

11

you need to escape some signs … / for example of > that should make it work.

12

how would you parse this message?
Aug 15 2018 10:46:33: %ASA-6-302014: Teardown TCP connection 1576899401 for outside:10.3.128.40/55258(LOCAL\james.smith) to inside:10.10.1.10/80 duration 0:03:15 bytes 916787 TCP Reset-I (james.smith)

because i have to brake everything into small thing. with grok extractors it’s easy, but with rule for pipelines doesn’t work…
for this one here is what i have as a grok extractor:
%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port}( (%{IPV4:src_mapped_ip}/%{INT:src_mapped_port}))?((%{DATA:src_fwuser}))? to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port}( (%{IPV4:dst_mapped_ip}/%{INT:dst_mapped_port}))?((%{DATA:dst_fwuser}))?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( (%{DATA:user}))?

can you figure out how can i have the same result with a rule for a pipeline?

13

the main advance of the pipeline is that you can have multiple rules one after each other that parse the message until it is in the final form.

But to return to your question - escape special characters, like in the search in your pattern and it will work.

14

Hi Jan,

that seems to work, but i will have quite a number of rules. once i will have everything finished, can you post the rule somewhere? maybe others will find it helpful

15

currently you can only write a community posting with proper markdown syntax - or something similar.

With Version 3.0 you would be able to create a content pack that includes pipelines and rules.

1 Like
16

hy guys,

@do you know when version 3 will be released? for example in 2 months time?
i have finishted the rule for the pipeline, i’ll continue working on it
this is what i have so far

rule “ASA firewall grok”
when
has_field(“message”)
then

let message_field = to_string($message.message);

let asa1 = grok(pattern: "%{CISCO_APP:application} client %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})", value: message_field, only_named_captures: true);
let asa2 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} from %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} due to %{CISCO_REASON:reason}", value: message_field, only_named_captures: true);
let asa3 = grok(pattern: "%{CISCO_ACTION:action} local-host %{DATA:dst_interface}:%{IPV4:dst_ip}", value: message_field, only_named_captures: true);
let asa4 = grok(pattern: "%{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IPV4:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port}", value: message_field, only_named_captures: true);
let asa5 = grok(pattern: "%{CISCO_IPS:Detected_by_IPS} requested to %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port}) to %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})", value: message_field, only_named_captures: true);
let asa6 = grok(pattern: "%{CISCO_REASON:reason} from %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port} with different initial sequence number", value: message_field, only_named_captures: true);
let asa7 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} from %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} on interface %{DST_ACCT:dst_interface}", value: message_field, only_named_captures: true);
let asa8 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port} duration %{TIME:duration} bytes %{INT:bytes}", value: message_field, only_named_captures: true);
let asa9 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port}( %{DATA:UNWANTED}) to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port}( %{DATA:UNWANTED})", value: message_field, only_named_captures: true);
let asa10 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IPV4:dst_ip}/%{INT:icmp_seq_num}(?:(%{DATA:user}))? gaddr %{IPV4:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IPV4:src_ip}/%{INT:icmp_code}( (%{DATA:user}))?", value: message_field, only_named_captures: true);
let asa11 = grok(pattern: "%{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IPV4:src_ip}/%{INT:src_port}(?:(%{DATA:user}))? to %{DATA:dst_interface}:%{IPV4:dst_ip}/%{INT:dst_port} duration %{TIME:duration} bytes %{INT:bytes}( (%{DATA:user}))?", value: message_field, only_named_captures: true);
let asa12 = grok(pattern: "%{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})?((%{DATA:user}))?dst %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})?((%{DATA:user})) by access-group %{DATA:access_group}", value: message_field, only_named_captures: true);
let asa13 = grok(pattern: "%{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IPV4:src_ip}(/%{INT:src_port})?((%{DATA:user}))? to %{DATA:src_xlated_interface}:%{IPV4:src_xlated_ip}/%{INT:src_xlated_port}", value: message_field, only_named_captures: true);
let asa14 = grok(pattern: "%{CISCO_ACTION:action} %{WORD:protocol} %{CISCO_REASON:reason} from %{IPV4:src_ip}/%{INT:src_port} to %{IPV4:dst_ip}/%{INT:dst_port} flags %{CISCO_FLAGS:tcp_flags}  on interface %{GREEDYDATA:interface}", value: message_field, only_named_captures: true);
let asa15 = grok(pattern: "%{IP:dst_ip} Accessed URL %{GREEDYDATA:url}?", value: message_field, only_named_captures: true);
let asa16 = grok(pattern: "Starting %{CISCO_SSL:ssl} with client %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})", value: message_field, only_named_captures: true);
let asa17 = grok(pattern: "%{CISCO_SSL:ssl} %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})", value: message_field, only_named_captures: true);
let asa18 = grok(pattern: "%{CISCO_SSL:ssl} with client %{DATA:dst_interface}:%{IPV4:dst_ip}(/%{INT:dst_port})", value: message_field, only_named_captures: true);

set_fields(asa1);
set_fields(asa2);
set_fields(asa3);
set_fields(asa4);
set_fields(asa5);
set_fields(asa6);
set_fields(asa7);
set_fields(asa8);
set_fields(asa9);
set_fields(asa10);
set_fields(asa11);
set_fields(asa12);
set_fields(asa13);
set_fields(asa14);
set_fields(asa15);
set_fields(asa16);
set_fields(asa17);
set_fields(asa18);

end

1 Like
17

we do not have offical ETA for Graylog 3 yet - as we want to have it working and stable. But it is not far away - so this year is save to say.

18

awesome, looking forward for that one :wink:

19

Hi gents,

i ran again into some troubles:

i have a lookup table and a data adapter csv file with the following:

“ipaddr”,“ASA_hostname”
“10.16.10.6”,“Jerry-01”
“10.16.20.6”,“Tom-01”

when i receive logs from ASA with a source of 10.16.10.6, i want to create another field using pipelines called ASA_host which has the firewall name of Jerry-01
and the same for firewall Tom

i have created the following pipeline:

rule “ASA lookup names”
when
has_field(“source”)
then

let ASA_host = lookup_value(to_string("source", $message.source), "source");
set_field("ASA_host");

end

but it’s not working, can you guys help me out with this one?

Thanks,
Marius.

20

i have managed to do this pipeline:

rule “ASA lookup names”
when
has_field(“source”)
then

let ASA_host = lookup_value("asa_lookup", $message.source);
set_field("ASA_host", ASA_host);

end

but the problem is that on the new ASA_host field created i see: {“value”:“Jerry-01”}

any idea where i’m doing wrong?