Hi,
at first the requested processing time graph.
I could not get it to display minutes (like in your version), but should give the overview.
Our Pipelines look like this:
The pipeline rules are exported to this content pack. Please note, I sanitized some of the contained IP addresses manually. The rest is mostly from other common available sources and content packs, anyway.
{
"v": 1,
"id": "36c526ed-7b6c-47b4-9a0a-61076686e85e",
"rev": 1,
"name": "Export of Pipelines",
"summary": "Export Pipelines for troubleshooting",
"description": "",
"vendor": "NicoS",
"url": "",
"parameters": [],
"entities": [
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "0364134b-ae00-4b2d-9586-3918a6628f84",
"data": {
"title": {
"@type": "string",
"@value": "FW__find_FWID_from_IP_in_REMOTE-IP"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "rule \"FW__find_FWID_from_IP_in_REMOTE-IP\"\nwhen\n not has_field(\"FW_ID\",$message)\nthen\n let SOURCE = to_string($message.gl2_remote_ip);\n let FW_ID = lookup_value(\"NameOrIP_to_FWID\", SOURCE);\n set_field(\"FW_ID\", FW_ID);\n //set_field(\"DEBUG_0724\", \"from remoteIP\");\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "a7542b6c-1390-42d6-a19a-d1c129f8a4c0",
"data": {
"title": {
"@type": "string",
"@value": "SRX IDS Fields SRC-DST-PORT"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "rule \"SRX IDS Fields SRC-DST-PORT\"\nwhen \n contains(to_string($message.message),\"RT_IDS\")\nthen\n let matches = grok(pattern: \"source: %{IPV4:src_ip}\", value: to_string($message.message));\n set_fields(matches);\n \n let matches = grok(pattern: \"destination: %{IPV4:dst_ip}\", value: to_string($message.message));\n set_fields(matches);\n \n set_field(\"FW_SRX_lsys\", \"root\");\n let matches = grok(pattern: \"Lsys: %{USERNAME:FW_SRX_lsys}\", value: to_string($message.message), only_named_captures: true);\n set_fields(matches);\n \nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "ff7225e9-9149-4011-89b7-fac61b30dac9",
"data": {
"title": {
"@type": "string",
"@value": "FW__Exclude_Scanners"
},
"description": {
"@type": "string",
"@value": "Exclude logs generated by Security Scanners"
},
"source": {
"@type": "string",
"@value": "rule \"FW__Exclude_Scanners\"\nwhen\n cidr_match(\"127.0.0.160/29\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.128/28\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.46/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.47/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.89/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.105/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.107/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.108/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.212/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.213/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.224/28\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.0/24\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.160/27\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.14/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.54/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.240/28\", to_ip($message.src_ip)) ||\n cidr_match(\"127.0.0.128/27\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.192/27\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.32/28\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.208/28\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.176/29\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.96/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.97/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.98/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.105/32\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.16/28\", to_ip($message.src_ip)) || \n cidr_match(\"127.0.0.160/29\", to_ip($message.src_ip))\nthen\n //set_field(\"DEBUG\", to_string(\"SCANNER \" + to_string($message.src_ip)));\n drop_message();\n \nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "0f9a483a-1072-42d0-a75f-0e4bd6b93397",
"data": {
"title": {
"@type": "string",
"@value": "ASA syslog/UDP raw header"
},
"description": {
"@type": "string",
"@value": "Cisco ASA Log header\n"
},
"source": {
"@type": "string",
"@value": "rule \"ASA syslog/UDP raw header\"\nwhen\n has_field(\"message\")\nthen\n let raw_log = to_string($message.message);\n let header = grok(pattern:\"%{CISCOTAG:ciscotag}: %{GREEDYDATA:cisco_message}\", value: raw_log,only_named_captures: true);\n set_fields(header);\nend\n"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline",
"version": "1"
},
"id": "38ee5c57-bb63-4646-a97c-8ec70d4d1633",
"data": {
"title": {
"@type": "string",
"@value": "Cisco ASA FieldsParser"
},
"description": {
"@type": "string",
"@value": "Pipeline for the Cisco ASA"
},
"source": {
"@type": "string",
"@value": "pipeline \"Cisco ASA FieldsParser\"\nstage 0 match either\nrule \"ASA syslog/UDP raw header\"\nstage 1 match either\nrule \"ASA syslog/UDP raw log\"\nstage 2 match either\nrule \"ASA authentication src_ip geoip lookup\"\nrule \"Threat Intelligence Lookups: src_ip\"\nend"
},
"connected_streams": [
{
"@type": "string",
"@value": "67c14ae6-7187-482a-b76e-40e2259fba58"
}
]
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline",
"version": "1"
},
"id": "e1614980-5d0b-4dba-a7f0-16b3ec9b6fe8",
"data": {
"title": {
"@type": "string",
"@value": "FW Fieldparser SRX"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "pipeline \"FW Fieldparser SRX\"\nstage 0 match either\nrule \"Is SRX FW\"\nstage 1 match either\nrule \"SRX IDS Fields SRC-DST-PORT\"\nrule \"SRX FW Fields SRC-DST-PORT\"\nend"
},
"connected_streams": [
{
"@type": "string",
"@value": "67c14ae6-7187-482a-b76e-40e2259fba58"
}
]
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "ca7f92ad-0888-49e7-a4b9-50ed3826098e",
"data": {
"title": {
"@type": "string",
"@value": "Netscreen fields parser"
},
"description": {
"@type": "string",
"@value": "parses firewall relevant fields from netscreen logs"
},
"source": {
"@type": "string",
"@value": "rule \"Netscreen fields parser\"\nwhen\n contains(to_string($message.message),\"netscreen\",true)\nthen\n let fields = grok(pattern: \"(src=%{IPV4:src_ip}|dst=%{IPV4:dst_ip})\", value: to_string($message.message));\n let raw_msg = to_string($message.message);\n\n set_fields(grok(pattern: \"src=%{IPV4:src_ip}\", value: raw_msg )); \n set_fields(grok(pattern: \"dst=%{IPV4:dst_ip}\", value: raw_msg ));\n set_fields(grok(pattern: \"src_port=%{INT:src_port}\", value: raw_msg ));\n set_fields(grok(pattern: \"dst_port=%{INT:dst_port}\", value: raw_msg ));\n set_fields(grok(pattern: \"action=%{WORD:action}\", value: raw_msg ));\n\n set_fields(fields);\n set_field(\"FW_Logtype\", \"netscreen\");\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline",
"version": "1"
},
"id": "8798b2c0-fff6-4db7-96d6-11bac980433f",
"data": {
"title": {
"@type": "string",
"@value": "FW Fieldparser Netscreen"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "pipeline \"FW Fieldparser Netscreen\"\nstage 0 match either\nrule \"Netscreen fields parser\"\nend"
},
"connected_streams": [
{
"@type": "string",
"@value": "67c14ae6-7187-482a-b76e-40e2259fba58"
}
]
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "f658a039-76ce-4e17-aeff-3ac5bf24b732",
"data": {
"title": {
"@type": "string",
"@value": "ASA authentication src_ip geoip lookup"
},
"description": {
"@type": "string",
"@value": "Authentication Attempt to the firewall"
},
"source": {
"@type": "string",
"@value": "rule \"ASA authentication src_ip geoip lookup\"\nwhen\n has_field(\"src_ip\") AND (regex(\"ASA-6-113005|ASA-6-113015\", to_string($message.ciscotag)).matches == true)\nthen\n let geo = lookup(\"geoip-lookup\", to_string($message.src_ip));\n set_field(\"src_ip_geolocation\", geo[\"coordinates\"]);\n set_field(\"src_ip_geo_country_code\", geo[\"country\"].iso_code);\n set_field(\"src_ip_geo_country_name\", geo[\"country\"].names.en);\n set_field(\"src_ip_geo_city_name\", geo[\"city\"].names.en); \n set_field(\"FW_Logtype\", \"ASA\");\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "8dd77d9f-caa0-4b64-acef-1c26954a27cd",
"data": {
"title": {
"@type": "string",
"@value": "Is SRX FW"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "rule \"Is SRX FW\"\nwhen \n has_field(\"message\") \n && ( \n contains(to_string($message.message),\"RT_FLOW_SESSION\") || \n contains(to_string($message.message),\"RT_IDS\") )\nthen\n set_field(\"FW_Logtype\", \"SRX\");\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "24b517ff-e068-4e60-9e4c-5d625b7a1165",
"data": {
"title": {
"@type": "string",
"@value": "Threat Intelligence Lookups: src_ip"
},
"description": {
"@type": "string",
"@value": "Threat Intelligence Lookups. By src_ip"
},
"source": {
"@type": "string",
"@value": "rule \"Threat Intelligence Lookups: src_ip\"\nwhen\n has_field(\"src_ip\") AND (regex(\"ASA-6-113005|ASA-6-113015\", to_string($message.ciscotag)).matches == true)\nthen\n set_fields(threat_intel_lookup_ip(to_string($message.src_ip), \"src_ip\"));\nEnd"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "cf5cba87-0d1e-49b1-b428-dd4a19c66027",
"data": {
"title": {
"@type": "string",
"@value": "SRX FW Fields SRC-DST-PORT"
},
"description": {
"@type": "string",
"@value": "Extracts SRX firewall specific fields"
},
"source": {
"@type": "string",
"@value": "rule \"SRX FW Fields SRC-DST-PORT\"\nwhen \n contains(to_string($message.message),\"RT_FLOW_SESSION\")\nthen\n let matches = grok(pattern: \"%{IPV4:src_ip}/%{INT:src_port}->%{IPV4:dst_ip}/%{INT:dst_port}\", value: to_string($message.message));\n set_fields(matches);\n \n set_field(\"FW_SRX_lsys\", \"root\");\n let matches = grok(pattern: \"Lsys %{USERNAME:FW_SRX_lsys}\", value: to_string($message.message), only_named_captures: true);\n set_fields(matches);\n \n\n set_field(\"FW_Logtype\", \"SRX\");\n //route_to_stream(name: \"Nicos_Teststream\");\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "63f375ef-6a79-463c-9b63-3115cfba1a67",
"data": {
"title": {
"@type": "string",
"@value": "FW__find_FWID_from_Name_or_IP_in_SOURCE"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "rule \"FW__find_FWID_from_Name_or_IP_in_SOURCE\"\nwhen\n not has_field(\"FW_ID\",$message)\nthen\n let SOURCE = to_string($message.\"source\");\n let FW_ID = lookup_value(\"NameOrIP_to_FWID\", SOURCE);\n set_field(\"FW_ID\", FW_ID);\n //set_field(\"DEBUG_0724\", \"from source\");\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "f3d8e9b9-fe64-4938-a004-6666b9e349fe",
"data": {
"title": {
"@type": "string",
"@value": "ASA syslog/UDP raw log"
},
"description": {
"@type": "string",
"@value": "Cisco ASA Log rules. "
},
"source": {
"@type": "string",
"@value": "rule \"ASA syslog/UDP raw log\"\nwhen\n has_field(\"cisco_message\")\nthen\n let raw_log = to_string($message.cisco_message);\n// let cisco_asa = grok(pattern:\"(%{CISCOFW104001}|%{CISCOFW104002}|%{CISCOFW104003}|%{CISCOFW104004}|%{CISCOFW105003}|%{CISCOFW105004}|%{CISCOFW105005}|%{CISCOFW105008}|%{CISCOFW106100_2_3}|%{CISCOFW106001}|%{CISCOFW106015}|%{CISCOFW106023}|%{CISCOFW113003}|%{CISCOFW113004}|%{CISCOFW113005}|%{CISCOFW113008}|%{CISCOFW113009_113011}|%{CISCOFW113014}|%{CISCOFW113015}|%{CISCOFW113019}|%{CISCOFW113022_3}|%{CISCOFW113039}|%{CISCOFW313005}|%{CISCOFW401004}|%{CISCOFW419001}|%{CISCOFW419002}|%{CISCOFW434002}|%{CISCOFW500004}|%{CISCOFW507003}|%{CISCOFW710001_710002_710003_710005_710006}|%{CISCOFW722037}|%{CISCOFW733100}|%{CISCOFW733100}|%{CISCOFW733102}|%{CISCOFW733103})\", value:raw_log,only_named_captures: true);\n let cisco_asa = grok(pattern:\"(%{CISCOFW104001}|%{CISCOFW104002}|%{CISCOFW104003}|%{CISCOFW104004}|%{CISCOFW105003}|%{CISCOFW105004}|%{CISCOFW105005}|%{CISCOFW105008}|%{CISCOFW106100_2_3}|%{CISCOFW106001}|%{CISCOFW106015}|%{CISCOFW106023}|%{CISCOFW113003}|%{CISCOFW113004}|%{CISCOFW113005}|%{CISCOFW113008}|%{CISCOFW113009_113011}|%{CISCOFW113014}|%{CISCOFW113015}|%{CISCOFW113019}|%{CISCOFW113022_3}|%{CISCOFW113039}|%{CISCOFW302013_302014_302015_302016}|%{CISCOFW302020_302021}|%{CISCOFW313005}|%{CISCOFW401004}|%{CISCOFW419001}|%{CISCOFW419002}|%{CISCOFW434002}|%{CISCOFW500004}|%{CISCOFW507003}|%{CISCOFW710001_710002_710003_710005_710006}|%{CISCOFW722037}|%{CISCOFW733100}|%{CISCOFW733100}|%{CISCOFW733102}|%{CISCOFW733103}|%{CiscoASA106100})\", value:raw_log,only_named_captures: true);\n set_fields(cisco_asa);\n remove_field(\"cisco_message\");\n set_field(\"FW_Logtype\", \"ASA\");\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "521fa014-d544-406a-adb4-fd9373c6782e",
"data": {
"title": {
"@type": "string",
"@value": "FW__add_additional_common_information"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "rule \"FW__add_additional_common_information\"\nwhen\n true\nthen\n set_field(\"FW_LogsourceIP\", to_string($message.gl2_remote_ip));\n \n //set_field(\"DEBUG_GLnode\", to_string($message.gl2_source_node));\n \n // set_field(\"XX_debug\",\"8\");\n // set_field(\"DEBUG_msglen\", to_string(length(to_string($message.message))));\n\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline",
"version": "1"
},
"id": "15714bce-4e2e-46f7-8c48-2b80ca0e2ec7",
"data": {
"title": {
"@type": "string",
"@value": "Scanner-Filter"
},
"description": {
"@type": "string",
"@value": "Filter out all logs generated from Siemens Scanner IPs"
},
"source": {
"@type": "string",
"@value": "pipeline \"Scanner-Filter\"\nstage 4 match pass\nrule \"FW__Exclude_Scanners\"\nend"
},
"connected_streams": [
{
"@type": "string",
"@value": "67c14ae6-7187-482a-b76e-40e2259fba58"
}
]
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline_rule",
"version": "1"
},
"id": "1952a36c-8421-44e3-ab62-b62558c32f62",
"data": {
"title": {
"@type": "string",
"@value": "FW__add_additional_information_by_FWID"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "rule \"FW__add_additional_information_by_FWID\"\nwhen\n has_field(\"FW_ID\", $message)\nthen\n let ADMINGROUP = lookup_value(\"ADMINGROUP_from_FWID\", $message.FW_ID);\n set_field(\"FW_Admingroup\", ADMINGROUP);\n \n let CLEARNAME = lookup_value(\"CLEARNAME_from_FWID\", $message.FW_ID);\n set_field(\"FW_Clearname\", CLEARNAME);\n\n let CUSTOMER = lookup_value(\"CUSTOMER_from_FWID\", $message.FW_ID);\n set_field(\"FW_Customer\", CUSTOMER);\n\nend"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "pipeline",
"version": "1"
},
"id": "4a58acaa-af88-4bc8-815a-14f8d453c9de",
"data": {
"title": {
"@type": "string",
"@value": "FW_Enricher"
},
"description": {
"@type": "string",
"@value": ""
},
"source": {
"@type": "string",
"@value": "pipeline \"FW_Enricher\"\nstage 6 match pass\nrule \"FW__find_FWID_from_Name_or_IP_in_SOURCE\"\nstage 7 match pass\nrule \"FW__find_FWID_from_IP_in_REMOTE-IP\"\nstage 8 match pass\nrule \"FW__add_additional_information_by_FWID\"\nstage 9 match pass\nrule \"FW__add_additional_common_information\"\nend"
},
"connected_streams": [
{
"@type": "string",
"@value": "67c14ae6-7187-482a-b76e-40e2259fba58"
}
]
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
},
{
"v": "1",
"type": {
"name": "stream_title",
"version": "1"
},
"id": "67c14ae6-7187-482a-b76e-40e2259fba58",
"data": {
"title": {
"@type": "string",
"@value": "Default Stream"
}
},
"constraints": [
{
"type": "server-version",
"version": ">=6.0.7+4779d72"
}
]
}
]
}