Before you post: Your responses to these questions will help the community help you. Please complete this template if you’re asking a support question.
Don’t forget to select tags to help index your topic!
1. Describe your incident:
Trying to enrich the logs associated with OPNSense filterlogs. I currently have netflow logs being enriched with GeoIP and OTX/Whois data with now issue. I basically modified the same rules to adjust to the field naming differences for source and destination IP between the filterlog and netflow, but the filterlog is not getting the updated enriched fields.
2. Describe your environment:
- OS Information:
Graylog Graylog 5.1.0+14ba491
OS: Ubuntu 22.04.2 LTS x86_64
Kernel: 5.15.0-72-generic
Shell: bash 5.1.16
CPU: AMD Ryzen 9 6900HX with Radeon Graphics (16) @ 3.300GHz
Memory: 4978MiB / 28839MiB
OPNsense 23.1.7_3-amd64
OS: FreeBSD 13.1-RELEASE-p7
CPU: Intel(R) Core™ i7-1165G7 @ 2.80GHz (4 cores, 8 threads)
Memory: 9970/32463 MB
-
Package Version:
Open Threat Exchange - Threat Intel Plugin
Whois Threat Intel Plugin
OPNSense Content Pack -
Service logs, configurations, and environment variables:
###Logs Anonymized###
Sample filterlog missing enrichment
{
"reason": "match",
"src-ip": "89.248.165.118",
"dst-ip": "x.x.x.109",
"gl2_remote_ip": "192.168.2.1",
"src-port": "52302",
"gl2_remote_port": 9269,
"dst-port": "2221",
"source": "x.x.x",
"gl2_source_input": "6465140d3f6c1762ab247d54",
"interface": "igc0",
"protocol-name": "tcp",
"f2": "1024",
"ip-flags": "none",
"rule-number": "112",
"tracker": "07c7d99fd89b2a05fb88b21a9a4348ca",
"action": "block",
"tos": "0x0",
"gl2_source_node": "143077a1-1f14-4cc9-8794-4b1a211f4aad",
"id": "39338",
"timestamp": "2023-05-21T20:46:06.000Z",
"direction": "in",
"datalength": "0",
"gl2_accounted_message_size": 799,
"offset": "0",
"level": 6,
"streams": [
"646a6fa1c811d25121010111",
"646515243f6c1762ab247fa7"
],
"length": "40",
"gl2_message_id": "01H100ZVNG007D0VW6KA8KTXP8",
"ip-version": "4",
"message": "x.x.x filterlog[15109]: 112,,,07c7d99fd89b2a05fb88b21a9a4348ca,igc0,match,block,in,4,0x0,,239,39338,0,none,6,tcp,40,89.248.165.118,x.x.x.109,52302,2221,0,S,2540593770,,1024,,",
"ttl": "239",
"tcp-flags": "S",
"sequence": "2540593770",
"facility_num": 16,
"protocol-id": "6",
"filterlog_ipv4_tcp": "112,,,07c7d99fd89b2a05fb88b21a9a4348ca,igc0,match,block,in,4,0x0,,239,39338,0,none,6,tcp,40,89.248.165.118,x.x.x.109,52302,2221,0,S,2540593770,,1024,,",
"_id": "8225eae0-f818-11ed-9376-0242cc279576",
"facility": "local0"
}
Sample filterlog source IP Pipeline GeoIP and Threat Intel Rule:
rule "filterlog src-ip geoip lookup"
when
has_field("src-ip") && ! in_private_net(to_string($message.src_ip))
then
let geo = lookup("geoip-lookup", to_string($message.src_ip));
set_field("src_ip_geolocation", geo["coordinates"]);
set_field("src_ip_geo_country_code", geo["country"].iso_code);
set_field("src_ip_geo_country_name", geo["country"].names.en);
set_field("src_ip_geo_city_name", geo["city"].names.en);
end
rule "filterlog src-ip threat intel"
when
has_field("src-ip") && ! in_private_net(to_string($message.src_ip))
then
let dst_addr_intel = threat_intel_lookup_ip(to_string($message.src_ip), "src_addr");
set_fields(dst_addr_intel);
let dns_question_intel = threat_intel_lookup_domain(to_string($message.dns_question), "dns_question");
set_fields(dns_question_intel);
let whois_intel = whois_lookup_ip(to_string($message.src_addr), "src_addr");
set_fields(whois_intel);
end
Sample netflow with enrichment
{
"nf_src_tos": 0,
"nf_version": 9,
"gl2_remote_ip": "192.168.2.1",
"dst_ip_geo_country_name": "United States",
"gl2_remote_port": 14115,
"nf_in_bytes": 79,
"nf_pkts": 1,
"source": "192.168.2.1",
"gl2_source_input": "64663b35b4873d45195d7e41",
"nf_tcp_flags": 0,
"nf_bytes": 79,
"nf_src_as": 0,
"nf_out_pkts": 0,
"nf_dst_address": "8.8.4.4",
"gl2_source_node": "143077a1-1f14-4cc9-8794-4b1a211f4aad",
"src_ip_geolocation": "33.1177,-96.6791",
"nf_ipv4_src_addr": "x.x.x.109",
"nf_src": "x.x.x.109:34174",
"dst_addr_threat_indicated": false,
"gl2_accounted_message_size": 1272,
"nf_flow_packet_id": 21709,
"dst_ip_geo_country_code": "US",
"nf_out_bytes": 0,
"streams": [
"64664465b4873d45195d917b"
],
"dst_ip_geolocation": "34.0544,-118.2441",
"gl2_message_id": "01H1014J20000J5J5H3G076NQJ",
"nf_ipv4_dst_addr": "8.8.4.4",
"nf_proto_name": "UDP",
"nf_src_addressr_whois_country_code": "US",
"nf_l4_dst_port": 53,
"src_ip_geo_city_name": "city",
"_id": "ddcaea87-f818-11ed-9376-0242cc279576",
"nf_start": "2023-05-21T20:48:22.000Z",
"nf_dst": "8.8.4.4:53",
"nf_last_switched": 1100487000,
"nf_stop": "2023-05-21T20:48:22.000Z",
"nf_protocol": 17,
"nf_l4_src_port": 34174,
"nf_dst_port": 53,
"nf_ipv4_next_hop": "x.x.x.1",
"nf_proto": 17,
"dst_ip_geo_city_name": "Los Angeles",
"nf_src_port": 34174,
"nf_src_address_threat_indicated": false,
"timestamp": "2023-05-21T20:48:40.000Z",
"src_ip_geo_country_name": "United States",
"nf_dst_mask": 0,
"nf_input_snmp": 11,
"message": "NetFlowV9 [x.x.x.109]:34174 <> [8.8.4.4]:53 proto:17 pkts:1 bytes:79",
"nf_dst_as": 0,
"nf_src_address": "x.x.x.109",
"nf_src_addressr_whois_organization": "AT&T Corp. (AC-3280)",
"nf_src_mask": 32,
"dns_question_threat_indicated": false,
"nf_in_pkts": 1,
"nf_first_switched": 1100487000,
"nf_snmp_input": 11,
"src_ip_geo_country_code": "US",
"nf_snmp_output": 1,
"nf_output_snmp": 1,
"nf_next_hop": "x.x.x.1"
}
Sample netflow source IP Pipeline GeoIP and Threat Intel Rule:
rule "src-ip geoip lookup"
when
has_field("nf_src_address") && ! in_private_net(to_string($message.src_ip))
then
let geo = lookup("geoip-lookup", to_string($message.nf_src_address));
set_field("src_ip_geolocation", geo["coordinates"]);
set_field("src_ip_geo_country_code", geo["country"].iso_code);
set_field("src_ip_geo_country_name", geo["country"].names.en);
set_field("src_ip_geo_city_name", geo["city"].names.en);
end
rule "src-ip threat intel"
when
has_field("nf_src_address") && ! in_private_net(to_string($message.src_ip))
then
let src_addr_intel = threat_intel_lookup_ip(to_string($message.nf_src_address), "nf_src_address");
set_fields(src_addr_intel);
let dns_question_intel = threat_intel_lookup_domain(to_string($message.dns_question), "dns_question");
set_fields(dns_question_intel);
let whois_intel = whois_lookup_ip(to_string($message.nf_src_address), "nf_src_addressr");
set_fields(whois_intel);
end
3. What steps have you already taken to try and solve the problem?
As stated above, the netflow enrichment works and just duplicated and modified the rules to account for the source and destination IPs for the OPNSense filterlog. I am basically at a loss as to why the logs within Graylog are not updating.
4. How can the community help?
Needing a second pair (or more) eyes to see what I am overlooking…or other recommendations…
Helpful Posting Tips: Tips for Posting Questions that Get Answers [Hold down CTRL and link on link to open tips documents in a separate tab]
