Hello Guys,
I have noticed that when I create a pipeline that processes logs and fields, these are no longer in the associated stream. However, if I send the stream via GELF TCP output to another Graylog instance, the logs processed by the pipeline can be seen there again.
Pipeline for parsing eset Logs:
{
"id": "66e91a7a401d3121e769e105",
"title": "Parsing Eset",
"description": "Extract Important Fields ",
"source": "pipeline \"Parsing Eset\"\nstage 0 match either\nrule \"Parse ESET log and extract msg\"\nstage 1 match pass\nrule \"Parse ESET log and extract dvchost\"\nrule \"Parse ESET log and extract act\"\nstage 2 match pass\nrule \"Parse CEF log and extract log_type\"\nstage 3 match pass\nrule \"Parse ESET log and extract requestet URL\"\nend",
"created_at": "2024-09-17T05:58:18.007Z",
"modified_at": "2024-09-17T09:13:48.817Z",
"stages": [
{
"stage": 0,
"match": "EITHER",
"rules": [
"Parse ESET log and extract msg"
]
},
{
"stage": 1,
"match": "PASS",
"rules": [
"Parse ESET log and extract dvchost",
"Parse ESET log and extract act"
]
},
{
"stage": 2,
"match": "PASS",
"rules": [
"Parse CEF log and extract log_type"
]
},
{
"stage": 3,
"match": "PASS",
"rules": [
"Parse ESET log and extract requestet URL"
]
}
],
"errors": null
}
Used Rules:
{
"title": "Parse CEF log and extract log_type",
"description": "",
"source": "rule \"Parse CEF log and extract log_type\"\nwhen\n has_field(\"message\") \nthen\n let message_field = to_string($message.message); \n \n// Extrahiere das Event (das fünfte Feld nach den Pipes '|' in der CEF Nachricht)\n let event_match = regex(\"\\\\|[^|]+\\\\|[^|]+\\\\|[^|]+\\\\|[^|]+\\\\|([^|]+)\", message_field);\n let event_value = event_match[\"0\"];\n set_field(\"log_type\", event_value); \nend",
"created_at": "2024-09-17T08:06:48.944Z",
"modified_at": "2024-09-17T08:07:06.526Z",
"errors": null,
"rule_builder": null,
"simulator_message": "{\n \"msg\": \"An\",\n \"process_id\": \"784\",\n \"gl2_accounted_message_size\": 976,\n \"gl2_receive_timestamp\": \"2024-09-17 06:27:00.567\",\n \"level\": 4,\n \"gl2_remote_ip\": \"10.0.142.148\",\n \"gl2_remote_port\": 42918,\n \"streams\": [\n \"66e91fc1fbf618573557c7d1\"\n ],\n \"gl2_message_id\": \"01J7ZAKDAJ00004S6VZW6J0NKA\",\n \"source\": \"EPP0\",\n \"message\": \"CEF:0|ESET|Protect|11.1.756.0|716|Filtered Website Event|5|dvc=172.29.0.15 dvchost=n0418.convales.intern deviceExternalId=b9a498ab-fa4e-43c6-b607-aba4b743792d ESETProtectDeviceGroupName=Alle/Convales.intern/Clients ESETProtectDeviceOsName=Microsoft Windows 10 Pro ESETProtectDeviceGroupDescription=CV Clients cat=ESET Filtered Website Event rt=Sep 17 2024 06:22:43 msg=An attempt to connect to URL dst=192.243.59.13 cs2=HTTP filter cs2Label=Scanner ID act=Blocked cn1=1 cn1Label=Handled requestUrl=https://www.topcreativeformat.com cs3=F178CFC97186B1D57A307EA25D978C68EF19DBD1 cs3Label=Hash suser=CONVALES\\\\\\\\TiKuge deviceProcessName=C:\\\\\\\\Program Files\\\\\\\\Mozilla Firefox\\\\\\\\firefox.exe cs1=Blocked by PUA blacklist cs1Label=Rule ID\",\n \"gl2_source_input\": \"66e13a71d17f950fb89d03e0\",\n \"gl2_processing_timestamp\": \"2024-09-17 06:27:00.567\",\n \"application_name\": \"ERAServer\",\n \"act\": \"Blocked\",\n \"dvchost\": \"n0418.convales.intern\",\n \"forwarder\": \"org.graylog2.outputs.GelfOutput\",\n \"gl2_source_node\": \"6b5c9092-6c51-4dcb-8d3b-9173b13b671e\",\n \"_id\": \"d8bcd672-74bd-11ef-8eda-005056a484a6\",\n \"id\": \"d8bc3a39-74bd-11ef-b3e8-005056a46741\",\n \"gl2_processing_duration_ms\": 0,\n \"timestamp\": \"2024-09-17T06:27:00.562Z\"\n}",
"id": "66e93898401d3121e76a2084"
},
{
"title": "Parse ESET log and extract act",
"description": "",
"source": "rule \"Parse ESET log and extract act\"\nwhen\n has_field(\"log_type\") && to_string($message.log_type) == \"Filtered Website Event\" \nthen\n let message_field = to_string($message.message);\n \n let requesturl_match = regex(\"requestUrl=([^ ]+)\", message_field);\n let requesturl_value = requesturl_match[\"0\"];\n set_field(\"requestUrl\", requesturl_value); \n \nend\n",
"created_at": "2024-09-17T06:16:07.926Z",
"modified_at": "2024-09-17T08:52:24.329Z",
"errors": null,
"rule_builder": null,
"simulator_message": "{\n \"msg\": \"An attempt to connect to URL\",\n \"process_id\": \"784\",\n \"gl2_accounted_message_size\": 1066,\n \"gl2_receive_timestamp\": \"2024-09-17 08:36:15.277\",\n \"level\": 4,\n \"gl2_remote_ip\": \"10.0.142.148\",\n \"gl2_remote_port\": 42092,\n \"streams\": [\n \"66e91fc1fbf618573557c7d1\"\n ],\n \"gl2_message_id\": \"01J7ZJ029A00006N7Z0CZW81N4\",\n \"source\": \"EPP0\",\n \"message\": \"CEF:0|ESET|Protect|11.1.756.0|716|Filtered Website Event|5|dvc=10.1.4.147 dvchost=n0017.dh.medipolis.intern deviceExternalId=b75cc491-0536-4e48-ad3d-b27d29d2fbb4 ESETProtectDeviceGroupName=Alle/Medipolis.intern/DH.medipolis.intern/Clients ESETProtectDeviceOsName=Microsoft Windows 10 Pro ESETProtectDeviceGroupDescription=DH Clients cat=ESET Filtered Website Event rt=Sep 17 2024 08:34:36 msg=An attempt to connect to URL dst=51.222.104.23 cs2=HTTP filter cs2Label=Scanner ID act=Blocked cn1=1 cn1Label=Handled requestUrl=https://www.machupicchu-tours-peru.com cs3=F178CFC97186B1D57A307EA25D978C68EF19DBD1 cs3Label=Hash suser=MEDIPOLIS\\\\\\\\LeSues deviceProcessName=C:\\\\\\\\Program Files\\\\\\\\Mozilla Firefox\\\\\\\\firefox.exe cs1=Blocked by PUA blacklist cs1Label=Rule ID\",\n \"gl2_source_input\": \"66e13a71d17f950fb89d03e0\",\n \"gl2_processing_timestamp\": \"2024-09-17 08:36:15.277\",\n \"log_type\": \"Filtered Website Event\",\n \"application_name\": \"ERAServer\",\n \"act\": \"Blocked\",\n \"dvchost\": \"n0017.dh.medipolis.intern\",\n \"forwarder\": \"org.graylog2.outputs.GelfOutput\",\n \"gl2_source_node\": \"4f9a67b1-0afd-46d3-a2bf-84defc6f3ba2\",\n \"_id\": \"e6e7add1-74cf-11ef-abc3-005056a4763b\",\n \"id\": \"e6e786c3-74cf-11ef-b3e8-005056a46741\",\n \"gl2_processing_duration_ms\": 0,\n \"timestamp\": \"2024-09-17T08:36:15.274Z\"\n}",
"id": "66e91ea7401d3121e769e5bf"
},
{
"title": "Parse ESET log and extract dvchost",
"description": "",
"source": "rule \"Parse ESET log and extract dvchost\"\nwhen\n has_field(\"message\")\nthen\n let message_field = to_string($message.message); \n \n \n let dvchost_match = regex(\"dvchost=([^ ]+)\", message_field);\n let dvchost_value = dvchost_match[\"0\"];\n set_field(\"dvchost\", dvchost_value); \nend\n",
"created_at": "2024-09-17T06:05:23.398Z",
"modified_at": "2024-09-17T06:18:05.159Z",
"errors": null,
"rule_builder": null,
"simulator_message": "message: test\nsource: unknown\n",
"id": "66e91c23401d3121e769e2cb"
},
{
"title": "Parse ESET log and extract msg",
"description": "",
"source": "rule \"Parse ESET log and extract msg\"\nwhen\n has_field(\"message\") \nthen\n let message_field = to_string($message.message); \n \n \n let msg_match = regex(\"msg=([^\\\\s]+.*?)(\\\\s\\\\w+=|$)\", message_field);\n let msg_value = msg_match[\"0\"];\n set_field(\"msg\", msg_value); \n \nend\n",
"created_at": "2024-09-17T06:17:09.419Z",
"modified_at": "2024-09-17T06:32:02.418Z",
"errors": null,
"rule_builder": null,
"simulator_message": "{\n \"process_id\": \"784\",\n \"gl2_accounted_message_size\": 893,\n \"level\": 4,\n \"gl2_remote_ip\": \"10.0.80.10\",\n \"gl2_remote_port\": 49995,\n \"streams\": [\n \"66deda58ed4c1a371cf37b55\",\n \"000000000000000000000001\"\n ],\n \"gl2_message_id\": \"01J7Z4VS6A00004MYEA2HR6JE4\",\n \"source\": \"EPP0\",\n \"message\": \"CEF:0|ESET|Protect|11.1.756.0|716|Filtered Website Event|5|dvc=10.1.20.53 dvchost=n0605.convales.intern deviceExternalId=bfc822d8-94c2-46ac-80e2-3d03f2c473ce ESETProtectDeviceGroupName=Alle/Convales.intern/Clients ESETProtectDeviceOsName=Microsoft Windows 10 Pro ESETProtectDeviceGroupDescription=CV Clients cat=ESET Filtered Website Event rt=Sep 17 2024 04:44:29 msg=An attempt to connect to URL dst=51.161.15.142 cs2=HTTP filter cs2Label=Scanner ID act=Blocked cn1=1 cn1Label=Handled requestUrl=https://image-optimizer.salessquad.co.uk cs3=356D7B2365F72C914EDA5150C8DAC3836A4CCFDC cs3Label=Hash suser=CONVALES\\\\\\\\JeBern deviceProcessName=C:\\\\\\\\Users\\\\\\\\JeBern\\\\\\\\AppData\\\\\\\\Local\\\\\\\\Programs\\\\\\\\Opera\\\\\\\\opera.exe cs1=Blocked by Anti-Phishing blacklist cs1Label=Rule ID\",\n \"gl2_source_input\": \"660feaf5322a4e5826d132aa\",\n \"application_name\": \"ERAServer\",\n \"gl2_source_node\": \"1311d675-37e6-4006-81e6-babe1fbb1359\",\n \"_id\": \"d63aa0c9-74af-11ef-b3e8-005056a46741\",\n \"timestamp\": \"2024-09-17T04:46:43.402Z\"\n}\n",
"id": "66e91ee5401d3121e769e5f5"
},
{
"title": "Parse ESET log and extract requestet URL",
"description": "",
"source": "rule \"Parse ESET log and extract requestet URL\"\nwhen\n has_field(\"log_type\") && to_string($message.log_type) == \"Filtered Website Event\"\nthen\n let message_field = to_string($message.message);\n \n let requesturl_match = regex(\"requestUrl=([^ ]+)\", message_field);\n let requesturl_value = requesturl_match[\"0\"];\n set_field(\"requestUrl\", requesturl_value); \n \nend\n",
"created_at": "2024-09-17T08:54:57.317Z",
"modified_at": "2024-09-17T09:20:53.025Z",
"errors": null,
"rule_builder": null,
"simulator_message": "{\n \"msg\": \"An attempt to connect to URL\",\n \"process_id\": \"784\",\n \"gl2_accounted_message_size\": 1027,\n \"gl2_receive_timestamp\": \"2024-09-17 09:10:46.988\",\n \"level\": 4,\n \"gl2_remote_ip\": \"10.0.142.148\",\n \"gl2_remote_port\": 36732,\n \"streams\": [\n \"66e91fc1fbf618573557c7d1\"\n ],\n \"gl2_message_id\": \"01J7ZKZ7QW00006XDB32MY6NQV\",\n \"source\": \"EPP0\",\n \"message\": \"CEF:0|ESET|Protect|11.1.756.0|716|Filtered Website Event|5|dvc=10.1.20.151 dvchost=n0587.convales.intern deviceExternalId=93dcaebf-26c6-478c-80a4-d9bb464a6dc6 ESETProtectDeviceGroupName=Alle/Convales.intern/Clients ESETProtectDeviceOsName=Microsoft Windows 10 Pro ESETProtectDeviceGroupDescription=CV Clients cat=ESET Filtered Website Event rt=Sep 17 2024 09:08:47 msg=An attempt to connect to URL dst=51.222.104.23 cs2=HTTP filter cs2Label=Scanner ID act=Blocked cn1=1 cn1Label=Handled requestUrl=https://www.machupicchu-tours-peru.com cs3=1CC3E09776F961A09F2E45AF9D6300993E1D45B7 cs3Label=Hash suser=CONVALES\\\\\\\\madeus deviceProcessName=C:\\\\\\\\Program Files\\\\\\\\Mozilla Firefox\\\\\\\\firefox.exe cs1=Blocked by PUA blacklist cs1Label=Rule ID\",\n \"gl2_source_input\": \"66e13a71d17f950fb89d03e0\",\n \"gl2_processing_timestamp\": \"2024-09-17 09:10:46.988\",\n \"log_type\": \"Filtered Website Event\",\n \"application_name\": \"ERAServer\",\n \"dvchost\": \"n0587.convales.intern\",\n \"forwarder\": \"org.graylog2.outputs.GelfOutput\",\n \"gl2_source_node\": \"4f9a67b1-0afd-46d3-a2bf-84defc6f3ba2\",\n \"_id\": \"b9bdb0c1-74d4-11ef-abc3-005056a4763b\",\n \"id\": \"b9bd89c4-74d4-11ef-b3e8-005056a46741\",\n \"gl2_processing_duration_ms\": 0,\n \"timestamp\": \"2024-09-17T09:10:45.244Z\"\n}\n",
"id": "66e943e1401d3121e76a2b9d"
},
Feel free to use… <3
The stages run through without any problems and in the 2nd graylog instance I can also see that they have been parsed correctly.
I would like to know what kind of mistake I made and how I can see my logs again in the actual instance
I running Graylog 5.2.7 with Elasticsearch 7.10.2 and MongoDB 6.0.17 on Debian10
Greetings
Marvin