Pipeline rule for Microsoft Exchange tracking logs?

Graylog Central (peer support)
1

Does anyone have a working example of a pipeline rule to extract Exchange 2016 message tracking logs?

I’m currently shipping the logs with filebeat and processing them with a rule similiar to the rule below. But this breaks when a , is in the subject. Is it possible to do a pipeline rule with a csv extractor or is GROK the preferred way? I’ve looked around but haven’t found much. Thinking someone must have a good way to do this with filebeat and a pipeline rule.

rule "type ex-msg-trk-transport"
when
 has_field("type") && to_string($message.type) == "ex-msg-trk-transport"
then
  // grok the message field

let message_field = to_string($message.message);
let action = grok(pattern: "(%{TIMESTAMP_ISO8601:date-time})?,(%{IPORHOST:client-ip})?,(%{IPORHOST:client-hostname})?,(%{IPORHOST:server-ip})?,(%{IPORHOST:server-hostname})?,(?<source-context>[^,]*)?,(?<connector-id>[^,]*)?,(%{WORD:source-component})?,(%{WORD:event-id})?,(%{NUMBER:internal-message-id})?,(?<message-id>[^,]*)?,(?<network-message-id>[^,]*)?,(?<recipient-address>[^,]*)?,(?<recipient-status>[^,]*)?,(%{NUMBER:total-bytes})?,(%{NUMBER:recipient-count})?,(?<related-recipient-address>[^,]*)?,(?<reference>[^,]*)?,(?<message-subject>[^,]*)?,(?<sender-address>[^,]*)?,(?<return-path>[^,]*)?,(?<message-info>[^,]*)?,(?<directionality>[^,]*)?,(?<tenant-id>[^,]*)?,(?<original-client-ip>[^,]*)?,(?<original-server-ip>[^,]*)?,(?<custom-data>[^,]*)?,(?<transport-traffic-type>[^,]*)?,(?<log-id>[^,]*)?,(?<schema-version>[^,]*)?", value: message_field,  only_named_captures: true);
  set_fields(action);
end

This would work sometimes, but not always. Depending on if the logs put “” around the subject.

rule "type ex-msg-trk-transport"
when
 has_field("type") && to_string($message.type) == "ex-msg-trk-transport"
then
  // grok the message field
let message_field = to_string($message.message);
let action = grok(pattern: "(%{TIMESTAMP_ISO8601:date-time})?,(%{IPORHOST:client-ip})?,(%{IPORHOST:client-hostname})?,(%{IPORHOST:server-ip})?,(%{IPORHOST:server-hostname})?,(?<source-context>[^,]*)?,(?<connector-id>[^,]*)?,(%{WORD:source-component})?,(%{WORD:event-id})?,(%{NUMBER:internal-message-id})?,(?<message-id>[^,]*)?,(?<network-message-id>[^,]*)?,(?<recipient-address>[^,]*)?,(?<recipient-status>[^,]*)?,(%{NUMBER:total-bytes})?,(%{NUMBER:recipient-count})?,(?<related-recipient-address>[^,]*)?,(?<reference>[^,]*)?,\"(%{GREEDYDATA:message-subject})?\",(?<sender-address>[^,]*)?,(?<return-path>[^,]*)?,(?<message-info>[^,]*)?,(?<directionality>[^,]*)?,(?<tenant-id>[^,]*)?,(?<original-client-ip>[^,]*)?,(?<original-server-ip>[^,]*)?,(?<custom-data>[^,]*)?,(?<transport-traffic-type>[^,]*)?,(?<log-id>[^,]*)?,(?<schema-version>[^,]*)?", value: message_field,  only_named_captures: true);
  set_fields(action);
end
2

What I ended up using to get Exchange 2016 message tracking logs working.
I used the csv plugin found at https://marketplace.graylog.org/addons/a635a144-b3e2-4214-a7e5-eb7fa8b2dd71

The pipeline rule is:

rule "type ex-msg-trk-transport-csv"
when
 has_field("type") && to_string($message.type) == "ex-msg-trk-transport"
then
  // process with csv
  let csv_fields = "date-time,client-ip,client-hostname,server-ip,server-hostname,source-context,connector-id,source,event-id,internal-message-id,message-id,network-message-id,recipient-address,recipient-status,total-bytes,recipient-count,related-recipient-address,reference,message-subject,sender-address,return-path,message-info,directionality,tenant-id,original-client-ip,original-server-ip,custom-data,transport-traffic-type,log-id,schema-version";
  let csv_parsed = csv(csv_fields: csv_fields, csv_text: to_string($message.message), csv_separator: ",", dummy_value:"not_used");
  set_fields(csv_parsed);
end

Hope this helps someone.

3

Looks like I need to figure out how to exclude the top four lines of the log file still that start with a #

#Software: Microsoft Exchange Server
#Version: 15.01.1531.003
#Log-type: Message Tracking Log
#Date: 2018-07-23T14:09:06.492Z
#Fields: date-time,client-ip,client-…

4

Here is what I ended up using. I placed both roles in the same Pipeline Stage

rule "type ex-msg-trk-csv-check"
when
contains(to_string($message.message), "#Software:", true) || contains(to_string($message.message), "#Version:", true) || contains(to_string($message.message), "#Log-type:", true) || contains(to_string($message.message), "#Date:", true) || contains(to_string($message.message), "#Fields:", true)
then
  // drop message
    drop_message();
end

rule "type ex-msg-trk-mailbox-csv"
when
 has_field("type") && to_string($message.type) == "ex-msg-trk-mailbox"
then
  // process with csv
  let csv_fields = "date-time,client-ip,client-hostname,server-ip,server-hostname,source-context,connector-id,source,event-id,internal-message-id,message-id,network-message-id,recipient-address,recipient-status,total-bytes,recipient-count,related-recipient-address,reference,message-subject,sender-address,return-path,message-info,directionality,tenant-id,original-client-ip,original-server-ip,custom-data,transport-traffic-type,log-id,schema-version";
  let csv_parsed = csv(csv_fields: csv_fields, csv_text: to_string($message.message), csv_separator: ",", dummy_value:"not_used");
  set_fields(csv_parsed);
end
1 Like
5

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.