Parsing and remove rule

Graylog Central (peer support)
1

I am trying to remove text out of an ADFS log, and my regex works to hightlight the text, but I want to remove it out of a message and leave the rest but its not working.

This is my rule:

rule "remove all before AuditBase"
when
has_field("message")
then
let ADFS = regex("(?i)(^The Federation Service.*|^Activity ID: .*|^Additional Data.*|^XML: .*|^<AuditBase.*|^</AuditBase>|^\\n)mg", to_string($message.message)); 
set_field("message", ADFS["0"]);
end

The Federation Service failed to validate a new credential. See XML for failure details. 

Activity ID: b7d7deb6-4323-40fd-9259-00800100007f 

Additional Data 
XML: <?xml version="1.0" encoding="utf-16"?>
<AuditBase xmlns:xsd="http://www.w3.org/2001/XMLSchema" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="FreshCredentialAudit">
  <AuditType>FreshCredentials</AuditType>
  <AuditResult>Failure</AuditResult>
  <FailureType>CredentialValidationError</FailureType>
  <ErrorCode>N/A</ErrorCode>
  <ContextComponents>
    <Component xsi:type="ResourceAuditComponent">
      <RelyingParty>https://weblogin.contoso.edu/adfs/services/trust</RelyingParty>
      <ClaimsProvider>N/A</ClaimsProvider>
      <UserId>contonso.edu\rachael</UserId>
    </Component>
    <Component xsi:type="AuthNAuditComponent">
      <PrimaryAuth>N/A</PrimaryAuth>
      <DeviceAuth>false</DeviceAuth>
      <DeviceId>N/A</DeviceId>
      <MfaPerformed>false</MfaPerformed>
      <MfaMethod>N/A</MfaMethod>
      <TokenBindingProvidedId>false</TokenBindingProvidedId>
      <TokenBindingReferredId>false</TokenBindingReferredId>
      <SsoBindingValidationLevel>NotSet</SsoBindingValidationLevel>
    </Component>
    <Component xsi:type="ProtocolAuditComponent">
      <OAuthClientId>N/A</OAuthClientId>
      <OAuthGrant>N/A</OAuthGrant>
    </Component>
    <Component xsi:type="RequestAuditComponent">
      <Server>https://weblogin.contoso.edu/adfs/services/trust</Server>
      <AuthProtocol>SAMLP</AuthProtocol>
      <NetworkLocation>Intranet</NetworkLocation>
      <IpAddress>199.199.94.22</IpAddress>
      <ForwardedIpAddress>65.129.84.94</ForwardedIpAddress>
      <ProxyIpAddress>N/A</ProxyIpAddress>
      <NetworkIpAddress>N/A</NetworkIpAddress>
      <ProxyServer>N/A</ProxyServer>
      <UserAgentString>Mozilla/5.0 (Linux; Android 7.1.1; SM-T350 Build/NMF26X; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/81.0.4044.138 Safari/537.36 MinuteMaid</UserAgentString>
      <Endpoint>/adfs/ls/</Endpoint>
    </Component>
  </ContextComponents>
</AuditBase>
2

Ended up creating a rule to handle the three Federation ADFS events

rule "ADFS-Parser2"
when
  has_field("message") AND contains(to_string($message.message), "The Federation Service")
then
  let adfs_audit_type = regex("(?i)(<AuditType>)(.*)(</AuditType>)", to_string($message.message));
  set_field("adfs_audit_type", adfs_audit_type["1"]);
  let adfs_audit_result = regex("(?i)(<AuditResult>)(.*)(</AuditResult>)", to_string($message.message));
  set_field("adfs_audit_result", adfs_audit_result["1"]);
  let adfs_failure_type = regex("(?i)(<FailureType>)(.*)(</FailureType>)", to_string($message.message));
  set_field("adfs_failure_type", adfs_failure_type["1"]);
  let adfs_errorCode = regex("(?i)(<ErrorCode>)(.*)(</ErrorCode>)", to_string($message.message));
  set_field("adfs_errorCode", adfs_errorCode["1"]);
  let adfs_relying_party = regex("(?i)(<RelyingParty>)(.*)(</RelyingParty>)", to_string($message.message));
  set_field("adfs_relying_party", adfs_relying_party["1"]);
  let adfs_claims_provider = regex("(?i)(<ClaimsProvider>)(.*)(</ClaimsProvider>)", to_string($message.message));
  set_field("adfs_claims_provider", adfs_claims_provider["1"]);
  let adfs_user_id = regex("(?i)(<UserId>)(.*)(</UserId)", to_string($message.message));
  set_field("adfs_user_id", adfs_user_id["1"]);
  let adfs_primary_auth = regex("(?i)(<PrimaryAuth>)(.*)(</PrimaryAuth)", to_string($message.message));
  set_field("adfs_primary_auth", adfs_primary_auth["1"]);
  let adfs_device_auth = regex("(?i)(<DeviceAuth>)(.*)(</DeviceAuth)", to_string($message.message));
  set_field("adfs_device_auth", adfs_device_auth["1"]);
  let adfs_device_id = regex("(?i)(<DeviceId>)(.*)(</DeviceId)", to_string($message.message));
  set_field("adfs_device_id", adfs_device_id["1"]);
  let adfs_mfa_performed = regex("(?i)(<MfaPerformed>)(.*)(</MfaPerformed)", to_string($message.message));
  set_field("adfs_mfa_performed", adfs_mfa_performed["1"]);
  let adfs_mfa_method= regex("(?i)(<MfaMethod>)(.*)(</MfaMethod)", to_string($message.message));
  set_field("adfs_mfa_method", adfs_mfa_method["1"]);
  let adfs_token_binding_provided_id= regex("(?i)(<TokenBindingProvidedId>)(.*)(</TokenBindingProvidedId)", to_string($message.message));
  set_field("adfs_token_binding_provided_id", adfs_token_binding_provided_id["1"]);
  let adfs_tokenbinding_referred_id= regex("(?i)(<TokenBindingReferredId>)(.*)(</TokenBindingReferredId)", to_string($message.message));
  set_field("adfs_tokenbinding_referred_id", adfs_tokenbinding_referred_id["1"]);
  let adfs_sso_binding_validation_level= regex("(?i)(<SsoBindingValidationLevel>)(.*)(</SsoBindingValidationLevel)", to_string($message.message));
  set_field("adfs_sso_binding_validation_level", adfs_sso_binding_validation_level["1"]);
  let adfs_oath_client_id= regex("(?i)(<OAuthClientId>)(.*)(</OAuthClientId)", to_string($message.message));
  set_field("adfs_oath_client_id", adfs_oath_client_id["1"]);
  let adfs_oath_grant= regex("(?i)(<OAuthGrant>)(.*)(</OAuthGrant)", to_string($message.message));
  set_field("adfs_oath_grant", adfs_oath_grant["1"]);
  let adfs_server= regex("(?i)(<Server>)(.*)(</Server)", to_string($message.message));
  set_field("adfs_server", adfs_server["1"]);
  let adfs_auth_protocol= regex("(?i)(<AuthProtocol>)(.*)(</AuthProtocol)", to_string($message.message));
  set_field("adfs_auth_protocol", adfs_auth_protocol["1"]);
  let adfs_network_location= regex("(?i)(<NetworkLocation>)(.*)(</NetworkLocation)", to_string($message.message));
  set_field("adfs_network_location", adfs_network_location["1"]);
  let adfs_ip_address= regex("(?i)(<IpAddress>)(.*)(</IpAddress)", to_string($message.message));
  set_field("adfs_ip_address", adfs_ip_address["1"]);
  let adfs_forwarded_ip_address= regex("(?i)(<ForwardedIpAddress>)(.*)(</ForwardedIpAddress)", to_string($message.message));
  set_field("adfs_forwarded_ip_address", adfs_forwarded_ip_address["1"]);
  let adfs_proxy_ip_address= regex("(?i)(<ProxyIpAddress>)(.*)(</ProxyIpAddress)", to_string($message.message));
  set_field("adfs_proxy_ip_address", adfs_proxy_ip_address["1"]);
  let adfs_network_ip_address= regex("(?i)(<NetworkIpAddress>)(.*)(</NetworkIpAddress)", to_string($message.message));
  set_field("adfs_network_ip_address", adfs_network_ip_address["1"]);
  let adfs_proxy_server= regex("(?i)(<ProxyServer>)(.*)(</ProxyServer)", to_string($message.message));
  set_field("adfs_proxy_server", adfs_proxy_server["1"]);
  let adfs_user_agent_string= regex("(?i)(<UserAgentString>)(.*)(</UserAgentString)", to_string($message.message));
  set_field("adfs_user_agent_string", adfs_user_agent_string["1"]);
  let adfs_endpoint= regex("(?i)(<Endpoint>)(.*)(</Endpoint)", to_string($message.message));
  set_field("adfs_endpoint", adfs_endpoint["1"]);
end
1 Like
3

Hey @giveen - I know you already have a solution but here area couple of thoughts:

regex is pretty inefficient at parsing xml You have ~22 regex commands in the rule, ran one of them at http://www.regex101.com to see how it goes and it took 348 steps just to find OAuthClientId data ~384 steps x ~22 is something like 8,500 steps in regex to parse each message. if you have low volume this is not an issue.

Possibly using winlogbeat to pick up the Windows ADFS logs, it will likely break out most of those fields for you - You can use the Configuration to modify or drop fields before sending to Graylog too.

In some instances with regex it looks as though you are not escaping the closing xml (i.e. /OAuthClient needs to have the forward slash escaped \/OAuthClient)

Looks like @jan mentioned regex but also suggested looking at winlogbeat - did you try that out?

1 Like
4

Heh @tmacgbay i haven’t given winlogbeat a try on that server, but i can certainly give it a try.

1 Like
5

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.