JSON parsing in pipeline

Hi.
Graylog 3.0.2+1686930

I’m trying to parse json fields in pipeline rule. In stage 0 I’m parsing “message” field, creating “Properties” field from it and parsing “Properties”:

when
    has_field("Properties") &&
    contains(to_string($message.message), "data", true)
then
    let msg = parse_json(to_string($message.message));
    let prop = select_jsonpath(msg, {Properties: "$.Properties"});
    set_field("Properties", to_string(prop.Properties));
    
    let props = parse_json(to_string($message.Properties));
    set_fields(to_map(props));
end

This part is working fine.
As result, new “IntegrationEvent” field appears.

In stage 1 I’m parsing json from “IntegrationEvent”. Simple script, like

let ints = parse_json(to_string($message.IntegrationEvent));
set_fields(to_map(ints));

does not work: nothing happens there, new fields does not appear.
I was trying to write this script:

when
    has_field("Properties") &&
    contains(to_string($message.Properties), "IntegrationEvent", true)
then
    let prop = parse_json(to_string($message.Properties));
    let int = select_jsonpath(prop, {IntegrationEvent: "$.IntegrationEvent"});

    let ints = parse_json(to_string(int.IntegrationEvent));
    set_fields(to_map(ints));
end

I’m checking for “Properties” field instead of “IntegrationEvent”, because didn’t see any result when was checking this field.
I was trying to set_field(“IntegrationEvent2”, to_string(int.IntegrationEvent)), and this field successfully appeared, so the first part of this script is working.
But the second part does not work: Messages, containing “IntegrationEvent” field, disappear.

Forgot to say: stage 0 didn’t work either, while I created extractor for “message” field, which is parsing it on first json level and creating “Properties” field, as well as some others, but it’s not in json format, so I’m rewriting it in this pipeline.
It’s looking weird though.

Check your Message processor configuration, your Message Filter chain should be before Pipeline Processor. If not, your pipeline rule couldn’t see extracted field, before it’s run before field extraction.

https://docs.graylog.org/en/3.2/pages/pipelines/usage.html#configure-the-message-processor

Thank You.

I found that on forums, when was trying to write the first script.
Now my Message processor config looks like this:

1 Like

Send your example message field, and desired result, so we can help you…

   {
	"Timestamp": "2020-03-31T10:10:11.3787618+03:00",
	"Level": "Information",
	"MessageTemplate": "Handling integration event: {IntegrationEventId} - {@IntegrationEvent}",
	"RenderedMessage": "Handling integration event: 0000 - ActualizeStoragePlaceEvent { StorageId: 1, ClientDateTime: 03/31/2020 07:10:11, FromStorageId: null, Id: 0000, CreationDate: 03/31/2020 07:10:11, CorrelationId: \"0000\" }",
	"Properties": {
		"IntegrationEventId": "0000",
		"IntegrationEvent": {
			"_typeTag": "ActualizeStoragePlaceEvent",
			"StorageId": 1,
			"ClientDateTime": "2020-03-31T07:10:11.7590000",
			"FromStorageId": null,
			"Id": "0000",
			"CreationDate": "2020-03-31T07:10:11.3817043Z",
			"CorrelationId": "0000"
		},
		"SourceContext": "Company.Data.Services.Impl.MesIntegrationService.EventHandlers.EventHandler`1[[Company.Data.Services.EventContracts.Domain.AccountingDomain.ActualizeStoragePlaceEvent, Company.Data.Services.EventContracts, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]]",
		"ProcessingEventId": "0000",
		"RequestId": "0000",
		"MachineName": "mesintegrationservice-548f4bf7cf-r5flx",
		"EnvironmentUserName": null,
		"ProcessId": 1,
		"ProcessName": "dotnet",
		"ThreadId": 38,
		"AssemblyName": "Company.Data.Services.Impl.MesIntegrationService",
		"AssemblyVersion": "1.0.0.0",
		"User-Agent": null,
		"Remote-IP": null,
		"EnvVar-ASPNETCORE_ENVIRONMENT": "Production",
		"Application": "MesIntegrationService"
	}
}

“Properties” field is taken from here and “IntegrationEvent” is taken from “Properties”.

So, the final message must look like:

IntegrationEvent
{
	"_typeTag": "ActualizeStoragePlaceEvent",
	"StorageId": 1,
	"ClientDateTime": "2020-03-31T07:10:11.7590000",
	"FromStorageId": null,
	"Id": "0000",
	"CreationDate": "2020-03-31T07:10:11.3817043Z",
	"CorrelationId": "0000"
}
Properties
{
	"IntegrationEventId": "0000",
	"IntegrationEvent": {
	"_typeTag": "ActualizeStoragePlaceEvent",
	"StorageId": 1,
	"ClientDateTime": "2020-03-31T07:10:11.7590000",
	"FromStorageId": null,
	"Id": "0000",
	"CreationDate": "2020-03-31T07:10:11.3817043Z",
	"CorrelationId": "0000"
	},
	"SourceContext": "Company.Data.Services.Impl.MesIntegrationService.EventHandlers.EventHandler`1[[Company.Data.Services.EventContracts.Domain.AccountingDomain.ActualizeStoragePlaceEvent, Company.Data.Services.EventContracts, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]]",
	"ProcessingEventId": "0000",
	"RequestId": "0000",
	"MachineName": "mesintegrationservice-548f4bf7cf-r5flx",
	"EnvironmentUserName": null,
	"ProcessId": 1,
	"ProcessName": "dotnet",
	"ThreadId": 38,
	"AssemblyName": "Company.Data.Services.Impl.MesIntegrationService",
	"AssemblyVersion": "1.0.0.0",
	"User-Agent": null,
	"Remote-IP": null,
	"EnvVar-ASPNETCORE_ENVIRONMENT": "Production",
	"Application": "MesIntegrationService"
}
message
{
	"Timestamp": "2020-03-31T10:10:11.3787618+03:00",
	"Level": "Information",
	"MessageTemplate": "Handling integration event: {IntegrationEventId} - {@IntegrationEvent}",
	"RenderedMessage": "Handling integration event: 0000 - ActualizeStoragePlaceEvent { StorageId: 1, ClientDateTime: 03/31/2020 07:10:11, FromStorageId: null, Id: 0000, CreationDate: 03/31/2020 07:10:11, CorrelationId: \"0000\" }",
	"Properties": {
		"IntegrationEventId": "0000",
		"IntegrationEvent": {
			"_typeTag": "ActualizeStoragePlaceEvent",
			"StorageId": 1,
			"ClientDateTime": "2020-03-31T07:10:11.7590000",
			"FromStorageId": null,
			"Id": "0000",
			"CreationDate": "2020-03-31T07:10:11.3817043Z",
			"CorrelationId": "0000"
		},
		"SourceContext": "Company.Data.Services.Impl.MesIntegrationService.EventHandlers.EventHandler`1[[Company.Data.Services.EventContracts.Domain.AccountingDomain.ActualizeStoragePlaceEvent, Company.Data.Services.EventContracts, Version=1.0.0.0, Culture=neutral, PublicKeyToken=null]]",
		"ProcessingEventId": "0000",
		"RequestId": "0000",
		"MachineName": "mesintegrationservice-548f4bf7cf-r5flx",
		"EnvironmentUserName": null,
		"ProcessId": 1,
		"ProcessName": "dotnet",
		"ThreadId": 38,
		"AssemblyName": "Company.Data.Services.Impl.MesIntegrationService",
		"AssemblyVersion": "1.0.0.0",
		"User-Agent": null,
		"Remote-IP": null,
		"EnvVar-ASPNETCORE_ENVIRONMENT": "Production",
		"Application": "MesIntegrationService"
	}
}

Found that after stage 0 “IntegrationEvent” field is not a string type.
Tried to set_field(“IntegrationEvent”, to_string($message.IntegrationEvent)). Got an error, maybe it refers to this message (new messages with “IntegrationEvent” stopped to appear):

{
    "type":"mapper_parsing_exception",
    "reason":"failed to parse field [Result] of type [keyword] in document with id 'ea4164a5-7419-11ea-b99f-005056b3bebe'",
    "caused_by":{
        "type":"illegal_state_exception","reason":"Can't get text on a START_OBJECT at 1:2167"
    }
}

Upd. No, it was not caused by my changes, it continued after I removed stage 1 and messages began to appear again.

Upd.2 I’ve checked, “IntegrationEvent” field has “map” type.

We could make it work, finally! All in stage 0:

rule "Data Parsing"

when
    has_field("Properties") &&
    contains(to_string($message.message), "data", true)
then
    let msg = parse_json(to_string($message.message));
    let prop = select_jsonpath(msg, {Properties: "$.Properties"});
    set_field("Properties", to_string(prop.Properties));
    
    let props = parse_json(to_string($message.Properties));
    set_fields(to_map(props));

    let int = select_jsonpath(msg, {IntEvent: "$.Properties.IntegrationEvent"});
    let ints = parse_json(to_string(int.IntEvent));
    set_fields(to_map(ints), "IntegrationEvent_");
end

Maybe there was some field in json, that concur with field in message.

1 Like

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.