Hi,
We have configure cloudtrail plugin in graylog to collect cloudtrail events from AWS which is working fine.
We need to output these cloudtrail events from Graylog to an external server.
The problem is the external server is receiving these cloudtrail events as a set of multiple events collectively into the external server log file as a single log entry.
We are expecting one log in external server per one cloudtrail event in graylog.
can you explain this?
Putting the sample of what’s received at the external server. This is just one entry in the external server which seems to have a collected records/log received as one log entry in external server log file
Oct 27 00:58:54 x.x.x.x {"version": "1.1","timestamp":1.666857167E9,"host":"aws-cloudtrail","short_message":"health.amazonaws.com:DescribeEventAggregates in us-east-1 by x.x.x.x / null","level":1,"full_message":"[aggregateField=eventTypeCategory, filter={eventTypeCategories=[scheduledChange], eventStatusCodes=[open, upcoming], startTimes=[{from=Oct 20, 2022 10:52:47 AM}]}]","_user_access_key_id":"ASIAWP3NCXCGJPA52HWL","_event_source":"health.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"x.x.x.x","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:47.000Z","_user_principal_id":"446368495756","_gl2_message_id":"01GGC76DSDW5W9S9TCYXAYNGXG","_recipient_account_id":"446368495756","_message":"health.amazonaws.com:DescribeEventAggregates in us-east-1 by x.x.x.x / null","_event_id":"6b9fbb30-8d9c-4a40-920e-65b209a334e8","_aws_region":"us-east-1","_full_message":"[aggregateField=eventTypeCategory, filter={eventTypeCategories=[scheduledChange], eventStatusCodes=[open, upcoming], startTimes=[{from=Oct 20, 2022 10:52:47 AM}]}]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"DescribeEventAggregates","_id":"281c8dd3-55cd-11ed-843d-52540069b6f4","_request_id":"c054e4a8-bcf6-413a-b8f0-678abf41bdab"}#000{"version":"1.1","timestamp":1.666857166E9,"host":"aws-cloudtrail","short_message":"health.amazonaws.com:DescribeEventAggregates in us-east-1 by AWS Internal / null","level":1,"full_message":"[aggregateField=eventTypeCategory, filter={eventStatusCodes=[open, upcoming], startTimes=[{from=Oct 20, 2022 7:52:46 AM}]}]","_user_access_key_id":"ASIAWP3NCXCGLEU3BXTU","_event_source":"health.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"AWS Internal","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"AWS Internal","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:46.000Z","_user_principal_id":"446368495756","_gl2_message_id":"01GGC76DSDZFWK9736K98XYN9X","_recipient_account_id":"446368495756","_message":"health.amazonaws.com:DescribeEventAggregates in us-east-1 by AWS Internal / null","_event_id":"14642efb-4b59-4a73-979f-06cd8d6c53cd","_aws_region":"us-east-1","_full_message":"[aggregateField=eventTypeCategory, filter={eventStatusCodes=[open, upcoming], startTimes=[{from=Oct 20, 2022 7:52:46 AM}]}]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"DescribeEventAggregates","_id":"281c8dd4-55cd-11ed-843d-52540069b6f4","_request_id":"f737a3ca-e1ef-4cb9-afd9-e3ee81078ddd"}#000{"version":"1.1","timestamp":1.666857167E9,"host":"aws-cloudtrail","short_message":"health.amazonaws.com:DescribeEventAggregates in us-east-1 by x.x.x.x / null","level":1,"full_message":"[aggregateField=eventTypeCategory, filter={eventTypeCategories=[accountNotification], eventStatusCodes=[open], startTimes=[{from=Oct 20, 2022 10:52:47 AM}]}]","_user_access_key_id":"ASIAWP3NCXCGJPA52HWL","_event_source":"health.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"x.x.x.x","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:47.000Z","_user_principal_id":"446368495756","_gl2_message_id":"01GGC76DSDTB08T8NFY87QYJME","_recipient_account_id":"446368495756","_message":"health.amazonaws.com:DescribeEventAggregates in us-east-1 by x.x.x.x / null","_event_id":"ade54ca6-135b-490d-b52c-6624b2f3132f","_aws_region":"us-east-1","_full_message":"[aggregateField=eventTypeCategory, filter={eventTypeCategories=[accountNotification], eventStatusCodes=[open], startTimes=[{from=Oct 20, 2022 10:52:47 AM}]}]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"DescribeEventAggregates","_id":"281c66db-55cd-11ed-843d-52540069b6f4","_request_id":"c39cc504-8550-4c12-9d3c-2f1a8e3a034a"}#000{"version":"1.1","timestamp":1.666857167E9,"host":"aws-cloudtrail","short_message":"ce.amazonaws.com:GetCostForecast in us-east-1 by x.x.x.x / null","level":1,"full_message":"[Filter={Not={Or=[{Dimensions={Key=RECORD_TYPE, Values=[Credit]}}, {Dimensions={Key=RECORD_TYPE, Values=[Refund]}}]}}, Granularity=MONTHLY, Metric=NET_UNBLENDED_COST, TimePeriod={End=2022-11-01, Start=2022-10-27}]","_user_access_key_id":"ASIAWP3NCXCGJPA52HWL","_event_source":"ce.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"x.x.x.x","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:47.000Z","_user_principal_id":"446368495756","_errorMessage":"User not enabled for cost explorer access","_gl2_message_id":"01GGC76DSD5CAKVAYDH8K57XPZ","_recipient_account_id":"446368495756","_message":"ce.amazonaws.com:GetCostForecast in us-east-1 by x.x.x.x / null","_event_id":"28b853c1-3837-45cd-8471-94c35e717a2c","_aws_region":"us-east-1","_full_message":"[Filter={Not={Or=[{Dimensions={Key=RECORD_TYPE, Values=[Credit]}}, {Dimensions={Key=RECORD_TYPE, Values=[Refund]}}]}}, Granularity=MONTHLY, Metric=NET_UNBLENDED_COST, TimePeriod={End=2022-11-01, Start=2022-10-27}]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"GetCostForecast","_id":"281c8dd6-55cd-11ed-843d-52540069b6f4","_request_id":"c30c15a3-b256-4dda-affd-5b2d332a1538"}#000{"version":"1.1","timestamp":1.666857167E9,"host":"aws-cloudtrail","short_message":"ce.amazonaws.com:GetCostAndUsage in us-east-1 by x.x.x.x / null","level":1,"full_message":"[Filter={Not={Or=[{Dimensions={Key=RECORD_TYPE, Values=[Credit]}}, {Dimensions={Key=RECORD_TYPE, Values=[Refund]}}]}}, Granularity=MONTHLY, GroupBy=[{Key=SERVICE, Type=DIMENSION}], Metrics=[UnblendedCost], TimePeriod={End=2022-11-01, Start=2022-09-01}]","_user_access_key_id":"ASIAWP3NCXCGJPA52HWL","_event_source":"ce.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"x.x.x.x","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:47.000Z","_user_principal_id":"446368495756","_errorMessage":"User not enabled for cost explorer access","_gl2_message_id":"01GGC76DSD4QQRT9E86CASH390","_recipient_account_id":"446368495756","_message":"ce.amazonaws.com:GetCostAndUsage in us-east-1 by x.x.x.x / null","_event_id":"a101d191-801a-4f66-9048-07aadd717061","_aws_region":"us-east-1","_full_message":"[Filter={Not={Or=[{Dimensions={Key=RECORD_TYPE, Values=[Credit]}}, {Dimensions={Key=RECORD_TYPE, Values=[Refund]}}]}}, Granularity=MONTHLY, GroupBy=[{Key=SERVICE, Type=DIMENSION}], Metrics=[UnblendedCost], TimePeriod={End=2022-11-01, Start=2022-09-01}]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"GetCostAndUsage","_id":"281c8dd5-55cd-11ed-843d-52540069b6f4","_request_id":"4e929774-39c3-4581-b80d-746a1df0628a"}#000{"version":"1.1","timestamp":1.666857175E9,"host":"aws-cloudtrail","short_message":"cloudtrail.amazonaws.com:DescribeTrails in us-east-1 by AWS Internal / null","level":1,"full_message":"[trailNameList=[], includeShadowTrails=true]","_user_access_key_id":"ASIAWP3NCXCGO6JOKY4B","_event_source":"cloudtrail.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"AWS Internal","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"AWS Internal","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:55.000Z","_user_principal_id":"446368495756","_gl2_message_id":"01GGC76DSRJNP86NN5CEW3W922","_recipient_account_id":"446368495756","_message":"cloudtrail.amazonaws.com:DescribeTrails in us-east-1 by AWS Internal / null","_event_id":"72aafe83-4289-4692-b60c-6bf447f959ab","_aws_region":"us-east-1","_full_message":"[trailNameList=[], includeShadowTrails=true]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"DescribeTrails","_id":"281e1470-55cd-11ed-843d-52540069b6f4","_request_id":"e48765d2-14b3-4428-aac3-f414e41efeed"}#000{"version":"1.1","timestamp":1.666857173E9,"host":"aws-cloudtrail","short_message":"cloudtrail.amazonaws.com:DescribeTrails in us-east-1 by AWS Internal / null","level":1,"full_message":"[trailNameList=[], includeShadowTrails=true]","_user_access_key_id":"ASIAWP3NCXCGO6JOKY4B","_event_source":"cloudtrail.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"AWS Internal","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"AWS Internal","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:53.000Z","_user_principal_id":"446368495756","_gl2_message_id":"01GGC76DSR12M10PK0KMRCZ6AB","_recipient_account_id":"446368495756","_message":"cloudtrail.amazonaws.com:DescribeTrails in us-east-1 by AWS Internal / null","_event_id":"6f69ced7-a23b-4fa6-aa1f-73805427ab07","_aws_region":"us-east-1","_full_message":"[trailNameList=[], includeShadowTrails=true]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"DescribeTrails","_id":"281e3b80-55cd-11ed-843d-52540069b6f4","_request_id":"125dea97-3f22-4593-9d0c-9284ebe61b64"}#000{"version":"1.1","timestamp":1.666857175E9,"host":"aws-cloudtrail","short_message":"cloudtrail.amazonaws.com:GetTrailStatus in us-east-1 by AWS Internal / null","level":1,"full_message":"[name=arn:aws:cloudtrail:us-east-1:446368495756:trail/xenex]","_user_access_key_id":"ASIAWP3NCXCGO6JOKY4B","_event_source":"cloudtrail.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"AWS Internal","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"AWS Internal","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:55.000Z","_user_principal_id":"446368495756","_gl2_message_id":"01GGC76DSRERW9EH09JVM1HN1E","_recipient_account_id":"446368495756","_message":"cloudtrail.amazonaws.com:GetTrailStatus in us-east-1 by AWS Internal / null","_event_id":"62957692-1699-4705-963f-024221d4d348","_aws_region":"us-east-1","_full_message":"[name=arn:aws:cloudtrail:us-east-1:446368495756:trail/xenex]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"GetTrailStatus","_id":"281e3b81-55cd-11ed-843d-52540069b6f4","_request_id":"6e1ed5a7-fa95-4ac1-9006-6bf5d9e0365f"}#000{"version":"1.1","timestamp":1.666857173E9,"host":"aws-cloudtrail","short_message":"s3.amazonaws.com:ListBuckets in us-east-1 by x.x.x.x / null","level":1,"full_message":"[Host=s3-external-1.amazonaws.com]","_user_access_key_id":"ASIAWP3NCXCGO6JOKY4B","_event_source":"s3.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"x.x.x.x","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"[AWSCloudTrail, aws-internal/3 aws-sdk-java/1.11.1030 Linux/5.10.135-105.570.amzn2int.x86_64 OpenJDK_64-Bit_Server_VM/25.322-b06 java/1.8.0_322 vendor/Oracle_Corporation cfg/retry-mode/standard]","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:53.000Z","_user_principal_id":"446368495756","_additional_event_data":"{SignatureVersion=SigV4, CipherSuite=ECDHE-RSA-AES128-GCM-SHA256, bytesTransferredIn=0, AuthenticationMethod=AuthHeader, x-amz-id-2=CjSoRkzGLlLQYYrEn7B6g7c4q+ipGxLzWvyhyC6RZh+doElGC8yEqWq5P3nL6JNvYfa05Ee5HUw=, bytesTransferredOut=525}","_gl2_message_id":"01GGC76DSRA5RF8C1AKT1MT1ST","_recipient_account_id":"446368495756","_message":"s3.amazonaws.com:ListBuckets in us-east-1 by x.x.x.x / null","_event_id":"59f02b8d-1f8e-41aa-be76-1635c59bc0fc","_aws_region":"us-east-1","_full_message":"[Host=s3-external-1.amazonaws.com]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"ListBuckets","_id":"281e3b82-55cd-11ed-843d-52540069b6f4","_request_id":"4VQ97TK6QV782SRD"}#000{"version":"1.1","timestamp":1.666857167E9,"host":"aws-cloudtrail","short_message":"ec2.amazonaws.com:DescribeAccountAttributes in us-east-1 by x.x.x.x / null","level":1,"full_message":"[accountAttributeNameSet={items=[{attributeName=supported-platforms}]}, filterSet={}]","_user_access_key_id":"ASIAWP3NCXCGAS2LB7DQ","_event_source":"ec2.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"x.x.x.x","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/106.0.0.0 Safari/537.36","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:47.000Z","_user_principal_id":"446368495756","_gl2_message_id":"01GGC76DSRQAQGD94AKAMRB0W9","_recipient_account_id":"446368495756","_message":"ec2.amazonaws.com:DescribeAccountAttributes in us-east-1 by x.x.x.x / null","_event_id":"7decb7c8-78b0-470a-b696-136ecc5998ae","_aws_region":"us-east-1","_full_message":"[accountAttributeNameSet={items=[{attributeName=supported-platforms}]}, filterSet={}]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"DescribeAccountAttributes","_id":"281c3fb0-55cd-11ed-843d-52540069b6f4","_request_id":"4978fc79-e311-4ea4-a643-86f6a0b6c892"}#000{"version":"1.1","timestamp":1.666857175E9,"host":"aws-cloudtrail","short_message":"cloudtrail.amazonaws.com:LookupEvents in us-east-1 by AWS Internal / null","level":1,"full_message":"[lookupAttributes=[{attributeKey=ReadOnly, attributeValue=false}], maxResults=5]","_user_access_key_id":"ASIAWP3NCXCGO6JOKY4B","_event_source":"cloudtrail.amazonaws.com","_source":"aws-cloudtrail","_gl2_source_input":"63484847e6135942b2e7d9b1","_event_type":"AwsApiCall","_user_type":"Root","_user_account_id":"446368495756","_user_session_creation_date":"2022-10-27T07:52:44Z","_aws_source":"true","_source_address":"AWS Internal","_gl2_source_node":"d1ea6f16-7635-432a-8bbf-242baba25a74","_user_agent":"AWS Internal","_user_principal_arn":"arn:aws:iam::446368495756:root","_timestamp":"2022-10-27T07:52:55.000Z","_user_principal_id":"446368495756","_gl2_message_id":"01GGC76DSRK9QBYR8TK8WZX8R1","_recipient_account_id":"446368495756","_message":"cloudtrail.amazonaws.com:LookupEvents in us-east-1 by AWS Internal / null","_event_id":"98f6a71d-d834-4417-83df-8c26f7dc8d6a","_aws_region":"us-east-1","_full_message":"[lookupAttributes=[{attributeKey=ReadOnly, attributeValue=false}], maxResults=5]","_user_session_mfa_authenticated":"false","_forwarder":"org.graylog2.outputs.GelfOutput","_event_name":"LookupEvents","_id":"281e3b83-55cd-11ed-843d-52540069b6f4","_request_id":"5f9af2d8-f61d-42e5-9a3e-499b26872454"}
Thanks in Advance.