Hey all, I’m new to Graylog and loving it!
I have fluent-bit set up on a Kubernetes cluster, so that all of my JSON logs are input to Greylog in GELF format.
Here is an example JSON log:
{
"level": 30,
"time": 1618445502823,
"pid": 1,
"hostname": "email-platform-api-6f5b9666b8-k4rkh",
"req": {
"id": 42785,
"method": "POST",
"url": "/leads/car-insurance",
"headers": {
"host": "email-platform-api.company.com",
"x-request-id": "c55105118199d8de688fae1bab9b4389",
"x-real-ip": "10.244.2.156",
"x-forwarded-for": "10.244.2.156",
"x-forwarded-host": "email-platform-api.company.com",
"x-forwarded-port": "443",
"x-forwarded-proto": "https",
"x-scheme": "https",
"x-original-forwarded-for": "159.65.66.12",
"content-length": "184",
"accept-encoding": "gzip",
"cf-ipcountry": "US",
"cf-ray": "6400f547d8273615-SJC",
"cf-visitor": "{\"scheme\":\"https\"}",
"accept": "application/json",
"content-type": "application/json",
"user-agent": "axios/0.19.2",
"cf-connecting-ip": "159.65.66.12",
"cdn-loop": "cloudflare",
"cf-request-id": "097477a0e50000361507239000000001"
},
"remoteAddress": "::ffff:10.244.0.126",
"remotePort": 56776
},
"context": "Http",
"res": {
"statusCode": 201,
"headers": {
"x-powered-by": "Express",
"vary": "Origin",
"access-control-allow-credentials": "true",
"content-type": "text/html; charset=utf-8",
"content-length": "4",
"etag": "W/\"4-X/5TO4MPCKAyY0ipFgr6/IraRNs\""
}
},
"responseTime": 77,
"msg": "request completed"
}
Evidently, fluent-bit or Greylog is flattening this structure, because the Elasticsearch mappings that Greylog is generating look like this:
{
"mappings": {
"_doc": {
"dynamic_templates": [
{
"internal_fields": {
"match": "gl2_*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"store_generic": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
],
"properties": {
"context": {
"type": "keyword"
},
"currentListId": {
"type": "long"
},
"currentListLeadExists": {
"type": "keyword"
},
"deliveryServerId": {
"type": "long"
},
"domains": {
"type": "keyword"
},
"dto_email": {
"type": "keyword"
},
"dto_firstName": {
"type": "keyword"
},
"dto_lastName": {
"type": "keyword"
},
"dto_offerEngineSessionId": {
"type": "keyword"
},
"emailId": {
"type": "keyword"
},
"end": {
"type": "date"
},
"firstLeadId": {
"type": "keyword"
},
"formLength": {
"type": "keyword"
},
"from": {
"type": "date"
},
"fromEmail": {
"type": "keyword"
},
"fullEmail": {
"type": "keyword"
},
"full_message": {
"type": "text",
"analyzer": "standard"
},
"gl2_accounted_message_size": {
"type": "long"
},
"gl2_message_id": {
"type": "keyword"
},
"gl2_processing_timestamp": {
"type": "date",
"format": "uuuu-MM-dd HH:mm:ss.SSS"
},
"gl2_receive_timestamp": {
"type": "date",
"format": "uuuu-MM-dd HH:mm:ss.SSS"
},
"gl2_remote_ip": {
"type": "keyword"
},
"gl2_remote_port": {
"type": "long"
},
"gl2_source_input": {
"type": "keyword"
},
"gl2_source_node": {
"type": "keyword"
},
"historyRecordExists": {
"type": "keyword"
},
"historyRecordId": {
"type": "keyword"
},
"hostname": {
"type": "keyword"
},
"html": {
"type": "keyword"
},
"innerMessage_complaint_arrivalDate": {
"type": "date"
},
"innerMessage_complaint_complainedRecipients": {
"type": "keyword"
},
"innerMessage_complaint_complaintSubType": {
"type": "keyword"
},
"innerMessage_complaint_feedbackId": {
"type": "keyword"
},
"innerMessage_complaint_timestamp": {
"type": "date"
},
"innerMessage_mail_destination": {
"type": "keyword"
},
"innerMessage_mail_messageId": {
"type": "keyword"
},
"innerMessage_mail_sendingAccountId": {
"type": "keyword"
},
"innerMessage_mail_source": {
"type": "keyword"
},
"innerMessage_mail_sourceArn": {
"type": "keyword"
},
"innerMessage_mail_sourceIp": {
"type": "keyword"
},
"innerMessage_mail_timestamp": {
"type": "date"
},
"innerMessage_notificationType": {
"type": "keyword"
},
"job_attemptsMade": {
"type": "long"
},
"job_data_deliveryServerIds": {
"type": "keyword"
},
"job_delay": {
"type": "long"
},
"job_deliveryServerIds": {
"type": "keyword"
},
"job_id": {
"type": "keyword"
},
"job_name": {
"type": "keyword"
},
"job_opts_attempts": {
"type": "long"
},
"job_opts_delay": {
"type": "long"
},
"job_opts_jobId": {
"type": "keyword"
},
"job_opts_removeOnComplete": {
"type": "keyword"
},
"job_opts_removeOnFail": {
"type": "keyword"
},
"job_opts_timestamp": {
"type": "long"
},
"job_processedOn": {
"type": "long"
},
"job_progress": {
"type": "long"
},
"job_timestamp": {
"type": "long"
},
"kubernetes_annotations_checksum_repositories": {
"type": "keyword"
},
"kubernetes_annotations_checksum_ssh": {
"type": "keyword"
},
"kubernetes_container_hash": {
"type": "keyword"
},
"kubernetes_container_image": {
"type": "keyword"
},
"kubernetes_container_name": {
"type": "keyword"
},
"kubernetes_docker_id": {
"type": "keyword"
},
"kubernetes_host": {
"type": "keyword"
},
"kubernetes_labels_app": {
"type": "keyword"
},
"kubernetes_labels_pod-template-hash": {
"type": "keyword"
},
"kubernetes_labels_release": {
"type": "keyword"
},
"kubernetes_namespace_name": {
"type": "keyword"
},
"kubernetes_pod_id": {
"type": "keyword"
},
"kubernetes_pod_name": {
"type": "keyword"
},
"lastLeadId": {
"type": "keyword"
},
"leadId": {
"type": "keyword"
},
"level": {
"type": "long"
},
"message": {
"type": "text",
"analyzer": "standard"
},
"moveLeadReason": {
"type": "keyword"
},
"msPerEmail": {
"type": "long"
},
"msg": {
"type": "keyword"
},
"now": {
"type": "date"
},
"pid": {
"type": "long"
},
"pulledTo": {
"type": "long"
},
"queuedMs": {
"type": "long"
},
"req_headers_accept": {
"type": "keyword"
},
"req_headers_accept-encoding": {
"type": "keyword"
},
"req_headers_accept-language": {
"type": "keyword"
},
"req_headers_authorization": {
"type": "keyword"
},
"req_headers_cdn-loop": {
"type": "keyword"
},
"req_headers_cf-connecting-ip": {
"type": "keyword"
},
"req_headers_cf-ipcountry": {
"type": "keyword"
},
"req_headers_cf-ray": {
"type": "keyword"
},
"req_headers_cf-request-id": {
"type": "keyword"
},
"req_headers_cf-visitor": {
"type": "keyword"
},
"req_headers_connection": {
"type": "keyword"
},
"req_headers_content-length": {
"type": "keyword"
},
"req_headers_content-type": {
"type": "keyword"
},
"req_headers_cookie": {
"type": "keyword"
},
"req_headers_dnt": {
"type": "keyword"
},
"req_headers_host": {
"type": "keyword"
},
"req_headers_if-modified-since": {
"type": "keyword"
},
"req_headers_if-none-match": {
"type": "keyword"
},
"req_headers_referer": {
"type": "keyword"
},
"req_headers_sec-ch-ua": {
"type": "keyword"
},
"req_headers_sec-ch-ua-mobile": {
"type": "keyword"
},
"req_headers_sec-fetch-dest": {
"type": "keyword"
},
"req_headers_sec-fetch-mode": {
"type": "keyword"
},
"req_headers_sec-fetch-site": {
"type": "keyword"
},
"req_headers_sec-fetch-user": {
"type": "keyword"
},
"req_headers_upgrade-insecure-requests": {
"type": "keyword"
},
"req_headers_user-agent": {
"type": "keyword"
},
"req_headers_x-forwarded-for": {
"type": "keyword"
},
"req_headers_x-forwarded-host": {
"type": "keyword"
},
"req_headers_x-forwarded-port": {
"type": "keyword"
},
"req_headers_x-forwarded-proto": {
"type": "keyword"
},
"req_headers_x-original-forwarded-for": {
"type": "keyword"
},
"req_headers_x-real-ip": {
"type": "keyword"
},
"req_headers_x-request-id": {
"type": "keyword"
},
"req_headers_x-requested-with": {
"type": "keyword"
},
"req_headers_x-scheme": {
"type": "keyword"
},
"req_id": {
"type": "long"
},
"req_method": {
"type": "keyword"
},
"req_remoteAddress": {
"type": "keyword"
},
"req_remotePort": {
"type": "long"
},
"req_url": {
"type": "keyword"
},
"res_headers_accept-ranges": {
"type": "keyword"
},
"res_headers_access-control-allow-credentials": {
"type": "keyword"
},
"res_headers_cache-control": {
"type": "keyword"
},
"res_headers_content-length": {
"type": "keyword"
},
"res_headers_content-type": {
"type": "keyword"
},
"res_headers_etag": {
"type": "keyword"
},
"res_headers_last-modified": {
"type": "keyword"
},
"res_headers_location": {
"type": "keyword"
},
"res_headers_vary": {
"type": "keyword"
},
"res_headers_x-powered-by": {
"type": "keyword"
},
"res_statusCode": {
"type": "long"
},
"responseTime": {
"type": "long"
},
"segmentId": {
"type": "long"
},
"sesMessageId": {
"type": "keyword"
},
"source": {
"type": "text",
"analyzer": "analyzer_keyword",
"fielddata": true
},
"start": {
"type": "date"
},
"stream": {
"type": "keyword"
},
"streams": {
"type": "keyword"
},
"subject": {
"type": "keyword"
},
"targetListId": {
"type": "long"
},
"targetListOwnerGroupId": {
"type": "long"
},
"targetListOwnerId": {
"type": "long"
},
"time": {
"type": "date"
},
"timestamp": {
"type": "date",
"format": "uuuu-MM-dd HH:mm:ss.SSS"
},
"to": {
"type": "date"
},
"toEmail": {
"type": "keyword"
},
"toIdeal": {
"type": "date"
},
"workflowBroadcastId": {
"type": "long"
},
"workflowId": {
"type": "long"
},
"workflowNodeId": {
"type": "long"
},
"workflowNodeLeadsLength": {
"type": "long"
}
}
}
}
}
Obviously, this won’t work, because when service A logs JSON with key “foo” as a date type, Elasticsearch makes a mapping for “foo” as a date type. Then later, service B logs JSON with key “foo” as a long type, the log from service B never makes it into Greylog because it fails to insert into the Elasticsearch index! So I’m seeing tons of this kind of error, and logs are not making it in.
How can I resolve this? Do I need to create my own Elasticsearch indices for each service in my entire system?
Lastly, it seems like someone could spam my service with a bunch of random headers, and since headers are a key in the JSON structure, this would create millions of ES mappings and destroy my logging setup, right? How do we prevent this? More generically, all of my services need to log arbitrary JSON without each key becoming an ES mapping, unless ES can support billions of mappings? I’m so confused.
Thanks so much in advance for any guidance here.