defined a new index in Graylog3 with “cloudfront” index prefix. I have a graylog pipeline configured to obtain geolocation information:
rule "cloudfrontGeoIP"
when
has_field("c-ip")
then
let geo = lookup("GeoipLookupTable",to_string($message."c-ip"));
set_field("src_ip_geolocation", geo["coordinates"]);
set_field("src_ip_geo_country_code", geo["country"].iso_code);
set_field("src_ip_geo_country_name", geo["country"].names.en);
set_field("src_ip_geo_city_name", geo["city"].names.en);
end
So I update the mapping to create a new field src_ip_location based on a copy of src_ip_geolocation:
{
"template": "cloudfront_*",
"mappings" : {
"message" : {
"properties" : {
"src_ip_geolocation": {
"type": "text",
"copy_to": "src_ip_location"
},
"src_ip_location": {
"type": "geo_point"
}
}
}
}
}
curl -X PUT -d @graylog-custom-mapping.json -H 'Content-Type: application/json' 'https://<es_url>/_template/graylog-custom-mapping?pretty'
{
"acknowledged" : true
}
The new template is created successfully in Elasticsearch domain:
curl -XGET 'https://<es_domain>/_template/graylog-custom-mapping?pretty'
{
"graylog-custom-mapping": {
"order": 0,
"index_patterns": [
"cloudfront_*"
],
"settings": {},
"mappings": {
"message": {
"properties": {
"src_ip_geolocation": {
"type": "text",
"copy_to": "src_ip_location"
},
"src_ip_location": {
"type": "geo_point"
}
}
}
},
"aliases": {}
}
}
Then I rotate the active index to force a new index recreation System>Indices> | Select “Cloudfront index” Maintenace>Rotate Active write index.
If I check now the cloudfront-template message properties are the default based on graylog-internal template and the custom properties are not applied:
curl -X GET 'https://<url_domain>/_template/cloudfront-template?pretty='
{
"cloudfront-template": {
"order": -1,
"index_patterns": [
"cloudfront_*"
],
"settings": {
"index": {
"analysis": {
"analyzer": {
"analyzer_keyword": {
"filter": "lowercase",
"tokenizer": "keyword"
}
}
}
}
},
"mappings": {
"message": {
"properties": {
"message": {
"type": "text",
"analyzer": "standard",
"fielddata": false
},
"full_message": {
"type": "text",
"analyzer": "standard",
"fielddata": false
},
"timestamp": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss.SSS"
},
"gl2_receive_timestamp": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss.SSS"
},
"gl2_processing_timestamp": {
"type": "date",
"format": "yyyy-MM-dd HH:mm:ss.SSS"
},
"source": {
"type": "text",
"analyzer": "analyzer_keyword",
"fielddata": true
},
"streams": {
"type": "keyword"
}
},
"dynamic_templates": [
{
"internal_fields": {
"match": "gl2_*",
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
},
{
"store_generic": {
"match_mapping_type": "string",
"mapping": {
"type": "keyword"
}
}
}
],
"_source": {
"enabled": true
}
}
},
"aliases": {}
}
}
What’s wrong ?
Graylog 3.1.3 + ES 6.4
Thank’s