Cannot apply new custom index template

defined a new index in Graylog3 with “cloudfront” index prefix. I have a graylog pipeline configured to obtain geolocation information:

rule "cloudfrontGeoIP"
when 
has_field("c-ip")
then 
let geo = lookup("GeoipLookupTable",to_string($message."c-ip"));
set_field("src_ip_geolocation", geo["coordinates"]);
set_field("src_ip_geo_country_code", geo["country"].iso_code);
set_field("src_ip_geo_country_name", geo["country"].names.en);
set_field("src_ip_geo_city_name", geo["city"].names.en); 
end

So I update the mapping to create a new field src_ip_location based on a copy of src_ip_geolocation:

{  
  "template": "cloudfront_*",  
  "mappings" : {
    "message" : {
      "properties" : {    
        "src_ip_geolocation": {
          "type": "text",
          "copy_to": "src_ip_location"
        },
        "src_ip_location": { 
           "type": "geo_point" 
        }        
      }
    }
  }
}
curl -X PUT -d @graylog-custom-mapping.json -H 'Content-Type: application/json' 'https://<es_url>/_template/graylog-custom-mapping?pretty'
{
  "acknowledged" : true
}

The new template is created successfully in Elasticsearch domain:

curl -XGET 'https://<es_domain>/_template/graylog-custom-mapping?pretty'

{
  "graylog-custom-mapping": {
    "order": 0,
    "index_patterns": [
      "cloudfront_*"
    ],
    "settings": {},
    "mappings": {
      "message": {
        "properties": {
          "src_ip_geolocation": {
            "type": "text",
            "copy_to": "src_ip_location"
          },
          "src_ip_location": {
            "type": "geo_point"
          }
        }
      }
    },
    "aliases": {}
  }
}

Then I rotate the active index to force a new index recreation System>Indices> | Select “Cloudfront index” Maintenace>Rotate Active write index.

If I check now the cloudfront-template message properties are the default based on graylog-internal template and the custom properties are not applied:

curl -X GET 'https://<url_domain>/_template/cloudfront-template?pretty=' 

{
  "cloudfront-template": {
    "order": -1,
    "index_patterns": [
      "cloudfront_*"
    ],
    "settings": {
      "index": {
        "analysis": {
          "analyzer": {
            "analyzer_keyword": {
              "filter": "lowercase",
              "tokenizer": "keyword"
            }
          }
        }
      }
    },
    "mappings": {
      "message": {
        "properties": {
          "message": {
            "type": "text",
            "analyzer": "standard",
            "fielddata": false
          },
          "full_message": {
            "type": "text",
            "analyzer": "standard",
            "fielddata": false
          },
          "timestamp": {
            "type": "date",
            "format": "yyyy-MM-dd HH:mm:ss.SSS"
          },
          "gl2_receive_timestamp": {
            "type": "date",
            "format": "yyyy-MM-dd HH:mm:ss.SSS"
          },
          "gl2_processing_timestamp": {
            "type": "date",
            "format": "yyyy-MM-dd HH:mm:ss.SSS"
          },
          "source": {
            "type": "text",
            "analyzer": "analyzer_keyword",
            "fielddata": true
          },
          "streams": {
            "type": "keyword"
          }
        },
        "dynamic_templates": [
          {
            "internal_fields": {
              "match": "gl2_*",
              "match_mapping_type": "string",
              "mapping": {
                "type": "keyword"
              }
            }
          },
          {
            "store_generic": {
              "match_mapping_type": "string",
              "mapping": {
                "type": "keyword"
              }
            }
          }
        ],
        "_source": {
          "enabled": true
        }
      }
    },
    "aliases": {}
  }
}

What’s wrong ?

Graylog 3.1.3 + ES 6.4

Thank’s

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.