Adding custom index mapping

Graylog Tech Challenges
1

Related to : Trouble restring a search to a field value - Daily Challenges - Graylog Community

I started looking at some of the fields I get from my unifi syslog entries running through a pipe line. I get a field called rule_name that I want to be able to search on, and I think to do this I should use a custom_index_mapper.

I’ve put everything from my unifi stream in it’s own index set, I believe it’s called unifi_0.

[root@elasticsearch elasticsearch]# curl -X GET 'http://localhost:9200/_template/unifi_0?pretty'
{ }

I think this is the index definition:

[root@elasticsearch elasticsearch]# curl -XGET -u graylog 'http://localhost:9200/unifi_0/?pretty'
Enter host password for user 'graylog':
{
  "unifi_0" : {
    "aliases" : {
      "unifi_deflector" : { }
    },
    "mappings" : {
      "dynamic_templates" : [
        {
          "internal_fields" : {
            "match" : "gl2_*",
            "match_mapping_type" : "string",
            "mapping" : {
              "type" : "keyword"
            }
          }
        },
        {
          "store_generic" : {
            "match_mapping_type" : "string",
            "mapping" : {
              "type" : "keyword"
            }
          }
        }
      ],
      "properties" : {
        "3way_hs" : {
          "type" : "keyword"
        },
        "MAC_addr" : {
          "type" : "keyword"
        },
        "dst_addr" : {
          "type" : "keyword"
        },
        "dst_port" : {
          "type" : "keyword"
        },
        "facility" : {
          "type" : "keyword"
        },
        "facility_num" : {
          "type" : "long"
        },
        "full_message" : {
          "type" : "text",
          "analyzer" : "standard"
        },
        "gl2_accounted_message_size" : {
          "type" : "long"
        },
        "gl2_message_id" : {
          "type" : "keyword"
        },
        "gl2_processing_timestamp" : {
          "type" : "date",
          "format" : "uuuu-MM-dd HH:mm:ss.SSS"
        },
        "gl2_receive_timestamp" : {
          "type" : "date",
          "format" : "uuuu-MM-dd HH:mm:ss.SSS"
        },
        "gl2_remote_ip" : {
          "type" : "keyword"
        },
        "gl2_remote_port" : {
          "type" : "long"
        },
        "gl2_source_input" : {
          "type" : "keyword"
        },
        "gl2_source_node" : {
          "type" : "keyword"
        },
        "id" : {
          "type" : "keyword"
        },
        "in_interface" : {
          "type" : "keyword"
        },
        "length_1" : {
          "type" : "keyword"
        },
        "length_2" : {
          "type" : "keyword"
        },
        "level" : {
          "type" : "long"
        },
        "message" : {
          "type" : "text",
          "analyzer" : "standard"
        },
        "out_interface" : {
          "type" : "keyword"
        },
        "prec" : {
          "type" : "keyword"
        },
        "protocol" : {
          "type" : "keyword"
        },
        "res" : {
          "type" : "keyword"
        },
        "rule_name" : {
          "type" : "keyword"
        },
        "source" : {
          "type" : "text",
          "analyzer" : "analyzer_keyword",
          "fielddata" : true
        },
        "src_addr" : {
          "type" : "keyword"
        },
        "src_port" : {
          "type" : "keyword"
        },
        "streams" : {
          "type" : "keyword"
        },
        "timestamp" : {
          "type" : "date",
          "format" : "uuuu-MM-dd HH:mm:ss.SSS"
        },
        "tos" : {
          "type" : "keyword"
        },
        "ttl" : {
          "type" : "keyword"
        },
        "urgp" : {
          "type" : "keyword"
        },
        "window" : {
          "type" : "keyword"
        }
      }
    },
    "settings" : {
      "index" : {
        "number_of_shards" : "4",
        "provided_name" : "unifi_0",
        "creation_date" : "1625101698685",
        "analysis" : {
          "analyzer" : {
            "analyzer_keyword" : {
              "filter" : "lowercase",
              "tokenizer" : "keyword"
            }
          }
        },
        "number_of_replicas" : "0",
        "uuid" : "",
        "version" : {
          "created" : "7100299"
        }
      }
    }
  }
}

So the field I want to initially index is rule_name

"rule_name" : {
          "type" : "keyword"
        },

I’ve got this is the custom mapping:

{
  "template": "unifi_0",
  "mappings" : {
    "message" : {
      "properties" : {
        "rule_name" : {
          "type" : "string",
          "index" : "not_analyzed"

        }
      }
    }
  }
}

And I’m getting the following when putting:

[root@elasticsearch elasticsearch]# curl -XPUT -d @graylog_unifi_template.json 'http://localhost:9200/_template/unifi_0?pretty' -H 'Content-Type: application/JSON'
{
  "error" : {
    "root_cause" : [
      {
        "type" : "mapper_parsing_exception",
        "reason" : "Root mapping definition has unsupported parameters:  [message : {properties={rule_name={index=not_analyzed, type=string}}}]"
      }
    ],
    "type" : "mapper_parsing_exception",
    "reason" : "Failed to parse mapping [_doc]: Root mapping definition has unsupported parameters:  [message : {properties={rule_name={index=not_analyzed, type=string}}}]",
    "caused_by" : {
      "type" : "mapper_parsing_exception",
      "reason" : "Root mapping definition has unsupported parameters:  [message : {properties={rule_name={index=not_analyzed, type=string}}}]"
    }
  },
  "status" : 400
}

I’m not sure I understand the message. I based it off this: Elasticsearch — Graylog 3.2.0 documentation and replaced ‘http_method’ with rule_name.

Any guidance appreciated.

3

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.