MapperParsingException EventDate

Graylog Central (peer support)
1

Hi, i am getting the indexing failure attempts for graylog,

[497]: index [graylog_391], type [message], id [23bc82e8-7d97-11e7-99b8-2cd02dbeb2ea], message [MapperParsingException[failed to parse [EventDate]]; nested: IllegalArgumentException[Invalid format: “2017/08/10” is malformed at “/08/10”];]
[498]: index [graylog_391], type [message], id [23bca9f0-7d97-11e7-99b8-2cd02dbeb2ea], message [MapperParsingException[failed to parse [EventDate]]; nested: IllegalArgumentException[Invalid format: “2017/08/10” is malformed at “/08/10”];]
[499]: index [graylog_391], type [message], id [23bca9f1-7d97-11e7-99b8-2cd02dbeb2ea], message [MapperParsingException[failed to parse [EventDate]]; nested: IllegalArgumentException[Invalid format: “2017/08/10” is malformed at “/08/10”];]

I followed this custom mapping “http://docs.graylog.org/en/2.2/pages/configuration/elasticsearch.html#custom-index-mappings” and rotated the indice but still i see the logs rolling with those messages.

We are using graylog 2.2.3 with ElasticSearch 2.4.4

Can you please assist.

Thanks, Hema.

2

What’s the index template and index mapping you’ve created?

3

Hi Jochen, Its the same as mentioned in the docs. Here is the output from our server,

curl -X GET 'http://localhost:9200/graylog_deflector/_mapping?pretty'
{
  "graylog_392" : {
    "mappings" : {
      "message" : {
        "dynamic_templates" : [ {
          "internal_fields" : {
            "mapping" : {
              "index" : "not_analyzed",
              "type" : "string"
            },
            "match" : "gl2_*"
          }
        }, {
          "store_generic" : {
            "mapping" : {
              "index" : "not_analyzed"
            },
            "match" : "*"
          }
        } ],
        "properties" : {
          "PA_Risk" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "PA_Threat" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "ReceiveTime" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "SerialNumber" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "Subtype" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "Type" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "application_name" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "connection_id" : {
            "type" : "long"
          },
          "connection_requests" : {
            "type" : "long"
          },
          "facility" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "from_nginx" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "full_message" : {
            "type" : "string",
            "analyzer" : "standard"
          },
          "gl2_remote_ip" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "gl2_remote_port" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "gl2_source_input" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "gl2_source_node" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "host" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "http_method" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "http_referer" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "http_response_code" : {
            "type" : "long"
          },
          "http_user_agent" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "http_version" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "ingest_time" : {
            "type" : "date",
            "format" : "strict_date_time"
          },
          "level" : {
            "type" : "long"
          },
          "message" : {
            "type" : "string",
            "analyzer" : "standard"
          },
          "millis" : {
            "type" : "double"
          },
          "nginx_access" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "nginx_error" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "process_id" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "remote_addr" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "remote_user" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "request_path" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "request_verb" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "response_bytes" : {
            "type" : "long"
          },
          "response_status" : {
            "type" : "long"
          },
          "server" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "source" : {
            "type" : "string",
            "analyzer" : "analyzer_keyword"
          },
          "streams" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "timestamp" : {
            "type" : "date",
            "format" : "yyyy-MM-dd HH:mm:ss.SSS"
          },
          "took_ms" : {
            "type" : "long"
          }
        }
      }
    }
  }
}
4

I don’t see any mapping for the “EventDate” field in your mapping…

5

Hi Jochen,
This was the custom mapping i used, could you help me where i should add the EventDate field, please.

graylog-custom-mapping.json
{
  "template": "graylog_*",
  "mappings" : {
    "message" : {
      "properties" : {
        "http_method" : {
          "type" : "string",
          "index" : "not_analyzed"
        },
        "http_response_code" : {
          "type" : "long"
        },
        "ingest_time" : {
          "type" : "date",
          "format": "strict_date_time"
        },
        "took_ms" : {
          "type" : "long"
        }
      }
    }
  }
}
6

First, please format your text snippets properly. This will make them much easier to read: Markdown Reference

Example:

```
Text
````

There also is no mapping for “EventDate”.

Since the automatically discovered type of “EventDate” is not identical to what Graylog is trying to write into it, you have to define a proper mapping for it, just like you did for the other fields.

7

Hi Jochen,
I have not idea where to use the EventDate in CustomMapping, could you help format it for me, please?

Thanks, Hema.

8

Hi Jochen,
I have changed the Json as below and after executing, i could not find the custom mapping in the index

{
  "template": "graylog_*",
  "mappings" : {
    "message" : {
      "properties" : {
        "EventDate" : {
          "type" : "date",
          "format": "strict_date_optional_time"
        }
      }
    }
  }
}

curl -X GET 'http://localhost:9200/graylog_deflector/_mapping?pretty'                                 {
  "graylog_404" : {
    "mappings" : {
      "message" : {
        "dynamic_templates" : [ {
          "internal_fields" : {
            "mapping" : {
              "index" : "not_analyzed",
              "type" : "string"
            },
            "match" : "gl2_*"
          }
        }, {
          "store_generic" : {
            "mapping" : {
              "index" : "not_analyzed"
            },
            "match" : "*"
          }
        } ],
        "properties" : {
          "Location" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "PA_Risk" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "PA_Threat" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "ReceiveTime" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "SerialNumber" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "Subtype" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "Type" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "application_name" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "connection_id" : {
            "type" : "long"
          },
          "connection_requests" : {
            "type" : "long"
          },
          "device_product" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "device_vendor" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "device_version" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "dvc" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "event_class_id" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "facility" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "from_nginx" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "full_message" : {
            "type" : "string",
            "analyzer" : "standard"
          },
          "gl2_remote_ip" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "gl2_remote_port" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "gl2_source_input" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "gl2_source_node" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "http_method" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "http_referer" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "http_response_code" : {
            "type" : "long"
          },
          "http_user_agent" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "http_version" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "ingest_time" : {
            "type" : "date",
            "format" : "strict_date_time"
          },
          "level" : {
            "type" : "long"
          },
          "message" : {
            "type" : "string",
            "analyzer" : "standard"
          },
          "millis" : {
            "type" : "double"
          },
          "msg" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "name" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "nginx_access" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "process_id" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "remote_addr" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "remote_user" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "request_path" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "request_verb" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "response_bytes" : {
            "type" : "long"
          },
          "response_status" : {
            "type" : "long"
          },
          "severity" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "severity_number" : {
            "type" : "long"
          },
          "source" : {
            "type" : "string",
            "analyzer" : "analyzer_keyword"
          },
          "src" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "streams" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "suser" : {
            "type" : "string",
            "index" : "not_analyzed"
          },
          "timestamp" : {
            "type" : "date",
            "format" : "yyyy-MM-dd HH:mm:ss.SSS"
          },
          "took_ms" : {
            "type" : "long"
          }
        }
      }
    }
  }
}

Can you please advise, if there is something wrong?

Thanks, Hema.

9

Hi Jochen,
Ignore my last message, i recycled the index and could see the template now, but the error still exist for the EventDate

2017-08-16T08:46:30.823-06:00 ERROR [Messages] Failed to index [60] messages. Please check the index error log in your web interface for the reason. Error: failure in bulk execution:

[65]: index [graylog_405], type [message], id [adaffc31-8291-11e7-99b8-2cd02dbeb2ea], message [MapperParsingException[failed to parse [EventDate]]; nested: IllegalArgumentException[Invalid format: "2017/08/16" is malformed at "/08/16"];]

[51]: index [graylog_405], type [message], id [adafae17-8291-11e7-99b8-2cd02dbeb2ea], message [MapperParsingException[failed to parse [generated_date]]; nested: IllegalArgumentException[Invalid format: "2017/08/16" is malformed at "/08/16"];]

"properties" : {
          "EventDate" : {
            "type" : "date",
            "format" : "strict_date_optional_time"
          },

Can you advise please.

Thanks, Hema.

10

Your EventDate (and generated_date) field still has the wrong format or you’ve configured the wrong date format in the index mapping, depending which way you look at it.

Please refer to https://www.elastic.co/guide/en/elasticsearch/reference/5.5/mapping-date-format.html#built-in-date-formats for details about the built-in date formats.

11

This topic was automatically closed 14 days after the last reply. New replies are no longer allowed.