1. Describe your incident:
At indices page I get “Could not retrieve Elasticsearch cluster health. Fetching Elasticsearch cluster health failed: There was an error fetching a resource: Internal Server Error. Additional information: Couldn’t read Elasticsearch cluster health” message
The whole story:
I migrate the graylog from a standalone server to docker with elasticsearch. There was no error.
I did upgrades in little steps from GL 2.X, Elastic 5.X, mongo 3.x.
After I tried to upgrade elasticsearch to opensearch, there was a problem, the opensearch container con’t open all elastic data, so the end “solution” was delete all data from elasticsearch, and start with a new empty opensearch. (It is a test system, settings are important, data not.)(I tied with compatibe and notmal mode)
The mongodb contains the old/migrated data.
Now the system does the following errors:
The mentioned error message,
Overwiev logs full with “There is no index target to point to. Creating one now.”
It writes out the logs, but it doesn’t delete the old indices.
The search doesn’t work, because it tied to access an old index. (The “recalculate index ranges” doesn’t work)
Can’t open an index set.
2. Describe your environment:
I run it in docker, here is the compose file.
docker-compose.yml
version: "3.8"
services:
graylog-mongodb:
image: "mongo:5.0.14"
container_name: graylog-mongodb
networks:
graylog_net:
ipv4_address: 172.20.0.2
volumes:
- "/srv/graylog/mongodb:/data/db"
environment:
TZ: "Europe/Budapest"
restart: "always"
graylog-opensearch:
# image: "docker.elastic.co/elasticsearch/elasticsearch:7.17.6"
image: "opensearchproject/opensearch:2.3.0"
# image: "opensearchproject/opensearch:1.3.2"
container_name: graylog-opensearch
networks:
graylog_net:
ipv4_address: 172.20.0.3
environment:
- "node.name=graylog-opensearch"
# - "compatibility.override_main_response_version=true"
- "cluster.name=graylog"
- "OPENSEARCH_JAVA_OPTS=-Xms1g -Xmx1g"
- "bootstrap.memory_lock=true"
- "discovery.type=single-node"
- "action.auto_create_index=false"
- "TZ=Europe/Budapest"
- "plugins.security.ssl.http.enabled=false"
- "plugins.security.disabled=true"
ulimits:
memlock:
hard: -1
soft: -1
volumes:
- "/srv/graylog/opensearch:/usr/share/opensearch/data"
# - "/srv/graylog/opensearch:/usr/share/elasticsearch/data"
restart: "always"
graylog-server:
hostname: "server"
image: "graylog/graylog:5.0.2"
container_name: graylog-server
depends_on:
graylog-opensearch:
condition: "service_started"
graylog-mongodb:
condition: "service_started"
networks:
graylog_net:
ipv4_address: 172.20.0.4
entrypoint: "/usr/bin/tini -- wait-for-it graylog-opensearch:9200 -- /docker-entrypoint.sh"
environment:
GRAYLOG_NODE_ID_FILE: "/usr/share/graylog/data/config/node-id"
GRAYLOG_PASSWORD_SECRET: "XX"
GRAYLOG_ROOT_PASSWORD_SHA2: "XX"
GRAYLOG_HTTP_BIND_ADDRESS: "0.0.0.0:9000"
GRAYLOG_HTTP_EXTERNAL_URI: "http://localhost:9000/"
GRAYLOG_ELASTICSEARCH_HOSTS: "http://graylog-opensearch:9200"
GRAYLOG_MONGODB_URI: "mongodb://graylog-mongodb:27017/graylog"
TZ: "Europe/Budapest"
GRAYLOG_TIMEZONE: "Europe/Budapest"
GRAYLOG_ROOT_TIMEZONE: "Europe/Budapest"
ports:
- "1514:1514/tcp" # Syslog
- "9000:9000/tcp" # Server API
volumes:
- "/srv/graylog/graylog/node-id:/usr/share/graylog/data/config/node-id:ro"
- "/srv/graylog/graylog/data:/usr/share/graylog/data/data"
- "/srv/graylog/graylog/journal:/usr/share/graylog/data/journal"
restart: "always"
networks:
graylog_net:
driver: bridge
driver_opts:
com.docker.network.bridge.host_binding_ipv4: "192.168.254.20"
ipam:
config:
- subnet: 172.20.0.0/24
I tried to check the elastic cluster status:
root@bds-docker:/srv/graylog/docker-graylog# docker exec -it graylog-server bash
graylog@server:~$ curl -XGET http://elasticsearch:9200/_cluster/health?pretty
{
"cluster_name" : "graylog",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"discovered_master" : true,
"discovered_cluster_manager" : true,
"active_primary_shards" : 40,
"active_shards" : 40,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
graylog@server:~$ curl -XGET http://graylog-opensearch:9200/_cluster/health?pretty
{
"cluster_name" : "graylog",
"status" : "green",
"timed_out" : false,
"number_of_nodes" : 1,
"number_of_data_nodes" : 1,
"discovered_master" : true,
"discovered_cluster_manager" : true,
"active_primary_shards" : 40,
"active_shards" : 40,
"relocating_shards" : 0,
"initializing_shards" : 0,
"unassigned_shards" : 0,
"delayed_unassigned_shards" : 0,
"number_of_pending_tasks" : 0,
"number_of_in_flight_fetch" : 0,
"task_max_waiting_in_queue_millis" : 0,
"active_shards_percent_as_number" : 100.0
}
Any idea?
, glad you resolved it without losing data.