1. Describe your incident:
I installed Graylog 5.1 with Opensearch 2.5, but Graylog can’t start with error message.
I used official OpenSearch ansible playbook with included roles to install OS cluster on 3 nodes (all data,master)
2. Describe your environment:
- OS Information:
root@hv1:/usr/share/opensearch/config# lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description: Ubuntu 22.04.3 LTS
Release: 22.04
Codename: jammy
- Package Version:
root@hv1:/usr/share/opensearch/config# apt policy graylog-server
graylog-server:
Установлен: 5.1.4-1
Кандидат: 5.1.4-1
Таблица версий:
*** 5.1.4-1 500
500 https://packages.graylog2.org/repo/debian stable/5.1 amd64 Packages
100 /var/lib/dpkg/status
5.1.3-1 500
500 https://packages.graylog2.org/repo/debian stable/5.1 amd64 Packages
5.1.2-1 500
- Service logs, configurations, and environment variables:
tail -f /var/log/graylog-server/server.log
2023-09-05T06:18:51.394Z INFO [VersionProbe] Elasticsearch is not available. Retry #4
2023-09-05T06:18:56.405Z ERROR [VersionProbe] Unable to retrieve version from Elasticsearch node: Unsupported or unrecognized SSL message. - Unsupported or unrecognized SSL message.
cat /etc/graylog/server/server.conf | grep -vE '^\s*(#|$)'
is_leader = true
node_id_file = /etc/graylog/server/node-id
password_secret = Pa$$w0rd
root_username = admin
root_email = "admin@example.local"
bin_dir = /usr/share/graylog-server/bin
data_dir = /var/lib/graylog-server
plugin_dir = /usr/share/graylog-server/plugin
http_bind_address = 127.0.0.1:9000
http_publish_uri = http://10.81.156.103:9000/
stream_aware_field_types=false
elasticsearch_hosts = https://admin:Pa$$w0rd@os1.example.local:9200,https://admin:Pa$$w0rd@os2.example.local:9200,https://admin:Pa$$w0rd@os3.example.local:9200
elasticsearch_disable_version_check = true
allow_leading_wildcard_searches = false
allow_highlighting = false
output_batch_size = 500
output_flush_interval = 1
output_fault_count_threshold = 5
output_fault_penalty_seconds = 30
processbuffer_processors = 5
outputbuffer_processors = 3
processor_wait_strategy = blocking
ring_size = 65536
inputbuffer_ring_size = 65536
inputbuffer_processors = 2
inputbuffer_wait_strategy = blocking
message_journal_enabled = true
message_journal_dir = /var/lib/graylog-server/journal
lb_recognition_period_seconds = 3
mongodb_max_connections = 1000
mongodb_uri = mongodb://graylog:Pa$$w0rd@mongodb1.example.local:27017,mongodb2.example.local:27017,mongodb3.example.local:27017/graylog?replicaSet=rs0
root_password_sha2 = superdupersecrethash
cat /usr/share/opensearch/config/opensearch.yml | grep -vE '^\s*(#|$)'
cluster.name: "os-cluster"
node.name: "os1"
network.host: "0.0.0.0"
http.port: 9200
bootstrap.memory_lock: true
discovery.seed_hosts: ["os1","os2","os3"]
node.roles: [data,master]
plugins.security.disabled: true
plugins.security.allow_default_init_securityindex: true
plugins.security.audit.type: internal_opensearch
plugins.security.enable_snapshot_restore_privilege: true
plugins.security.check_snapshot_restore_write_privileges: true
plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
plugins.security.ssl.transport.pemcert_filepath: os1.pem
plugins.security.ssl.transport.pemkey_filepath: os1.key
plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem
plugins.security.ssl.transport.enforce_hostname_verification: false
plugins.security.ssl.transport.resolve_hostname: false
plugins.security.ssl.http.enabled: true
plugins.security.ssl.http.pemcert_filepath: os1_http.pem
plugins.security.ssl.http.pemkey_filepath: os1_http.key
plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
plugins.security.nodes_dn:
- CN=os1.example.local,OU=Ops,O=example.local\, Inc.,DC=example.local
- CN=os2.example.local,OU=Ops,O=example.local\, Inc.,DC=example.local
- CN=os3.example.local,OU=Ops,O=example.local\, Inc.,DC=example.local
plugins.security.authcz.admin_dn:
- CN=admin.example.local,OU=Ops,O=example.local\, Inc.,DC=example.local
curl -XGET 'http://os1.example.local:9200/_nodes/_all?pretty=true'
"_nodes" : {
"total" : 1,
"successful" : 1,
"failed" : 0
},
"cluster_name" : "os-cluster",
"nodes" : {
"kR6jiCd9SyyR6zkTy8o5FQ" : {
"name" : "os1",
"transport_address" : "10.81.156.103:9300",
"host" : "10.81.156.103",
"ip" : "10.81.156.103",
"version" : "2.5.0",
"build_type" : "tar",
"build_hash" : "b8a8b6c4d7fc7a7e32eb2cb68ecad8057a4636ad",
"total_indexing_buffer" : 214748364,
"roles" : [
"data",
"master"
],
"attributes" : {
"shard_indexing_pressure_enabled" : "true"
},
"settings" : {
"cluster" : {
"name" : "os-cluster"
},
"node" : {
"name" : "os1",
"pidfile" : "/usr/share/opensearch/opensearch.pid",
"attr" : {
"shard_indexing_pressure_enabled" : "true"
},
"roles" : [
"data",
"master"
]
},
"path" : {
"logs" : "/usr/share/opensearch/logs",
"home" : "/usr/share/opensearch"
},
"discovery" : {
"seed_hosts" : [
"os1",
"os2",
"os3"
]
},
"plugins" : {
"security" : {
"restapi" : {
"roles_enabled" : [
"all_access",
"security_rest_api_access"
]
},
"authcz" : {
"admin_dn" : [
"CN=admin.example.local,OU=Ops,O=example.local\\, Inc.,DC=example.local"
]
},
"nodes_dn" : [
"CN=os1.example.local,OU=Ops,O=example.local\\, Inc.,DC=example.local",
"CN=os2.example.local,OU=Ops,O=example.local\\, Inc.,DC=example.local",
"CN=os3.example.local,OU=Ops,O=example.local\\, Inc.,DC=example.local"
]
}
},
"client" : {
"type" : "node"
},
"http" : {
"type" : {
"default" : "netty4"
},
"port" : "9200"
},
"index" : {
"store" : {
"hybrid" : {
"mmap" : {
"extensions" : [
"nvd",
"dvd",
"tim",
"tip",
"dim",
"kdd",
"kdi",
"cfs",
"doc",
"vec",
"vex"
]
}
}
}
},
"bootstrap" : {
"memory_lock" : "true"
},
"transport" : {
"type" : {
"default" : "netty4"
}
},
"network" : {
"host" : "0.0.0.0"
}
},
"os" : {
"refresh_interval_in_millis" : 1000,
"name" : "Linux",
"pretty_name" : "Ubuntu 22.04.3 LTS",
"arch" : "amd64",
"version" : "5.15.0-79-generic",
"available_processors" : 32,
"allocated_processors" : 32
},
"process" : {
"refresh_interval_in_millis" : 1000,
"id" : 449615,
"mlockall" : true
},
"jvm" : {
"pid" : 449615,
"version" : "17.0.5",
"vm_name" : "OpenJDK 64-Bit Server VM",
"vm_version" : "17.0.5+8",
"vm_vendor" : "Eclipse Adoptium",
"bundled_jdk" : true,
"using_bundled_jdk" : true,
"start_time_in_millis" : 1693892272125,
"mem" : {
"heap_init_in_bytes" : 2147483648,
"heap_max_in_bytes" : 2147483648,
"non_heap_init_in_bytes" : 7667712,
"non_heap_max_in_bytes" : 0,
"direct_max_in_bytes" : 0
},
"gc_collectors" : [
"G1 Young Generation",
"G1 Old Generation"
],
"memory_pools" : [
"CodeHeap 'non-nmethods'",
"Metaspace",
"CodeHeap 'profiled nmethods'",
"Compressed Class Space",
"G1 Eden Space",
"G1 Old Gen",
"G1 Survivor Space",
"CodeHeap 'non-profiled nmethods'"
],
"using_compressed_ordinary_object_pointers" : "true",
"input_arguments" : [
"-Xshare:auto",
"-Dopensearch.networkaddress.cache.ttl=60",
"-Dopensearch.networkaddress.cache.negative.ttl=10",
"-XX:+AlwaysPreTouch",
"-Xss1m",
"-Djava.awt.headless=true",
"-Dfile.encoding=UTF-8",
"-Djna.nosys=true",
"-XX:-OmitStackTraceInFastThrow",
"-XX:+ShowCodeDetailsInExceptionMessages",
"-Dio.netty.noUnsafe=true",
"-Dio.netty.noKeySetOptimization=true",
"-Dio.netty.recycler.maxCapacityPerThread=0",
"-Dio.netty.allocator.numDirectArenas=0",
"-Dlog4j.shutdownHookEnabled=false",
"-Dlog4j2.disable.jmx=true",
"-Djava.locale.providers=SPI,COMPAT",
"-Xms2g",
"-Xmx2g",
"-XX:+UseG1GC",
"-XX:G1ReservePercent=25",
"-XX:InitiatingHeapOccupancyPercent=30",
"-Djava.io.tmpdir=/tmp/opensearch-7709752078852015358",
"-XX:+HeapDumpOnOutOfMemoryError",
"-XX:HeapDumpPath=data",
"-XX:ErrorFile=logs/hs_err_pid%p.log",
"-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m",
"-XX:MaxDirectMemorySize=1073741824",
"-Dopensearch.path.home=/usr/share/opensearch",
"-Dopensearch.path.conf=/usr/share/opensearch/config",
"-Dopensearch.distribution.type=tar",
"-Dopensearch.bundled_jdk=true"
]
},
}
}
3. What steps have you already taken to try and solve the problem?
I tryed to remove password from OS connection string:
elasticsearch_hosts = https://os1.example.local:9200,https://os2.example.local:9200,https:/
then i tryed change to localhost ip address:
elasticsearch_hosts = https://127.0.0.1:9200,https://127.0.0.1:9200,https://127.0.0.1:9200
and i tryed to change only one node:
elasticsearch_hosts = https://127.0.0.1:9200
but i get same error.
4. How can the community help?
Help me please, where i have mistake? Thansk!
Helpful Posting Tips: Tips for Posting Questions that Get Answers [Hold down CTRL and link on link to open tips documents in a separate tab]