Hi,
I am trying to run Graylog on Kubernetes. I am using a mongodb and elasticsearch hosted outside Kubernetes and trying to start the containers. Although on container starting up, the graylog database is created and the node config is pushed to MongoDB. Graylog container fails to start.
It throws the following error:
2019-01-15 23:24:43,975 INFO : org.graylog2.bootstrap.ServerBootstrap - Graylog server 2.4.6+ceaa7e4 starting up 2019-01-15 23:24:43,976 INFO : org.graylog2.bootstrap.ServerBootstrap - JRE: Oracle Corporation 1.8.0_181 on Linux 4.4.148-k8s 2019-01-15 23:24:43,976 INFO : org.graylog2.bootstrap.ServerBootstrap - Deployment: docker 2019-01-15 23:24:43,976 INFO : org.graylog2.bootstrap.ServerBootstrap - OS: Debian GNU/Linux 9 (stretch) (debian) 2019-01-15 23:24:43,976 INFO : org.graylog2.bootstrap.ServerBootstrap - Arch: amd64 Exception in thread "main" com.mongodb.MongoQueryException: Query failed with error code 2 and error message 'error processing query: ns=graylog.nodesTree: $and is_master == true type == "SERVER" last_seen $gte 1547594681 $not node_id == "2568a867-4398-44bf-a740-b0a160e10a66" Sort: {} Proj: {} No query solutions' on server 172.30.17.9:27017 at com.mongodb.operation.FindOperation$1.call(FindOperation.java:720) at com.mongodb.operation.FindOperation$1.call(FindOperation.java:709) at com.mongodb.operation.OperationHelper.withConnectionSource(OperationHelper.java:433) at com.mongodb.operation.OperationHelper.withConnection(OperationHelper.java:406) at com.mongodb.operation.FindOperation.execute(FindOperation.java:709) at com.mongodb.operation.FindOperation.execute(FindOperation.java:81) at com.mongodb.Mongo.execute(Mongo.java:810) at com.mongodb.Mongo$2.execute(Mongo.java:797) at com.mongodb.DBCursor.initializeCursor(DBCursor.java:871) at com.mongodb.DBCursor.hasNext(DBCursor.java:142) at com.google.common.collect.Iterators.addAll(Iterators.java:372) at com.google.common.collect.Lists.newArrayList(Lists.java:165) at com.google.common.collect.Lists.newArrayList(Lists.java:150) at org.graylog2.database.PersistedServiceImpl.cursorToList(PersistedServiceImpl.java:121) at org.graylog2.database.PersistedServiceImpl.query(PersistedServiceImpl.java:68) at org.graylog2.database.PersistedServiceImpl.query(PersistedServiceImpl.java:72) at org.graylog2.cluster.NodeServiceImpl.isOnlyMaster(NodeServiceImpl.java:154) at org.graylog2.commands.Server.startNodeRegistration(Server.java:159) at org.graylog2.bootstrap.ServerBootstrap.startCommand(ServerBootstrap.java:120) at org.graylog2.bootstrap.CmdLineTool.run(CmdLineTool.java:209) at org.graylog2.bootstrap.Main.main(Main.java:44)
This is the statelessset using to bringup graylog containers:
apiVersion: v1
kind: ConfigMap
metadata:
name: graylog
namespace: graylog
labels:
component: graylog
data:
master-override.conf: |
is_master=true
slave-override.conf: |
is_master=false
apiVersion: apps/v1beta2
kind: StatefulSet
metadata:
name: graylog
namespace: graylog
labels:
component: graylog
spec:
selector:
matchLabels:
component: graylog
updateStrategy:
type: RollingUpdate
serviceName: graylog-svc
podManagementPolicy: OrderedReady
replicas: 1
template:
metadata:
labels:
component: graylog
spec:
initContainers:
- name: graylog-init
image: graylog/graylog:2.4
imagePullPolicy: IfNotPresent
command:
- bash
- “-c”
- |
updateMainConfigFile () {
overrideFile=$1
mainFile=$2
cat $overrideFile | egrep -v “[1]$|[2]#” |
while read line; do
key=${line%%=}
cat $mainFile | sed "s/^$key./$line/" > /tmp/config.tmp
mv /tmp/config.tmp $mainFile
done
}
# Retrieve all files
cp -R /usr/share/graylog/data/config/* /tmp/graylog/conf/
# Override main config file
if [[ ${HOSTNAME##*-} == 0 ]]; then
updateMainConfigFile /tmp/config-map/master-override.conf /tmp/graylog/conf/graylog.conf
else
updateMainConfigFile /tmp/config-map/slave-override.conf /tmp/graylog/conf/graylog.conf
fi
volumeMounts:
- name: conf
mountPath: /tmp/graylog/conf
- name: config-map
mountPath: /tmp/config-map
terminationGracePeriodSeconds: 10
containers:
- name: graylog
image: graylog/graylog:2.4
imagePullPolicy: IfNotPresent
readinessProbe:
httpGet:
path: /api/system/lbstatus
port: 12900
periodSeconds: 2
initialDelaySeconds: 20
failureThreshold: 2
successThreshold: 1
timeoutSeconds: 5
ports:
- containerPort: 12900
- containerPort: 9000
volumeMounts:
- name: conf
mountPath: /usr/share/graylog/data/config
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: “GRAYLOG_ELASTICSEARCH_HOSTS”
value: “”
- name: “GRAYLOG_ELASTICSEARCH_DISCOVERY_ENABLED”
value: “true”
- name: “GRAYLOG_MONGODB_URI”
value: “mongodb://:27017/graylog”
- name: “GRAYLOG_PASSWORD_SECRET”
value: “somepasswordpepper”
- name: “GRAYLOG_ROOT_PASSWORD_SHA2”
value: “8c6976e5b5410415bde908bd4dee15dfb167a9c873fc4bb8a81f6f2ab448a918”
- name: “GRAYLOG_WEB_ENDPOINT_URI”
value: “domain-url”
- name: “GRAYLOG_REST_TRANSPORT_URI”
value: “http://graylog-svc.graylog:9000/api/”
- name: “GRAYLOG_REST_LISTEN_URI”
value: “http://graylog-svc.graylog:12900/api/”
volumes:
- name: conf
emptyDir: {}
- name: config-map
configMap:
name: graylog
apiVersion: v1
kind: Service
metadata:
labels:
component: graylog
name: graylog-svc
namespace: graylog
spec:
selector:
component: graylog
ports:
port: 9000
targetPort: 9000
protocol: TCP
name: web-ui
port: 12900
targetPort: 12900
protocol: TCP
name: api
port: 5044
protocol: TCP
targetPort: 5044
name: input-beat
port: 12201
protocol: UDP
targetPort: 12201
name: input-gelf-udp
sessionAffinity: None
type: ClusterIP
apiVersion: extensions/v1beta1
kind: Ingress
metadata:
name: graylog-ingress
namespace: graylog
spec:
rules:
host: mydomain-url
http:
paths:
path: /api
backend:
serviceName: graylog-svc
servicePort: 12900
path: /
backend:
serviceName: graylog-svc
servicePort: 9000
Elasticsearch version : 5.6
MongoDB version: 3.4.18