Hi Team,
I have Graylog 2.5 running with MongoDb & Elastic Search on a single server. I am trying to ingest logs into Graylog using Filebeat. I have set up Beats as input in the Web UI and Logstash as output in the Filebeat config.
Also I have a beats plugin “graylog-plugin-beats-2.4.7” on Graylog.
My team’s APIs are running in Kubernetes and we are trying to pull the logs using Filebeat. I am getting the log messages into Graylog but metadata was missing.
If I write the Filebeat logs to a file I can see the metadata but not seen when ingested into the Graylog. It is giving only the default fields and missing all other meta data fields.
The above image shows the logs ingested with basic fields.
If I write the logs to a file then I can see all the metadata fileds without issue.
{
"@timestamp": "2019-07-09T19:34:44.718Z",
"@metadata": {
"beat": "filebeat",
"type": "doc",
"version": "6.6.1"
},
"stream": "stdout",
"docker": {
"container": {
"id": "cb80801475e2c961ccfe4068650daf304ed539d0c1682cd6ad3f2ed20acf90ee",
"labels": {
"io": {
"kubernetes": {
"sandbox": {
"id": "bfa2189ebce0d96657ad39f72ea024edfecf690403cd2a79acad0400a9208970"
},
"docker": {
"type": "container"
},
"pod": {
"name": "cdt-api-service-deployment-79fb644df5-jc54w",
"uid": "6e6cfbcf-9f4f-11e9-9d71-62e031173cb1",
"namespace": "cdt-deployment"
},
"container": {
"logpath": "/var/log/pods/6e6cfbcf-9f4f-11e9-9d71-62e031173cb1/api-service/0.log",
"name": "api-service"
}
}
},
"annotation": {
"io": {
"kubernetes": {
"container": {
"ports": "[{\"containerPort\":8080,\"protocol\":\"TCP\"}]",
"hash": "c4e2d745",
"terminationMessagePath": "/dev/termination-log",
"restartCount": "0",
"terminationMessagePolicy": "File"
},
"pod": {
"terminationGracePeriod": "30"
}
}
}
}
},
"image": "sha256:63eeb3e1527ac0b892f5cd9101fb3a258016d19d429f90cf95a569f6c3179788",
"name": "k8s_api-service_cdt-api-service-deployment-79fb644df5-jc54w_cdt-deployment_6e6cfbcf-9f4f-11e9-9d71-62e031173cb1_0"
}
},
"offset": 29970357,
"log": {
"file": {
"path": "/var/lib/docker/containers/cb80801475e2c961ccfe4068650daf304ed539d0c1682cd6ad3f2ed20acf90ee/cb80801475e2c961ccfe4068650daf304ed539d0c1682cd6ad3f2ed20acf90ee-json.log"
}
},
"prospector": {
"type": "docker"
},
"input": {
"type": "docker"
},
"beat": {
"name": "filebeat-55xd8",
"hostname": "filebeat-55xd8",
"version": "6.6.1"
},
"host": {
"name": "filebeat-55xd8"
},
"source": "/var/lib/docker/containers/cb80801475e2c961ccfe4068650daf304ed539d0c1682cd6ad3f2ed20acf90ee/cb80801475e2c961ccfe4068650daf304ed539d0c1682cd6ad3f2ed20acf90ee-json.log",
"message": "2019-07-09 19:34:44.718 [http-nio-8080-exec-1] [] [] DEBUG o.s.s.w.a.ExceptionTranslationFilter - Chain processed normally"
}
filebeat.yaml
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-config
namespace: kube-system
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
data:
filebeat.yml: |-
filebeat.config:
prospectors:
enabled: true
# Mounted `filebeat-prospectors` configmap:
path: ${path.config}/prospectors.d/*.yml
# Reload prospectors configs as they change:
reload.enabled: false
modules:
path: ${path.config}/modules.d/*.yml
# Reload module configs as they change:
reload.enabled: false
output.logstash:
enabled: true
hosts: ['mydns.com:5044']
---
apiVersion: v1
kind: ConfigMap
metadata:
name: filebeat-prospectors
namespace: kube-system
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
data:
kubernetes.yml: |-
- type: docker
combine_partial: true
containers:
path: "/var/lib/docker/containers"
containers.ids:
- "*"
# json.keys_under_root: true
# json.add_error_key: false
# json.message_key: log
# json.ignore_decoding_error: true
multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2}'
multiline.negate: true
multiline.match: after
processors:
- add_kubernetes_metadata:
in_cluster: true
namespace: true
- add_docker_metadata: ~
- drop_event.when.regexp:
or:
- kubernetes.pod.name: "external-dns.*"
- kubernetes.pod.name: "filebeat*"
- kubernetes.namespace: "msp*"
---
apiVersion: extensions/v1beta1
kind: DaemonSet
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
spec:
template:
metadata:
labels:
k8s-app: filebeat
kubernetes.io/cluster-service: "true"
spec:
serviceAccountName: filebeat
terminationGracePeriodSeconds: 30
containers:
- name: filebeat
image: docker.elastic.co/beats/filebeat:6.6.1
args: [
"-c", "/etc/filebeat.yml",
"-e",
]
env:
- name: ELASTICSEARCH_HOST
value: mydns.com
- name: ELASTICSEARCH_PORT
value: "5044"
- name: ELASTIC_CLOUD_ID
value:
- name: ELASTIC_CLOUD_AUTH
value:
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
securityContext:
runAsUser: 0
resources:
limits:
memory: 500Mi
requests:
cpu: 100m
memory: 200Mi
volumeMounts:
- name: config
mountPath: /etc/filebeat.yml
readOnly: true
subPath: filebeat.yml
- name: prospectors
mountPath: /usr/share/filebeat/prospectors.d
readOnly: true
- name: dockersock
mountPath: /var/run/docker.sock
readOnly: true
- name: data
mountPath: /usr/share/filebeat/data
- name: varlibdockercontainers
mountPath: /var/lib/docker/containers
readOnly: true
- name: varlogcontainers
mountPath: /var/log/containers
readOnly: true
- name: varlogpods
mountPath: /var/log/pods
readOnly: true
volumes:
- name: config
configMap:
defaultMode: 0600
name: filebeat-config
- name: varlibdockercontainers
hostPath:
path: /var/lib/docker/containers
- name: varlogcontainers
hostPath:
path: /var/log/containers
- name: varlogpods
hostPath:
path: /var/log/pods
- name: dockersock
hostPath:
path: /var/run/docker.sock
- name: prospectors
configMap:
defaultMode: 0600
name: filebeat-prospectors
# data folder stores a registry of read status for all files, so we don't send everything again on a Filebeat pod restart
- name: data
hostPath:
path: /var/lib/filebeat-data
type: DirectoryOrCreate
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRoleBinding
metadata:
name: filebeat
subjects:
- kind: ServiceAccount
name: filebeat
namespace: kube-system
roleRef:
kind: ClusterRole
name: filebeat
apiGroup: rbac.authorization.k8s.io
---
apiVersion: rbac.authorization.k8s.io/v1beta1
kind: ClusterRole
metadata:
name: filebeat
labels:
k8s-app: filebeat
rules:
- apiGroups: [""] # "" indicates the core API group
resources:
- namespaces
- pods
verbs:
- get
- watch
- list
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: filebeat
namespace: kube-system
labels:
k8s-app: filebeat
---
I have setup Beats as input in WebUI:
Any help is appreciated.