logs-streaming/filebeat-config.yml


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34

kind: ConfigMap
metadata:
  name: filebeat-config
  namespace: logs-kafka
apiVersion: v1
data:
  filebeat.yml: |+

    filebeat.prospectors:
    - type: log
      enabled: true
      paths:
        - /var/lib/docker/containers/*/*.log
      symlinks: true
      # reduce the risk for aggregation recursion: match pod names to exclude own's logs + test logs
      exclude_files: ['^(.*\/)?logs-','^(.*\/)?test-']

    processors:
    - add_kubernetes_metadata:
        in_cluster: true
        namespace: logs-kafka

    output.kafka:
      hosts: ["kafka-0.broker.kafka.svc.cluster.local:9092", "kafka-1.broker.kafka.svc.cluster.local:9092", "kafka-2.broker.kafka.svc.cluster.local:9092"]
      topic: ${TOPIC}

      partition.round_robin:
        reachable_only: false

      client_id: filebeat-kubernetes
      version: 0.11.0.0
      required_acks: 1
      compression: gzip
      max_message_bytes: 1000000