aboutsummaryrefslogtreecommitdiff
path: root/logs-streaming/logs-kube-kafka-raw.yml
blob: 4470ed564c4b03561eef87ff018109f6ed5ebc69 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
apiVersion: apps/v1beta2
kind: DaemonSet
metadata:
  name: logs-kafka-raw
  namespace: logs-kafka
spec:
  selector:
    matchLabels:
      k8s-app: logs-kafka
      log-wrapping: none
      container-delimiters: tail
      version: v1
      kubernetes.io/cluster-service: "true"
  template:
    metadata:
      labels:
        k8s-app: logs-kafka
        log-wrapping: none
        container-delimiters: tail
        version: v1
        kubernetes.io/cluster-service: "true"
    spec:
      # maybe the /bin/sh does not forward signals?
      terminationGracePeriodSeconds: 5
      containers:
      - name: kafkacat
        image: solsson/kafkacat@sha256:b32eedf936f3cde44cd164ddc77dfcf7565a8af4e357ff6de1abe4389ca530c9
        env:
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              fieldPath: spec.nodeName
        command:
        - sh
        - -ec
        - >
          find /var/log/containers/ -maxdepth 1 -name *.log | grep -v "/$HOSTNAME" | tee /tmp/start-list;
          cd /var/log/containers/;
          tail
          -n 0
          --follow=name
          *.log
          |
          kafkacat
          -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
          -t ops-kube-logs-raw-001
          -P
          -z snappy
          -v
          -d broker,topic
          -k $NODE_NAME
        livenessProbe:
          exec:
            command:
            - /bin/sh
            - -c
            - find /var/log/containers/ -maxdepth 1 -name *.log | grep -v "/$HOSTNAME" | diff - /tmp/start-list
          failureThreshold: 1
          initialDelaySeconds: 5
          periodSeconds: 5
        resources:
          requests:
            cpu: 2m
            memory: 4Mi
          limits:
            cpu: 10m
            memory: 4Mi
        volumeMounts:
        - name: varlog
          mountPath: /var/log
          readOnly: true
        - name: varlibdockercontainers
          mountPath: /var/lib/docker/containers
          readOnly: true
      terminationGracePeriodSeconds: 10
      volumes:
      - name: varlog
        hostPath:
          path: /var/log
      - name: varlibdockercontainers
        hostPath:
          path: /var/lib/docker/containers