From 3a0b097a2d695f0aea26bc5bad245c3c74809714 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Sat, 4 Nov 2017 16:18:51 +0100
Subject: Copies the result of the original filebeat branch ...

but changes naming and namespace, to run alongside -raw.
---
 logs-streaming/00namespace.yml                  |  5 ++
 logs-streaming/filebeat-config.yml              | 33 +++++++++++++
 logs-streaming/filebeat-logs-kube-kafka.yml     | 63 +++++++++++++++++++++++++
 logs-streaming/topic-ops-kube-logs-filebeat.yml | 31 ++++++++++++
 4 files changed, 132 insertions(+)
 create mode 100644 logs-streaming/00namespace.yml
 create mode 100644 logs-streaming/filebeat-config.yml
 create mode 100644 logs-streaming/filebeat-logs-kube-kafka.yml
 create mode 100644 logs-streaming/topic-ops-kube-logs-filebeat.yml

diff --git a/logs-streaming/00namespace.yml b/logs-streaming/00namespace.yml
new file mode 100644
index 0000000..d420941
--- /dev/null
+++ b/logs-streaming/00namespace.yml
@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: logs-kafka
diff --git a/logs-streaming/filebeat-config.yml b/logs-streaming/filebeat-config.yml
new file mode 100644
index 0000000..ba7a4b1
--- /dev/null
+++ b/logs-streaming/filebeat-config.yml
@@ -0,0 +1,33 @@
+kind: ConfigMap
+metadata:
+  name: filebeat-config
+  namespace: logs-kafka
+apiVersion: v1
+data:
+  filebeat.yml: |+
+
+    filebeat.prospectors:
+    - type: log
+      enabled: true
+      paths:
+        - /var/log/containers/*.log
+      symlinks: true
+      # reduce the risk for aggregation recursion: match pod names to exclude own's logs + test logs
+      exclude_files: ['^(.*\/)?logs-','^(.*\/)?test-']
+
+    processors:
+    - add_kubernetes_metadata:
+        in_cluster: true
+
+    output.kafka:
+      hosts: ["kafka-0.broker.kafka.svc.cluster.local:9092", "kafka-1.broker.kafka.svc.cluster.local:9092", "kafka-2.broker.kafka.svc.cluster.local:9092"]
+      topic: ${TOPIC}
+
+      partition.round_robin:
+        reachable_only: false
+
+      client_id: filebeat-kubernetes
+      version: 0.11.0.0
+      required_acks: 1
+      compression: gzip
+      max_message_bytes: 1000000
\ No newline at end of file
diff --git a/logs-streaming/filebeat-logs-kube-kafka.yml b/logs-streaming/filebeat-logs-kube-kafka.yml
new file mode 100644
index 0000000..e1b931d
--- /dev/null
+++ b/logs-streaming/filebeat-logs-kube-kafka.yml
@@ -0,0 +1,63 @@
+apiVersion: apps/v1beta2
+kind: DaemonSet
+metadata:
+  name: filebeat-kube-kafka
+  namespace: logs-kafka
+spec:
+  selector:
+    matchLabels:
+      k8s-app: filebeat-kube-kafka
+      version: v1
+      kubernetes.io/cluster-service: "true"
+  template:
+    metadata:
+      labels:
+        k8s-app: filebeat-kube-kafka
+        version: v1
+        kubernetes.io/cluster-service: "true"
+    spec:
+      containers:
+      - name: filebeat
+        image: solsson/kafka-filebeat-kubernetes@sha256:79b40d81b892ffb3b917eb249f4b3050badf57fadffbdc35272060c9b377209e
+        command:
+        - filebeat
+        - -e
+        - -c
+        - /etc/filebeat/filebeat.yml
+        - -d
+        - "service,beat"
+        env:
+        - name: TOPIC
+          value: ops-kube-logs-filebeat-001
+        resources:
+          limits:
+            memory: 100Mi
+          requests:
+            cpu: 100m
+            memory: 100Mi
+        volumeMounts:
+        - name: config
+          mountPath: /etc/filebeat
+          readOnly: true
+        - name: data
+          mountPath: /data
+        - name: varlog
+          mountPath: /var/log
+          readOnly: true
+        - name: varlibdockercontainers
+          mountPath: /var/lib/docker/containers
+          readOnly: true
+      terminationGracePeriodSeconds: 10
+      volumes:
+      - name: config
+        configMap:
+          name: filebeat-config
+      - name: data
+        hostPath:
+          path: /kubernets-filebeat-state
+      - name: varlog
+        hostPath:
+          path: /var/log
+      - name: varlibdockercontainers
+        hostPath:
+          path: /var/lib/docker/containers
\ No newline at end of file
diff --git a/logs-streaming/topic-ops-kube-logs-filebeat.yml b/logs-streaming/topic-ops-kube-logs-filebeat.yml
new file mode 100644
index 0000000..d4aaf58
--- /dev/null
+++ b/logs-streaming/topic-ops-kube-logs-filebeat.yml
@@ -0,0 +1,31 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: topic-ops-kube-logs-filebeat
+  namespace: logs-kafka
+spec:
+  template:
+    metadata:
+      labels:
+        app: topic-create
+        topic-id: ops-kube-logs-filebeat
+        topic-gen: "001"
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        command:
+        - ./bin/kafka-topics.sh
+        - --zookeeper
+        - zookeeper.kafka:2181
+        - --create
+        - --if-not-exists
+        - --topic
+        - ops-kube-logs-filebeat-001
+        - --partitions
+        - "1"
+        - --replication-factor
+        - "2"
+        - --config
+        - retention.ms=6912500000
+      restartPolicy: Never
-- 
cgit v1.2.3