From 30b888a01f5f2f02522fb376a4fc827ce3882405 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 09:42:11 +0200 Subject: Creates a pod for expermentation with kubectl and kafkacat --- addon-events/events-kube-kafka.yml | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 addon-events/events-kube-kafka.yml diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml new file mode 100644 index 0000000..9eeed58 --- /dev/null +++ b/addon-events/events-kube-kafka.yml @@ -0,0 +1,27 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: events-kube-kafka + namespace: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # prefer duplicate events over missed + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: events + from: kube + to: kafka + spec: + containers: + - name: kubectl-kafkacat + image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b + command: + - sh + - -ec + - 'tail -f /dev/null' -- cgit v1.2.3 From 4bdfb2b606a9db2fc9d1bba07e29f482b28ae82d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 6 Jul 2017 11:52:15 +0200 Subject: Produces and reads events --- addon-events/events-kube-kafka.yml | 12 ++++++++-- addon-events/test-event-consumer.yml | 30 +++++++++++++++++++++++ addon-events/topic-ops-kube-events-all-json.yml | 32 +++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 2 deletions(-) create mode 100644 addon-events/test-event-consumer.yml create mode 100644 addon-events/topic-ops-kube-events-all-json.yml diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 9eeed58..95d0ad2 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -22,6 +22,14 @@ spec: - name: kubectl-kafkacat image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b command: - - sh + - /bin/bash - -ec - - 'tail -f /dev/null' + - > + kubectl + get events + -w + | + kafkacat + -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + -t ops-kube-events-all-json-001 + -P diff --git a/addon-events/test-event-consumer.yml b/addon-events/test-event-consumer.yml new file mode 100644 index 0000000..d98f88c --- /dev/null +++ b/addon-events/test-event-consumer.yml @@ -0,0 +1,30 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: test-events-consumer + namespace: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # prefer duplicate events over missed + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: events + type: test + from: kafka-ops + spec: + containers: + - name: kafkacat + image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 + command: + - kafkacat + - -b + - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - -t + - ops-kube-events-all-json-001 + - -C diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml new file mode 100644 index 0000000..e6396a4 --- /dev/null +++ b/addon-events/topic-ops-kube-events-all-json.yml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-ops-kube-events-all-json + namespace: kafka +spec: + template: + metadata: + labels: + app: topic-create + topic-id: ops-kube-events-all-json + topic-gen: "001" + spec: + containers: + - name: kafka + image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756 + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper:2181 + - --create + - --if-not-exists + - --topic + - ops-kube-events-all-json-001 + - --partitions + - "1" + - --replication-factor + - "1" + - --config + # this might be eight days + - retention.ms=69125000 + restartPolicy: Never -- cgit v1.2.3 From ccdd8bec306601a82b8faea056a82e50c4b4c1de Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:11:40 +0200 Subject: wip --- addon-events/events-kube-kafka.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 95d0ad2..d45d480 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -19,15 +19,16 @@ spec: to: kafka spec: containers: - - name: kubectl-kafkacat - image: solsson/kubectl-kafkacat@sha256:e496cb9bca667c5cf629bcedd3a5788affa4c68e9a6a0198d521e9dd6fcaf89b + - name: kafkacat-curl + image: solsson/kafkacat-curl@sha256:26b81296ba1d2c6b2cbce81c644b3b780ab5beab57bebbbb11f0d0a99e2d0d2b command: - /bin/bash - -ec - > - kubectl - get events - -w + curl + --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt + --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccont/token)" + https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events | kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -- cgit v1.2.3 From 5ee97a2ef2e7bb451db3d36c6767b5e6a4e49e89 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:48:51 +0200 Subject: Quite interesting way to test a command from yaml - see it in logs --- addon-events/events-kube-kafka.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index d45d480..4529785 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -25,12 +25,14 @@ spec: - /bin/bash - -ec - > + echo ' curl --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt - --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccont/token)" + --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events | kafkacat -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P + '; tail -f /dev/null -- cgit v1.2.3 From 4da60f8453f128c9b466a51f6a2cdbb9f9f32c92 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 07:50:24 +0200 Subject: Produces proper json. Let's see when curl gives up and the container restarts. --- addon-events/events-kube-kafka.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 4529785..ed2bde4 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -25,8 +25,8 @@ spec: - /bin/bash - -ec - > - echo ' curl + -s --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events @@ -35,4 +35,3 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - '; tail -f /dev/null -- cgit v1.2.3 From efacc180a893cd7ce83ecb99e2f9c0d70339d64b Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 08:06:17 +0200 Subject: Testing compression but solsson/kafka:0.11.0.0 throws ... Caused by: java.lang.NoClassDefFoundError: Could not initialize class org.xerial.snappy.Snappy at org.xerial.snappy.SnappyInputStream.readFully(SnappyInputStream.java:145) at org.xerial.snappy.SnappyInputStream.readHeader(SnappyInputStream.java:99) at org.xerial.snappy.SnappyInputStream.(SnappyInputStream.java:59) at org.apache.kafka.common.record.CompressionType$3.wrapForInput(CompressionType.java:82) ... 40 more --- addon-events/events-kube-kafka.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index ed2bde4..e188a92 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,3 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P + -z snappy -- cgit v1.2.3 From b6575a536fea27a36972da2d599c8d2f29f773b6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 7 Jul 2017 08:08:57 +0200 Subject: Works with my current kafka image, and the test consumer still sees JSON --- addon-events/events-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index e188a92..5cc7883 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,4 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - -z snappy + -z gzip -- cgit v1.2.3 From a30f22e324594ab1c6d14db1bdfd4f2ff1f7698e Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 27 Jul 2017 09:17:27 +0200 Subject: Latest build suppors snappy compression --- addon-events/events-kube-kafka.yml | 2 +- addon-events/topic-ops-kube-events-all-json.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 5cc7883..e188a92 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -35,4 +35,4 @@ spec: -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -t ops-kube-events-all-json-001 -P - -z gzip + -z snappy diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml index e6396a4..8eb3b66 100644 --- a/addon-events/topic-ops-kube-events-all-json.yml +++ b/addon-events/topic-ops-kube-events-all-json.yml @@ -13,7 +13,7 @@ spec: spec: containers: - name: kafka - image: solsson/kafka:0.11.0.0@sha256:4c194db2ec15698aca6f1aa8a2fd5e5c566caed82b4bf43446c388f315397756 + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce command: - ./bin/kafka-topics.sh - --zookeeper -- cgit v1.2.3 From f626a7764513fd85bdf721be766646312fd848b1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 5 Aug 2017 06:24:38 +0200 Subject: Adds RBAC policy for curl, default service account --- rbac-namespace-default/events-watcher.yml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 rbac-namespace-default/events-watcher.yml diff --git a/rbac-namespace-default/events-watcher.yml b/rbac-namespace-default/events-watcher.yml new file mode 100644 index 0000000..c8384b6 --- /dev/null +++ b/rbac-namespace-default/events-watcher.yml @@ -0,0 +1,30 @@ +# If events-kube-kafka-* goes crashlooping you probably need this +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: events-watcher + labels: + origin: github.com_Yolean_kubernetes-kafka +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kafka-events-watcher + labels: + origin: github.com_Yolean_kubernetes-kafka +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: events-watcher +subjects: +- kind: ServiceAccount + name: default + namespace: kafka -- cgit v1.2.3 From 4f13cf571b18c06074f5216852b722cdb3d5dd67 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 5 Aug 2017 06:59:29 +0200 Subject: Moves the example comsumer to the test namespace --- addon-events/test-event-consumer.yml | 30 ------------------------------ test/events-topic.yml | 29 +++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 30 deletions(-) delete mode 100644 addon-events/test-event-consumer.yml create mode 100644 test/events-topic.yml diff --git a/addon-events/test-event-consumer.yml b/addon-events/test-event-consumer.yml deleted file mode 100644 index d98f88c..0000000 --- a/addon-events/test-event-consumer.yml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: test-events-consumer - namespace: kafka -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - # prefer duplicate events over missed - maxUnavailable: 0 - maxSurge: 1 - template: - metadata: - labels: - app: events - type: test - from: kafka-ops - spec: - containers: - - name: kafkacat - image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 - command: - - kafkacat - - -b - - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - - -t - - ops-kube-events-all-json-001 - - -C diff --git a/test/events-topic.yml b/test/events-topic.yml new file mode 100644 index 0000000..6d11d77 --- /dev/null +++ b/test/events-topic.yml @@ -0,0 +1,29 @@ +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: events-topic + namespace: test-kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # prefer duplicate events over missed + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + test-target: events-topic + test-type: readiness + spec: + containers: + - name: kafkacat + image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 + command: + - kafkacat + - -b + - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - -t + - ops-kube-events-all-json-001 + - -C -- cgit v1.2.3 From 8935cf549857b9b4f6dfb68209681789483b15d8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 5 Aug 2017 07:01:09 +0200 Subject: Not a proper test yet, but the PR documents how to use it to see events --- test/events-topic.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/test/events-topic.yml b/test/events-topic.yml index 6d11d77..68ca919 100644 --- a/test/events-topic.yml +++ b/test/events-topic.yml @@ -5,17 +5,12 @@ metadata: namespace: test-kafka spec: replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - # prefer duplicate events over missed - maxUnavailable: 0 - maxSurge: 1 template: metadata: labels: test-target: events-topic - test-type: readiness + # Would be a valid testcase if it verifies that >0 messages per some time unit have been produced + #test-type: readiness spec: containers: - name: kafkacat -- cgit v1.2.3 From 8d0d99302b80e960b592d1d8779b3c502815fc19 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 21:34:19 +0200 Subject: Use an image with new kafkacat and curl versions --- addon-events/events-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index e188a92..2d6913a 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -20,7 +20,7 @@ spec: spec: containers: - name: kafkacat-curl - image: solsson/kafkacat-curl@sha256:26b81296ba1d2c6b2cbce81c644b3b780ab5beab57bebbbb11f0d0a99e2d0d2b + image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0 command: - /bin/bash - -ec -- cgit v1.2.3 From b9e7b8a02e328309907bd024619008d8d7cdfc07 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 21:56:03 +0200 Subject: Makes things a bit less obscure if for example RBAC fails --- addon-events/events-kube-kafka.yml | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 2d6913a..062d097 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -21,10 +21,18 @@ spec: containers: - name: kafkacat-curl image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0 + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - name: TOPIC + value: ops-kube-events-all-json-001 command: - /bin/bash - -ec + # curl errors will go to kafka, kafkacat errors will got to log - > + echo "Started at $(date -u +%FT%TZ) producing to $TOPIC" + ; curl -s --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt @@ -32,7 +40,9 @@ spec: https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events | kafkacat - -b kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - -t ops-kube-events-all-json-001 + -b $BOOTSTRAP + -t $TOPIC -P -z snappy + -v + -d broker,topic -- cgit v1.2.3 From e4c517984066f8e76909ec8a38fd414ada2c10c4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 22:02:53 +0200 Subject: Argh, curl will pretty print the json ... and kafkacat will produce each line separately --- addon-events/events-kube-kafka.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 062d097..a707b1c 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -28,7 +28,7 @@ spec: value: ops-kube-events-all-json-001 command: - /bin/bash - - -ec + - -c # curl errors will go to kafka, kafkacat errors will got to log - > echo "Started at $(date -u +%FT%TZ) producing to $TOPIC" @@ -46,3 +46,13 @@ spec: -z snappy -v -d broker,topic + ; + echo "Last message on topic:" + ; + kafkacat + -b $BOOTSTRAP + -t $TOPIC + -C + -o -1 + -c 1 + -z snappy -- cgit v1.2.3 From d628275ff253366f93c2533c7bdd191c909ebd69 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 22:07:21 +0200 Subject: bash -e is important, in combination with curl -f, so ... we don't send error messages to the topic (even if it's json -- format is different and it's pretty printed) --- addon-events/events-kube-kafka.yml | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index a707b1c..92387aa 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -28,12 +28,12 @@ spec: value: ops-kube-events-all-json-001 command: - /bin/bash - - -c - # curl errors will go to kafka, kafkacat errors will got to log + - -ec - > echo "Started at $(date -u +%FT%TZ) producing to $TOPIC" ; curl + -f -s --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" @@ -46,13 +46,3 @@ spec: -z snappy -v -d broker,topic - ; - echo "Last message on topic:" - ; - kafkacat - -b $BOOTSTRAP - -t $TOPIC - -C - -o -1 - -c 1 - -z snappy -- cgit v1.2.3 From fc5a8362abda5e5e92df00d7bce21b3f5a7ee280 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 26 Oct 2017 22:34:30 +0200 Subject: kafkacat's debug output solves the silence, but ... to see the error message (like missing RBAC) you basically have to run the curl command. I'd like startup to be as fast as possible, to not miss events, so I hesitate to add checks. --- addon-events/events-kube-kafka.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml index 92387aa..2cd1891 100644 --- a/addon-events/events-kube-kafka.yml +++ b/addon-events/events-kube-kafka.yml @@ -30,8 +30,6 @@ spec: - /bin/bash - -ec - > - echo "Started at $(date -u +%FT%TZ) producing to $TOPIC" - ; curl -f -s -- cgit v1.2.3 From 071f973fd2210cb5dc3edca17682e4b520b68b71 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 27 Oct 2017 08:33:39 +0200 Subject: Manifests updated to 1.8 and v3.0.0 pre-merge --- addon-events/events-kube-kafka.yml | 46 ---------------------- addon-events/topic-ops-kube-events-all-json.yml | 32 ---------------- events-kube/events-kube-kafka.yml | 51 +++++++++++++++++++++++++ events-kube/test/events-topic.yml | 27 +++++++++++++ events-kube/topic-ops-kube-events-all-json.yml | 32 ++++++++++++++++ test/events-topic.yml | 24 ------------ 6 files changed, 110 insertions(+), 102 deletions(-) delete mode 100644 addon-events/events-kube-kafka.yml delete mode 100644 addon-events/topic-ops-kube-events-all-json.yml create mode 100644 events-kube/events-kube-kafka.yml create mode 100644 events-kube/test/events-topic.yml create mode 100644 events-kube/topic-ops-kube-events-all-json.yml delete mode 100644 test/events-topic.yml diff --git a/addon-events/events-kube-kafka.yml b/addon-events/events-kube-kafka.yml deleted file mode 100644 index 2cd1891..0000000 --- a/addon-events/events-kube-kafka.yml +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: events-kube-kafka - namespace: kafka -spec: - replicas: 1 - strategy: - type: RollingUpdate - rollingUpdate: - # prefer duplicate events over missed - maxUnavailable: 0 - maxSurge: 1 - template: - metadata: - labels: - app: events - from: kube - to: kafka - spec: - containers: - - name: kafkacat-curl - image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0 - env: - - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - - name: TOPIC - value: ops-kube-events-all-json-001 - command: - - /bin/bash - - -ec - - > - curl - -f - -s - --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt - --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" - https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events - | - kafkacat - -b $BOOTSTRAP - -t $TOPIC - -P - -z snappy - -v - -d broker,topic diff --git a/addon-events/topic-ops-kube-events-all-json.yml b/addon-events/topic-ops-kube-events-all-json.yml deleted file mode 100644 index 8eb3b66..0000000 --- a/addon-events/topic-ops-kube-events-all-json.yml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: topic-ops-kube-events-all-json - namespace: kafka -spec: - template: - metadata: - labels: - app: topic-create - topic-id: ops-kube-events-all-json - topic-gen: "001" - spec: - containers: - - name: kafka - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce - command: - - ./bin/kafka-topics.sh - - --zookeeper - - zookeeper:2181 - - --create - - --if-not-exists - - --topic - - ops-kube-events-all-json-001 - - --partitions - - "1" - - --replication-factor - - "1" - - --config - # this might be eight days - - retention.ms=69125000 - restartPolicy: Never diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml new file mode 100644 index 0000000..769fad1 --- /dev/null +++ b/events-kube/events-kube-kafka.yml @@ -0,0 +1,51 @@ +apiVersion: extensions/v1beta2 +kind: Deployment +metadata: + name: events-kube-kafka + namespace: kafka +spec: + replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + # prefer duplicate events over missed + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app: events + from: kube + to: kafka + template: + metadata: + labels: + app: events + from: kube + to: kafka + spec: + containers: + - name: kafkacat-curl + image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0 + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - name: TOPIC + value: ops-kube-events-all-json-001 + command: + - /bin/bash + - -ec + - > + curl + -f + -s + --cacert /run/secrets/kubernetes.io/serviceaccount/ca.crt + --header "Authorization: Bearer $(cat /run/secrets/kubernetes.io/serviceaccount/token)" + https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/watch/events + | + kafkacat + -b $BOOTSTRAP + -t $TOPIC + -P + -z snappy + -v + -d broker,topic diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml new file mode 100644 index 0000000..bbed4fa --- /dev/null +++ b/events-kube/test/events-topic.yml @@ -0,0 +1,27 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: events-topic + namespace: test-kafka +spec: + replicas: 1 + selector: + matchLabels: + test-target: events-topic + template: + metadata: + labels: + test-target: events-topic + # Would be a valid testcase if it verifies that >0 messages per some time unit have been produced + #test-type: readiness + spec: + containers: + - name: kafkacat + image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 + command: + - kafkacat + - -b + - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - -t + - ops-kube-events-all-json-001 + - -C diff --git a/events-kube/topic-ops-kube-events-all-json.yml b/events-kube/topic-ops-kube-events-all-json.yml new file mode 100644 index 0000000..8eb3b66 --- /dev/null +++ b/events-kube/topic-ops-kube-events-all-json.yml @@ -0,0 +1,32 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: topic-ops-kube-events-all-json + namespace: kafka +spec: + template: + metadata: + labels: + app: topic-create + topic-id: ops-kube-events-all-json + topic-gen: "001" + spec: + containers: + - name: kafka + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper:2181 + - --create + - --if-not-exists + - --topic + - ops-kube-events-all-json-001 + - --partitions + - "1" + - --replication-factor + - "1" + - --config + # this might be eight days + - retention.ms=69125000 + restartPolicy: Never diff --git a/test/events-topic.yml b/test/events-topic.yml deleted file mode 100644 index 68ca919..0000000 --- a/test/events-topic.yml +++ /dev/null @@ -1,24 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: events-topic - namespace: test-kafka -spec: - replicas: 1 - template: - metadata: - labels: - test-target: events-topic - # Would be a valid testcase if it verifies that >0 messages per some time unit have been produced - #test-type: readiness - spec: - containers: - - name: kafkacat - image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 - command: - - kafkacat - - -b - - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - - -t - - ops-kube-events-all-json-001 - - -C -- cgit v1.2.3 From 1de4d2cb4452b47f2755cbe94b655d3aad2ead13 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 27 Oct 2017 10:08:41 +0200 Subject: An actual assertion --- events-kube/test/events-topic.yml | 82 ++++++++++++++++++++++++++++++++++----- 1 file changed, 72 insertions(+), 10 deletions(-) diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml index bbed4fa..126d4fb 100644 --- a/events-kube/test/events-topic.yml +++ b/events-kube/test/events-topic.yml @@ -1,3 +1,34 @@ +--- +kind: ConfigMap +metadata: + name: events-topic + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + tail -f /tmp/testlog + + test.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + PREVIOUS=$(sha1sum /tmp/event || echo "") + kafkacat -b $BOOTSTRAP -t $TOPIC -C -o -1 -c 1 | tee /tmp/event + CURRENT=$(sha1sum /tmp/event) + [ "$PREVIOUS" == "$CURRENT" ] && echo "{\"test-status\”: \"No new event in $TOPIC\"}" && exit 1 + + exit 0 + + quit-on-nonzero-exit.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + exit 0 + +--- apiVersion: apps/v1beta2 kind: Deployment metadata: @@ -5,23 +36,54 @@ metadata: namespace: test-kafka spec: replicas: 1 + strategy: + type: Recreate selector: matchLabels: test-target: events-topic + test-type: readiness template: metadata: labels: test-target: events-topic - # Would be a valid testcase if it verifies that >0 messages per some time unit have been produced - #test-type: readiness + test-type: readiness + # for example: + # readonly - can be used in production + # isolated - read/write but in a manner that does not affect other services + # load - unsuitable for production because it uses significant resources + # chaos - unsuitable for production because it injects failure modes + #test-use: spec: containers: - - name: kafkacat - image: solsson/kafkacat@sha256:36d1f191cc33a8365074280279205e6b4f52cd8cc8fb1b896bb4c943c9dee8f8 + - name: testcase + image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef + env: + - name: BOOTSTRAP + value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + - name: TOPIC + value: ops-kube-events-all-json-001 command: - - kafkacat - - -b - - kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - - -t - - ops-kube-events-all-json-001 - - -C + - /bin/bash + - -e + - /test/setup.sh + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/test.sh + initialDelaySeconds: 10 + periodSeconds: 10 + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/quit-on-nonzero-exit.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: events-topic -- cgit v1.2.3 From 8eb4a4d272a78a8f75e29a7ab4339b6bbc3787a7 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 27 Oct 2017 11:04:29 +0200 Subject: Needs to be tweaked depending on the churn --- events-kube/test/events-topic.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml index 126d4fb..844c136 100644 --- a/events-kube/test/events-topic.yml +++ b/events-kube/test/events-topic.yml @@ -73,7 +73,7 @@ spec: - -e - /test/test.sh initialDelaySeconds: 10 - periodSeconds: 10 + periodSeconds: 60 livenessProbe: exec: command: -- cgit v1.2.3 From 8ee3bebca5f979b4f99b213042181c1d9a1a3098 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 27 Oct 2017 13:04:52 +0200 Subject: Fixes JSON syntax, topic is jq:able now --- events-kube/test/events-topic.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml index 844c136..29891f7 100644 --- a/events-kube/test/events-topic.yml +++ b/events-kube/test/events-topic.yml @@ -15,10 +15,10 @@ data: exec >> /tmp/testlog exec 2>&1 - PREVIOUS=$(sha1sum /tmp/event || echo "") + PREVIOUS=$(sha1sum /tmp/event 2>/dev/null || echo "") kafkacat -b $BOOTSTRAP -t $TOPIC -C -o -1 -c 1 | tee /tmp/event CURRENT=$(sha1sum /tmp/event) - [ "$PREVIOUS" == "$CURRENT" ] && echo "{\"test-status\”: \"No new event in $TOPIC\"}" && exit 1 + [ "$PREVIOUS" == "$CURRENT" ] && echo "{\"test-result\": \"No new event in $TOPIC\"}" && exit 1 exit 0 -- cgit v1.2.3 From 97f03a469bf4911da00256da880ec31779d85ae1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 27 Oct 2017 13:07:32 +0200 Subject: Aligns RBAC with addons->folders for v3.0.0 --- events-kube/rbac/cluster-events-watcher.yml | 30 +++++++++++++++++++++++++++++ rbac-namespace-default/events-watcher.yml | 30 ----------------------------- 2 files changed, 30 insertions(+), 30 deletions(-) create mode 100644 events-kube/rbac/cluster-events-watcher.yml delete mode 100644 rbac-namespace-default/events-watcher.yml diff --git a/events-kube/rbac/cluster-events-watcher.yml b/events-kube/rbac/cluster-events-watcher.yml new file mode 100644 index 0000000..c8384b6 --- /dev/null +++ b/events-kube/rbac/cluster-events-watcher.yml @@ -0,0 +1,30 @@ +# If events-kube-kafka-* goes crashlooping you probably need this +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: events-watcher + labels: + origin: github.com_Yolean_kubernetes-kafka +rules: +- apiGroups: + - "" + resources: + - events + verbs: + - watch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1beta1 +metadata: + name: kafka-events-watcher + labels: + origin: github.com_Yolean_kubernetes-kafka +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: events-watcher +subjects: +- kind: ServiceAccount + name: default + namespace: kafka diff --git a/rbac-namespace-default/events-watcher.yml b/rbac-namespace-default/events-watcher.yml deleted file mode 100644 index c8384b6..0000000 --- a/rbac-namespace-default/events-watcher.yml +++ /dev/null @@ -1,30 +0,0 @@ -# If events-kube-kafka-* goes crashlooping you probably need this ---- -kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: events-watcher - labels: - origin: github.com_Yolean_kubernetes-kafka -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - watch ---- -kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 -metadata: - name: kafka-events-watcher - labels: - origin: github.com_Yolean_kubernetes-kafka -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: events-watcher -subjects: -- kind: ServiceAccount - name: default - namespace: kafka -- cgit v1.2.3 From d4a3bc6565e14b2f068a2e0a40733c319274bd95 Mon Sep 17 00:00:00 2001 From: Anton Lindgren Date: Mon, 30 Oct 2017 08:58:43 +0100 Subject: Fix retention from 0.8 days to 8 days Silly change, but 0.8 days is very short. Google helped me verify the math. --- events-kube/topic-ops-kube-events-all-json.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/events-kube/topic-ops-kube-events-all-json.yml b/events-kube/topic-ops-kube-events-all-json.yml index 8eb3b66..62f2b80 100644 --- a/events-kube/topic-ops-kube-events-all-json.yml +++ b/events-kube/topic-ops-kube-events-all-json.yml @@ -27,6 +27,6 @@ spec: - --replication-factor - "1" - --config - # this might be eight days - - retention.ms=69125000 + # 8 days + - retention.ms=691200000 restartPolicy: Never -- cgit v1.2.3 From 3fca1e7ee736d3e8d3647453de07e9986bcfe2fa Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 30 Oct 2017 22:21:37 +0100 Subject: Fixes manifest. I must have had a local edit last time. --- events-kube/events-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml index 769fad1..086137d 100644 --- a/events-kube/events-kube-kafka.yml +++ b/events-kube/events-kube-kafka.yml @@ -1,4 +1,4 @@ -apiVersion: extensions/v1beta2 +apiVersion: apps/v1beta2 kind: Deployment metadata: name: events-kube-kafka -- cgit v1.2.3 From 907467ca1d262581cf5754a3a6fd8cd3de1b4a1a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 9 Nov 2017 16:58:08 +0100 Subject: Uses kafkacat build as in tests and logs-streaming --- events-kube/events-kube-kafka.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml index 086137d..0b8bf10 100644 --- a/events-kube/events-kube-kafka.yml +++ b/events-kube/events-kube-kafka.yml @@ -25,7 +25,7 @@ spec: spec: containers: - name: kafkacat-curl - image: solsson/kafkacat-curl@sha256:6ad61f2e6343359c3972d7a86815568c0a1d0560068134c5d702a152eb5123a0 + image: solsson/kafkacat-curl@sha256:65568eb5b0c22d360546c7591d1efdee7ce919e4b299be33c0e4b47ff997c1e8 env: - name: BOOTSTRAP value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 -- cgit v1.2.3 From a861237ed8b2ff6aaff46c9404f1bb5a05d82edb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 18 Dec 2017 08:29:58 +0100 Subject: Modernize based on * https://github.com/Yolean/kubernetes-kafka/pull/107 * https://github.com/Yolean/kubernetes-kafka/issues/101 * https://github.com/Yolean/fluent-bit-kubernetes-kafka/commit/e045ef1ef03f9fd299575a7aa9ca312e6687e441 * https://github.com/Yolean/kubernetes-kafka/pull/52 --- events-kube/events-kube-kafka.yml | 6 ++--- events-kube/test/events-topic.yml | 6 ++--- events-kube/topic-ops-kube-events-all-json.yml | 32 -------------------------- 3 files changed, 6 insertions(+), 38 deletions(-) delete mode 100644 events-kube/topic-ops-kube-events-all-json.yml diff --git a/events-kube/events-kube-kafka.yml b/events-kube/events-kube-kafka.yml index 0b8bf10..31703b4 100644 --- a/events-kube/events-kube-kafka.yml +++ b/events-kube/events-kube-kafka.yml @@ -25,12 +25,12 @@ spec: spec: containers: - name: kafkacat-curl - image: solsson/kafkacat-curl@sha256:65568eb5b0c22d360546c7591d1efdee7ce919e4b299be33c0e4b47ff997c1e8 + image: solsson/kafkacat-curl@sha256:b5484379301937129e67550331782a0f7ac030a4b913a254d084faea4bcf44a2 env: - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + value: bootstrap.kafka:9092 - name: TOPIC - value: ops-kube-events-all-json-001 + value: ops.kube-events-all.stream.json.001 command: - /bin/bash - -ec diff --git a/events-kube/test/events-topic.yml b/events-kube/test/events-topic.yml index 29891f7..0c48c36 100644 --- a/events-kube/test/events-topic.yml +++ b/events-kube/test/events-topic.yml @@ -56,12 +56,12 @@ spec: spec: containers: - name: testcase - image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef + image: solsson/kafkacat@sha256:2c539e4f58960ab7872976ebc664dd92de18cf27e7cbbeb296d654a2351f6ca4 env: - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + value: bootstrap.kafka:9092 - name: TOPIC - value: ops-kube-events-all-json-001 + value: ops.kube-events-all.stream.json.001 command: - /bin/bash - -e diff --git a/events-kube/topic-ops-kube-events-all-json.yml b/events-kube/topic-ops-kube-events-all-json.yml deleted file mode 100644 index 62f2b80..0000000 --- a/events-kube/topic-ops-kube-events-all-json.yml +++ /dev/null @@ -1,32 +0,0 @@ -apiVersion: batch/v1 -kind: Job -metadata: - name: topic-ops-kube-events-all-json - namespace: kafka -spec: - template: - metadata: - labels: - app: topic-create - topic-id: ops-kube-events-all-json - topic-gen: "001" - spec: - containers: - - name: kafka - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce - command: - - ./bin/kafka-topics.sh - - --zookeeper - - zookeeper:2181 - - --create - - --if-not-exists - - --topic - - ops-kube-events-all-json-001 - - --partitions - - "1" - - --replication-factor - - "1" - - --config - # 8 days - - retention.ms=691200000 - restartPolicy: Never -- cgit v1.2.3