diff options
author | solsson <solsson@gmail.com> | 2017-11-09 15:56:07 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-11-09 15:56:07 +0100 |
commit | 2257cd12c0fb8b4908382d565ea8e74dfce43c4d (patch) | |
tree | 5d8e0dae3561136f03be9dd8f42d41657e10f657 | |
parent | 6b24807de8286ecfb9bf47fe1c269816920dec73 (diff) | |
parent | b8f6078ac45465eff37df9acd67d1bf2c41ae448 (diff) | |
download | kubernetes-kafka-2257cd12c0fb8b4908382d565ea8e74dfce43c4d.tar.gz kubernetes-kafka-2257cd12c0fb8b4908382d565ea8e74dfce43c4d.tar.bz2 kubernetes-kafka-2257cd12c0fb8b4908382d565ea8e74dfce43c4d.zip |
Merge pull request #90 from Yolean/1.8-manifestsv3.0.0
Updates manifests to beta2 workload API, k8s 1.8
-rw-r--r-- | README.md | 14 | ||||
-rw-r--r-- | kafka/00namespace.yml (renamed from 00namespace.yml) | 0 | ||||
-rw-r--r-- | kafka/10broker-config.yml (renamed from 10broker-config.yml) | 0 | ||||
-rw-r--r-- | kafka/20dns.yml (renamed from 20dns.yml) | 0 | ||||
-rw-r--r-- | kafka/30bootstrap-service.yml (renamed from 30bootstrap-service.yml) | 0 | ||||
-rw-r--r-- | kafka/50kafka.yml (renamed from 50kafka.yml) | 6 | ||||
-rw-r--r-- | kafka/test/00namespace.yml (renamed from test/00namespace.yml) | 0 | ||||
-rw-r--r-- | kafka/test/kafkacat.yml (renamed from test/kafkacat.yml) | 2 | ||||
-rw-r--r-- | kafka/test/produce-consume.yml (renamed from test/produce-consume.yml) | 0 | ||||
-rwxr-xr-x | qa-yolean.sh | 51 | ||||
-rw-r--r-- | rbac-namespace-default/node-reader.yml | 4 | ||||
-rw-r--r-- | yahoo-kafka-manager/kafka-manager.yml | 2 | ||||
-rw-r--r-- | zookeeper/50pzoo.yml | 7 | ||||
-rw-r--r-- | zookeeper/51zoo.yml | 6 |
14 files changed, 27 insertions, 65 deletions
@@ -1,4 +1,5 @@ - +_Manifests here require Kubernetes 1.8 now. +On earlier versions use [v2.1.0](https://github.com/Yolean/kubernetes-kafka/tree/v2.1.0)._ # Kafka on Kubernetes @@ -49,7 +50,7 @@ To support automatic migration in the face of availability zone unavailability w ## Start Kafka ``` -kubectl apply -f ./ +kubectl apply -f ./kafka/ ``` You might want to verify in logs that Kafka found its own DNS name(s) correctly. Look for records like: @@ -61,7 +62,6 @@ kubectl -n kafka logs kafka-0 | grep "Registered broker" That's it. Just add business value :wink:. For clients we tend to use [librdkafka](https://github.com/edenhill/librdkafka)-based drivers like [node-rdkafka](https://github.com/Blizzard/node-rdkafka). To use [Kafka Connect](http://kafka.apache.org/documentation/#connect) and [Kafka Streams](http://kafka.apache.org/documentation/streams/) you may want to take a look at our [sample](https://github.com/solsson/dockerfiles/tree/master/connect-files) [Dockerfile](https://github.com/solsson/dockerfiles/tree/master/streams-logfilter)s. -And don't forget the [addon](https://github.com/Yolean/kubernetes-kafka/labels/addon)s. ## RBAC @@ -75,8 +75,6 @@ kubectl apply -f rbac-namespace-default/ Tests are based on the [kube-test](https://github.com/Yolean/kube-test) concept. Like the rest of this repo they have `kubectl` as the only local dependency. -``` -kubectl apply -f test/ -# Anything that isn't READY here is a failed test -kubectl get pods -l test-type=readiness -w --namespace=test-kafka -``` +Run self-tests or not. They do generate some load, but indicate if the platform is working or not. + * To include tests, replace `apply -f` with `apply -R -f` in your `kubectl`s above. + * Anything that isn't READY in `kubectl get pods -l test-type=readiness --namespace=test-kafka` is a failed test. diff --git a/00namespace.yml b/kafka/00namespace.yml index a6cf001..a6cf001 100644 --- a/00namespace.yml +++ b/kafka/00namespace.yml diff --git a/10broker-config.yml b/kafka/10broker-config.yml index c0c4c8c..c0c4c8c 100644 --- a/10broker-config.yml +++ b/kafka/10broker-config.yml diff --git a/20dns.yml b/kafka/20dns.yml index 4088c31..4088c31 100644 --- a/20dns.yml +++ b/kafka/20dns.yml diff --git a/30bootstrap-service.yml b/kafka/30bootstrap-service.yml index 7c2a337..7c2a337 100644 --- a/30bootstrap-service.yml +++ b/kafka/30bootstrap-service.yml diff --git a/50kafka.yml b/kafka/50kafka.yml index c39ba9e..dff4a79 100644 --- a/50kafka.yml +++ b/kafka/50kafka.yml @@ -1,9 +1,13 @@ -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: StatefulSet metadata: name: kafka namespace: kafka spec: +spec: + selector: + matchLabels: + app: kafka serviceName: "broker" replicas: 3 template: diff --git a/test/00namespace.yml b/kafka/test/00namespace.yml index fbb6e0e..fbb6e0e 100644 --- a/test/00namespace.yml +++ b/kafka/test/00namespace.yml diff --git a/test/kafkacat.yml b/kafka/test/kafkacat.yml index 61a265c..8d6c64e 100644 --- a/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -23,8 +23,10 @@ data: echo "${UNIQUE: -41:5}:Test $UNIQUE" >> /shared/produce.tmp sleep $PC_WAIT LAST=$(tail -n 1 /shared/consumed.tmp) + [ -z "$LAST" ] && echo "Nothing consumed" && exit 1 LAST_TS=$(echo $LAST | awk -F';' '{print $1}') + [ -z "$LAST_TS" ] && echo "Failed to get timestamp for message: $LAST" && exit 1 LAST_MSG=$(echo $LAST | awk -F';' '{print $4}') NOW=$(date -u +%s%3N) DIFF_S=$((($NOW - $LAST_TS)/1000)) diff --git a/test/produce-consume.yml b/kafka/test/produce-consume.yml index a326f01..a326f01 100644 --- a/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml diff --git a/qa-yolean.sh b/qa-yolean.sh deleted file mode 100755 index dcdd6f2..0000000 --- a/qa-yolean.sh +++ /dev/null @@ -1,51 +0,0 @@ -#!/bin/bash -# Combines addons into what we 'kubectl apply -f' to production -set -ex - -ANNOTATION_PREFIX='yolean.se/kubernetes-kafka-' -BUILD=$(basename $0) -REMOTE=origin -FROM="$REMOTE/" -START=master - -[ ! -z "$(git status --untracked-files=no -s)" ] && echo "Working copy must be clean" && exit 1 - -function annotate { - key=$1 - value=$2 - file=$3 - case $(uname) in - Darwin*) - sed -i '' 's| annotations:| annotations:\ - --next-annotation--|' $file - sed -i '' "s|--next-annotation--|${ANNOTATION_PREFIX}$key: '$value'|" $file - ;; - *) - sed -i "s| annotations:| annotations:\n ${ANNOTATION_PREFIX}$key: '$value'|" $file - ;; - esac -} - -git checkout ${FROM}$START -REVS="$START:$(git rev-parse --short ${FROM}$START)" - -git checkout -b qa-yolean-$(date +"%Y%m%dT%H%M%S") - -for BRANCH in \ - addon-storage-classes \ - rolling-update \ - addon-rest \ - addon-metrics \ - addon-kube-events-topic -do - git merge --no-ff ${FROM}$BRANCH -m "qa-yolean merge ${FROM}$BRANCH" && \ - REVS="$REVS $BRANCH:$(git rev-parse --short ${FROM}$BRANCH)" -done - -END_BRANCH_GIT=$(git rev-parse --abbrev-ref HEAD) - -for F in ./50kafka.yml ./zookeeper/50pzoo.yml ./zookeeper/51zoo.yml -do - annotate revs "$REVS" $F - annotate build "$END_BRANCH_GIT" $F -done diff --git a/rbac-namespace-default/node-reader.yml b/rbac-namespace-default/node-reader.yml index edf3dde..1d0cd51 100644 --- a/rbac-namespace-default/node-reader.yml +++ b/rbac-namespace-default/node-reader.yml @@ -8,7 +8,7 @@ # --- kind: ClusterRole -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: node-reader labels: @@ -22,7 +22,7 @@ rules: - get --- kind: ClusterRoleBinding -apiVersion: rbac.authorization.k8s.io/v1beta1 +apiVersion: rbac.authorization.k8s.io/v1 metadata: name: kafka-node-reader labels: diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml index 678c174..77319e4 100644 --- a/yahoo-kafka-manager/kafka-manager.yml +++ b/yahoo-kafka-manager/kafka-manager.yml @@ -1,4 +1,4 @@ -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: Deployment metadata: name: kafka-manager diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index a166a91..874d6df 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -1,9 +1,14 @@ -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: StatefulSet metadata: name: pzoo namespace: kafka spec: +spec: + selector: + matchLabels: + app: zookeeper + storage: persistent serviceName: "pzoo" replicas: 3 template: diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index a2922ef..084e8f3 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -1,9 +1,13 @@ -apiVersion: apps/v1beta1 +apiVersion: apps/v1beta2 kind: StatefulSet metadata: name: zoo namespace: kafka spec: + selector: + matchLabels: + app: zookeeper + storage: ephemeral serviceName: "zoo" replicas: 2 template: |