Merge pull request #90 from Yolean/1.8-manifestsv3.0.0

Updates manifests to beta2 workload API, k8s 1.8

14 files changed, 27 insertions, 65 deletions

diff --git a/README.md b/README.md
index 2f007cc..526b7d5 100644
--- a/README.md
+++ b/README.md
@@ -1,4 +1,5 @@
-
+_Manifests here require Kubernetes 1.8 now.
+On earlier versions use [v2.1.0](https://github.com/Yolean/kubernetes-kafka/tree/v2.1.0)._
 
 # Kafka on Kubernetes
 
@@ -49,7 +50,7 @@ To support automatic migration in the face of availability zone unavailability w
 ## Start Kafka
 
 ```
-kubectl apply -f ./
+kubectl apply -f ./kafka/
 ```
 
 You might want to verify in logs that Kafka found its own DNS name(s) correctly. Look for records like:
@@ -61,7 +62,6 @@ kubectl -n kafka logs kafka-0 | grep "Registered broker"
 That's it. Just add business value :wink:.
 For clients we tend to use [librdkafka](https://github.com/edenhill/librdkafka)-based drivers like [node-rdkafka](https://github.com/Blizzard/node-rdkafka).
 To use [Kafka Connect](http://kafka.apache.org/documentation/#connect) and [Kafka Streams](http://kafka.apache.org/documentation/streams/) you may want to take a look at our [sample](https://github.com/solsson/dockerfiles/tree/master/connect-files) [Dockerfile](https://github.com/solsson/dockerfiles/tree/master/streams-logfilter)s.
-And don't forget the [addon](https://github.com/Yolean/kubernetes-kafka/labels/addon)s.
 
 ## RBAC
 
@@ -75,8 +75,6 @@ kubectl apply -f rbac-namespace-default/
 Tests are based on the [kube-test](https://github.com/Yolean/kube-test) concept.
 Like the rest of this repo they have `kubectl` as the only local dependency.
 
-```
-kubectl apply -f test/
-# Anything that isn't READY here is a failed test
-kubectl get pods -l test-type=readiness -w --namespace=test-kafka
-```
+Run self-tests or not. They do generate some load, but indicate if the platform is working or not.
+ * To include tests, replace `apply -f` with `apply -R -f` in your `kubectl`s above.
+ * Anything that isn't READY in `kubectl get pods -l test-type=readiness --namespace=test-kafka` is a failed test.
diff --git a/00namespace.yml b/kafka/00namespace.yml
index a6cf001..a6cf001 100644
--- a/00namespace.yml
+++ b/kafka/00namespace.yml
diff --git a/10broker-config.yml b/kafka/10broker-config.yml
index c0c4c8c..c0c4c8c 100644
--- a/10broker-config.yml
+++ b/kafka/10broker-config.yml
diff --git a/20dns.yml b/kafka/20dns.yml
index 4088c31..4088c31 100644
--- a/20dns.yml
+++ b/kafka/20dns.yml
diff --git a/30bootstrap-service.yml b/kafka/30bootstrap-service.yml
index 7c2a337..7c2a337 100644
--- a/30bootstrap-service.yml
+++ b/kafka/30bootstrap-service.yml
diff --git a/50kafka.yml b/kafka/50kafka.yml
index c39ba9e..dff4a79 100644
--- a/50kafka.yml
+++ b/kafka/50kafka.yml
@@ -1,9 +1,13 @@
-apiVersion: apps/v1beta1
+apiVersion: apps/v1beta2
 kind: StatefulSet
 metadata:
   name: kafka
   namespace: kafka
 spec:
+spec:
+  selector:
+    matchLabels:
+      app: kafka
   serviceName: "broker"
   replicas: 3
   template:
diff --git a/test/00namespace.yml b/kafka/test/00namespace.yml
index fbb6e0e..fbb6e0e 100644
--- a/test/00namespace.yml
+++ b/kafka/test/00namespace.yml
diff --git a/test/kafkacat.yml b/kafka/test/kafkacat.yml
index 61a265c..8d6c64e 100644
--- a/test/kafkacat.yml
+++ b/kafka/test/kafkacat.yml
@@ -23,8 +23,10 @@ data:
     echo "${UNIQUE: -41:5}:Test $UNIQUE" >> /shared/produce.tmp
     sleep $PC_WAIT
     LAST=$(tail -n 1 /shared/consumed.tmp)
+    [ -z "$LAST" ] && echo "Nothing consumed" && exit 1
 
     LAST_TS=$(echo $LAST | awk -F';' '{print $1}')
+    [ -z "$LAST_TS" ] && echo "Failed to get timestamp for message: $LAST" && exit 1
     LAST_MSG=$(echo $LAST | awk -F';' '{print $4}')
     NOW=$(date -u +%s%3N)
     DIFF_S=$((($NOW - $LAST_TS)/1000))
diff --git a/test/produce-consume.yml b/kafka/test/produce-consume.yml
index a326f01..a326f01 100644
--- a/test/produce-consume.yml
+++ b/kafka/test/produce-consume.yml
diff --git a/qa-yolean.sh b/qa-yolean.sh
deleted file mode 100755
index dcdd6f2..0000000
--- a/qa-yolean.sh
+++ /dev/null
@@ -1,51 +0,0 @@
-#!/bin/bash
-# Combines addons into what we 'kubectl apply -f' to production
-set -ex
-
-ANNOTATION_PREFIX='yolean.se/kubernetes-kafka-'
-BUILD=$(basename $0)
-REMOTE=origin
-FROM="$REMOTE/"
-START=master
-
-[ ! -z "$(git status --untracked-files=no -s)" ] && echo "Working copy must be clean" && exit 1
-
-function annotate {
-  key=$1
-  value=$2
-  file=$3
-  case $(uname) in
-    Darwin*)
-      sed -i '' 's|      annotations:|      annotations:\
-        --next-annotation--|' $file
-      sed -i '' "s|--next-annotation--|${ANNOTATION_PREFIX}$key: '$value'|" $file
-      ;;
-    *)
-      sed -i "s|      annotations:|      annotations:\n        ${ANNOTATION_PREFIX}$key: '$value'|" $file
-      ;;
-  esac
-}
-
-git checkout ${FROM}$START
-REVS="$START:$(git rev-parse --short ${FROM}$START)"
-
-git checkout -b qa-yolean-$(date +"%Y%m%dT%H%M%S")
-
-for BRANCH in \
-  addon-storage-classes \
-  rolling-update \
-  addon-rest \
-  addon-metrics \
-  addon-kube-events-topic
-do
-  git merge --no-ff ${FROM}$BRANCH -m "qa-yolean merge ${FROM}$BRANCH" && \
-    REVS="$REVS $BRANCH:$(git rev-parse --short ${FROM}$BRANCH)"
-done
-
-END_BRANCH_GIT=$(git rev-parse --abbrev-ref HEAD)
-
-for F in ./50kafka.yml ./zookeeper/50pzoo.yml ./zookeeper/51zoo.yml
-do
-  annotate revs "$REVS" $F
-  annotate build "$END_BRANCH_GIT" $F
-done
diff --git a/rbac-namespace-default/node-reader.yml b/rbac-namespace-default/node-reader.yml
index edf3dde..1d0cd51 100644
--- a/rbac-namespace-default/node-reader.yml
+++ b/rbac-namespace-default/node-reader.yml
@@ -8,7 +8,7 @@
 #
 ---
 kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: node-reader
   labels:
@@ -22,7 +22,7 @@ rules:
   - get
 ---
 kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1beta1
+apiVersion: rbac.authorization.k8s.io/v1
 metadata:
   name: kafka-node-reader
   labels:
diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml
index 678c174..77319e4 100644
--- a/yahoo-kafka-manager/kafka-manager.yml
+++ b/yahoo-kafka-manager/kafka-manager.yml
@@ -1,4 +1,4 @@
-apiVersion: apps/v1beta1
+apiVersion: apps/v1beta2
 kind: Deployment
 metadata:
   name: kafka-manager
diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml
index a166a91..874d6df 100644
--- a/zookeeper/50pzoo.yml
+++ b/zookeeper/50pzoo.yml
@@ -1,9 +1,14 @@
-apiVersion: apps/v1beta1
+apiVersion: apps/v1beta2
 kind: StatefulSet
 metadata:
   name: pzoo
   namespace: kafka
 spec:
+spec:
+  selector:
+    matchLabels:
+      app: zookeeper
+      storage: persistent
   serviceName: "pzoo"
   replicas: 3
   template:
diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml
index a2922ef..084e8f3 100644
--- a/zookeeper/51zoo.yml
+++ b/zookeeper/51zoo.yml
@@ -1,9 +1,13 @@
-apiVersion: apps/v1beta1
+apiVersion: apps/v1beta2
 kind: StatefulSet
 metadata:
   name: zoo
   namespace: kafka
 spec:
+  selector:
+    matchLabels:
+      app: zookeeper
+      storage: ephemeral
   serviceName: "zoo"
   replicas: 2
   template: