From c60d99ce5690fe5eb64c7e89d995f4b9f55852de Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 26 Dec 2016 00:12:43 +0100 Subject: Manages to set zk node id without altering the official image, but leader-election port bind fails when using DNS name instead of 0.0.0.0 --- zookeeper/00namespace.yml | 5 ++ zookeeper/20zoo-service.yml | 15 +++++ zookeeper/30service.yml | 12 ++++ zookeeper/50zoo.yml | 52 +++++++++++++++ zookeeper/README.md | 85 ------------------------ zookeeper/bootstrap/pv-template.yml | 48 -------------- zookeeper/bootstrap/pv.sh | 11 ---- zookeeper/bootstrap/pvc.yml | 48 -------------- zookeeper/init/Dockerfile | 29 -------- zookeeper/init/Makefile | 27 -------- zookeeper/init/install.sh | 76 --------------------- zookeeper/init/on-change.sh | 49 -------------- zookeeper/init/on-start.sh | 73 -------------------- zookeeper/service.yml | 12 ---- zookeeper/zookeeper.yaml | 128 ------------------------------------ 15 files changed, 84 insertions(+), 586 deletions(-) create mode 100644 zookeeper/00namespace.yml create mode 100644 zookeeper/20zoo-service.yml create mode 100644 zookeeper/30service.yml create mode 100644 zookeeper/50zoo.yml delete mode 100644 zookeeper/README.md delete mode 100644 zookeeper/bootstrap/pv-template.yml delete mode 100755 zookeeper/bootstrap/pv.sh delete mode 100644 zookeeper/bootstrap/pvc.yml delete mode 100644 zookeeper/init/Dockerfile delete mode 100644 zookeeper/init/Makefile delete mode 100755 zookeeper/init/install.sh delete mode 100755 zookeeper/init/on-change.sh delete mode 100755 zookeeper/init/on-start.sh delete mode 100644 zookeeper/service.yml delete mode 100644 zookeeper/zookeeper.yaml diff --git a/zookeeper/00namespace.yml b/zookeeper/00namespace.yml new file mode 100644 index 0000000..a6cf001 --- /dev/null +++ b/zookeeper/00namespace.yml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: kafka diff --git a/zookeeper/20zoo-service.yml b/zookeeper/20zoo-service.yml new file mode 100644 index 0000000..460b63e --- /dev/null +++ b/zookeeper/20zoo-service.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Service +metadata: + name: zoo + labels: + app: zookeeper +spec: + ports: + - port: 2888 + name: peer + - port: 3888 + name: leader-election + clusterIP: None + selector: + app: zookeeper diff --git a/zookeeper/30service.yml b/zookeeper/30service.yml new file mode 100644 index 0000000..f33f393 --- /dev/null +++ b/zookeeper/30service.yml @@ -0,0 +1,12 @@ +# the headless service is for PetSet DNS, this one is for clients +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + namespace: kafka +spec: + ports: + - port: 2181 + name: client + selector: + app: zookeeper diff --git a/zookeeper/50zoo.yml b/zookeeper/50zoo.yml new file mode 100644 index 0000000..b73d622 --- /dev/null +++ b/zookeeper/50zoo.yml @@ -0,0 +1,52 @@ +apiVersion: apps/v1beta1 +kind: StatefulSet +metadata: + name: zoo + namespace: kafka +spec: + serviceName: "zoo" + replicas: 5 + template: + metadata: + labels: + app: zookeeper + annotations: + pod.beta.kubernetes.io/init-containers: '[ + { + "name": "install", + "image": "alpine:3.4", + "command": ["sh", "-c", "echo $(($(hostname | sed s/zoo-//) + 1)) > /data/myid" ], + "volumeMounts": [ + { + "name": "datadir", + "mountPath": "/data" + } + ] + } + ]' + spec: + terminationGracePeriodSeconds: 10 + containers: + - name: zookeeper + image: zookeeper:3.4.9@sha256:50cfe2c77fe203ab528ffb808f1a8b505db73b1f85aedbc52e4fdde69e2ebfe8 + env: + - name: ZOO_SERVERS + value: server.1=zoo-0.zoo:2888:3888:participant server.2=zoo-1.zoo:2888:3888:participant server.3=zoo-2.zoo:2888:3888:participant server.4=zoo-3.zoo:2888:3888:participant server.5=zoo-4.zoo:2888:3888:participant + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: peer + - containerPort: 3888 + name: leader-election + volumeMounts: + - name: datadir + mountPath: /data + # There's defaults in this folder, such as logging config + #- name: conf + # mountPath: /conf + volumes: + #- name: conf + # emptyDir: {} + - name: datadir + emptyDir: {} diff --git a/zookeeper/README.md b/zookeeper/README.md deleted file mode 100644 index f485148..0000000 --- a/zookeeper/README.md +++ /dev/null @@ -1,85 +0,0 @@ -# Zookeeper - -This example runs zookeeper through a petset. - -## Bootstrap - -Create the petset in this directory -``` -$ kubetl create -f zookeeper.yaml -``` - -Once you have all 3 nodes in Running, you can run the "test.sh" script in this directory. - -## Failover - -You can test failover by killing the leader. Insert a key: -```console -$ kubectl -n kafka exec zoo-0 -- /opt/zookeeper/bin/zkCli.sh create /foo bar; -$ kubectl -n kafka exec zoo-2 -- /opt/zookeeper/bin/zkCli.sh get /foo; - -Watch existing members: -```console -$ kubectl -n kafka run --attach bbox --image=busybox --restart=Never -- sh -c 'while true; do for i in 0 1 2; do echo zoo-$i $(echo stats | nc zoo-$i.zk:2181 | grep Mode); sleep 1; done; done'; -zoo-2 Mode: follower -zoo-0 Mode: follower -zoo-1 Mode: leader -zoo-2 Mode: follower -``` - -Delete pets and wait for the petset controller to bring the back up: -```console -$ kubectl -n kafka delete po -l app=zk -$ kubectl -n kafka get po --watch-only -NAME READY STATUS RESTARTS AGE -zoo-0 0/1 Init:0/2 0 16s -zoo-0 0/1 Init:0/2 0 21s -zoo-0 0/1 PodInitializing 0 23s -zoo-0 1/1 Running 0 41s -zoo-1 0/1 Pending 0 0s -zoo-1 0/1 Init:0/2 0 0s -zoo-1 0/1 Init:0/2 0 14s -zoo-1 0/1 PodInitializing 0 17s -zoo-1 0/1 Running 0 18s -zoo-2 0/1 Pending 0 0s -zoo-2 0/1 Init:0/2 0 0s -zoo-2 0/1 Init:0/2 0 12s -zoo-2 0/1 Init:0/2 0 28s -zoo-2 0/1 PodInitializing 0 31s -zoo-2 0/1 Running 0 32s -... - -zoo-0 Mode: follower -zoo-1 Mode: leader -zoo-2 Mode: follower -``` - -Check the previously inserted key: -```console -$ kubectl -n kafka exec zoo-1 -- /opt/zookeeper/bin/zkCli.sh get /foo -ionid = 0x354887858e80035, negotiated timeout = 30000 - -WATCHER:: - -WatchedEvent state:SyncConnected type:None path:null -bar -``` - -## Scaling - -You can scale up by modifying the number of replicas on the PetSet. - -## Image Upgrade - -TODO: Add details - -## Maintenance - -TODO: Add details - -## Limitations -* Both petset and init containers are in alpha -* Look through the on-start and on-change scripts for TODOs -* Doesn't support the addition of observers through the petset -* Only supports storage options that have backends for persistent volume claims - diff --git a/zookeeper/bootstrap/pv-template.yml b/zookeeper/bootstrap/pv-template.yml deleted file mode 100644 index 61d84ee..0000000 --- a/zookeeper/bootstrap/pv-template.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: datadir-zoo-0 - namespace: kafka - labels: - app: zk - petindex: "0" -spec: - accessModes: - - ReadWriteOnce - capacity: - storage: 100Mi - hostPath: - path: /tmp/k8s-data/datadir-zoo-0 ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: datadir-zoo-1 - namespace: kafka - labels: - app: zk - petindex: "1" -spec: - accessModes: - - ReadWriteOnce - capacity: - storage: 100Mi - hostPath: - path: /tmp/k8s-data/datadir-zoo-1 ---- -apiVersion: v1 -kind: PersistentVolume -metadata: - name: datadir-zoo-2 - namespace: kafka - labels: - app: zk - petindex: "2" -spec: - accessModes: - - ReadWriteOnce - capacity: - storage: 100Mi - hostPath: - path: /tmp/k8s-data/datadir-zoo-2 diff --git a/zookeeper/bootstrap/pv.sh b/zookeeper/bootstrap/pv.sh deleted file mode 100755 index 7bbe121..0000000 --- a/zookeeper/bootstrap/pv.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -echo "Note that in for example GKE a PetSet will have PersistentVolume(s) and PersistentVolumeClaim(s) created for it automatically" - -dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && cd .. && pwd )" -path="$dir/data" -echo "Please enter a path where to store data during local testing: ($path)" -read newpath -[ -n "$newpath" ] && path=$newpath - -cat zookeeper/bootstrap/pv-template.yml | sed "s|/tmp/k8s-data|$path|" | kubectl create -f - diff --git a/zookeeper/bootstrap/pvc.yml b/zookeeper/bootstrap/pvc.yml deleted file mode 100644 index 84d0e46..0000000 --- a/zookeeper/bootstrap/pvc.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: datadir-zoo-0 - namespace: kafka -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi - selector: - matchLabels: - app: zk - petindex: "0" ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: datadir-zoo-1 - namespace: kafka -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi - selector: - matchLabels: - app: zk - petindex: "1" ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: datadir-zoo-2 - namespace: kafka -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi - selector: - matchLabels: - app: zk - petindex: "2" diff --git a/zookeeper/init/Dockerfile b/zookeeper/init/Dockerfile deleted file mode 100644 index d3e8aae..0000000 --- a/zookeeper/init/Dockerfile +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# TODO: get rid of bash dependency and switch to plain busybox. -# The tar in busybox also doesn't seem to understand compression. -FROM debian:jessie -MAINTAINER Prashanth.B - -RUN apt-get update && apt-get install -y wget netcat - -ADD on-start.sh / -ADD on-change.sh / -# See contrib/pets/peer-finder for details -RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder - -ADD install.sh / -RUN chmod -c 755 /install.sh /on-start.sh /on-change.sh /peer-finder -Entrypoint ["/install.sh"] diff --git a/zookeeper/init/Makefile b/zookeeper/init/Makefile deleted file mode 100644 index bfb6978..0000000 --- a/zookeeper/init/Makefile +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -all: push - -TAG = 0.1 -PREFIX = gcr.io/google_containers/zookeeper-install - -container: - docker build -t $(PREFIX):$(TAG) . - -push: container - gcloud docker push $(PREFIX):$(TAG) - -clean: - docker rmi $(PREFIX):$(TAG) diff --git a/zookeeper/init/install.sh b/zookeeper/init/install.sh deleted file mode 100755 index 6ed7269..0000000 --- a/zookeeper/init/install.sh +++ /dev/null @@ -1,76 +0,0 @@ -#! /bin/bash - -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This volume is assumed to exist and is shared with parent of the init -# container. It contains the zookeeper installation. -INSTALL_VOLUME="/opt" - -# This volume is assumed to exist and is shared with the peer-finder -# init container. It contains on-start/change configuration scripts. -WORKDIR_VOLUME="/work-dir" - -# As of April-2016 is 3.4.8 is the latest stable, but versions 3.5.0 onward -# allow dynamic reconfiguration. -VERSION="3.5.0-alpha" - -for i in "$@" -do -case $i in - -v=*|--version=*) - VERSION="${i#*=}" - shift - ;; - -i=*|--install-into=*) - INSTALL_VOLUME="${i#*=}" - shift - ;; - -w=*|--work-dir=*) - WORKDIR_VOLUME="${i#*=}" - shift - ;; - *) - # unknown option - ;; -esac -done - -echo installing config scripts into "${WORKDIR_VOLUME}" -mkdir -p "${WORKDIR_VOLUME}" -cp /on-start.sh "${WORKDIR_VOLUME}"/ -cp /on-change.sh "${WORKDIR_VOLUME}"/ -cp /peer-finder "${WORKDIR_VOLUME}"/ - -echo installing zookeeper-"${VERSION}" into "${INSTALL_VOLUME}" -mkdir -p "${INSTALL_VOLUME}" -wget -q -O - http://apache.mirrors.pair.com/zookeeper/zookeeper-"${VERSION}"/zookeeper-"${VERSION}".tar.gz | tar -xzf - -C "${INSTALL_VOLUME}" -mv "${INSTALL_VOLUME}"/zookeeper-"${VERSION}" "${INSTALL_VOLUME}"/zookeeper -cp "${INSTALL_VOLUME}"/zookeeper/conf/zoo_sample.cfg "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg - -# TODO: Should dynamic config be tied to the version? -IFS="." read -ra RELEASE <<< "${VERSION}" -if [ $(expr "${RELEASE[1]}") -gt 4 ]; then - echo zookeeper-"${VERSION}" supports dynamic reconfiguration, enabling it - echo "standaloneEnabled=false" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg - echo "dynamicConfigFile="${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg.dynamic" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg -fi - -# TODO: This is a hack, netcat is convenient to have in the zookeeper container -# I want to avoid using a custom zookeeper image just for this. So copy it. -NC=$(which nc) -if [ "${NC}" != "" ]; then - echo copying nc into "${INSTALL_VOLUME}" - cp "${NC}" "${INSTALL_VOLUME}" -fi diff --git a/zookeeper/init/on-change.sh b/zookeeper/init/on-change.sh deleted file mode 100755 index bb9a504..0000000 --- a/zookeeper/init/on-change.sh +++ /dev/null @@ -1,49 +0,0 @@ -#! /bin/bash - -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script configures zookeeper cluster member ship for version of zookeeper -# < 3.5.0. It should not be used with the on-start.sh script in this example. -# As of April-2016 is 3.4.8 is the latest stable. - -CFG=/opt/zookeeper/conf/zoo.cfg -CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak -MY_ID=/tmp/zookeeper/myid - -# write myid -IFS='-' read -ra ADDR <<< "$(hostname)" -echo $(expr "1" + "${ADDR[1]}") > "${MY_ID}" - -# TODO: This is a dumb way to reconfigure zookeeper because it allows dynamic -# reconfig, but it's simple. -i=0 -echo " -tickTime=2000 -initLimit=10 -syncLimit=5 -dataDir=/tmp/zookeeper -clientPort=2181 -" > "${CFG_BAK}" - -while read -ra LINE; do - let i=i+1 - echo "server.${i}=${LINE}:2888:3888" >> "${CFG_BAK}" -done -cp ${CFG_BAK} ${CFG} - -# TODO: Typically one needs to first add a new member as an "observer" then -# promote it to "participant", but that requirement is relaxed if we never -# start > 1 at a time. -/opt/zookeeper/bin/zkServer.sh restart diff --git a/zookeeper/init/on-start.sh b/zookeeper/init/on-start.sh deleted file mode 100755 index 392b82a..0000000 --- a/zookeeper/init/on-start.sh +++ /dev/null @@ -1,73 +0,0 @@ -#! /bin/bash - -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This script configures zookeeper cluster member ship for version of zookeeper -# >= 3.5.0. It should not be used with the on-change.sh script in this example. -# As of April-2016 is 3.4.8 is the latest stable. - -# Both /opt and /tmp/zookeeper are assumed to be volumes shared with the parent. -CFG=/opt/zookeeper/conf/zoo.cfg.dynamic -CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak -MY_ID_FILE=/tmp/zookeeper/myid -HOSTNAME=$(hostname) - -while read -ra LINE; do - PEERS=("${PEERS[@]}" $LINE) -done - -# Don't add the first member as an observer -if [ ${#PEERS[@]} -eq 1 ]; then - # We need to write our index in this list of servers into MY_ID_FILE. - # Note that this may not always coincide with the hostname id. - echo 1 > "${MY_ID_FILE}" - echo "server.1=${PEERS[0]}:2888:3888;2181" > "${CFG}" - # TODO: zkServer-initialize is the safe way to handle changes to datadir - # because simply starting will create a new datadir, BUT if the user changed - # pod template they might end up with 2 datadirs and brief split brain. - exit -fi - -# Every subsequent member is added as an observer and promoted to a participant -echo "" > "${CFG_BAK}" -i=0 -for peer in "${PEERS[@]}"; do - let i=i+1 - if [[ "${peer}" == *"${HOSTNAME}"* ]]; then - MY_ID=$i - MY_NAME=${peer} - echo $i > "${MY_ID_FILE}" - echo "server.${i}=${peer}:2888:3888:observer;2181" >> "${CFG_BAK}" - else - echo "server.${i}=${peer}:2888:3888:participant;2181" >> "${CFG_BAK}" - fi -done - -# Once the dynamic config file is written it shouldn't be modified, so the final -# reconfigure needs to happen through the "reconfig" command. -cp ${CFG_BAK} ${CFG} - -# TODO: zkServer-initialize is the safe way to handle changes to datadir -# because simply starting will create a new datadir, BUT if the user changed -# pod template they might end up with 2 datadirs and brief split brain. -/opt/zookeeper/bin/zkServer.sh start - -# TODO: We shouldn't need to specify the address of the master as long as -# there's quorum. According to the docs the new server is just not allowed to -# vote, it's still allowed to propose config changes, and it knows the -# existing members of the ensemble from *its* config. This works as expected, -# but we should correlate with more satisfying empirical evidence. -/opt/zookeeper/bin/zkCli.sh reconfig -add "server.$MY_ID=$MY_NAME:2888:3888:participant;2181" -/opt/zookeeper/bin/zkServer.sh stop diff --git a/zookeeper/service.yml b/zookeeper/service.yml deleted file mode 100644 index e11e791..0000000 --- a/zookeeper/service.yml +++ /dev/null @@ -1,12 +0,0 @@ -# the headless service is for PetSet DNS, this one is for clients -apiVersion: v1 -kind: Service -metadata: - name: zookeeper - namespace: kafka -spec: - ports: - - port: 2181 - name: client - selector: - app: zk diff --git a/zookeeper/zookeeper.yaml b/zookeeper/zookeeper.yaml deleted file mode 100644 index ee2b237..0000000 --- a/zookeeper/zookeeper.yaml +++ /dev/null @@ -1,128 +0,0 @@ -# A headless service to create DNS records -apiVersion: v1 -kind: Service -metadata: - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" - name: zk - namespace: kafka - labels: - app: zk -spec: - ports: - - port: 2888 - name: peer - - port: 3888 - name: leader-election - # *.zk.kafka.svc.cluster.local - clusterIP: None - selector: - app: zk ---- -apiVersion: apps/v1alpha1 -kind: PetSet -metadata: - name: zoo - namespace: kafka -spec: - serviceName: "zk" - replicas: 3 - template: - metadata: - labels: - app: zk - annotations: - pod.alpha.kubernetes.io/initialized: "true" - pod.alpha.kubernetes.io/init-containers: '[ - { - "name": "install", - "image": "gcr.io/google_containers/zookeeper-install:0.1", - "imagePullPolicy": "Always", - "args": ["--version=3.5.2-alpha", "--install-into=/opt", "--work-dir=/work-dir"], - "volumeMounts": [ - { - "name": "opt", - "mountPath": "/opt/" - }, - { - "name": "workdir", - "mountPath": "/work-dir" - } - ] - }, - { - "name": "bootstrap", - "image": "java:openjdk-8-jre", - "command": ["/work-dir/peer-finder"], - "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=zk"], - "env": [ - { - "name": "POD_NAMESPACE", - "valueFrom": { - "fieldRef": { - "apiVersion": "v1", - "fieldPath": "metadata.namespace" - } - } - } - ], - "volumeMounts": [ - { - "name": "opt", - "mountPath": "/opt/" - }, - { - "name": "workdir", - "mountPath": "/work-dir" - }, - { - "name": "datadir", - "mountPath": "/tmp/zookeeper" - } - ] - } - ]' - spec: - containers: - - name: zk - image: java:openjdk-8-jre - ports: - - containerPort: 2181 - name: client - - containerPort: 2888 - name: peer - - containerPort: 3888 - name: leader-election - command: - - /opt/zookeeper/bin/zkServer.sh - args: - - start-foreground - readinessProbe: - exec: - command: - - sh - - -c - - "/opt/zookeeper/bin/zkCli.sh ls /" - initialDelaySeconds: 15 - timeoutSeconds: 5 - volumeMounts: - - name: datadir - mountPath: /tmp/zookeeper - - name: opt - mountPath: /opt/ - volumes: - - name: opt - emptyDir: {} - - name: workdir - emptyDir: {} - volumeClaimTemplates: - - metadata: - name: datadir - namespace: kafka - annotations: - volume.alpha.kubernetes.io/storage-class: anything - spec: - accessModes: [ "ReadWriteOnce" ] - resources: - requests: - storage: 20Gi -- cgit v1.2.3 From 2507955be781d15f9e17f2cb53fcbd2e3a785e27 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 26 Dec 2016 00:17:28 +0100 Subject: Adds automatic id through a /docker-entrypoint.sh modification in https://github.com/solsson/zookeeper-docker --- zookeeper/50zoo.yml | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/zookeeper/50zoo.yml b/zookeeper/50zoo.yml index b73d622..5cb7c02 100644 --- a/zookeeper/50zoo.yml +++ b/zookeeper/50zoo.yml @@ -10,25 +10,11 @@ spec: metadata: labels: app: zookeeper - annotations: - pod.beta.kubernetes.io/init-containers: '[ - { - "name": "install", - "image": "alpine:3.4", - "command": ["sh", "-c", "echo $(($(hostname | sed s/zoo-//) + 1)) > /data/myid" ], - "volumeMounts": [ - { - "name": "datadir", - "mountPath": "/data" - } - ] - } - ]' spec: terminationGracePeriodSeconds: 10 containers: - name: zookeeper - image: zookeeper:3.4.9@sha256:50cfe2c77fe203ab528ffb808f1a8b505db73b1f85aedbc52e4fdde69e2ebfe8 + image: solsson/zookeeper-statefulset:3.4.9@sha256:d32b44b32009a69b3450a5216f459e504f1041f587596895219fc04cf22f5546 env: - name: ZOO_SERVERS value: server.1=zoo-0.zoo:2888:3888:participant server.2=zoo-1.zoo:2888:3888:participant server.3=zoo-2.zoo:2888:3888:participant server.4=zoo-3.zoo:2888:3888:participant server.5=zoo-4.zoo:2888:3888:participant -- cgit v1.2.3 From 72ec5ab97e864f5781598c97ffa60e481fe4ec17 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 26 Dec 2016 01:41:40 +0100 Subject: Converts Kafka to a StatefulSet, nothing but http://kubernetes.io/docs/concepts/abstractions/controllers/statefulsets/ --- 10pvc.yml | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 20dns.yml | 2 -- 50kafka.yml | 16 +++++----------- bootstrap/pvc.yml | 48 ------------------------------------------------ 4 files changed, 53 insertions(+), 61 deletions(-) create mode 100644 10pvc.yml delete mode 100644 bootstrap/pvc.yml diff --git a/10pvc.yml b/10pvc.yml new file mode 100644 index 0000000..cecd3f2 --- /dev/null +++ b/10pvc.yml @@ -0,0 +1,48 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: datadir-kafka-0 + namespace: kafka +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Gi + selector: + matchLabels: + app: kafka + petindex: "0" +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: datadir-kafka-1 + namespace: kafka +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Gi + selector: + matchLabels: + app: kafka + petindex: "1" +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: datadir-kafka-2 + namespace: kafka +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 200Gi + selector: + matchLabels: + app: kafka + petindex: "2" diff --git a/20dns.yml b/20dns.yml index bb92d87..4088c31 100644 --- a/20dns.yml +++ b/20dns.yml @@ -3,8 +3,6 @@ apiVersion: v1 kind: Service metadata: - annotations: - service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" name: broker namespace: kafka spec: diff --git a/50kafka.yml b/50kafka.yml index fe45627..1715c6d 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -1,5 +1,5 @@ -apiVersion: apps/v1alpha1 -kind: PetSet +apiVersion: apps/v1beta1 +kind: StatefulSet metadata: name: kafka namespace: kafka @@ -10,14 +10,11 @@ spec: metadata: labels: app: kafka - annotations: - pod.alpha.kubernetes.io/initialized: "true" - pod.alpha.kubernetes.io/init-containers: '[ - ]' spec: + terminationGracePeriodSeconds: 10 containers: - name: broker - image: solsson/kafka-persistent:0.10.1.0 + image: solsson/kafka-persistent:0.10.1@sha256:110f9e866acd4fb9e059b45884c34a210b2f40d6e2f8afe98ded616f43b599f9 ports: - containerPort: 9092 command: @@ -30,11 +27,8 @@ spec: volumeClaimTemplates: - metadata: name: datadir - namespace: kafka - annotations: - volume.alpha.kubernetes.io/storage-class: anything spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 100Mi + storage: 200Gi diff --git a/bootstrap/pvc.yml b/bootstrap/pvc.yml deleted file mode 100644 index 9729060..0000000 --- a/bootstrap/pvc.yml +++ /dev/null @@ -1,48 +0,0 @@ ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: datadir-kafka-0 - namespace: kafka -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi - selector: - matchLabels: - app: kafka - petindex: "0" ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: datadir-kafka-1 - namespace: kafka -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi - selector: - matchLabels: - app: kafka - petindex: "1" ---- -kind: PersistentVolumeClaim -apiVersion: v1 -metadata: - name: datadir-kafka-2 - namespace: kafka -spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 100Mi - selector: - matchLabels: - app: kafka - petindex: "2" -- cgit v1.2.3 From df1238594ecf7589112fd2f8ae9ffa28e35c1a91 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 2 Jan 2017 08:37:24 +0100 Subject: Use podindex label instead of petindex, because statefulindex sounds weird --- 10pvc.yml | 6 +++--- bootstrap/pv-template.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/10pvc.yml b/10pvc.yml index cecd3f2..51de19c 100644 --- a/10pvc.yml +++ b/10pvc.yml @@ -13,7 +13,7 @@ spec: selector: matchLabels: app: kafka - petindex: "0" + podindex: "0" --- kind: PersistentVolumeClaim apiVersion: v1 @@ -29,7 +29,7 @@ spec: selector: matchLabels: app: kafka - petindex: "1" + podindex: "1" --- kind: PersistentVolumeClaim apiVersion: v1 @@ -45,4 +45,4 @@ spec: selector: matchLabels: app: kafka - petindex: "2" + podindex: "2" diff --git a/bootstrap/pv-template.yml b/bootstrap/pv-template.yml index 73b4f99..e58bfb2 100644 --- a/bootstrap/pv-template.yml +++ b/bootstrap/pv-template.yml @@ -5,7 +5,7 @@ metadata: name: datadir-kafka-0 labels: app: kafka - petindex: "0" + podindex: "0" spec: accessModes: - ReadWriteOnce @@ -20,7 +20,7 @@ metadata: name: datadir-kafka-1 labels: app: kafka - petindex: "1" + podindex: "1" spec: accessModes: - ReadWriteOnce @@ -35,7 +35,7 @@ metadata: name: datadir-kafka-2 labels: app: kafka - petindex: "2" + podindex: "2" spec: accessModes: - ReadWriteOnce -- cgit v1.2.3 From e631d5856532aecd2dfc0a7662d6d7310a482c40 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 2 Jan 2017 11:54:37 +0100 Subject: Updates readme for statefulset and the new zookeeper approach --- README.md | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index c2b5427..4af7310 100644 --- a/README.md +++ b/README.md @@ -1,23 +1,19 @@ # Kafka as Kubernetes PetSet -Example of three Kafka brokers depending on three Zookeeper instances. +Example of three Kafka brokers depending on five Zookeeper instances. -To get consistent service DNS names `kafka-N.broker.kafka`(`.svc.cluster.local`), run everything in a namespace: +To get consistent service DNS names `kafka-N.broker.kafka`(`.svc.cluster.local`), run everything in a [namespace](http://kubernetes.io/docs/admin/namespaces/walkthrough/): ``` kubectl create -f 00namespace.yml ``` ## Set up volume claims -This step can be skipped in clusters that support automatic volume provisioning, such as GKE. +You may add [storage class](http://kubernetes.io/docs/user-guide/persistent-volumes/#storageclasses) +to the kafka StatefulSet declaration to enable automatic volume provisioning. -You need this step in Minikube. - -``` -./zookeeper/bootstrap/pv.sh -kubectl create -f ./zookeeper/bootstrap/pvc.yml -``` +Alternatively create [PV](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistent-volumes)s and [PVC](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims)s manually. For example in Minikube. ``` ./bootstrap/pv.sh @@ -26,21 +22,27 @@ kubectl create -f ./bootstrap/pvc.yml kubectl get pvc ``` -The volume size in the example is very small. The numbers don't really matter as long as they match. Minimal size on GKE is 1 GB. - ## Set up Zookeeper -This module contains a copy of `pets/zookeeper/` from https://github.com/kubernetes/contrib. +There is a Zookeeper+StatefulSet [blog post](http://blog.kubernetes.io/2016/12/statefulset-run-scale-stateful-applications-in-kubernetes.html) and [example](https://github.com/kubernetes/contrib/tree/master/statefulsets/zookeeper), +but it appears tuned for workloads heavier than Kafka topic metadata. -See the `./zookeeper` folder and follow the README there. +The Kafka book (Definitive Guide, O'Reilly 2016) recommends that Kafka has its own Zookeeper cluster, +so we use the [official docker image](https://hub.docker.com/_/zookeeper/) +but with a [startup script change to guess node id from hostname](https://github.com/solsson/zookeeper-docker/commit/df9474f858ad548be8a365cb000a4dd2d2e3a217). -An additional service has been added here, create using: +Zookeeper runs as a [Deployment](http://kubernetes.io/docs/user-guide/deployments/) without persistent storage: ``` -kubectl create -f ./zookeeper/service.yml +kubectl create -f ./zookeeper/ ``` +If you lose your zookeeper cluster, kafka will be unaware that persisted topics exist. +The data is still there, but you need to re-create topics. + ## Start Kafka +Assuming you have your PVCs `Bound`, or enabled automatic provisioning (see above), go ahead and: + ``` kubectl create -f ./ ``` -- cgit v1.2.3