diff options
Diffstat (limited to 'zookeeper')
-rw-r--r-- | zookeeper/README.md | 85 | ||||
-rw-r--r-- | zookeeper/init/Dockerfile | 29 | ||||
-rw-r--r-- | zookeeper/init/Makefile | 27 | ||||
-rwxr-xr-x | zookeeper/init/install.sh | 76 | ||||
-rwxr-xr-x | zookeeper/init/on-change.sh | 49 | ||||
-rwxr-xr-x | zookeeper/init/on-start.sh | 73 | ||||
-rwxr-xr-x | zookeeper/test.sh | 19 | ||||
-rw-r--r-- | zookeeper/zookeeper.yaml | 123 |
8 files changed, 481 insertions, 0 deletions
diff --git a/zookeeper/README.md b/zookeeper/README.md new file mode 100644 index 0000000..52e1561 --- /dev/null +++ b/zookeeper/README.md @@ -0,0 +1,85 @@ +# Zookeeper + +This example runs zookeeper through a petset. + +## Bootstrap + +Create the petset in this directory +``` +$ kubetl create -f zookeeper.yaml +``` + +Once you have all 3 nodes in Running, you can run the "test.sh" script in this directory. + +## Failover + +You can test failover by killing the leader. Insert a key: +```console +$ kubectl exec zoo-0 -- /opt/zookeeper/bin/zkCli.sh create /foo bar; +$ kubectl exec zoo-2 -- /opt/zookeeper/bin/zkCli.sh get /foo; + +Watch existing members: +```console +$ kubectl run --attach bbox --image=busybox --restart=Never -- sh -c 'while true; do for i in 0 1 2; do echo zoo-$i $(echo stats | nc zoo-$i.zk:2181 | grep Mode); sleep 1; done; done'; +zoo-2 Mode: follower +zoo-0 Mode: follower +zoo-1 Mode: leader +zoo-2 Mode: follower +``` + +Delete pets and wait for the petset controller to bring the back up: +```console +$ kubectl delete po -l app=zk +$ kubectl get po --watch-only +NAME READY STATUS RESTARTS AGE +zoo-0 0/1 Init:0/2 0 16s +zoo-0 0/1 Init:0/2 0 21s +zoo-0 0/1 PodInitializing 0 23s +zoo-0 1/1 Running 0 41s +zoo-1 0/1 Pending 0 0s +zoo-1 0/1 Init:0/2 0 0s +zoo-1 0/1 Init:0/2 0 14s +zoo-1 0/1 PodInitializing 0 17s +zoo-1 0/1 Running 0 18s +zoo-2 0/1 Pending 0 0s +zoo-2 0/1 Init:0/2 0 0s +zoo-2 0/1 Init:0/2 0 12s +zoo-2 0/1 Init:0/2 0 28s +zoo-2 0/1 PodInitializing 0 31s +zoo-2 0/1 Running 0 32s +... + +zoo-0 Mode: follower +zoo-1 Mode: leader +zoo-2 Mode: follower +``` + +Check the previously inserted key: +```console +$ kubectl exec zoo-1 -- /opt/zookeeper/bin/zkCli.sh get /foo +ionid = 0x354887858e80035, negotiated timeout = 30000 + +WATCHER:: + +WatchedEvent state:SyncConnected type:None path:null +bar +``` + +## Scaling + +You can scale up by modifying the number of replicas on the PetSet. + +## Image Upgrade + +TODO: Add details + +## Maintenance + +TODO: Add details + +## Limitations +* Both petset and init containers are in alpha +* Look through the on-start and on-change scripts for TODOs +* Doesn't support the addition of observers through the petset +* Only supports storage options that have backends for persistent volume claims + diff --git a/zookeeper/init/Dockerfile b/zookeeper/init/Dockerfile new file mode 100644 index 0000000..d3e8aae --- /dev/null +++ b/zookeeper/init/Dockerfile @@ -0,0 +1,29 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# TODO: get rid of bash dependency and switch to plain busybox. +# The tar in busybox also doesn't seem to understand compression. +FROM debian:jessie +MAINTAINER Prashanth.B <beeps@google.com> + +RUN apt-get update && apt-get install -y wget netcat + +ADD on-start.sh / +ADD on-change.sh / +# See contrib/pets/peer-finder for details +RUN wget -qO /peer-finder https://storage.googleapis.com/kubernetes-release/pets/peer-finder + +ADD install.sh / +RUN chmod -c 755 /install.sh /on-start.sh /on-change.sh /peer-finder +Entrypoint ["/install.sh"] diff --git a/zookeeper/init/Makefile b/zookeeper/init/Makefile new file mode 100644 index 0000000..bfb6978 --- /dev/null +++ b/zookeeper/init/Makefile @@ -0,0 +1,27 @@ +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +all: push + +TAG = 0.1 +PREFIX = gcr.io/google_containers/zookeeper-install + +container: + docker build -t $(PREFIX):$(TAG) . + +push: container + gcloud docker push $(PREFIX):$(TAG) + +clean: + docker rmi $(PREFIX):$(TAG) diff --git a/zookeeper/init/install.sh b/zookeeper/init/install.sh new file mode 100755 index 0000000..6ed7269 --- /dev/null +++ b/zookeeper/init/install.sh @@ -0,0 +1,76 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This volume is assumed to exist and is shared with parent of the init +# container. It contains the zookeeper installation. +INSTALL_VOLUME="/opt" + +# This volume is assumed to exist and is shared with the peer-finder +# init container. It contains on-start/change configuration scripts. +WORKDIR_VOLUME="/work-dir" + +# As of April-2016 is 3.4.8 is the latest stable, but versions 3.5.0 onward +# allow dynamic reconfiguration. +VERSION="3.5.0-alpha" + +for i in "$@" +do +case $i in + -v=*|--version=*) + VERSION="${i#*=}" + shift + ;; + -i=*|--install-into=*) + INSTALL_VOLUME="${i#*=}" + shift + ;; + -w=*|--work-dir=*) + WORKDIR_VOLUME="${i#*=}" + shift + ;; + *) + # unknown option + ;; +esac +done + +echo installing config scripts into "${WORKDIR_VOLUME}" +mkdir -p "${WORKDIR_VOLUME}" +cp /on-start.sh "${WORKDIR_VOLUME}"/ +cp /on-change.sh "${WORKDIR_VOLUME}"/ +cp /peer-finder "${WORKDIR_VOLUME}"/ + +echo installing zookeeper-"${VERSION}" into "${INSTALL_VOLUME}" +mkdir -p "${INSTALL_VOLUME}" +wget -q -O - http://apache.mirrors.pair.com/zookeeper/zookeeper-"${VERSION}"/zookeeper-"${VERSION}".tar.gz | tar -xzf - -C "${INSTALL_VOLUME}" +mv "${INSTALL_VOLUME}"/zookeeper-"${VERSION}" "${INSTALL_VOLUME}"/zookeeper +cp "${INSTALL_VOLUME}"/zookeeper/conf/zoo_sample.cfg "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg + +# TODO: Should dynamic config be tied to the version? +IFS="." read -ra RELEASE <<< "${VERSION}" +if [ $(expr "${RELEASE[1]}") -gt 4 ]; then + echo zookeeper-"${VERSION}" supports dynamic reconfiguration, enabling it + echo "standaloneEnabled=false" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg + echo "dynamicConfigFile="${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg.dynamic" >> "${INSTALL_VOLUME}"/zookeeper/conf/zoo.cfg +fi + +# TODO: This is a hack, netcat is convenient to have in the zookeeper container +# I want to avoid using a custom zookeeper image just for this. So copy it. +NC=$(which nc) +if [ "${NC}" != "" ]; then + echo copying nc into "${INSTALL_VOLUME}" + cp "${NC}" "${INSTALL_VOLUME}" +fi diff --git a/zookeeper/init/on-change.sh b/zookeeper/init/on-change.sh new file mode 100755 index 0000000..bb9a504 --- /dev/null +++ b/zookeeper/init/on-change.sh @@ -0,0 +1,49 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script configures zookeeper cluster member ship for version of zookeeper +# < 3.5.0. It should not be used with the on-start.sh script in this example. +# As of April-2016 is 3.4.8 is the latest stable. + +CFG=/opt/zookeeper/conf/zoo.cfg +CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak +MY_ID=/tmp/zookeeper/myid + +# write myid +IFS='-' read -ra ADDR <<< "$(hostname)" +echo $(expr "1" + "${ADDR[1]}") > "${MY_ID}" + +# TODO: This is a dumb way to reconfigure zookeeper because it allows dynamic +# reconfig, but it's simple. +i=0 +echo " +tickTime=2000 +initLimit=10 +syncLimit=5 +dataDir=/tmp/zookeeper +clientPort=2181 +" > "${CFG_BAK}" + +while read -ra LINE; do + let i=i+1 + echo "server.${i}=${LINE}:2888:3888" >> "${CFG_BAK}" +done +cp ${CFG_BAK} ${CFG} + +# TODO: Typically one needs to first add a new member as an "observer" then +# promote it to "participant", but that requirement is relaxed if we never +# start > 1 at a time. +/opt/zookeeper/bin/zkServer.sh restart diff --git a/zookeeper/init/on-start.sh b/zookeeper/init/on-start.sh new file mode 100755 index 0000000..392b82a --- /dev/null +++ b/zookeeper/init/on-start.sh @@ -0,0 +1,73 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This script configures zookeeper cluster member ship for version of zookeeper +# >= 3.5.0. It should not be used with the on-change.sh script in this example. +# As of April-2016 is 3.4.8 is the latest stable. + +# Both /opt and /tmp/zookeeper are assumed to be volumes shared with the parent. +CFG=/opt/zookeeper/conf/zoo.cfg.dynamic +CFG_BAK=/opt/zookeeper/conf/zoo.cfg.bak +MY_ID_FILE=/tmp/zookeeper/myid +HOSTNAME=$(hostname) + +while read -ra LINE; do + PEERS=("${PEERS[@]}" $LINE) +done + +# Don't add the first member as an observer +if [ ${#PEERS[@]} -eq 1 ]; then + # We need to write our index in this list of servers into MY_ID_FILE. + # Note that this may not always coincide with the hostname id. + echo 1 > "${MY_ID_FILE}" + echo "server.1=${PEERS[0]}:2888:3888;2181" > "${CFG}" + # TODO: zkServer-initialize is the safe way to handle changes to datadir + # because simply starting will create a new datadir, BUT if the user changed + # pod template they might end up with 2 datadirs and brief split brain. + exit +fi + +# Every subsequent member is added as an observer and promoted to a participant +echo "" > "${CFG_BAK}" +i=0 +for peer in "${PEERS[@]}"; do + let i=i+1 + if [[ "${peer}" == *"${HOSTNAME}"* ]]; then + MY_ID=$i + MY_NAME=${peer} + echo $i > "${MY_ID_FILE}" + echo "server.${i}=${peer}:2888:3888:observer;2181" >> "${CFG_BAK}" + else + echo "server.${i}=${peer}:2888:3888:participant;2181" >> "${CFG_BAK}" + fi +done + +# Once the dynamic config file is written it shouldn't be modified, so the final +# reconfigure needs to happen through the "reconfig" command. +cp ${CFG_BAK} ${CFG} + +# TODO: zkServer-initialize is the safe way to handle changes to datadir +# because simply starting will create a new datadir, BUT if the user changed +# pod template they might end up with 2 datadirs and brief split brain. +/opt/zookeeper/bin/zkServer.sh start + +# TODO: We shouldn't need to specify the address of the master as long as +# there's quorum. According to the docs the new server is just not allowed to +# vote, it's still allowed to propose config changes, and it knows the +# existing members of the ensemble from *its* config. This works as expected, +# but we should correlate with more satisfying empirical evidence. +/opt/zookeeper/bin/zkCli.sh reconfig -add "server.$MY_ID=$MY_NAME:2888:3888:participant;2181" +/opt/zookeeper/bin/zkServer.sh stop diff --git a/zookeeper/test.sh b/zookeeper/test.sh new file mode 100755 index 0000000..fbe12ff --- /dev/null +++ b/zookeeper/test.sh @@ -0,0 +1,19 @@ +#! /bin/bash + +# Copyright 2016 The Kubernetes Authors All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +kubectl exec zoo-0 -- /opt/zookeeper/bin/zkCli.sh create /foo bar; +kubectl exec zoo-2 -- /opt/zookeeper/bin/zkCli.sh get /foo; + diff --git a/zookeeper/zookeeper.yaml b/zookeeper/zookeeper.yaml new file mode 100644 index 0000000..ada1af3 --- /dev/null +++ b/zookeeper/zookeeper.yaml @@ -0,0 +1,123 @@ +# A headless service to create DNS records +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: zk + labels: + app: zk +spec: + ports: + - port: 2888 + name: peer + - port: 3888 + name: leader-election + # *.zk.default.svc.cluster.local + clusterIP: None + selector: + app: zk +--- +apiVersion: apps/v1alpha1 +kind: PetSet +metadata: + name: zoo +spec: + serviceName: "zk" + replicas: 3 + template: + metadata: + labels: + app: zk + annotations: + pod.alpha.kubernetes.io/initialized: "true" + pod.alpha.kubernetes.io/init-containers: '[ + { + "name": "install", + "image": "gcr.io/google_containers/zookeeper-install:0.1", + "imagePullPolicy": "Always", + "args": ["--version=3.5.0-alpha", "--install-into=/opt", "--work-dir=/work-dir"], + "volumeMounts": [ + { + "name": "opt", + "mountPath": "/opt/" + }, + { + "name": "workdir", + "mountPath": "/work-dir" + } + ] + }, + { + "name": "bootstrap", + "image": "java:openjdk-8-jre", + "command": ["/work-dir/peer-finder"], + "args": ["-on-start=\"/work-dir/on-start.sh\"", "-service=zk"], + "env": [ + { + "name": "POD_NAMESPACE", + "valueFrom": { + "fieldRef": { + "apiVersion": "v1", + "fieldPath": "metadata.namespace" + } + } + } + ], + "volumeMounts": [ + { + "name": "opt", + "mountPath": "/opt/" + }, + { + "name": "workdir", + "mountPath": "/work-dir" + }, + { + "name": "datadir", + "mountPath": "/tmp/zookeeper" + } + ] + } + ]' + spec: + containers: + - name: zk + image: java:openjdk-8-jre + ports: + - containerPort: 2888 + name: peer + - containerPort: 3888 + name: leader-election + command: + - /opt/zookeeper/bin/zkServer.sh + args: + - start-foreground + readinessProbe: + exec: + command: + - sh + - -c + - "/opt/zookeeper/bin/zkCli.sh ls /" + initialDelaySeconds: 15 + timeoutSeconds: 5 + volumeMounts: + - name: datadir + mountPath: /tmp/zookeeper + - name: opt + mountPath: /opt/ + volumes: + - name: opt + emptyDir: {} + - name: workdir + emptyDir: {} + volumeClaimTemplates: + - metadata: + name: datadir + annotations: + volume.alpha.kubernetes.io/storage-class: anything + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 20Gi |