From 76c0b71c2bd22035c18eee11ec23e97d653db2a6 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 28 Jul 2017 08:16:38 +0200 Subject: Adds storage class for Zookeeper, with sample manifests This reverts commit efb1019fd9881f316e0ee355cbafc26e3d146be2. --- README.md | 6 ++++++ configure-gke/storageclass-zookeeper-gke.yml | 7 +++++++ configure-minikube/storageclass-zookeeper-minikube.yml | 5 +++++ zookeeper/50pzoo.yml | 4 +++- 4 files changed, 21 insertions(+), 1 deletion(-) create mode 100644 configure-gke/storageclass-zookeeper-gke.yml create mode 100644 configure-minikube/storageclass-zookeeper-minikube.yml diff --git a/README.md b/README.md index 66f329f..7c0a7f6 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,12 @@ The goal is to provide [Bootstrap servers](http://kafka.apache.org/documentation Zookeeper at `zookeeper.kafka.svc.cluster.local:2181`. +## Prepare storage classes + +For Minikube run `kubectl create -f configure-minikube/`. + +There's a similar setup for GKE, in `configure-gke` of course. You might want to tweak it before creating. + ## Start Zookeeper The [Kafka book](https://www.confluent.io/resources/kafka-definitive-guide-preview-edition/) recommends that Kafka has its own Zookeeper cluster with at least 5 instances. diff --git a/configure-gke/storageclass-zookeeper-gke.yml b/configure-gke/storageclass-zookeeper-gke.yml new file mode 100644 index 0000000..44891ba --- /dev/null +++ b/configure-gke/storageclass-zookeeper-gke.yml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd diff --git a/configure-minikube/storageclass-zookeeper-minikube.yml b/configure-minikube/storageclass-zookeeper-minikube.yml new file mode 100644 index 0000000..ba89eb4 --- /dev/null +++ b/configure-minikube/storageclass-zookeeper-minikube.yml @@ -0,0 +1,5 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: k8s.io/minikube-hostpath diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 446748a..fea60e0 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -61,8 +61,10 @@ spec: volumeClaimTemplates: - metadata: name: data + annotations: + volume.beta.kubernetes.io/storage-class: kafka-zookeeper spec: accessModes: [ "ReadWriteOnce" ] resources: requests: - storage: 10Gi + storage: 1Gi -- cgit v1.2.3 From ee795bd0a1aa097a2a5dfbb78f59a9af22b5a738 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 28 Jul 2017 08:21:23 +0200 Subject: Adds storage class for kafka brokers --- 50kafka.yml | 2 ++ configure-gke/storageclass-broker-gke.yml | 7 +++++++ configure-minikube/storageclass-broker-minikube.yml | 5 +++++ 3 files changed, 14 insertions(+) create mode 100644 configure-gke/storageclass-broker-gke.yml create mode 100644 configure-minikube/storageclass-broker-minikube.yml diff --git a/50kafka.yml b/50kafka.yml index e4451fd..5fcd943 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -77,6 +77,8 @@ spec: volumeClaimTemplates: - metadata: name: data + annotations: + volume.beta.kubernetes.io/storage-class: kafka-broker spec: accessModes: [ "ReadWriteOnce" ] resources: diff --git a/configure-gke/storageclass-broker-gke.yml b/configure-gke/storageclass-broker-gke.yml new file mode 100644 index 0000000..d436148 --- /dev/null +++ b/configure-gke/storageclass-broker-gke.yml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard diff --git a/configure-minikube/storageclass-broker-minikube.yml b/configure-minikube/storageclass-broker-minikube.yml new file mode 100644 index 0000000..3cff3b2 --- /dev/null +++ b/configure-minikube/storageclass-broker-minikube.yml @@ -0,0 +1,5 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: k8s.io/minikube-hostpath -- cgit v1.2.3 From 0880d9f7478d253ccbba53f572bfd22071bdd567 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 25 Oct 2017 08:45:51 +0200 Subject: Suggests a single configuration/, as a collection of examples --- README.md | 2 +- configure-gke/storageclass-broker-gke.yml | 7 ------- configure-gke/storageclass-zookeeper-gke.yml | 7 ------- configure/gke-storageclass-broker-pd.yml | 7 +++++++ configure/gke-storageclass-zookeeper-ssd.yml | 7 +++++++ 5 files changed, 15 insertions(+), 15 deletions(-) delete mode 100644 configure-gke/storageclass-broker-gke.yml delete mode 100644 configure-gke/storageclass-zookeeper-gke.yml create mode 100644 configure/gke-storageclass-broker-pd.yml create mode 100644 configure/gke-storageclass-zookeeper-ssd.yml diff --git a/README.md b/README.md index 7c0a7f6..fd37ccd 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Zookeeper at `zookeeper.kafka.svc.cluster.local:2181`. For Minikube run `kubectl create -f configure-minikube/`. -There's a similar setup for GKE, in `configure-gke` of course. You might want to tweak it before creating. +There's a similar setup for GKE, `configure/gke-*`. You might want to tweak it before creating. ## Start Zookeeper diff --git a/configure-gke/storageclass-broker-gke.yml b/configure-gke/storageclass-broker-gke.yml deleted file mode 100644 index d436148..0000000 --- a/configure-gke/storageclass-broker-gke.yml +++ /dev/null @@ -1,7 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: kafka-broker -provisioner: kubernetes.io/gce-pd -parameters: - type: pd-standard diff --git a/configure-gke/storageclass-zookeeper-gke.yml b/configure-gke/storageclass-zookeeper-gke.yml deleted file mode 100644 index 44891ba..0000000 --- a/configure-gke/storageclass-zookeeper-gke.yml +++ /dev/null @@ -1,7 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: kafka-zookeeper -provisioner: kubernetes.io/gce-pd -parameters: - type: pd-ssd diff --git a/configure/gke-storageclass-broker-pd.yml b/configure/gke-storageclass-broker-pd.yml new file mode 100644 index 0000000..d436148 --- /dev/null +++ b/configure/gke-storageclass-broker-pd.yml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-standard diff --git a/configure/gke-storageclass-zookeeper-ssd.yml b/configure/gke-storageclass-zookeeper-ssd.yml new file mode 100644 index 0000000..44891ba --- /dev/null +++ b/configure/gke-storageclass-zookeeper-ssd.yml @@ -0,0 +1,7 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: kubernetes.io/gce-pd +parameters: + type: pd-ssd -- cgit v1.2.3 From 0575d572d5d63f596d7e57a6ac424cdbebd7bb90 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 25 Oct 2017 08:56:53 +0200 Subject: Uses the new storage class field, requires 1.8 --- 50kafka.yml | 3 +-- README.md | 8 -------- configure/gke-storageclass-broker-pd.yml | 1 + configure/gke-storageclass-zookeeper-ssd.yml | 1 + zookeeper/50pzoo.yml | 3 +-- 5 files changed, 4 insertions(+), 12 deletions(-) diff --git a/50kafka.yml b/50kafka.yml index 5fcd943..0f0956e 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -77,10 +77,9 @@ spec: volumeClaimTemplates: - metadata: name: data - annotations: - volume.beta.kubernetes.io/storage-class: kafka-broker spec: accessModes: [ "ReadWriteOnce" ] + storageClassName: kafka-broker resources: requests: storage: 200Gi diff --git a/README.md b/README.md index fd37ccd..236b139 100644 --- a/README.md +++ b/README.md @@ -70,14 +70,6 @@ For clusters that enfoce [RBAC](https://kubernetes.io/docs/admin/authorization/r kubectl apply -f rbac-namespace-default/ ``` -## Caution: `Delete` Reclaim Policy is default - -In production you likely want to [manually set Reclaim Policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/), -or your data will be gone if the generated [volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)s are deleted. - -This can't be done [in manifests](https://github.com/Yolean/kubernetes-kafka/pull/50), -at least not [until Kubernetes 1.8](https://github.com/kubernetes/features/issues/352). - ## Tests Tests are based on the [kube-test](https://github.com/Yolean/kube-test) concept. diff --git a/configure/gke-storageclass-broker-pd.yml b/configure/gke-storageclass-broker-pd.yml index d436148..dbb7203 100644 --- a/configure/gke-storageclass-broker-pd.yml +++ b/configure/gke-storageclass-broker-pd.yml @@ -3,5 +3,6 @@ apiVersion: storage.k8s.io/v1 metadata: name: kafka-broker provisioner: kubernetes.io/gce-pd +reclaimPolicy: Retain parameters: type: pd-standard diff --git a/configure/gke-storageclass-zookeeper-ssd.yml b/configure/gke-storageclass-zookeeper-ssd.yml index 44891ba..5d6673a 100644 --- a/configure/gke-storageclass-zookeeper-ssd.yml +++ b/configure/gke-storageclass-zookeeper-ssd.yml @@ -3,5 +3,6 @@ apiVersion: storage.k8s.io/v1 metadata: name: kafka-zookeeper provisioner: kubernetes.io/gce-pd +reclaimPolicy: Retain parameters: type: pd-ssd diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index fea60e0..a166a91 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -61,10 +61,9 @@ spec: volumeClaimTemplates: - metadata: name: data - annotations: - volume.beta.kubernetes.io/storage-class: kafka-zookeeper spec: accessModes: [ "ReadWriteOnce" ] + storageClassName: kafka-zookeeper resources: requests: storage: 1Gi -- cgit v1.2.3 From 5cfda3c3dd77454960e8f5d64beae1eff9f98213 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 9 Nov 2017 13:26:43 +0100 Subject: The configure/ folder is meant as a collection of examples, for the choices you'll have to make for any kafka cluster. --- README.md | 2 +- configure-minikube/storageclass-broker-minikube.yml | 5 ----- configure-minikube/storageclass-zookeeper-minikube.yml | 5 ----- configure/minikube-storageclass-broker.yml | 5 +++++ configure/minikube-storageclass-zookeeper.yml | 5 +++++ 5 files changed, 11 insertions(+), 11 deletions(-) delete mode 100644 configure-minikube/storageclass-broker-minikube.yml delete mode 100644 configure-minikube/storageclass-zookeeper-minikube.yml create mode 100644 configure/minikube-storageclass-broker.yml create mode 100644 configure/minikube-storageclass-zookeeper.yml diff --git a/README.md b/README.md index 236b139..9d382c8 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Zookeeper at `zookeeper.kafka.svc.cluster.local:2181`. ## Prepare storage classes -For Minikube run `kubectl create -f configure-minikube/`. +For Minikube run `kubectl apply -f configure/minikube-storageclass-broker.yml; kubectl apply -f configure/minikube-storageclass-zookeeper.yml`. There's a similar setup for GKE, `configure/gke-*`. You might want to tweak it before creating. diff --git a/configure-minikube/storageclass-broker-minikube.yml b/configure-minikube/storageclass-broker-minikube.yml deleted file mode 100644 index 3cff3b2..0000000 --- a/configure-minikube/storageclass-broker-minikube.yml +++ /dev/null @@ -1,5 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: kafka-broker -provisioner: k8s.io/minikube-hostpath diff --git a/configure-minikube/storageclass-zookeeper-minikube.yml b/configure-minikube/storageclass-zookeeper-minikube.yml deleted file mode 100644 index ba89eb4..0000000 --- a/configure-minikube/storageclass-zookeeper-minikube.yml +++ /dev/null @@ -1,5 +0,0 @@ -kind: StorageClass -apiVersion: storage.k8s.io/v1 -metadata: - name: kafka-zookeeper -provisioner: k8s.io/minikube-hostpath diff --git a/configure/minikube-storageclass-broker.yml b/configure/minikube-storageclass-broker.yml new file mode 100644 index 0000000..3cff3b2 --- /dev/null +++ b/configure/minikube-storageclass-broker.yml @@ -0,0 +1,5 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: k8s.io/minikube-hostpath diff --git a/configure/minikube-storageclass-zookeeper.yml b/configure/minikube-storageclass-zookeeper.yml new file mode 100644 index 0000000..ba89eb4 --- /dev/null +++ b/configure/minikube-storageclass-zookeeper.yml @@ -0,0 +1,5 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: k8s.io/minikube-hostpath -- cgit v1.2.3 From 9c6f0e14c2dc271701a84e0230276badcd66a79d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 9 Nov 2017 13:28:57 +0100 Subject: I don't know if Retain makes any difference in minikube, but it's a sensible recommendation for storage classes. --- configure/minikube-storageclass-broker.yml | 1 + configure/minikube-storageclass-zookeeper.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/configure/minikube-storageclass-broker.yml b/configure/minikube-storageclass-broker.yml index 3cff3b2..ae930b4 100644 --- a/configure/minikube-storageclass-broker.yml +++ b/configure/minikube-storageclass-broker.yml @@ -3,3 +3,4 @@ apiVersion: storage.k8s.io/v1 metadata: name: kafka-broker provisioner: k8s.io/minikube-hostpath +reclaimPolicy: Retain diff --git a/configure/minikube-storageclass-zookeeper.yml b/configure/minikube-storageclass-zookeeper.yml index ba89eb4..48c0f35 100644 --- a/configure/minikube-storageclass-zookeeper.yml +++ b/configure/minikube-storageclass-zookeeper.yml @@ -3,3 +3,4 @@ apiVersion: storage.k8s.io/v1 metadata: name: kafka-zookeeper provisioner: k8s.io/minikube-hostpath +reclaimPolicy: Retain -- cgit v1.2.3