diff options
| -rw-r--r-- | 50kafka.yml | 9 | ||||
| -rw-r--r-- | README.md | 14 | ||||
| -rw-r--r-- | configure/gke-storageclass-broker-pd.yml | 8 | ||||
| -rw-r--r-- | configure/gke-storageclass-zookeeper-ssd.yml | 8 | ||||
| -rw-r--r-- | configure/minikube-storageclass-broker.yml | 6 | ||||
| -rw-r--r-- | configure/minikube-storageclass-zookeeper.yml | 6 | ||||
| -rw-r--r-- | yahoo-kafka-manager/kafka-manager-service.yml | 12 | ||||
| -rw-r--r-- | yahoo-kafka-manager/kafka-manager.yml | 26 | ||||
| -rw-r--r-- | zookeeper/50pzoo.yml | 3 |
9 files changed, 78 insertions, 14 deletions
diff --git a/50kafka.yml b/50kafka.yml index e4451fd..c39ba9e 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -60,11 +60,9 @@ spec: cpu: 100m memory: 512Mi readinessProbe: - exec: - command: - - /bin/sh - - -c - - 'echo "" | nc -w 1 127.0.0.1 9092' + tcpSocket: + port: 9092 + timeoutSeconds: 1 volumeMounts: - name: config mountPath: /etc/kafka @@ -79,6 +77,7 @@ spec: name: data spec: accessModes: [ "ReadWriteOnce" ] + storageClassName: kafka-broker resources: requests: storage: 200Gi @@ -30,6 +30,12 @@ The goal is to provide [Bootstrap servers](http://kafka.apache.org/documentation Zookeeper at `zookeeper.kafka.svc.cluster.local:2181`. +## Prepare storage classes + +For Minikube run `kubectl apply -f configure/minikube-storageclass-broker.yml; kubectl apply -f configure/minikube-storageclass-zookeeper.yml`. + +There's a similar setup for GKE, `configure/gke-*`. You might want to tweak it before creating. + ## Start Zookeeper The [Kafka book](https://www.confluent.io/resources/kafka-definitive-guide-preview-edition/) recommends that Kafka has its own Zookeeper cluster with at least 5 instances. @@ -64,14 +70,6 @@ For clusters that enfoce [RBAC](https://kubernetes.io/docs/admin/authorization/r kubectl apply -f rbac-namespace-default/ ``` -## Caution: `Delete` Reclaim Policy is default - -In production you likely want to [manually set Reclaim Policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/), -or your data will be gone if the generated [volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)s are deleted. - -This can't be done [in manifests](https://github.com/Yolean/kubernetes-kafka/pull/50), -at least not [until Kubernetes 1.8](https://github.com/kubernetes/features/issues/352). - ## Tests Tests are based on the [kube-test](https://github.com/Yolean/kube-test) concept. diff --git a/configure/gke-storageclass-broker-pd.yml b/configure/gke-storageclass-broker-pd.yml new file mode 100644 index 0000000..dbb7203 --- /dev/null +++ b/configure/gke-storageclass-broker-pd.yml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Retain +parameters: + type: pd-standard diff --git a/configure/gke-storageclass-zookeeper-ssd.yml b/configure/gke-storageclass-zookeeper-ssd.yml new file mode 100644 index 0000000..5d6673a --- /dev/null +++ b/configure/gke-storageclass-zookeeper-ssd.yml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Retain +parameters: + type: pd-ssd diff --git a/configure/minikube-storageclass-broker.yml b/configure/minikube-storageclass-broker.yml new file mode 100644 index 0000000..ae930b4 --- /dev/null +++ b/configure/minikube-storageclass-broker.yml @@ -0,0 +1,6 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: k8s.io/minikube-hostpath +reclaimPolicy: Retain diff --git a/configure/minikube-storageclass-zookeeper.yml b/configure/minikube-storageclass-zookeeper.yml new file mode 100644 index 0000000..48c0f35 --- /dev/null +++ b/configure/minikube-storageclass-zookeeper.yml @@ -0,0 +1,6 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: k8s.io/minikube-hostpath +reclaimPolicy: Retain diff --git a/yahoo-kafka-manager/kafka-manager-service.yml b/yahoo-kafka-manager/kafka-manager-service.yml new file mode 100644 index 0000000..3d26adf --- /dev/null +++ b/yahoo-kafka-manager/kafka-manager-service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: kafka-manager + namespace: kafka +spec: + selector: + app: kafka-manager + ports: + - protocol: TCP + port: 80 + targetPort: 80 diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml new file mode 100644 index 0000000..678c174 --- /dev/null +++ b/yahoo-kafka-manager/kafka-manager.yml @@ -0,0 +1,26 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: kafka-manager + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-manager + template: + metadata: + labels: + app: kafka-manager + spec: + containers: + - name: kafka-manager + image: solsson/kafka-manager@sha256:e07b5c50b02c761b3471ebb62ede88afc0625e325fe428316e32fec7f227ff9b + ports: + - containerPort: 80 + env: + - name: ZK_HOSTS + value: zookeeper.kafka:2181 + command: + - ./bin/kafka-manager + - -Dhttp.port=80
\ No newline at end of file diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 446748a..a166a91 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -63,6 +63,7 @@ spec: name: data spec: accessModes: [ "ReadWriteOnce" ] + storageClassName: kafka-zookeeper resources: requests: - storage: 10Gi + storage: 1Gi |