diff options
| -rw-r--r-- | 10broker-config.yml | 17 | ||||
| -rw-r--r-- | 30bootstrap-service.yml | 11 | ||||
| -rw-r--r-- | 50kafka.yml | 16 | ||||
| -rw-r--r-- | README.md | 14 | ||||
| -rw-r--r-- | configure/gke-storageclass-broker-pd.yml | 8 | ||||
| -rw-r--r-- | configure/gke-storageclass-zookeeper-ssd.yml | 8 | ||||
| -rw-r--r-- | configure/minikube-storageclass-broker.yml | 6 | ||||
| -rw-r--r-- | configure/minikube-storageclass-zookeeper.yml | 6 | ||||
| -rw-r--r-- | outside-services/outside-0.yml | 15 | ||||
| -rw-r--r-- | outside-services/outside-1.yml | 15 | ||||
| -rw-r--r-- | outside-services/outside-2.yml | 15 | ||||
| -rw-r--r-- | test/basic-produce-consume.yml | 28 | ||||
| -rw-r--r-- | test/basic-with-kafkacat.yml | 5 | ||||
| -rw-r--r-- | yahoo-kafka-manager/kafka-manager-service.yml | 12 | ||||
| -rw-r--r-- | yahoo-kafka-manager/kafka-manager.yml | 26 | ||||
| -rw-r--r-- | zookeeper/50pzoo.yml | 7 | ||||
| -rw-r--r-- | zookeeper/51zoo.yml | 4 |
17 files changed, 190 insertions, 23 deletions
diff --git a/10broker-config.yml b/10broker-config.yml index 7d296bf..c0c4c8c 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -22,6 +22,16 @@ data: else sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties fi + + kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID + + OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + if [ $? -ne 0 ]; then + echo "Outside (i.e. cluster-external access) host lookup command failed" + else + OUTSIDE_HOST=${OUTSIDE_HOST}:3240${KAFKA_BROKER_ID} + sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}|" /etc/kafka/server.properties + fi } server.properties: |- @@ -49,9 +59,6 @@ data: #init#broker.rack=#init# - # Switch to enable topic deletion or not, default value is false - delete.topic.enable=true - ############################# Socket Server Settings ############################# # The address the socket server listens on. It will get the value returned from @@ -61,14 +68,18 @@ data: # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 #listeners=PLAINTEXT://:9092 + listeners=OUTSIDE://:9094,PLAINTEXT://:9092 # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value # returned from java.net.InetAddress.getCanonicalHostName(). #advertised.listeners=PLAINTEXT://your.host.name:9092 + #init#advertised.listeners=OUTSIDE://#init#,PLAINTEXT://:9092 # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT + inter.broker.listener.name=PLAINTEXT # The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 diff --git a/30bootstrap-service.yml b/30bootstrap-service.yml new file mode 100644 index 0000000..7c2a337 --- /dev/null +++ b/30bootstrap-service.yml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: bootstrap + namespace: kafka +spec: + ports: + - port: 9092 + selector: + app: kafka diff --git a/50kafka.yml b/50kafka.yml index 50d0228..c39ba9e 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -21,18 +21,29 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace command: ['/bin/bash', '/etc/kafka/init.sh'] volumeMounts: - name: config mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties ports: - - containerPort: 9092 + - name: inside + containerPort: 9092 + - name: outside + containerPort: 9094 command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties @@ -66,6 +77,7 @@ spec: name: data spec: accessModes: [ "ReadWriteOnce" ] + storageClassName: kafka-broker resources: requests: storage: 200Gi @@ -30,6 +30,12 @@ The goal is to provide [Bootstrap servers](http://kafka.apache.org/documentation Zookeeper at `zookeeper.kafka.svc.cluster.local:2181`. +## Prepare storage classes + +For Minikube run `kubectl apply -f configure/minikube-storageclass-broker.yml; kubectl apply -f configure/minikube-storageclass-zookeeper.yml`. + +There's a similar setup for GKE, `configure/gke-*`. You might want to tweak it before creating. + ## Start Zookeeper The [Kafka book](https://www.confluent.io/resources/kafka-definitive-guide-preview-edition/) recommends that Kafka has its own Zookeeper cluster with at least 5 instances. @@ -64,14 +70,6 @@ For clusters that enfoce [RBAC](https://kubernetes.io/docs/admin/authorization/r kubectl apply -f rbac-namespace-default/ ``` -## Caution: `Delete` Reclaim Policy is default - -In production you likely want to [manually set Reclaim Policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/), -or your data will be gone if the generated [volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)s are deleted. - -This can't be done [in manifests](https://github.com/Yolean/kubernetes-kafka/pull/50), -at least not [until Kubernetes 1.8](https://github.com/kubernetes/features/issues/352). - ## Tests Tests are based on the [kube-test](https://github.com/Yolean/kube-test) concept. diff --git a/configure/gke-storageclass-broker-pd.yml b/configure/gke-storageclass-broker-pd.yml new file mode 100644 index 0000000..dbb7203 --- /dev/null +++ b/configure/gke-storageclass-broker-pd.yml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Retain +parameters: + type: pd-standard diff --git a/configure/gke-storageclass-zookeeper-ssd.yml b/configure/gke-storageclass-zookeeper-ssd.yml new file mode 100644 index 0000000..5d6673a --- /dev/null +++ b/configure/gke-storageclass-zookeeper-ssd.yml @@ -0,0 +1,8 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: kubernetes.io/gce-pd +reclaimPolicy: Retain +parameters: + type: pd-ssd diff --git a/configure/minikube-storageclass-broker.yml b/configure/minikube-storageclass-broker.yml new file mode 100644 index 0000000..ae930b4 --- /dev/null +++ b/configure/minikube-storageclass-broker.yml @@ -0,0 +1,6 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: k8s.io/minikube-hostpath +reclaimPolicy: Retain diff --git a/configure/minikube-storageclass-zookeeper.yml b/configure/minikube-storageclass-zookeeper.yml new file mode 100644 index 0000000..48c0f35 --- /dev/null +++ b/configure/minikube-storageclass-zookeeper.yml @@ -0,0 +1,6 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: k8s.io/minikube-hostpath +reclaimPolicy: Retain diff --git a/outside-services/outside-0.yml b/outside-services/outside-0.yml new file mode 100644 index 0000000..7bc12bd --- /dev/null +++ b/outside-services/outside-0.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-0 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "0" + ports: + - protocol: TCP + targetPort: 9094 + port: 32400 + nodePort: 32400 + type: NodePort
\ No newline at end of file diff --git a/outside-services/outside-1.yml b/outside-services/outside-1.yml new file mode 100644 index 0000000..1642ee0 --- /dev/null +++ b/outside-services/outside-1.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-1 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "1" + ports: + - protocol: TCP + targetPort: 9094 + port: 32401 + nodePort: 32401 + type: NodePort
\ No newline at end of file diff --git a/outside-services/outside-2.yml b/outside-services/outside-2.yml new file mode 100644 index 0000000..78c313c --- /dev/null +++ b/outside-services/outside-2.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-2 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "2" + ports: + - protocol: TCP + targetPort: 9094 + port: 32402 + nodePort: 32402 + type: NodePort
\ No newline at end of file diff --git a/test/basic-produce-consume.yml b/test/basic-produce-consume.yml index 20bb276..32ec4ec 100644 --- a/test/basic-produce-consume.yml +++ b/test/basic-produce-consume.yml @@ -36,7 +36,31 @@ data: #pkill java exit 0 - +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: basic-produce-consume + namespace: test-kafka +spec: + template: + spec: + containers: + - name: topic-create + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka.svc.cluster.local:2181 + - --create + - --if-not-exists + - --topic + - test-basic-produce-consume + - --partitions + - "1" + - --replication-factor + - "2" + restartPolicy: Never --- apiVersion: apps/v1beta1 kind: Deployment @@ -53,7 +77,7 @@ spec: spec: containers: - name: testcase - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: BOOTSTRAP value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 diff --git a/test/basic-with-kafkacat.yml b/test/basic-with-kafkacat.yml index c5a2503..78e423c 100644 --- a/test/basic-with-kafkacat.yml +++ b/test/basic-with-kafkacat.yml @@ -23,7 +23,6 @@ data: kafkacat -C -b $BOOTSTRAP -t test-basic-with-kafkacat -o -1 -e | grep $unique exit 0 - --- apiVersion: batch/v1 kind: Job @@ -35,7 +34,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: - ./bin/kafka-topics.sh - --zookeeper @@ -47,7 +46,7 @@ spec: - --partitions - "1" - --replication-factor - - "1" + - "2" restartPolicy: Never --- apiVersion: apps/v1beta1 diff --git a/yahoo-kafka-manager/kafka-manager-service.yml b/yahoo-kafka-manager/kafka-manager-service.yml new file mode 100644 index 0000000..3d26adf --- /dev/null +++ b/yahoo-kafka-manager/kafka-manager-service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: kafka-manager + namespace: kafka +spec: + selector: + app: kafka-manager + ports: + - protocol: TCP + port: 80 + targetPort: 80 diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml new file mode 100644 index 0000000..678c174 --- /dev/null +++ b/yahoo-kafka-manager/kafka-manager.yml @@ -0,0 +1,26 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: kafka-manager + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: kafka-manager + template: + metadata: + labels: + app: kafka-manager + spec: + containers: + - name: kafka-manager + image: solsson/kafka-manager@sha256:e07b5c50b02c761b3471ebb62ede88afc0625e325fe428316e32fec7f227ff9b + ports: + - containerPort: 80 + env: + - name: ZK_HOSTS + value: zookeeper.kafka:2181 + command: + - ./bin/kafka-manager + - -Dhttp.port=80
\ No newline at end of file diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 5848b2f..a166a91 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -16,7 +16,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: ['/bin/bash', '/etc/kafka/init.sh'] volumeMounts: - name: config @@ -25,7 +25,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties @@ -63,6 +63,7 @@ spec: name: data spec: accessModes: [ "ReadWriteOnce" ] + storageClassName: kafka-zookeeper resources: requests: - storage: 10Gi + storage: 1Gi diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index fb2b741..a2922ef 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -16,7 +16,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: ['/bin/bash', '/etc/kafka/init.sh'] env: - name: ID_OFFSET @@ -28,7 +28,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties |