diff options
| -rw-r--r-- | 10broker-config.yml | 17 | ||||
| -rw-r--r-- | 50kafka.yml | 17 | ||||
| -rw-r--r-- | README.md | 12 | ||||
| -rw-r--r-- | outside-services/outside-0.yml | 15 | ||||
| -rw-r--r-- | outside-services/outside-1.yml | 15 | ||||
| -rw-r--r-- | outside-services/outside-2.yml | 15 | ||||
| -rw-r--r-- | test/basic-produce-consume.yml | 28 | ||||
| -rw-r--r-- | test/basic-with-kafkacat.yml | 7 | ||||
| -rw-r--r-- | zookeeper/50pzoo.yml | 6 | ||||
| -rw-r--r-- | zookeeper/51zoo.yml | 6 |
10 files changed, 117 insertions, 21 deletions
diff --git a/10broker-config.yml b/10broker-config.yml index 7d296bf..c0c4c8c 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -22,6 +22,16 @@ data: else sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties fi + + kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID + + OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + if [ $? -ne 0 ]; then + echo "Outside (i.e. cluster-external access) host lookup command failed" + else + OUTSIDE_HOST=${OUTSIDE_HOST}:3240${KAFKA_BROKER_ID} + sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}|" /etc/kafka/server.properties + fi } server.properties: |- @@ -49,9 +59,6 @@ data: #init#broker.rack=#init# - # Switch to enable topic deletion or not, default value is false - delete.topic.enable=true - ############################# Socket Server Settings ############################# # The address the socket server listens on. It will get the value returned from @@ -61,14 +68,18 @@ data: # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 #listeners=PLAINTEXT://:9092 + listeners=OUTSIDE://:9094,PLAINTEXT://:9092 # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value # returned from java.net.InetAddress.getCanonicalHostName(). #advertised.listeners=PLAINTEXT://your.host.name:9092 + #init#advertised.listeners=OUTSIDE://#init#,PLAINTEXT://:9092 # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT + inter.broker.listener.name=PLAINTEXT # The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 diff --git a/50kafka.yml b/50kafka.yml index c564ffe..e4451fd 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -15,24 +15,35 @@ spec: terminationGracePeriodSeconds: 30 initContainers: - name: init-config - image: solsson/kafka-initutils@sha256:c275d681019a0d8f01295dbd4a5bae3cfa945c8d0f7f685ae1f00f2579f08c7d + image: solsson/kafka-initutils@sha256:c98d7fb5e9365eab391a5dcd4230fc6e72caf929c60f29ff091e3b0215124713 env: - name: NODE_NAME valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace command: ['/bin/bash', '/etc/kafka/init.sh'] volumeMounts: - name: config mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties ports: - - containerPort: 9092 + - name: inside + containerPort: 9092 + - name: outside + containerPort: 9094 command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties @@ -6,12 +6,15 @@ Transparent Kafka setup that you can grow with. Good for both experiments and production. How to use: + * Good to know: you'll likely want to fork this repo. It prioritizes clarity over configurability, using plain manifests and .propeties files; no client side logic. * Run a Kubernetes cluster, [minikube](https://github.com/kubernetes/minikube) or real. * Quickstart: use the `kubectl apply`s below. - * Kafka for real: fork and have a look at [addon](https://github.com/Yolean/kubernetes-kafka/labels/addon)s. + * Have a look at [addon](https://github.com/Yolean/kubernetes-kafka/labels/addon)s, or the official forks: + - [kubernetes-kafka-small](https://github.com/Reposoft/kubernetes-kafka-small) for single-node clusters like Minikube. + - [StreamingMicroservicesPlatform](https://github.com/StreamingMicroservicesPlatform/kubernetes-kafka) Like Confluent's [platform quickstart](https://docs.confluent.io/current/connect/quickstart.html) but for Kubernetes. * Join the discussion in issues and PRs. -No readable readme can properly introduce both [Kafka](http://kafka.apache.org/) and [Kubernets](https://kubernetes.io/), +No readable readme can properly introduce both [Kafka](http://kafka.apache.org/) and [Kubernetes](https://kubernetes.io/), but we think the combination of the two is a great backbone for microservices. Back when we read [Newman](http://samnewman.io/books/building_microservices/) we were beginners with both. Now we've read [Kleppmann](http://dataintensive.net/), [Confluent](https://www.confluent.io/blog/) and [SRE](https://landing.google.com/sre/book.html) and enjoy this "Streaming Platform" lock-in :smile:. @@ -64,13 +67,16 @@ kubectl apply -f rbac-namespace-default/ ## Caution: `Delete` Reclaim Policy is default In production you likely want to [manually set Reclaim Policy](https://kubernetes.io/docs/tasks/administer-cluster/change-pv-reclaim-policy/), -our your data will be gone if the generated [volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)s are deleted. +or your data will be gone if the generated [volume claim](https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims)s are deleted. This can't be done [in manifests](https://github.com/Yolean/kubernetes-kafka/pull/50), at least not [until Kubernetes 1.8](https://github.com/kubernetes/features/issues/352). ## Tests +Tests are based on the [kube-test](https://github.com/Yolean/kube-test) concept. +Like the rest of this repo they have `kubectl` as the only local dependency. + ``` kubectl apply -f test/ # Anything that isn't READY here is a failed test diff --git a/outside-services/outside-0.yml b/outside-services/outside-0.yml new file mode 100644 index 0000000..7bc12bd --- /dev/null +++ b/outside-services/outside-0.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-0 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "0" + ports: + - protocol: TCP + targetPort: 9094 + port: 32400 + nodePort: 32400 + type: NodePort
\ No newline at end of file diff --git a/outside-services/outside-1.yml b/outside-services/outside-1.yml new file mode 100644 index 0000000..1642ee0 --- /dev/null +++ b/outside-services/outside-1.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-1 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "1" + ports: + - protocol: TCP + targetPort: 9094 + port: 32401 + nodePort: 32401 + type: NodePort
\ No newline at end of file diff --git a/outside-services/outside-2.yml b/outside-services/outside-2.yml new file mode 100644 index 0000000..78c313c --- /dev/null +++ b/outside-services/outside-2.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-2 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "2" + ports: + - protocol: TCP + targetPort: 9094 + port: 32402 + nodePort: 32402 + type: NodePort
\ No newline at end of file diff --git a/test/basic-produce-consume.yml b/test/basic-produce-consume.yml index fdacea0..32ec4ec 100644 --- a/test/basic-produce-consume.yml +++ b/test/basic-produce-consume.yml @@ -36,7 +36,31 @@ data: #pkill java exit 0 - +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: basic-produce-consume + namespace: test-kafka +spec: + template: + spec: + containers: + - name: topic-create + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka.svc.cluster.local:2181 + - --create + - --if-not-exists + - --topic + - test-basic-produce-consume + - --partitions + - "1" + - --replication-factor + - "2" + restartPolicy: Never --- apiVersion: apps/v1beta1 kind: Deployment @@ -53,7 +77,7 @@ spec: spec: containers: - name: testcase - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: BOOTSTRAP value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 diff --git a/test/basic-with-kafkacat.yml b/test/basic-with-kafkacat.yml index a8974e8..78e423c 100644 --- a/test/basic-with-kafkacat.yml +++ b/test/basic-with-kafkacat.yml @@ -23,7 +23,6 @@ data: kafkacat -C -b $BOOTSTRAP -t test-basic-with-kafkacat -o -1 -e | grep $unique exit 0 - --- apiVersion: batch/v1 kind: Job @@ -35,7 +34,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: - ./bin/kafka-topics.sh - --zookeeper @@ -47,7 +46,7 @@ spec: - --partitions - "1" - --replication-factor - - "1" + - "2" restartPolicy: Never --- apiVersion: apps/v1beta1 @@ -67,7 +66,7 @@ spec: - name: testcase # common test images #image: solsson/curl@sha256:8b0927b81d10043e70f3e05e33e36fb9b3b0cbfcbccdb9f04fd53f67a270b874 - image: solsson/kafkacat@sha256:1266d140c52cb39bf314b6f22b6d7a01c4c9084781bc779fdfade51214a713a8 + image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef #image: solsson/kubectl-kafkacat@sha256:3715a7ede3f168f677ee6faf311ff6887aff31f660cfeecad5d87b4f18516321 env: - name: BOOTSTRAP diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 7fd373c..446748a 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -16,7 +16,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: ['/bin/bash', '/etc/kafka/init.sh'] volumeMounts: - name: config @@ -25,7 +25,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties @@ -48,7 +48,7 @@ spec: command: - /bin/sh - -c - - '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]' + - '[ "imok" = "$(echo ruok | nc -w 1 -q 1 127.0.0.1 2181)" ]' volumeMounts: - name: config mountPath: /etc/kafka diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index f5d1f91..a2922ef 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -16,7 +16,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: ['/bin/bash', '/etc/kafka/init.sh'] env: - name: ID_OFFSET @@ -28,7 +28,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties @@ -51,7 +51,7 @@ spec: command: - /bin/sh - -c - - '[ "imok" = "$(echo ruok | nc -w 1 127.0.0.1 2181)" ]' + - '[ "imok" = "$(echo ruok | nc -w 1 -q 1 127.0.0.1 2181)" ]' volumeMounts: - name: config mountPath: /etc/kafka |