diff options
-rw-r--r-- | 10broker-config.yml | 17 | ||||
-rw-r--r-- | 30bootstrap-service.yml | 11 | ||||
-rw-r--r-- | 50kafka.yml | 15 | ||||
-rw-r--r-- | outside-services/outside-0.yml | 15 | ||||
-rw-r--r-- | outside-services/outside-1.yml | 15 | ||||
-rw-r--r-- | outside-services/outside-2.yml | 15 | ||||
-rw-r--r-- | test/basic-produce-consume.yml | 28 | ||||
-rw-r--r-- | test/basic-with-kafkacat.yml | 5 | ||||
-rw-r--r-- | zookeeper/50pzoo.yml | 4 | ||||
-rw-r--r-- | zookeeper/51zoo.yml | 4 |
10 files changed, 115 insertions, 14 deletions
diff --git a/10broker-config.yml b/10broker-config.yml index 7d296bf..c0c4c8c 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -22,6 +22,16 @@ data: else sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties fi + + kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID + + OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + if [ $? -ne 0 ]; then + echo "Outside (i.e. cluster-external access) host lookup command failed" + else + OUTSIDE_HOST=${OUTSIDE_HOST}:3240${KAFKA_BROKER_ID} + sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}|" /etc/kafka/server.properties + fi } server.properties: |- @@ -49,9 +59,6 @@ data: #init#broker.rack=#init# - # Switch to enable topic deletion or not, default value is false - delete.topic.enable=true - ############################# Socket Server Settings ############################# # The address the socket server listens on. It will get the value returned from @@ -61,14 +68,18 @@ data: # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 #listeners=PLAINTEXT://:9092 + listeners=OUTSIDE://:9094,PLAINTEXT://:9092 # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value # returned from java.net.InetAddress.getCanonicalHostName(). #advertised.listeners=PLAINTEXT://your.host.name:9092 + #init#advertised.listeners=OUTSIDE://#init#,PLAINTEXT://:9092 # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT + inter.broker.listener.name=PLAINTEXT # The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 diff --git a/30bootstrap-service.yml b/30bootstrap-service.yml new file mode 100644 index 0000000..7c2a337 --- /dev/null +++ b/30bootstrap-service.yml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: bootstrap + namespace: kafka +spec: + ports: + - port: 9092 + selector: + app: kafka diff --git a/50kafka.yml b/50kafka.yml index 430423a..e4451fd 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -21,18 +21,29 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace command: ['/bin/bash', '/etc/kafka/init.sh'] volumeMounts: - name: config mountPath: /etc/kafka containers: - name: broker - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties ports: - - containerPort: 9092 + - name: inside + containerPort: 9092 + - name: outside + containerPort: 9094 command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties diff --git a/outside-services/outside-0.yml b/outside-services/outside-0.yml new file mode 100644 index 0000000..7bc12bd --- /dev/null +++ b/outside-services/outside-0.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-0 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "0" + ports: + - protocol: TCP + targetPort: 9094 + port: 32400 + nodePort: 32400 + type: NodePort
\ No newline at end of file diff --git a/outside-services/outside-1.yml b/outside-services/outside-1.yml new file mode 100644 index 0000000..1642ee0 --- /dev/null +++ b/outside-services/outside-1.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-1 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "1" + ports: + - protocol: TCP + targetPort: 9094 + port: 32401 + nodePort: 32401 + type: NodePort
\ No newline at end of file diff --git a/outside-services/outside-2.yml b/outside-services/outside-2.yml new file mode 100644 index 0000000..78c313c --- /dev/null +++ b/outside-services/outside-2.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-2 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "2" + ports: + - protocol: TCP + targetPort: 9094 + port: 32402 + nodePort: 32402 + type: NodePort
\ No newline at end of file diff --git a/test/basic-produce-consume.yml b/test/basic-produce-consume.yml index 20bb276..32ec4ec 100644 --- a/test/basic-produce-consume.yml +++ b/test/basic-produce-consume.yml @@ -36,7 +36,31 @@ data: #pkill java exit 0 - +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: basic-produce-consume + namespace: test-kafka +spec: + template: + spec: + containers: + - name: topic-create + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka.svc.cluster.local:2181 + - --create + - --if-not-exists + - --topic + - test-basic-produce-consume + - --partitions + - "1" + - --replication-factor + - "2" + restartPolicy: Never --- apiVersion: apps/v1beta1 kind: Deployment @@ -53,7 +77,7 @@ spec: spec: containers: - name: testcase - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: BOOTSTRAP value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 diff --git a/test/basic-with-kafkacat.yml b/test/basic-with-kafkacat.yml index c5a2503..78e423c 100644 --- a/test/basic-with-kafkacat.yml +++ b/test/basic-with-kafkacat.yml @@ -23,7 +23,6 @@ data: kafkacat -C -b $BOOTSTRAP -t test-basic-with-kafkacat -o -1 -e | grep $unique exit 0 - --- apiVersion: batch/v1 kind: Job @@ -35,7 +34,7 @@ spec: spec: containers: - name: topic-create - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: - ./bin/kafka-topics.sh - --zookeeper @@ -47,7 +46,7 @@ spec: - --partitions - "1" - --replication-factor - - "1" + - "2" restartPolicy: Never --- apiVersion: apps/v1beta1 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 5848b2f..446748a 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -16,7 +16,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: ['/bin/bash', '/etc/kafka/init.sh'] volumeMounts: - name: config @@ -25,7 +25,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index fb2b741..a2922ef 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -16,7 +16,7 @@ spec: terminationGracePeriodSeconds: 10 initContainers: - name: init-config - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d command: ['/bin/bash', '/etc/kafka/init.sh'] env: - name: ID_OFFSET @@ -28,7 +28,7 @@ spec: mountPath: /var/lib/zookeeper/data containers: - name: zookeeper - image: solsson/kafka:0.11.0.1@sha256:1b6b25ea30a93d0d9dc69eb4531d7246d72df4015bbb5baa5bb551a77011bed5 + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties |