From 198833ad4c0393299a23fef71b843fc627469775 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 11:17:55 +0200 Subject: Gives init container access to pod name and namespace --- 50kafka.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/50kafka.yml b/50kafka.yml index 430423a..0de12dc 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -21,6 +21,14 @@ spec: valueFrom: fieldRef: fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace command: ['/bin/bash', '/etc/kafka/init.sh'] volumeMounts: - name: config -- cgit v1.2.3 From 40488e65d35be93dad26b52a7d32054ba4a08610 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 11:18:37 +0200 Subject: Uses pod name for broker id, instead of the host=pod assumption --- 10broker-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/10broker-config.yml b/10broker-config.yml index 7d296bf..0010ff7 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -8,7 +8,7 @@ data: #!/bin/bash set -x - KAFKA_BROKER_ID=${HOSTNAME##*-} + KAFKA_BROKER_ID=${POD_NAME##*-} sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties hash kubectl 2>/dev/null || { -- cgit v1.2.3 From c92d50cc6822861e0bc6951ef82a032f6992483f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 12:12:08 +0200 Subject: Labels each statefulset pod with the index, to allow individual services --- 10broker-config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/10broker-config.yml b/10broker-config.yml index 0010ff7..6e43e06 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -22,6 +22,8 @@ data: else sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties fi + + kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID } server.properties: |- -- cgit v1.2.3 From dd9ce9533684492e80c45de9c48f38e168fd7840 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 12:52:24 +0200 Subject: Exemplifies external access using host lookup that works with minikube Listener ports from https://hub.docker.com/r/wurstmeister/kafka/ Config based on KIP-103 --- 10broker-config.yml | 12 ++++++++++++ 50kafka.yml | 1 + outside-services/outside-0.yml | 15 +++++++++++++++ outside-services/outside-1.yml | 15 +++++++++++++++ outside-services/outside-2.yml | 15 +++++++++++++++ 5 files changed, 58 insertions(+) create mode 100644 outside-services/outside-0.yml create mode 100644 outside-services/outside-1.yml create mode 100644 outside-services/outside-2.yml diff --git a/10broker-config.yml b/10broker-config.yml index 6e43e06..b899401 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -24,6 +24,14 @@ data: fi kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID + + OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') + if [ $? -ne 0 ]; then + echo "Outside (i.e. cluster-external access) host lookup command failed" + else + OUTSIDE_HOST=${OUTSIDE_HOST}:3240${KAFKA_BROKER_ID} + sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}|" /etc/kafka/server.properties + fi } server.properties: |- @@ -63,14 +71,18 @@ data: # EXAMPLE: # listeners = PLAINTEXT://your.host.name:9092 #listeners=PLAINTEXT://:9092 + listeners=OUTSIDE://:9094,PLAINTEXT://:9092 # Hostname and port the broker will advertise to producers and consumers. If not set, # it uses the value for "listeners" if configured. Otherwise, it will use the value # returned from java.net.InetAddress.getCanonicalHostName(). #advertised.listeners=PLAINTEXT://your.host.name:9092 + #init#advertised.listeners=OUTSIDE://#init#,PLAINTEXT://:9092 # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + listener.security.protocol.map=OUTSIDE:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + inter.broker.listener.name=PLAINTEXT # The number of threads that the server uses for receiving requests from the network and sending responses to the network num.network.threads=3 diff --git a/50kafka.yml b/50kafka.yml index 0de12dc..2f50332 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -41,6 +41,7 @@ spec: value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties ports: - containerPort: 9092 + - containerPort: 9094 command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties diff --git a/outside-services/outside-0.yml b/outside-services/outside-0.yml new file mode 100644 index 0000000..7bc12bd --- /dev/null +++ b/outside-services/outside-0.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-0 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "0" + ports: + - protocol: TCP + targetPort: 9094 + port: 32400 + nodePort: 32400 + type: NodePort \ No newline at end of file diff --git a/outside-services/outside-1.yml b/outside-services/outside-1.yml new file mode 100644 index 0000000..1642ee0 --- /dev/null +++ b/outside-services/outside-1.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-1 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "1" + ports: + - protocol: TCP + targetPort: 9094 + port: 32401 + nodePort: 32401 + type: NodePort \ No newline at end of file diff --git a/outside-services/outside-2.yml b/outside-services/outside-2.yml new file mode 100644 index 0000000..78c313c --- /dev/null +++ b/outside-services/outside-2.yml @@ -0,0 +1,15 @@ +kind: Service +apiVersion: v1 +metadata: + name: outside-2 + namespace: kafka +spec: + selector: + app: kafka + kafka-broker-id: "2" + ports: + - protocol: TCP + targetPort: 9094 + port: 32402 + nodePort: 32402 + type: NodePort \ No newline at end of file -- cgit v1.2.3 From 73e0e5b7a0d262e828147321679efc2a3710f705 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sun, 15 Oct 2017 13:06:05 +0200 Subject: Revert "Uses pod name for broker id"; keep doing like before and like zoo This reverts commit dcdac0fb36a4476f494af419527ba42f279c485b. --- 10broker-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/10broker-config.yml b/10broker-config.yml index b899401..312df3b 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -8,7 +8,7 @@ data: #!/bin/bash set -x - KAFKA_BROKER_ID=${POD_NAME##*-} + KAFKA_BROKER_ID=${HOSTNAME##*-} sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties hash kubectl 2>/dev/null || { -- cgit v1.2.3 From 4c202f482d6adda147e406bed6879625425e0a6d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 25 Oct 2017 09:18:50 +0200 Subject: Keeps internal listener first in protocol map, as ... documented in https://cwiki.apache.org/confluence/display/KAFKA/KIP-103%3A+Separation+of+Internal+and+External+traffic#KIP-103:SeparationofInternalandExternaltraffic-Compatibility,Deprecation,andMigrationPlan "ZooKeeper-based consumers will use the first listener with PLAINTEXT as the security protocol, so listener ordering is important in such cases." Confirmed this using kafka-manager (#83) --- 10broker-config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/10broker-config.yml b/10broker-config.yml index 312df3b..2af1ae1 100644 --- a/10broker-config.yml +++ b/10broker-config.yml @@ -81,7 +81,7 @@ data: # Maps listener names to security protocols, the default is for them to be the same. See the config documentation for more details #listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL - listener.security.protocol.map=OUTSIDE:PLAINTEXT,PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL + listener.security.protocol.map=PLAINTEXT:PLAINTEXT,SSL:SSL,SASL_PLAINTEXT:SASL_PLAINTEXT,SASL_SSL:SASL_SSL,OUTSIDE:PLAINTEXT inter.broker.listener.name=PLAINTEXT # The number of threads that the server uses for receiving requests from the network and sending responses to the network -- cgit v1.2.3 From 5f27972bb315877148a9782a4e572e8d8a5d00a1 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 25 Oct 2017 10:36:39 +0200 Subject: Adds port names as documentation in kafka manifest --- 50kafka.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/50kafka.yml b/50kafka.yml index 2f50332..34f68df 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -40,8 +40,10 @@ spec: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties ports: - - containerPort: 9092 - - containerPort: 9094 + - name: inside + containerPort: 9092 + - name: outside + containerPort: 9094 command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties -- cgit v1.2.3