diff options
Diffstat (limited to 'kafka')
| -rw-r--r-- | kafka/10broker-config.yml | 88 | ||||
| -rw-r--r-- | kafka/50kafka.yml | 15 | ||||
| -rw-r--r-- | kafka/test/kafkacat.yml | 6 | ||||
| -rw-r--r-- | kafka/test/produce-consume.yml | 8 |
4 files changed, 47 insertions, 70 deletions
diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index c0c4c8c..1fe67c0 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -11,6 +11,9 @@ data: KAFKA_BROKER_ID=${HOSTNAME##*-} sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties + LABELS="kafka-broker-id=$KAFKA_BROKER_ID" + ANNOTATIONS="" + hash kubectl 2>/dev/null || { sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties } && { @@ -21,36 +24,47 @@ data: sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# zone label not found for node $NODE_NAME/" /etc/kafka/server.properties else sed -i "s/#init#broker.rack=#init#/broker.rack=$ZONE/" /etc/kafka/server.properties + LABELS="$LABELS kafka-broker-rack=$ZONE" fi - kubectl -n $POD_NAMESPACE label pod $POD_NAME kafka-broker-id=$KAFKA_BROKER_ID - OUTSIDE_HOST=$(kubectl get node "$NODE_NAME" -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}') if [ $? -ne 0 ]; then echo "Outside (i.e. cluster-external access) host lookup command failed" else - OUTSIDE_HOST=${OUTSIDE_HOST}:3240${KAFKA_BROKER_ID} - sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}|" /etc/kafka/server.properties + OUTSIDE_PORT=3240${KAFKA_BROKER_ID} + sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|" /etc/kafka/server.properties + ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" + fi + + if [ ! -z "$LABELS" ]; then + kubectl -n $POD_NAMESPACE label pod $POD_NAME $LABELS || echo "Failed to label $POD_NAMESPACE.$POD_NAME - RBAC issue?" + fi + if [ ! -z "$ANNOTATIONS" ]; then + kubectl -n $POD_NAMESPACE annotate pod $POD_NAME $ANNOTATIONS || echo "Failed to annotate $POD_NAMESPACE.$POD_NAME - RBAC issue?" fi } server.properties: |- - # Licensed to the Apache Software Foundation (ASF) under one or more - # contributor license agreements. See the NOTICE file distributed with - # this work for additional information regarding copyright ownership. - # The ASF licenses this file to You under the Apache License, Version 2.0 - # (the "License"); you may not use this file except in compliance with - # the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - # see kafka.server.KafkaConfig for additional details and defaults + ############################# Log Basics ############################# + + # A comma seperated list of directories under which to store log files + # Overrides log.dir + log.dirs=/var/lib/kafka/data/topics + + # The default number of log partitions per topic. More partitions allow greater + # parallelism for consumption, but this will also result in more files across + # the brokers. + num.partitions=1 + + default.replication.factor=3 + + min.insync.replicas=2 + + auto.create.topics.enable=true + + # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. + # This value is recommended to be increased for installations with data dirs located in RAID array. + num.recovery.threads.per.data.dir=1 ############################# Server Basics ############################# @@ -96,21 +110,6 @@ data: # The maximum size of a request that the socket server will accept (protection against OOM) socket.request.max.bytes=104857600 - - ############################# Log Basics ############################# - - # A comma seperated list of directories under which to store log files - log.dirs=/tmp/kafka-logs - - # The default number of log partitions per topic. More partitions allow greater - # parallelism for consumption, but this will also result in more files across - # the brokers. - num.partitions=1 - - # The number of threads per data directory to be used for log recovery at startup and flushing at shutdown. - # This value is recommended to be increased for installations with data dirs located in RAID array. - num.recovery.threads.per.data.dir=1 - ############################# Internal Topic Settings ############################# # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" # For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. @@ -143,7 +142,7 @@ data: # from the end of the log. # The minimum age of a log file to be eligible for deletion due to age - log.retention.hours=168 + log.retention.hours=-1 # A size-based retention policy for logs. Segments are pruned from the log as long as the remaining # segments don't drop below log.retention.bytes. Functions independently of log.retention.hours. @@ -163,7 +162,7 @@ data: # server. e.g. "127.0.0.1:3000,127.0.0.1:3001,127.0.0.1:3002". # You can also append an optional chroot string to the urls to specify the # root directory for all kafka znodes. - zookeeper.connect=localhost:2181 + zookeeper.connect=zookeeper:2181 # Timeout in ms for connecting to zookeeper zookeeper.connection.timeout.ms=6000 @@ -179,21 +178,6 @@ data: group.initial.rebalance.delay.ms=0 log4j.properties: |- - # Licensed to the Apache Software Foundation (ASF) under one or more - # contributor license agreements. See the NOTICE file distributed with - # this work for additional information regarding copyright ownership. - # The ASF licenses this file to You under the Apache License, Version 2.0 - # (the "License"); you may not use this file except in compliance with - # the License. You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - # Unspecified loggers and loggers with additivity=true output to server.log and stdout # Note that INFO only applies to unspecified loggers, the log level of the child logger is used otherwise log4j.rootLogger=INFO, stdout diff --git a/kafka/50kafka.yml b/kafka/50kafka.yml index dff4a79..229bc4d 100644 --- a/kafka/50kafka.yml +++ b/kafka/50kafka.yml @@ -4,12 +4,13 @@ metadata: name: kafka namespace: kafka spec: -spec: selector: matchLabels: app: kafka serviceName: "broker" replicas: 3 + updateStrategy: + type: OnDelete template: metadata: labels: @@ -43,22 +44,18 @@ spec: env: - name: KAFKA_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka/log4j.properties + - name: JMX_PORT + value: "5555" ports: - name: inside containerPort: 9092 - name: outside containerPort: 9094 + - name: jmx + containerPort: 5555 command: - ./bin/kafka-server-start.sh - /etc/kafka/server.properties - - --override - - zookeeper.connect=zookeeper:2181 - - --override - - log.retention.hours=-1 - - --override - - log.dirs=/var/lib/kafka/data/topics - - --override - - auto.create.topics.enable=false resources: requests: cpu: 100m diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index 8d6c64e..98410bc 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -83,8 +83,6 @@ spec: - test-kafkacat - --partitions - "1" - - --replication-factor - - "2" restartPolicy: Never --- apiVersion: apps/v1beta2 @@ -115,7 +113,7 @@ spec: image: solsson/kafkacat@sha256:b32eedf936f3cde44cd164ddc77dfcf7565a8af4e357ff6de1abe4389ca530c9 env: - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092 + value: bootstrap.kafka:9092 command: - /bin/bash - -cex @@ -134,7 +132,7 @@ spec: image: solsson/kafkacat@sha256:b32eedf936f3cde44cd164ddc77dfcf7565a8af4e357ff6de1abe4389ca530c9 env: - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092 + value: bootstrap.kafka:9092 command: - /bin/bash - -cex diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index a326f01..71228b8 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -66,8 +66,6 @@ spec: - test-produce-consume - --partitions - "1" - - --replication-factor - - "2" restartPolicy: Never --- apiVersion: apps/v1beta2 @@ -98,7 +96,7 @@ spec: image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092 + value: bootstrap.kafka:9092 command: - /bin/bash - -cex @@ -117,7 +115,7 @@ spec: image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d env: - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092 + value: bootstrap.kafka:9092 command: - /bin/bash - -cex @@ -134,7 +132,7 @@ spec: image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef env: - name: BOOTSTRAP - value: kafka-0.broker.kafka.svc.cluster.local:9092 + value: bootstrap.kafka:9092 command: - /bin/bash - -e |