diff options
Diffstat (limited to 'kafka')
-rw-r--r-- | kafka/10broker-config.yml | 12 | ||||
-rw-r--r-- | kafka/test/kafkacat.yml | 13 | ||||
-rw-r--r-- | kafka/test/produce-consume.yml | 2 | ||||
-rw-r--r-- | kafka/test/replication-config.yml | 98 |
4 files changed, 110 insertions, 15 deletions
diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 2846232..debfda1 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -12,6 +12,7 @@ data: sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties LABELS="kafka-broker-id=$KAFKA_BROKER_ID" + ANNOTATIONS="" hash kubectl 2>/dev/null || { sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties @@ -32,12 +33,15 @@ data: else OUTSIDE_PORT=3240${KAFKA_BROKER_ID} sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|" /etc/kafka/server.properties - LABELS="$LABELS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" + ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" fi if [ ! -z "$LABELS" ]; then kubectl -n $POD_NAMESPACE label pod $POD_NAME $LABELS || echo "Failed to label $POD_NAMESPACE.$POD_NAME - RBAC issue?" fi + if [ ! -z "$ANNOTATIONS" ]; then + kubectl -n $POD_NAMESPACE annotate pod $POD_NAME $ANNOTATIONS || echo "Failed to annotate $POD_NAMESPACE.$POD_NAME - RBAC issue?" + fi } server.properties: |- @@ -109,9 +113,9 @@ data: ############################# Internal Topic Settings ############################# # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" # For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. - offsets.topic.replication.factor=1 - transaction.state.log.replication.factor=1 - transaction.state.log.min.isr=1 + #offsets.topic.replication.factor=1 + #transaction.state.log.replication.factor=1 + #transaction.state.log.min.isr=1 ############################# Log Flush Policy ############################# diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index f73735d..d3c2766 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -83,8 +83,6 @@ spec: - test-kafkacat - --partitions - "1" - - --replication-factor - - "2" restartPolicy: Never --- apiVersion: apps/v1beta2 @@ -93,6 +91,7 @@ metadata: name: kafkacat namespace: test-kafka spec: + # Note that this test sets a consumer group, but asserts assume that the tests gets its own messages replicas: 1 selector: matchLabels: @@ -103,12 +102,6 @@ spec: labels: test-target: kafka-client-kafkacat test-type: readiness - # for example: - # readonly - can be used in production - # isolated - read/write but in a manner that does not affect other services - # load - unsuitable for production because it uses significant resources - # chaos - unsuitable for production because it injects failure modes - #test-use: spec: containers: - name: producer @@ -135,11 +128,13 @@ spec: env: - name: BOOTSTRAP value: bootstrap.kafka:9092 + - name: CONSUMER_GROUP_ID + value: test-kafkacat-group command: - /bin/bash - -cex - > - kafkacat -C -b $BOOTSTRAP -t test-kafkacat -o -1 -f '%T;%k:%p;%o;%s\n' -u -d broker | + kafkacat -b $BOOTSTRAP -G $CONSUMER_GROUP_ID test-kafkacat -o -1 -f '%T;%k:%p;%o;%s\n' -u -d broker | tee /shared/consumed.tmp ; volumeMounts: diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index 4b1435b..71228b8 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -66,8 +66,6 @@ spec: - test-produce-consume - --partitions - "1" - - --replication-factor - - "2" restartPolicy: Never --- apiVersion: apps/v1beta2 diff --git a/kafka/test/replication-config.yml b/kafka/test/replication-config.yml new file mode 100644 index 0000000..abaea49 --- /dev/null +++ b/kafka/test/replication-config.yml @@ -0,0 +1,98 @@ +# https://github.com/Yolean/kubernetes-kafka/pull/140 +--- +kind: ConfigMap +metadata: + name: replication-config + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + tail -f /tmp/testlog + + test.sh: |- + exec >> /tmp/testlog + exec 2>&1 + set -e + + kafkacat -L -b $BOOTSTRAP > /tmp/metadata + BROKERS=$(cat /tmp/metadata | grep -E ' [0-9]+ brokers:' | awk '{print $1}') + if (( $BROKERS == 1 )); then + echo "Only one broker; no need to check for >1 replication config." + exit 0 + fi + + WITH_TWO_OR_MORE=$(cat /tmp/metadata | grep -E ' replicas: [0-9]+,[0-9]+' | wc -l) + WITH_ONE=$(cat /tmp/metadata | grep -E ' replicas: [0-9]+, ' | wc -l) + if (( $WITH_TWO_OR_MORE == 0 )); then + echo "No partitions have >1 replica, so this is probably normal." + exit 0 + fi + + if (( $WITH_ONE > $MAX_SINGLE_REPLICA_PARTITIONS )); then + echo "$(date --iso-8601='ns') There are $WITH_ONE partitions with only one replica. Alerts for under-replicated partitions won't catch that." + exit 10 + fi + + echo "$(date --iso-8601='ns') $WITH_ONE partitions have one replica and WITH_TWO_OR_MORE have more" + exit 0 + + quit-on-nonzero-exit.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + exit 0 +--- +apiVersion: apps/v1beta2 +kind: ReplicaSet +metadata: + name: replication-config + namespace: test-kafka +spec: + # Note that this test sets a consumer group, but asserts assume that the tests gets its own messages + replicas: 1 + selector: + matchLabels: + test-target: kafka-replication-config + test-type: readiness + template: + metadata: + labels: + test-target: kafka-replication-config + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/kafkacat@sha256:b32eedf936f3cde44cd164ddc77dfcf7565a8af4e357ff6de1abe4389ca530c9 + env: + - name: BOOTSTRAP + value: bootstrap.kafka:9092 + - name: MAX_SINGLE_REPLICA_PARTITIONS + value: "0" + command: + - /bin/bash + - -e + - /test/setup.sh + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/test.sh + initialDelaySeconds: 30 + periodSeconds: 60 + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/quit-on-nonzero-exit.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: replication-config |