diff options
28 files changed, 946 insertions, 37 deletions
@@ -35,7 +35,7 @@ Zookeeper at `zookeeper.kafka.svc.cluster.local:2181`. For Minikube run `kubectl apply -f configure/minikube-storageclass-broker.yml; kubectl apply -f configure/minikube-storageclass-zookeeper.yml`. -There's a similar setup for GKE, `configure/gke-*`. You might want to tweak it before creating. +There's a similar setup for AKS under `configure/aks-*` and for GKE under `configure/gke-*`. You might want to tweak it before creating. ## Start Zookeeper @@ -63,7 +63,7 @@ That's it. Just add business value :wink:. ## RBAC -For clusters that enfoce [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) there's a minimal set of policies in +For clusters that enforce [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) there's a minimal set of policies in ``` kubectl apply -f rbac-namespace-default/ ``` diff --git a/avro-tools/avro-tools-config.yml b/avro-tools/avro-tools-config.yml new file mode 100644 index 0000000..5f5d342 --- /dev/null +++ b/avro-tools/avro-tools-config.yml @@ -0,0 +1,43 @@ +kind: ConfigMap +metadata: + name: avro-tools-config + namespace: kafka +apiVersion: v1 +data: + schema-registry.properties: |- + port=80 + listeners=http://0.0.0.0:80 + kafkastore.bootstrap.servers=PLAINTEXT://bootstrap.kafka:9092 + kafkastore.topic=_schemas + debug=false + + # https://github.com/Landoop/schema-registry-ui#prerequisites + access.control.allow.methods=GET,POST,PUT,OPTIONS + access.control.allow.origin=* + + kafka-rest.properties: |- + #id=kafka-rest-test-server + listeners=http://0.0.0.0:80 + bootstrap.servers=PLAINTEXT://bootstrap.kafka:9092 + schema.registry.url=http://avro-schemas.kafka:80 + + # https://github.com/Landoop/kafka-topics-ui#common-issues + access.control.allow.methods=GET,POST,PUT,DELETE,OPTIONS + access.control.allow.origin=* + + log4j.properties: |- + log4j.rootLogger=INFO, stdout + + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + + log4j.logger.kafka=WARN, stdout + log4j.logger.org.apache.zookeeper=WARN, stdout + log4j.logger.org.apache.kafka=WARN, stdout + log4j.logger.org.I0Itec.zkclient=WARN, stdout + log4j.additivity.kafka.server=false + log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false + + log4j.logger.org.apache.kafka.clients.Metadata=DEBUG, stdout + log4j.logger.org.apache.kafka.clients.consumer.internals.AbstractCoordinator=INFO, stdout diff --git a/avro-tools/rest-service.yml b/avro-tools/rest-service.yml new file mode 100644 index 0000000..bf683a3 --- /dev/null +++ b/avro-tools/rest-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: avro-rest + namespace: kafka +spec: + ports: + - port: 80 + selector: + app: rest-proxy diff --git a/avro-tools/rest.yml b/avro-tools/rest.yml new file mode 100644 index 0000000..be299e6 --- /dev/null +++ b/avro-tools/rest.yml @@ -0,0 +1,46 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: avro-rest + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: rest-proxy + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: rest-proxy + spec: + containers: + - name: cp + image: solsson/kafka-cp@sha256:2797da107f477ede2e826c29b2589f99f22d9efa2ba6916b63e07c7045e15044 + env: + - name: KAFKAREST_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties + command: + - kafka-rest-start + - /etc/kafka-rest/kafka-rest.properties + readinessProbe: + httpGet: + path: / + port: 80 + livenessProbe: + httpGet: + path: / + port: 80 + ports: + - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/kafka-rest + volumes: + - name: config + configMap: + name: avro-tools-config diff --git a/avro-tools/schemas-service.yml b/avro-tools/schemas-service.yml new file mode 100644 index 0000000..9a9cd4e --- /dev/null +++ b/avro-tools/schemas-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: avro-schemas + namespace: kafka +spec: + ports: + - port: 80 + selector: + app: schema-registry diff --git a/avro-tools/schemas.yml b/avro-tools/schemas.yml new file mode 100644 index 0000000..b811d17 --- /dev/null +++ b/avro-tools/schemas.yml @@ -0,0 +1,47 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: avro-schemas + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: schema-registry + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + template: + metadata: + labels: + app: schema-registry + spec: + containers: + - name: cp + image: solsson/kafka-cp@sha256:2797da107f477ede2e826c29b2589f99f22d9efa2ba6916b63e07c7045e15044 + env: + - name: SCHEMA_REGISTRY_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties + command: + - schema-registry-start + - /etc/schema-registry/schema-registry.properties + readinessProbe: + httpGet: + path: / + port: 80 + livenessProbe: + httpGet: + path: / + port: 80 + initialDelaySeconds: 60 + ports: + - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/schema-registry + volumes: + - name: config + configMap: + name: avro-tools-config diff --git a/avro-tools/test/70rest-test1.yml b/avro-tools/test/70rest-test1.yml new file mode 100644 index 0000000..f26cdc9 --- /dev/null +++ b/avro-tools/test/70rest-test1.yml @@ -0,0 +1,43 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: rest-test1 + namespace: kafka +spec: + backoffLimit: 1 + template: + metadata: + name: rest-test1 + spec: + containers: + - name: curl + image: solsson/curl@sha256:523319afd39573746e8f5a7c98d4a6cd4b8cbec18b41eb30c8baa13ede120ce3 + env: + - name: REST + value: http://rest.kafka.svc.cluster.local + - name: TOPIC + value: test1 + command: + - /bin/bash + - -ce + - > + curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST; + + curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics; + + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test1; + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date)\"}]}" $REST/topics/$TOPIC -v; + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test2; + + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v; + + curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v; + + curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v; + + sleep 300 + restartPolicy: Never diff --git a/avro-tools/test/rest-curl.yml b/avro-tools/test/rest-curl.yml new file mode 100644 index 0000000..486392c --- /dev/null +++ b/avro-tools/test/rest-curl.yml @@ -0,0 +1,186 @@ +--- +kind: ConfigMap +metadata: + name: rest-curl + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + # Keep starting up until rest proxy is up and running + curl --retry 10 --retry-delay 30 --retry-connrefused -I -s $REST + + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/brokers | egrep '."brokers":.0' + + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics + echo "" + + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC + echo "" + + curl -X POST \ + -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ + --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ + $REST/topics/$TOPIC + echo "" + + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions + echo "" + + curl -X POST \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' \ + $REST/consumers/my_json_consumer + echo "" + + curl -X POST \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + --data "{\"topics\":[\"$TOPIC\"]}" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription \ + -w "%{http_code}" + echo "" + + curl -X GET \ + -H "Accept: application/vnd.kafka.json.v2+json" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance/records + + curl -X DELETE \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance + + # schema-registry + + curl -X GET $SCHEMAS/subjects + echo "" + + curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \ + --data '{"schema": "{\"type\": \"string\"}"}' \ + $SCHEMAS/subjects/$TOPIC-key/versions + echo "" + + curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \ + --data '{"schema": "{\"type\": \"string\"}"}' \ + $SCHEMAS/subjects/$TOPIC-value/versions + echo "" + + curl -X GET $SCHEMAS/schemas/ids/1 + echo "" + + curl -X GET $SCHEMAS/subjects/$TOPIC-value/versions/1 + echo "" + + # rest + schema + # TODO new topic needed because this breaks json consumer above + + curl -X POST -H "Content-Type: application/vnd.kafka.avro.v2+json" \ + -H "Accept: application/vnd.kafka.v2+json" \ + --data '{"value_schema": "{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"name\": \"name\", \"type\": \"string\"}]}", "records": [{"value": {"name": "testUser"}}]}' \ + $REST/topics/$TOPIC + echo "" + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" \ + --data '{"name": "my_consumer_instance", "format": "avro", "auto.offset.reset": "earliest"}' \ + $REST/consumers/my_avro_consumer + echo "" + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" \ + --data "{\"topics\":[\"$TOPIC\"]}" \ + $REST/consumers/my_avro_consumer/instances/my_consumer_instance/subscription + + curl -X GET -H "Accept: application/vnd.kafka.avro.v2+json" \ + $REST/consumers/my_avro_consumer/instances/my_consumer_instance/records + + tail -f /tmp/testlog + + continue.sh: |- + exit 0 + + run.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + exit 0 + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rest-curl + namespace: test-kafka +spec: + template: + spec: + containers: + - name: topic-create + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka.svc.cluster.local:2181 + - --create + - --if-not-exists + - --topic + - test-rest-curl + - --partitions + - "1" + - --replication-factor + - "1" + restartPolicy: Never +--- +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: rest-curl + namespace: test-kafka +spec: + replicas: 1 + selector: + matchLabels: + test-target: kafka-confluent-rest + test-type: readiness + template: + metadata: + labels: + test-target: kafka-confluent-rest + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/curl@sha256:523319afd39573746e8f5a7c98d4a6cd4b8cbec18b41eb30c8baa13ede120ce3 + env: + - name: SCHEMAS + value: http://schemas.kafka.svc.cluster.local + - name: REST + value: http://rest.kafka.svc.cluster.local + - name: TOPIC + value: test-rest-curl + # Test set up + command: + - /bin/bash + - -e + - /test/setup.sh + # Test run, again and again + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/run.sh + # We haven't worked on timing + periodSeconds: 60 + # Test quit on nonzero exit + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/continue.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: rest-curl diff --git a/configure/aks-storageclass-broker-managed.yml b/configure/aks-storageclass-broker-managed.yml new file mode 100644 index 0000000..1c9b3c6 --- /dev/null +++ b/configure/aks-storageclass-broker-managed.yml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: kafka-broker +provisioner: kubernetes.io/azure-disk +reclaimPolicy: Retain +parameters: + kind: "Managed" + storageaccounttype: Premium_LRS
\ No newline at end of file diff --git a/configure/aks-storageclass-zookeeper-managed.yml b/configure/aks-storageclass-zookeeper-managed.yml new file mode 100644 index 0000000..6963dcc --- /dev/null +++ b/configure/aks-storageclass-zookeeper-managed.yml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: kafka-zookeeper +provisioner: kubernetes.io/azure-disk +reclaimPolicy: Retain +parameters: + kind: "Managed" + storageaccounttype: Premium_LRS
\ No newline at end of file diff --git a/configure/docker-storageclass-broker.yml b/configure/docker-storageclass-broker.yml new file mode 100644 index 0000000..4a99776 --- /dev/null +++ b/configure/docker-storageclass-broker.yml @@ -0,0 +1,6 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-broker +provisioner: docker.io/hostpath +reclaimPolicy: Retain diff --git a/configure/docker-storageclass-zookeeper.yml b/configure/docker-storageclass-zookeeper.yml new file mode 100644 index 0000000..038f2c8 --- /dev/null +++ b/configure/docker-storageclass-zookeeper.yml @@ -0,0 +1,6 @@ +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + name: kafka-zookeeper +provisioner: docker.io/hostpath +reclaimPolicy: Retain diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 2846232..debfda1 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -12,6 +12,7 @@ data: sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties LABELS="kafka-broker-id=$KAFKA_BROKER_ID" + ANNOTATIONS="" hash kubectl 2>/dev/null || { sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties @@ -32,12 +33,15 @@ data: else OUTSIDE_PORT=3240${KAFKA_BROKER_ID} sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|" /etc/kafka/server.properties - LABELS="$LABELS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" + ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" fi if [ ! -z "$LABELS" ]; then kubectl -n $POD_NAMESPACE label pod $POD_NAME $LABELS || echo "Failed to label $POD_NAMESPACE.$POD_NAME - RBAC issue?" fi + if [ ! -z "$ANNOTATIONS" ]; then + kubectl -n $POD_NAMESPACE annotate pod $POD_NAME $ANNOTATIONS || echo "Failed to annotate $POD_NAMESPACE.$POD_NAME - RBAC issue?" + fi } server.properties: |- @@ -109,9 +113,9 @@ data: ############################# Internal Topic Settings ############################# # The replication factor for the group metadata internal topics "__consumer_offsets" and "__transaction_state" # For anything other than development testing, a value greater than 1 is recommended for to ensure availability such as 3. - offsets.topic.replication.factor=1 - transaction.state.log.replication.factor=1 - transaction.state.log.min.isr=1 + #offsets.topic.replication.factor=1 + #transaction.state.log.replication.factor=1 + #transaction.state.log.min.isr=1 ############################# Log Flush Policy ############################# diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index f73735d..d3c2766 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -83,8 +83,6 @@ spec: - test-kafkacat - --partitions - "1" - - --replication-factor - - "2" restartPolicy: Never --- apiVersion: apps/v1beta2 @@ -93,6 +91,7 @@ metadata: name: kafkacat namespace: test-kafka spec: + # Note that this test sets a consumer group, but asserts assume that the tests gets its own messages replicas: 1 selector: matchLabels: @@ -103,12 +102,6 @@ spec: labels: test-target: kafka-client-kafkacat test-type: readiness - # for example: - # readonly - can be used in production - # isolated - read/write but in a manner that does not affect other services - # load - unsuitable for production because it uses significant resources - # chaos - unsuitable for production because it injects failure modes - #test-use: spec: containers: - name: producer @@ -135,11 +128,13 @@ spec: env: - name: BOOTSTRAP value: bootstrap.kafka:9092 + - name: CONSUMER_GROUP_ID + value: test-kafkacat-group command: - /bin/bash - -cex - > - kafkacat -C -b $BOOTSTRAP -t test-kafkacat -o -1 -f '%T;%k:%p;%o;%s\n' -u -d broker | + kafkacat -b $BOOTSTRAP -G $CONSUMER_GROUP_ID test-kafkacat -o -1 -f '%T;%k:%p;%o;%s\n' -u -d broker | tee /shared/consumed.tmp ; volumeMounts: diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index 4b1435b..71228b8 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -66,8 +66,6 @@ spec: - test-produce-consume - --partitions - "1" - - --replication-factor - - "2" restartPolicy: Never --- apiVersion: apps/v1beta2 diff --git a/kafka/test/replication-config.yml b/kafka/test/replication-config.yml new file mode 100644 index 0000000..abaea49 --- /dev/null +++ b/kafka/test/replication-config.yml @@ -0,0 +1,98 @@ +# https://github.com/Yolean/kubernetes-kafka/pull/140 +--- +kind: ConfigMap +metadata: + name: replication-config + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + tail -f /tmp/testlog + + test.sh: |- + exec >> /tmp/testlog + exec 2>&1 + set -e + + kafkacat -L -b $BOOTSTRAP > /tmp/metadata + BROKERS=$(cat /tmp/metadata | grep -E ' [0-9]+ brokers:' | awk '{print $1}') + if (( $BROKERS == 1 )); then + echo "Only one broker; no need to check for >1 replication config." + exit 0 + fi + + WITH_TWO_OR_MORE=$(cat /tmp/metadata | grep -E ' replicas: [0-9]+,[0-9]+' | wc -l) + WITH_ONE=$(cat /tmp/metadata | grep -E ' replicas: [0-9]+, ' | wc -l) + if (( $WITH_TWO_OR_MORE == 0 )); then + echo "No partitions have >1 replica, so this is probably normal." + exit 0 + fi + + if (( $WITH_ONE > $MAX_SINGLE_REPLICA_PARTITIONS )); then + echo "$(date --iso-8601='ns') There are $WITH_ONE partitions with only one replica. Alerts for under-replicated partitions won't catch that." + exit 10 + fi + + echo "$(date --iso-8601='ns') $WITH_ONE partitions have one replica and WITH_TWO_OR_MORE have more" + exit 0 + + quit-on-nonzero-exit.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + exit 0 +--- +apiVersion: apps/v1beta2 +kind: ReplicaSet +metadata: + name: replication-config + namespace: test-kafka +spec: + # Note that this test sets a consumer group, but asserts assume that the tests gets its own messages + replicas: 1 + selector: + matchLabels: + test-target: kafka-replication-config + test-type: readiness + template: + metadata: + labels: + test-target: kafka-replication-config + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/kafkacat@sha256:b32eedf936f3cde44cd164ddc77dfcf7565a8af4e357ff6de1abe4389ca530c9 + env: + - name: BOOTSTRAP + value: bootstrap.kafka:9092 + - name: MAX_SINGLE_REPLICA_PARTITIONS + value: "0" + command: + - /bin/bash + - -e + - /test/setup.sh + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/test.sh + initialDelaySeconds: 30 + periodSeconds: 60 + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/quit-on-nonzero-exit.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: replication-config diff --git a/linkedin-burrow/burrow-config.yml b/linkedin-burrow/burrow-config.yml new file mode 100644 index 0000000..5fe6dda --- /dev/null +++ b/linkedin-burrow/burrow-config.yml @@ -0,0 +1,36 @@ +kind: ConfigMap +metadata: + name: burrow-config + namespace: kafka +apiVersion: v1 +data: + burrow.toml: |- + [zookeeper] + servers=[ "zookeeper:2181" ] + timeout=6 + root-path="/burrow" + + [cluster.local] + class-name="kafka" + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] + topic-refresh=60 + offset-refresh=30 + + [consumer.local] + class-name="kafka" + cluster="local" + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] + group-blacklist="" + group-whitelist="" + + [consumer.local_zk] + class-name="kafka_zk" + cluster="local" + servers=[ "zookeeper:2181" ] + zookeeper-path="/local" + zookeeper-timeout=30 + group-blacklist="" + group-whitelist="" + + [httpserver.default] + address=":8000" diff --git a/linkedin-burrow/burrow-service.yml b/linkedin-burrow/burrow-service.yml new file mode 100644 index 0000000..15eac06 --- /dev/null +++ b/linkedin-burrow/burrow-service.yml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: burrow + namespace: kafka +spec: + selector: + app: burrow + ports: + - name: web + protocol: TCP + port: 80 + - name: api + protocol: TCP + port: 8000 + - name: prometheus + protocol: TCP + port: 8080 diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml new file mode 100644 index 0000000..2e8c6b5 --- /dev/null +++ b/linkedin-burrow/burrow.yml @@ -0,0 +1,60 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: burrow + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: burrow + template: + metadata: + labels: + app: burrow + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + containers: + - name: burrow + image: solsson/burrow@sha256:a6fe5d1268a7e7d58f7f960697fc8fda3e6861a2653fe71135a2207c9cf18cf0 + ports: + - name: api + containerPort: 8000 + readinessProbe: + httpGet: + path: /burrow/admin + port: 8000 + livenessProbe: + httpGet: + path: /burrow/admin + port: 8000 + volumeMounts: + - name: config + mountPath: /etc/burrow + - name: prom + image: solsson/burrow-exporter:api-v3@sha256:dd690c04ae31c62e4d7d5398f24f368fb2c48046ec3ae68bbc7582c114819a7b + ports: + - name: prometheus + containerPort: 8080 + env: + - name: BURROW_ADDR + value: http://localhost:8000 + - name: METRICS_ADDR + value: 0.0.0.0:8080 + - name: INTERVAL + value: "30" + - name: dashboard + image: joway/burrow-dashboard@sha256:b4cce87c8264119d73f9a6df5d787ea811ce2138d39ca6cd56525bcfbb5169bf + env: + - name: BURROW_BACKEND + value: http://localhost:8000 + ports: + - name: web + containerPort: 80 + protocol: TCP + volumes: + - name: config + configMap: + name: burrow-config diff --git a/maintenance/README.md b/maintenance/README.md new file mode 100644 index 0000000..5830d79 --- /dev/null +++ b/maintenance/README.md @@ -0,0 +1,32 @@ + +## Re-assign Leadership + +This is one of the cases where this repo begs to differ from traditional Kafka setups. +In Kubernetes the restart of a pod, and subsequent start on a different node, should be a non-event. + +> ”when a broker is stopped and restarted, it does not resume leadership of any partitions automatically” + +_-- Neha Narkhede, Gwen Shapira, and Todd Palino. ”Kafka: The Definitive Guide”_ + +Create the `preferred-replica-election-job.yml` resource, after deleting any previous one. + +## Change a Partition's Replicas + +> ”From time to time, it may be necessary to change the replica assignments for a partition. Some examples of when this might be needed are: +> * If a topic’s partitions are not balanced across the cluster, causing uneven load on brokers +> * If a broker is taken offline and the partition is under-replicated +> * If a new broker is added and needs to receive a share of the cluster load” + +_-- Neha Narkhede, Gwen Shapira, and Todd Palino. ”Kafka: The Definitive Guide”_ + +Use the `reassign-paritions-job.yml`, after editing `TOPICS` and `BROKERS`. + +## Increase a topic's replication factor + +See https://github.com/Yolean/kubernetes-kafka/pull/140 + +Use the `replication-factor-increase-job.yml`, after editing `TOPICS` and `BROKERS`. + +The affected topics may end up without a preferred replica. See above to fix that, +or to affect only your selected topics use [Kafka Manager](https://github.com/Yolean/kubernetes-kafka/pull/83)'s topic screen, +Generate Partition Assignments followed by Reassign Partitions. diff --git a/maintenance/preferred-replica-election-job.yml b/maintenance/preferred-replica-election-job.yml new file mode 100644 index 0000000..ac4f13a --- /dev/null +++ b/maintenance/preferred-replica-election-job.yml @@ -0,0 +1,19 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: preferred-replica-election + namespace: kafka +spec: + template: + metadata: + name: preferred-replica-election + spec: + containers: + - name: kafka + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - ./bin/kafka-preferred-replica-election.sh + - --zookeeper + - zookeeper:2181 + restartPolicy: Never + backoffLimit: 3 diff --git a/maintenance/reassign-paritions-job.yml b/maintenance/reassign-paritions-job.yml new file mode 100644 index 0000000..e9e184e --- /dev/null +++ b/maintenance/reassign-paritions-job.yml @@ -0,0 +1,51 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: reassign-partitions + namespace: kafka +spec: + template: + metadata: + name: reassign-partitions + spec: + containers: + - name: kafka + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + env: + - name: ZOOKEEPER + value: zookeeper.kafka:2181 + # the following must be edited per job + - name: TOPICS + value: test-produce-consume,test-kafkacat + - name: BROKERS + value: 0,2 + command: + - /bin/bash + - -ce + - > + echo '{"topics":[' > /tmp/reassign-topics.json; + echo -n ' {"topic":"' >> /tmp/reassign-topics.json; + echo -n $TOPICS | sed 's/,/"},\n {"topic":"/g' >> /tmp/reassign-topics.json; + echo '"}' >> /tmp/reassign-topics.json; + echo ']}' >> /tmp/reassign-topics.json; + + echo "# reassign-topics.json"; + cat /tmp/reassign-topics.json; + + ./bin/kafka-reassign-partitions.sh + --zookeeper=$ZOOKEEPER + --generate + --topics-to-move-json-file=/tmp/reassign-topics.json + --broker-list=$BROKERS > /tmp/generated.txt; + + tail -n 1 /tmp/generated.txt > /tmp/proposed-reassignment.json; + + echo "# proposed-reassignment.json"; + cat /tmp/proposed-reassignment.json; + + ./bin/kafka-reassign-partitions.sh + --zookeeper=$ZOOKEEPER + --execute + --reassignment-json-file=/tmp/proposed-reassignment.json; + restartPolicy: Never + backoffLimit: 3 diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml new file mode 100644 index 0000000..de35987 --- /dev/null +++ b/maintenance/replication-factor-increase-job.yml @@ -0,0 +1,65 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: replication-factor-increase + namespace: kafka +spec: + template: + metadata: + name: replication-factor-increase + spec: + containers: + - name: kafka + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + env: + - name: ZOOKEEPER + value: zookeeper.kafka:2181 + # the following must be edited per job + - name: TOPICS + value: "" + - name: BROKERS + value: 0,1,2 + command: + - /bin/bash + - -ce + - > + if [ -z "$TOPICS" ]; then + echo "Please set the TOPICS env (comma-separated) and re-create the job" + tail -f /dev/null + fi + + echo '{"topics":[' > /tmp/reassign-topics.json; + echo -n ' {"topic":"' >> /tmp/reassign-topics.json; + echo -n $TOPICS | sed 's/,/"},\n {"topic":"/g' >> /tmp/reassign-topics.json; + echo '"}' >> /tmp/reassign-topics.json; + echo ']}' >> /tmp/reassign-topics.json; + + echo "# reassign-topics.json"; + cat /tmp/reassign-topics.json; + + ./bin/kafka-reassign-partitions.sh + --zookeeper=$ZOOKEEPER + --generate + --topics-to-move-json-file=/tmp/reassign-topics.json + --broker-list=$BROKERS > /tmp/generated.txt; + + tail -n 1 /tmp/generated.txt > /tmp/proposed-reassignment.json; + + sleep 1; + echo "# proposed-reassignment.json"; + cat /tmp/proposed-reassignment.json; + + sed -i 's/"replicas":\[.\]/"replicas":[0,1,2]/g' /tmp/proposed-reassignment.json; + sed -i 's/,"log_dirs":\["any"\]//g' /tmp/proposed-reassignment.json; + echo "# proposed-reassignment.json modified to affect replication factor"; + cat /tmp/proposed-reassignment.json; + + echo "# Triggering kafka-reassign-partitions.sh" + ./bin/kafka-reassign-partitions.sh + --zookeeper=$ZOOKEEPER + --execute + --reassignment-json-file=/tmp/proposed-reassignment.json; + + echo "# Reassignment exited. Upon success you may want to run preferred-replica-election." + restartPolicy: Never + backoffLimit: 3 diff --git a/maintenance/test/replicated-partitions.yml b/maintenance/test/replicated-partitions.yml new file mode 100644 index 0000000..a1f8158 --- /dev/null +++ b/maintenance/test/replicated-partitions.yml @@ -0,0 +1,50 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: replicated-partitions + namespace: test-kafka +spec: + replicas: 1 + selector: + matchLabels: + test-type: readiness + test-target: under-replicated-partitions + template: + metadata: + labels: + test-type: readiness + test-target: under-replicated-partitions + spec: + containers: + - name: kafka + image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d + command: + - /bin/bash + - -ec + - > + touch /tmp/testlog; + tail -f /tmp/testlog + readinessProbe: + exec: + command: + - /bin/bash + - -c + - > + echo "### $(date -Ins -u) ###" >> /tmp/testlog + && + [ + $( + ./bin/kafka-topics.sh + --zookeeper zookeeper.kafka:2181 + --describe + --under-replicated-partitions + | + tee -a /tmp/testlog + | + wc -l + ) + -eq + 0 + ] + periodSeconds: 30 + timeoutSeconds: 29 diff --git a/prometheus/10-metrics-config.yml b/prometheus/10-metrics-config.yml new file mode 100644 index 0000000..345e192 --- /dev/null +++ b/prometheus/10-metrics-config.yml @@ -0,0 +1,45 @@ +kind: ConfigMap +metadata: + name: jmx-config + namespace: kafka +apiVersion: v1 +data: + + jmx-kafka-prometheus.yml: |+ + lowercaseOutputName: true + jmxUrl: service:jmx:rmi:///jndi/rmi://127.0.0.1:5555/jmxrmi + ssl: false + whitelistObjectNames: ["kafka.server:*","kafka.controller:*","java.lang:*"] + rules: + - pattern : kafka.server<type=ReplicaFetcherManager, name=MaxLag, clientId=(.+)><>Value + - pattern : kafka.server<type=BrokerTopicMetrics, name=(BytesInPerSec|BytesOutPerSec|MessagesInPerSec), topic=(.+)><>OneMinuteRate + - pattern : kafka.server<type=KafkaRequestHandlerPool, name=RequestHandlerAvgIdlePercent><>OneMinuteRate + - pattern : kafka.server<type=Produce><>queue-size + - pattern : kafka.server<type=ReplicaManager, name=(PartitionCount|UnderReplicatedPartitions)><>(Value|OneMinuteRate) + - pattern : kafka.server<type=controller-channel-metrics, broker-id=(.+)><>(.*) + - pattern : kafka.server<type=socket-server-metrics, networkProcessor=(.+)><>(.*) + - pattern : kafka.server<type=Fetch><>queue-size + - pattern : kafka.server<type=SessionExpireListener, name=(.+)><>OneMinuteRate + - pattern : kafka.controller<type=KafkaController, name=(.+)><>Value + - pattern : java.lang<type=OperatingSystem><>SystemCpuLoad + - pattern : java.lang<type=Memory><HeapMemoryUsage>used + - pattern : java.lang<type=OperatingSystem><>FreePhysicalMemorySize + + jmx-zookeeper-prometheus.yaml: |+ + rules: + - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d)><>(\\w+)" + name: "zookeeper_$2" + - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d)><>(\\w+)" + name: "zookeeper_$3" + labels: + replicaId: "$2" + - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d), name2=(\\w+)><>(\\w+)" + name: "zookeeper_$4" + labels: + replicaId: "$2" + memberType: "$3" + - pattern: "org.apache.ZooKeeperService<name0=ReplicatedServer_id(\\d), name1=replica.(\\d), name2=(\\w+), name3=(\\w+)><>(\\w+)" + name: "zookeeper_$4_$5" + labels: + replicaId: "$2" + memberType: "$3"
\ No newline at end of file diff --git a/prometheus/50-kafka-jmx-exporter-patch.yml b/prometheus/50-kafka-jmx-exporter-patch.yml new file mode 100644 index 0000000..7876ef4 --- /dev/null +++ b/prometheus/50-kafka-jmx-exporter-patch.yml @@ -0,0 +1,42 @@ +# meant to be applied using +# kubectl --namespace kafka patch statefulset kafka --patch "$(cat prometheus/50-kafka-jmx-exporter-patch.yml )" +apiVersion: apps/v1beta2 +kind: StatefulSet +metadata: + name: kafka + namespace: kafka +spec: + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "5556" + spec: + containers: + - name: metrics + image: solsson/kafka-prometheus-jmx-exporter@sha256:a23062396cd5af1acdf76512632c20ea6be76885dfc20cd9ff40fb23846557e8 + command: + - java + - -XX:+UnlockExperimentalVMOptions + - -XX:+UseCGroupMemoryLimitForHeap + - -XX:MaxRAMFraction=1 + - -XshowSettings:vm + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - /etc/jmx-kafka/jmx-kafka-prometheus.yml + ports: + - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 60Mi + limits: + memory: 120Mi + volumeMounts: + - name: jmx-config + mountPath: /etc/jmx-kafka + volumes: + - name: jmx-config + configMap: + name: jmx-config diff --git a/yahoo-kafka-manager/kafka-manager.yml b/yahoo-kafka-manager/kafka-manager.yml index 77319e4..e176981 100644 --- a/yahoo-kafka-manager/kafka-manager.yml +++ b/yahoo-kafka-manager/kafka-manager.yml @@ -15,7 +15,7 @@ spec: spec: containers: - name: kafka-manager - image: solsson/kafka-manager@sha256:e07b5c50b02c761b3471ebb62ede88afc0625e325fe428316e32fec7f227ff9b + image: solsson/kafka-manager@sha256:5db7d54cdb642ec5a92f37a869fdcf2aa479b2552e900b2d2b83b38a1806c2de ports: - containerPort: 80 env: diff --git a/zookeeper/test.sh b/zookeeper/test.sh deleted file mode 100755 index fbe12ff..0000000 --- a/zookeeper/test.sh +++ /dev/null @@ -1,19 +0,0 @@ -#! /bin/bash - -# Copyright 2016 The Kubernetes Authors All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -kubectl exec zoo-0 -- /opt/zookeeper/bin/zkCli.sh create /foo bar; -kubectl exec zoo-2 -- /opt/zookeeper/bin/zkCli.sh get /foo; - |