diff options
| -rw-r--r-- | README.md | 4 | ||||
| -rw-r--r-- | configure/aks-storageclass-broker-managed.yml | 9 | ||||
| -rw-r--r-- | configure/aks-storageclass-zookeeper-managed.yml | 9 | ||||
| -rw-r--r-- | kafka/10broker-config.yml | 6 | ||||
| -rw-r--r-- | kafka/test/kafkacat.yml | 2 | ||||
| -rw-r--r-- | kafka/test/produce-consume.yml | 2 | ||||
| -rw-r--r-- | linkedin-burrow/burrow-config.yml | 36 | ||||
| -rw-r--r-- | linkedin-burrow/burrow-service.yml | 18 | ||||
| -rw-r--r-- | linkedin-burrow/burrow.yml | 60 |
9 files changed, 139 insertions, 7 deletions
@@ -35,7 +35,7 @@ Zookeeper at `zookeeper.kafka.svc.cluster.local:2181`. For Minikube run `kubectl apply -f configure/minikube-storageclass-broker.yml; kubectl apply -f configure/minikube-storageclass-zookeeper.yml`. -There's a similar setup for GKE, `configure/gke-*`. You might want to tweak it before creating. +There's a similar setup for AKS under `configure/aks-*` and for GKE under `configure/gke-*`. You might want to tweak it before creating. ## Start Zookeeper @@ -63,7 +63,7 @@ That's it. Just add business value :wink:. ## RBAC -For clusters that enfoce [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) there's a minimal set of policies in +For clusters that enforce [RBAC](https://kubernetes.io/docs/admin/authorization/rbac/) there's a minimal set of policies in ``` kubectl apply -f rbac-namespace-default/ ``` diff --git a/configure/aks-storageclass-broker-managed.yml b/configure/aks-storageclass-broker-managed.yml new file mode 100644 index 0000000..1c9b3c6 --- /dev/null +++ b/configure/aks-storageclass-broker-managed.yml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: kafka-broker +provisioner: kubernetes.io/azure-disk +reclaimPolicy: Retain +parameters: + kind: "Managed" + storageaccounttype: Premium_LRS
\ No newline at end of file diff --git a/configure/aks-storageclass-zookeeper-managed.yml b/configure/aks-storageclass-zookeeper-managed.yml new file mode 100644 index 0000000..6963dcc --- /dev/null +++ b/configure/aks-storageclass-zookeeper-managed.yml @@ -0,0 +1,9 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: kafka-zookeeper +provisioner: kubernetes.io/azure-disk +reclaimPolicy: Retain +parameters: + kind: "Managed" + storageaccounttype: Premium_LRS
\ No newline at end of file diff --git a/kafka/10broker-config.yml b/kafka/10broker-config.yml index 2846232..1fe67c0 100644 --- a/kafka/10broker-config.yml +++ b/kafka/10broker-config.yml @@ -12,6 +12,7 @@ data: sed -i "s/#init#broker.id=#init#/broker.id=$KAFKA_BROKER_ID/" /etc/kafka/server.properties LABELS="kafka-broker-id=$KAFKA_BROKER_ID" + ANNOTATIONS="" hash kubectl 2>/dev/null || { sed -i "s/#init#broker.rack=#init#/#init#broker.rack=# kubectl not found in path/" /etc/kafka/server.properties @@ -32,12 +33,15 @@ data: else OUTSIDE_PORT=3240${KAFKA_BROKER_ID} sed -i "s|#init#advertised.listeners=OUTSIDE://#init#|advertised.listeners=OUTSIDE://${OUTSIDE_HOST}:${OUTSIDE_PORT}|" /etc/kafka/server.properties - LABELS="$LABELS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" + ANNOTATIONS="$ANNOTATIONS kafka-listener-outside-host=$OUTSIDE_HOST kafka-listener-outside-port=$OUTSIDE_PORT" fi if [ ! -z "$LABELS" ]; then kubectl -n $POD_NAMESPACE label pod $POD_NAME $LABELS || echo "Failed to label $POD_NAMESPACE.$POD_NAME - RBAC issue?" fi + if [ ! -z "$ANNOTATIONS" ]; then + kubectl -n $POD_NAMESPACE annotate pod $POD_NAME $ANNOTATIONS || echo "Failed to annotate $POD_NAMESPACE.$POD_NAME - RBAC issue?" + fi } server.properties: |- diff --git a/kafka/test/kafkacat.yml b/kafka/test/kafkacat.yml index f73735d..98410bc 100644 --- a/kafka/test/kafkacat.yml +++ b/kafka/test/kafkacat.yml @@ -83,8 +83,6 @@ spec: - test-kafkacat - --partitions - "1" - - --replication-factor - - "2" restartPolicy: Never --- apiVersion: apps/v1beta2 diff --git a/kafka/test/produce-consume.yml b/kafka/test/produce-consume.yml index 4b1435b..71228b8 100644 --- a/kafka/test/produce-consume.yml +++ b/kafka/test/produce-consume.yml @@ -66,8 +66,6 @@ spec: - test-produce-consume - --partitions - "1" - - --replication-factor - - "2" restartPolicy: Never --- apiVersion: apps/v1beta2 diff --git a/linkedin-burrow/burrow-config.yml b/linkedin-burrow/burrow-config.yml new file mode 100644 index 0000000..5fe6dda --- /dev/null +++ b/linkedin-burrow/burrow-config.yml @@ -0,0 +1,36 @@ +kind: ConfigMap +metadata: + name: burrow-config + namespace: kafka +apiVersion: v1 +data: + burrow.toml: |- + [zookeeper] + servers=[ "zookeeper:2181" ] + timeout=6 + root-path="/burrow" + + [cluster.local] + class-name="kafka" + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] + topic-refresh=60 + offset-refresh=30 + + [consumer.local] + class-name="kafka" + cluster="local" + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] + group-blacklist="" + group-whitelist="" + + [consumer.local_zk] + class-name="kafka_zk" + cluster="local" + servers=[ "zookeeper:2181" ] + zookeeper-path="/local" + zookeeper-timeout=30 + group-blacklist="" + group-whitelist="" + + [httpserver.default] + address=":8000" diff --git a/linkedin-burrow/burrow-service.yml b/linkedin-burrow/burrow-service.yml new file mode 100644 index 0000000..15eac06 --- /dev/null +++ b/linkedin-burrow/burrow-service.yml @@ -0,0 +1,18 @@ +kind: Service +apiVersion: v1 +metadata: + name: burrow + namespace: kafka +spec: + selector: + app: burrow + ports: + - name: web + protocol: TCP + port: 80 + - name: api + protocol: TCP + port: 8000 + - name: prometheus + protocol: TCP + port: 8080 diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml new file mode 100644 index 0000000..2e8c6b5 --- /dev/null +++ b/linkedin-burrow/burrow.yml @@ -0,0 +1,60 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: burrow + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: burrow + template: + metadata: + labels: + app: burrow + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + containers: + - name: burrow + image: solsson/burrow@sha256:a6fe5d1268a7e7d58f7f960697fc8fda3e6861a2653fe71135a2207c9cf18cf0 + ports: + - name: api + containerPort: 8000 + readinessProbe: + httpGet: + path: /burrow/admin + port: 8000 + livenessProbe: + httpGet: + path: /burrow/admin + port: 8000 + volumeMounts: + - name: config + mountPath: /etc/burrow + - name: prom + image: solsson/burrow-exporter:api-v3@sha256:dd690c04ae31c62e4d7d5398f24f368fb2c48046ec3ae68bbc7582c114819a7b + ports: + - name: prometheus + containerPort: 8080 + env: + - name: BURROW_ADDR + value: http://localhost:8000 + - name: METRICS_ADDR + value: 0.0.0.0:8080 + - name: INTERVAL + value: "30" + - name: dashboard + image: joway/burrow-dashboard@sha256:b4cce87c8264119d73f9a6df5d787ea811ce2138d39ca6cd56525bcfbb5169bf + env: + - name: BURROW_BACKEND + value: http://localhost:8000 + ports: + - name: web + containerPort: 80 + protocol: TCP + volumes: + - name: config + configMap: + name: burrow-config |