diff options
author | solsson <solsson@gmail.com> | 2017-06-27 12:24:01 +0200 |
---|---|---|
committer | GitHub <noreply@github.com> | 2017-06-27 12:24:01 +0200 |
commit | e5ba57eba0131adb3bc7a20f1323489e63aeb715 (patch) | |
tree | cb7ac03f5d9dd3bc1421f5efc6843559a02004df | |
parent | 49865bca8b3fd5fb214f4b9e88d490f9dd09104b (diff) | |
parent | 13442382d2be57fc9e368ceb67f392c5494fdfa9 (diff) | |
download | kubernetes-kafka-e5ba57eba0131adb3bc7a20f1323489e63aeb715.tar.gz kubernetes-kafka-e5ba57eba0131adb3bc7a20f1323489e63aeb715.tar.bz2 kubernetes-kafka-e5ba57eba0131adb3bc7a20f1323489e63aeb715.zip |
Merge pull request #35 from Yolean/resource-limits
Apply resource limits based on tests with little load
-rw-r--r-- | 50kafka.yml | 11 | ||||
-rw-r--r-- | README.md | 14 | ||||
-rw-r--r-- | test/monitoring-test.yml | 35 | ||||
-rw-r--r-- | zookeeper/50pzoo.yml | 17 | ||||
-rw-r--r-- | zookeeper/51zoo.yml | 17 |
5 files changed, 74 insertions, 20 deletions
diff --git a/50kafka.yml b/50kafka.yml index 372eba8..2a3faf6 100644 --- a/50kafka.yml +++ b/50kafka.yml @@ -20,6 +20,13 @@ spec: image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 80Mi - name: broker image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -36,6 +43,10 @@ spec: --override log.retention.hours=-1 --override log.dirs=/var/lib/kafka/data/topics --override broker.id=${HOSTNAME##*-} + resources: + requests: + cpu: 100m + memory: 512Mi volumeMounts: - name: data mountPath: /var/lib/kafka/data @@ -8,20 +8,6 @@ To get consistent service DNS names `kafka-N.broker.kafka`(`.svc.cluster.local`) kubectl create -f 00namespace.yml ``` -## Set up volume claims - -You may add [storage class](http://kubernetes.io/docs/user-guide/persistent-volumes/#storageclasses) -to the kafka StatefulSet declaration to enable automatic volume provisioning. - -Alternatively create [PV](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistent-volumes)s and [PVC](http://kubernetes.io/docs/user-guide/persistent-volumes/#persistentvolumeclaims)s manually. For example in Minikube. - -``` -./bootstrap/pv.sh -kubectl create -f ./10pvc.yml -# check that claims are bound -kubectl -n kafka get pvc -``` - ## Set up Zookeeper The Kafka book (Definitive Guide, O'Reilly 2016) recommends that Kafka has its own Zookeeper cluster with at least 5 instances. diff --git a/test/monitoring-test.yml b/test/monitoring-test.yml new file mode 100644 index 0000000..4b5f230 --- /dev/null +++ b/test/monitoring-test.yml @@ -0,0 +1,35 @@ +# Sets up a pod that monitors itself, to test resource usage etc. +# kubectl exec test-metrics-... -- /bin/sh -c 'apk add --no-cache curl && curl http://localhost:5556/metrics' +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: monitoring-test + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: monitoring-test + # Uncomment to test with prometheus + #annotations: + # prometheus.io/scrape: "true" + # prometheus.io/port: "5556" + spec: + containers: + - name: monitor + image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d + command: + - java + - -Dcom.sun.management.jmxremote.ssl=false + - -Dcom.sun.management.jmxremote.authenticate=false + - -Dcom.sun.management.jmxremote.port=5555 + - -jar + - jmx_prometheus_httpserver.jar + - "5556" + - example_configs/httpserver_sample_config.yml + ports: + - name: jmx + containerPort: 5555 + - name: slashmetrics + containerPort: 5556 diff --git a/zookeeper/50pzoo.yml b/zookeeper/50pzoo.yml index 25e2ceb..da1ba81 100644 --- a/zookeeper/50pzoo.yml +++ b/zookeeper/50pzoo.yml @@ -20,13 +20,20 @@ spec: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d command: - - "java" - - "-jar" - - "jmx_prometheus_httpserver.jar" + - java + - -jar + - jmx_prometheus_httpserver.jar - "5556" - example_configs/zookeeper.yaml ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 80Mi - name: zookeeper image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -49,6 +56,10 @@ spec: name: peer - containerPort: 3888 name: leader-election + resources: + requests: + cpu: 10m + memory: 100Mi volumeMounts: - name: config mountPath: /usr/local/kafka/config diff --git a/zookeeper/51zoo.yml b/zookeeper/51zoo.yml index f041a9a..f4f6298 100644 --- a/zookeeper/51zoo.yml +++ b/zookeeper/51zoo.yml @@ -20,13 +20,20 @@ spec: - name: metrics image: solsson/kafka-prometheus-jmx-exporter@sha256:1f7c96c287a2dbec1d909cd8f96c0656310239b55a9a90d7fd12c81f384f1f7d command: - - "java" - - "-jar" - - "jmx_prometheus_httpserver.jar" + - java + - -jar + - jmx_prometheus_httpserver.jar - "5556" - example_configs/zookeeper.yaml ports: - containerPort: 5556 + resources: + requests: + cpu: 0m + memory: 40Mi + limits: + cpu: 10m + memory: 80Mi - name: zookeeper image: solsson/kafka:0.11.0.0-rc2@sha256:c1316e0131f4ec83bc645ca2141e4fda94e0d28f4fb5f836e15e37a5e054bdf1 env: @@ -49,6 +56,10 @@ spec: name: peer - containerPort: 3888 name: leader-election + resources: + requests: + cpu: 10m + memory: 100Mi volumeMounts: - name: config mountPath: /usr/local/kafka/config |