From a45297dbfc21fc6459dd9af5503806e5356bf062 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:15:42 +0100 Subject: Adds linkedin/Burrow for use with port-forward 8000 --- linkedin-burrow/10-burrow-config.yml | 36 ++++++++++++++++++++++++++++++++++++ linkedin-burrow/50-burrow.yml | 27 +++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 linkedin-burrow/10-burrow-config.yml create mode 100644 linkedin-burrow/50-burrow.yml diff --git a/linkedin-burrow/10-burrow-config.yml b/linkedin-burrow/10-burrow-config.yml new file mode 100644 index 0000000..62ed62e --- /dev/null +++ b/linkedin-burrow/10-burrow-config.yml @@ -0,0 +1,36 @@ +kind: ConfigMap +metadata: + name: burrow-config + namespace: kafka +apiVersion: v1 +data: + burrow.toml: |- + [zookeeper] + servers=[ "zookeeper:2181" ] + timeout=6 + root-path="/burrow" + + [cluster.local] + class-name="kafka" + servers=[ "kafka:9092" ] + topic-refresh=60 + offset-refresh=30 + + [consumer.local] + class-name="kafka" + cluster="local" + servers=[ "kafka:9092" ] + group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-whitelist="" + + [consumer.local_zk] + class-name="kafka_zk" + cluster="local" + servers=[ "zookeeper:2181" ] + zookeeper-path="/local" + zookeeper-timeout=30 + group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-whitelist="" + + [httpserver.default] + address=":8000" diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml new file mode 100644 index 0000000..c7d88d2 --- /dev/null +++ b/linkedin-burrow/50-burrow.yml @@ -0,0 +1,27 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: burrow + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: burrow + template: + metadata: + labels: + app: burrow + spec: + containers: + - name: kafka-manager + image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef + ports: + - containerPort: 8000 + volumeMounts: + - name: config + mountPath: /etc/burrow + volumes: + - name: config + configMap: + name: burrow-config -- cgit v1.2.3 From 43c277ddb0b2f038a4facd407aadc2553724eb1d Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:16:34 +0100 Subject: Lists actual brokers --- linkedin-burrow/10-burrow-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkedin-burrow/10-burrow-config.yml b/linkedin-burrow/10-burrow-config.yml index 62ed62e..53db4e6 100644 --- a/linkedin-burrow/10-burrow-config.yml +++ b/linkedin-burrow/10-burrow-config.yml @@ -12,14 +12,14 @@ data: [cluster.local] class-name="kafka" - servers=[ "kafka:9092" ] + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] topic-refresh=60 offset-refresh=30 [consumer.local] class-name="kafka" cluster="local" - servers=[ "kafka:9092" ] + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" group-whitelist="" -- cgit v1.2.3 From f84bd724a1a23d995a34850ae34c887ec7dc34bf Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:17:15 +0100 Subject: fixes container name --- linkedin-burrow/50-burrow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml index c7d88d2..57ff62a 100644 --- a/linkedin-burrow/50-burrow.yml +++ b/linkedin-burrow/50-burrow.yml @@ -14,7 +14,7 @@ spec: app: burrow spec: containers: - - name: kafka-manager + - name: burrow image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef ports: - containerPort: 8000 -- cgit v1.2.3 From 545193d425b809b1cec64670d0366ac0ca301542 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:25:52 +0100 Subject: Adds prometheus exporter, but sadly incompatible with current Burrow https://github.com/jirwin/burrow_exporter/issues/8 --- linkedin-burrow/50-burrow.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml index 57ff62a..aa7ce6c 100644 --- a/linkedin-burrow/50-burrow.yml +++ b/linkedin-burrow/50-burrow.yml @@ -12,6 +12,9 @@ spec: metadata: labels: app: burrow + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" spec: containers: - name: burrow @@ -21,6 +24,17 @@ spec: volumeMounts: - name: config mountPath: /etc/burrow + - name: prom + image: solsson/burrow-exporter@sha256:2358412bc4757b7bd459452a9d759a923fc5ba93b26f5fb5b3d1ed8a0189417c + ports: + - containerPort: 8080 + env: + - name: BURROW_ADDR + value: http://localhost:8000 + - name: METRICS_ADDR + value: 0.0.0.0:8080 + - name: INTERVAL + value: "30" volumes: - name: config configMap: -- cgit v1.2.3 From 8b21daf46aff1b4d54dc25e49584194c9593c228 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 17 Jan 2018 21:46:38 +0100 Subject: The build from search-and-replace in github.com/jirwin/burrow_exporter/pull/9 --- linkedin-burrow/50-burrow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml index aa7ce6c..7af6922 100644 --- a/linkedin-burrow/50-burrow.yml +++ b/linkedin-burrow/50-burrow.yml @@ -25,7 +25,7 @@ spec: - name: config mountPath: /etc/burrow - name: prom - image: solsson/burrow-exporter@sha256:2358412bc4757b7bd459452a9d759a923fc5ba93b26f5fb5b3d1ed8a0189417c + image: solsson/burrow-exporter:api-v3@sha256:4407616c5720e8a3397f668eec1aaa31df751b3614c0adfc56cae231f2eaeee2 ports: - containerPort: 8080 env: -- cgit v1.2.3 From b3950835b49d2444b1c390fb0d9033c24b52693a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 19 Jan 2018 06:30:14 +0100 Subject: Adds a service. Ordering is ok with plain file names here. --- linkedin-burrow/10-burrow-config.yml | 36 ------------------------------- linkedin-burrow/50-burrow.yml | 41 ------------------------------------ linkedin-burrow/burrow-config.yml | 36 +++++++++++++++++++++++++++++++ linkedin-burrow/burrow-service.yml | 12 +++++++++++ linkedin-burrow/burrow.yml | 41 ++++++++++++++++++++++++++++++++++++ 5 files changed, 89 insertions(+), 77 deletions(-) delete mode 100644 linkedin-burrow/10-burrow-config.yml delete mode 100644 linkedin-burrow/50-burrow.yml create mode 100644 linkedin-burrow/burrow-config.yml create mode 100644 linkedin-burrow/burrow-service.yml create mode 100644 linkedin-burrow/burrow.yml diff --git a/linkedin-burrow/10-burrow-config.yml b/linkedin-burrow/10-burrow-config.yml deleted file mode 100644 index 53db4e6..0000000 --- a/linkedin-burrow/10-burrow-config.yml +++ /dev/null @@ -1,36 +0,0 @@ -kind: ConfigMap -metadata: - name: burrow-config - namespace: kafka -apiVersion: v1 -data: - burrow.toml: |- - [zookeeper] - servers=[ "zookeeper:2181" ] - timeout=6 - root-path="/burrow" - - [cluster.local] - class-name="kafka" - servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] - topic-refresh=60 - offset-refresh=30 - - [consumer.local] - class-name="kafka" - cluster="local" - servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] - group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" - group-whitelist="" - - [consumer.local_zk] - class-name="kafka_zk" - cluster="local" - servers=[ "zookeeper:2181" ] - zookeeper-path="/local" - zookeeper-timeout=30 - group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" - group-whitelist="" - - [httpserver.default] - address=":8000" diff --git a/linkedin-burrow/50-burrow.yml b/linkedin-burrow/50-burrow.yml deleted file mode 100644 index 7af6922..0000000 --- a/linkedin-burrow/50-burrow.yml +++ /dev/null @@ -1,41 +0,0 @@ -apiVersion: apps/v1beta2 -kind: Deployment -metadata: - name: burrow - namespace: kafka -spec: - replicas: 1 - selector: - matchLabels: - app: burrow - template: - metadata: - labels: - app: burrow - annotations: - prometheus.io/scrape: "true" - prometheus.io/port: "8080" - spec: - containers: - - name: burrow - image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef - ports: - - containerPort: 8000 - volumeMounts: - - name: config - mountPath: /etc/burrow - - name: prom - image: solsson/burrow-exporter:api-v3@sha256:4407616c5720e8a3397f668eec1aaa31df751b3614c0adfc56cae231f2eaeee2 - ports: - - containerPort: 8080 - env: - - name: BURROW_ADDR - value: http://localhost:8000 - - name: METRICS_ADDR - value: 0.0.0.0:8080 - - name: INTERVAL - value: "30" - volumes: - - name: config - configMap: - name: burrow-config diff --git a/linkedin-burrow/burrow-config.yml b/linkedin-burrow/burrow-config.yml new file mode 100644 index 0000000..53db4e6 --- /dev/null +++ b/linkedin-burrow/burrow-config.yml @@ -0,0 +1,36 @@ +kind: ConfigMap +metadata: + name: burrow-config + namespace: kafka +apiVersion: v1 +data: + burrow.toml: |- + [zookeeper] + servers=[ "zookeeper:2181" ] + timeout=6 + root-path="/burrow" + + [cluster.local] + class-name="kafka" + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] + topic-refresh=60 + offset-refresh=30 + + [consumer.local] + class-name="kafka" + cluster="local" + servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] + group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-whitelist="" + + [consumer.local_zk] + class-name="kafka_zk" + cluster="local" + servers=[ "zookeeper:2181" ] + zookeeper-path="/local" + zookeeper-timeout=30 + group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-whitelist="" + + [httpserver.default] + address=":8000" diff --git a/linkedin-burrow/burrow-service.yml b/linkedin-burrow/burrow-service.yml new file mode 100644 index 0000000..f9fc7b4 --- /dev/null +++ b/linkedin-burrow/burrow-service.yml @@ -0,0 +1,12 @@ +kind: Service +apiVersion: v1 +metadata: + name: burrow + namespace: kafka +spec: + selector: + app: burrow + ports: + - protocol: TCP + port: 80 + targetPort: 8000 diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml new file mode 100644 index 0000000..7af6922 --- /dev/null +++ b/linkedin-burrow/burrow.yml @@ -0,0 +1,41 @@ +apiVersion: apps/v1beta2 +kind: Deployment +metadata: + name: burrow + namespace: kafka +spec: + replicas: 1 + selector: + matchLabels: + app: burrow + template: + metadata: + labels: + app: burrow + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + containers: + - name: burrow + image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef + ports: + - containerPort: 8000 + volumeMounts: + - name: config + mountPath: /etc/burrow + - name: prom + image: solsson/burrow-exporter:api-v3@sha256:4407616c5720e8a3397f668eec1aaa31df751b3614c0adfc56cae231f2eaeee2 + ports: + - containerPort: 8080 + env: + - name: BURROW_ADDR + value: http://localhost:8000 + - name: METRICS_ADDR + value: 0.0.0.0:8080 + - name: INTERVAL + value: "30" + volumes: + - name: config + configMap: + name: burrow-config -- cgit v1.2.3 From 18322c3d72f6e31042f2d3d0765111a15d04fc28 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 19 Jan 2018 07:38:35 +0100 Subject: Lets us see all consumers, for now --- linkedin-burrow/burrow-config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/linkedin-burrow/burrow-config.yml b/linkedin-burrow/burrow-config.yml index 53db4e6..5fe6dda 100644 --- a/linkedin-burrow/burrow-config.yml +++ b/linkedin-burrow/burrow-config.yml @@ -20,7 +20,7 @@ data: class-name="kafka" cluster="local" servers=[ "kafka-0.broker:9092", "kafka-1.broker:9092", "kafka-2.broker:9092" ] - group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-blacklist="" group-whitelist="" [consumer.local_zk] @@ -29,7 +29,7 @@ data: servers=[ "zookeeper:2181" ] zookeeper-path="/local" zookeeper-timeout=30 - group-blacklist="^(console-consumer-|python-kafka-consumer-).*$" + group-blacklist="" group-whitelist="" [httpserver.default] -- cgit v1.2.3 From 181432c4fbd2281887398067661dda70b23a143a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 19 Jan 2018 11:50:49 +0100 Subject: Adds a frontend container. No need to scale it indivudually. --- linkedin-burrow/burrow-service.yml | 10 ++++++++-- linkedin-burrow/burrow.yml | 23 +++++++++++++++++++++-- 2 files changed, 29 insertions(+), 4 deletions(-) diff --git a/linkedin-burrow/burrow-service.yml b/linkedin-burrow/burrow-service.yml index f9fc7b4..15eac06 100644 --- a/linkedin-burrow/burrow-service.yml +++ b/linkedin-burrow/burrow-service.yml @@ -7,6 +7,12 @@ spec: selector: app: burrow ports: - - protocol: TCP + - name: web + protocol: TCP port: 80 - targetPort: 8000 + - name: api + protocol: TCP + port: 8000 + - name: prometheus + protocol: TCP + port: 8080 diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml index 7af6922..bb43dd1 100644 --- a/linkedin-burrow/burrow.yml +++ b/linkedin-burrow/burrow.yml @@ -20,14 +20,24 @@ spec: - name: burrow image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef ports: - - containerPort: 8000 + - name: api + containerPort: 8000 + readinessProbe: + httpGet: + path: /burrow/admin + port: 8000 + livenessProbe: + httpGet: + path: /burrow/admin + port: 8000 volumeMounts: - name: config mountPath: /etc/burrow - name: prom image: solsson/burrow-exporter:api-v3@sha256:4407616c5720e8a3397f668eec1aaa31df751b3614c0adfc56cae231f2eaeee2 ports: - - containerPort: 8080 + - name: prometheus + containerPort: 8080 env: - name: BURROW_ADDR value: http://localhost:8000 @@ -35,6 +45,15 @@ spec: value: 0.0.0.0:8080 - name: INTERVAL value: "30" + - name: dashboard + image: joway/burrow-dashboard@sha256:b4cce87c8264119d73f9a6df5d787ea811ce2138d39ca6cd56525bcfbb5169bf + env: + - name: BURROW_BACKEND + value: http://localhost:8000 + ports: + - name: web + containerPort: 80 + protocol: TCP volumes: - name: config configMap: -- cgit v1.2.3 From 1dbe418d5561be9ae8f4cb2a3edd77fa50b0ea43 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Fri, 19 Jan 2018 12:17:58 +0100 Subject: Applies the fix for v3 API so now we export five metrics https://github.com/jirwin/burrow_exporter/pull/9 --- linkedin-burrow/burrow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml index bb43dd1..bdaebb6 100644 --- a/linkedin-burrow/burrow.yml +++ b/linkedin-burrow/burrow.yml @@ -34,7 +34,7 @@ spec: - name: config mountPath: /etc/burrow - name: prom - image: solsson/burrow-exporter:api-v3@sha256:4407616c5720e8a3397f668eec1aaa31df751b3614c0adfc56cae231f2eaeee2 + image: solsson/burrow-exporter:api-v3@sha256:dd690c04ae31c62e4d7d5398f24f368fb2c48046ec3ae68bbc7582c114819a7b ports: - name: prometheus containerPort: 8080 -- cgit v1.2.3 From 29eca40589a083fc06d77dca285ad13eb4f15423 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Wed, 31 Jan 2018 16:16:05 +0100 Subject: Upgrades to current Burrow master, a91cf4d --- linkedin-burrow/burrow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/linkedin-burrow/burrow.yml b/linkedin-burrow/burrow.yml index bdaebb6..2e8c6b5 100644 --- a/linkedin-burrow/burrow.yml +++ b/linkedin-burrow/burrow.yml @@ -18,7 +18,7 @@ spec: spec: containers: - name: burrow - image: solsson/burrow@sha256:1f17cb48ba4062b2f796fdf250dce6b6a9d5127b42e15ef2fbc79c68c34ad0ef + image: solsson/burrow@sha256:a6fe5d1268a7e7d58f7f960697fc8fda3e6861a2653fe71135a2207c9cf18cf0 ports: - name: api containerPort: 8000 -- cgit v1.2.3