From 3a12d2ef3912e11131b8a7f36beef58f405e0e4f Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Jul 2017 07:25:59 +0200 Subject: Adds schema registry --- 60schema-registry.yml | 22 ++++++++++++++++++++++ 61schema-registry-service.yml | 10 ++++++++++ 2 files changed, 32 insertions(+) create mode 100644 60schema-registry.yml create mode 100644 61schema-registry-service.yml diff --git a/60schema-registry.yml b/60schema-registry.yml new file mode 100644 index 0000000..e02fe8b --- /dev/null +++ b/60schema-registry.yml @@ -0,0 +1,22 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: schema-registry + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: schema-registry + spec: + containers: + - name: schema-registry + image: confluentinc/cp-schema-registry@sha256:ac1eb34d9a60ce8904eb1bc01fd94bf1f6513924ca507734679d4b513133714c + env: + - name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL + value: zookeeper:2181 + - name: SCHEMA_REGISTRY_HOST_NAME + value: schema-registry + ports: + - containerPort: 8081 diff --git a/61schema-registry-service.yml b/61schema-registry-service.yml new file mode 100644 index 0000000..652b224 --- /dev/null +++ b/61schema-registry-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: schema-registry + namespace: kafka +spec: + ports: + - port: 8081 + selector: + app: schema-registry -- cgit v1.2.3 From c14d4cce6c391443ba0dd4345d06426c510fd233 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Jul 2017 07:43:33 +0200 Subject: Avoids the schema-registry service name, becuase it interferes ... with container configuration, as Kubernetes creates the env SCHEMA_REGISTRY_PORT if the service has been created first. --- 60schema-registry.yml | 22 ---------------------- 60schemas-service.yml | 10 ++++++++++ 61schema-registry-service.yml | 10 ---------- 61schemas.yml | 22 ++++++++++++++++++++++ 4 files changed, 32 insertions(+), 32 deletions(-) delete mode 100644 60schema-registry.yml create mode 100644 60schemas-service.yml delete mode 100644 61schema-registry-service.yml create mode 100644 61schemas.yml diff --git a/60schema-registry.yml b/60schema-registry.yml deleted file mode 100644 index e02fe8b..0000000 --- a/60schema-registry.yml +++ /dev/null @@ -1,22 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: schema-registry - namespace: kafka -spec: - replicas: 1 - template: - metadata: - labels: - app: schema-registry - spec: - containers: - - name: schema-registry - image: confluentinc/cp-schema-registry@sha256:ac1eb34d9a60ce8904eb1bc01fd94bf1f6513924ca507734679d4b513133714c - env: - - name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL - value: zookeeper:2181 - - name: SCHEMA_REGISTRY_HOST_NAME - value: schema-registry - ports: - - containerPort: 8081 diff --git a/60schemas-service.yml b/60schemas-service.yml new file mode 100644 index 0000000..2d1894c --- /dev/null +++ b/60schemas-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: schemas + namespace: kafka +spec: + ports: + - port: 8081 + selector: + app: schema-registry diff --git a/61schema-registry-service.yml b/61schema-registry-service.yml deleted file mode 100644 index 652b224..0000000 --- a/61schema-registry-service.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: schema-registry - namespace: kafka -spec: - ports: - - port: 8081 - selector: - app: schema-registry diff --git a/61schemas.yml b/61schemas.yml new file mode 100644 index 0000000..2423e91 --- /dev/null +++ b/61schemas.yml @@ -0,0 +1,22 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: schemas + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: schema-registry + spec: + containers: + - name: cp-schema-registry + image: confluentinc/cp-schema-registry@sha256:ac1eb34d9a60ce8904eb1bc01fd94bf1f6513924ca507734679d4b513133714c + env: + - name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL + value: zookeeper:2181 + - name: SCHEMA_REGISTRY_HOST_NAME + value: schemas + ports: + - containerPort: 8081 -- cgit v1.2.3 From 55f5b317df9cdf85d1f8027b00650e51694a372c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Jul 2017 07:48:16 +0200 Subject: This is http so there's no reason to not use port 80 --- 60schemas-service.yml | 2 +- 61schemas.yml | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/60schemas-service.yml b/60schemas-service.yml index 2d1894c..2c30905 100644 --- a/60schemas-service.yml +++ b/60schemas-service.yml @@ -5,6 +5,6 @@ metadata: namespace: kafka spec: ports: - - port: 8081 + - port: 80 selector: app: schema-registry diff --git a/61schemas.yml b/61schemas.yml index 2423e91..20d1868 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -18,5 +18,7 @@ spec: value: zookeeper:2181 - name: SCHEMA_REGISTRY_HOST_NAME value: schemas + - name: SCHEMA_REGISTRY_LISTENERS + value: http://0.0.0.0:80 ports: - - containerPort: 8081 + - containerPort: 80 -- cgit v1.2.3 From dfc65f1ce19fb93d6a5af4ce52a3cc35acd35e25 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 25 Jul 2017 07:51:14 +0200 Subject: Adds kafka-rest proxy as rest:80 --- 70rest-service.yml | 10 ++++++++++ 71rest.yml | 26 ++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 70rest-service.yml create mode 100644 71rest.yml diff --git a/70rest-service.yml b/70rest-service.yml new file mode 100644 index 0000000..a3ffda6 --- /dev/null +++ b/70rest-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: rest + namespace: kafka +spec: + ports: + - port: 80 + selector: + app: kafka-rest diff --git a/71rest.yml b/71rest.yml new file mode 100644 index 0000000..8f52b42 --- /dev/null +++ b/71rest.yml @@ -0,0 +1,26 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rest + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: kafka-rest + spec: + containers: + - name: cp-kafka-rest + image: confluentinc/cp-kafka-rest@sha256:aa213c1a67eae6ce9836b52a9b5ecee4d6a0b44f2b9cc69f4e4de85131462f1d + env: + - name: KAFKA_REST_ZOOKEEPER_CONNECT + value: zookeeper:2181 + - name: KAFKA_REST_HOST_NAME + value: rest + - name: KAFKA_REST_LISTENERS + value: http://0.0.0.0:80 + - name: KAFKA_REST_SCHEMA_REGISTRY_URL + value: http://schemas.kafka.svc.cluster.local:80 + ports: + - containerPort: 80 -- cgit v1.2.3 From d340d41cc7eda1c44dd069257fa7eabfbc01ec03 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 27 Jul 2017 05:52:14 +0200 Subject: Tests rest-proxy, but hangs on GET /consumers/.../records --- test/70rest-test1.yml | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 test/70rest-test1.yml diff --git a/test/70rest-test1.yml b/test/70rest-test1.yml new file mode 100644 index 0000000..e52da30 --- /dev/null +++ b/test/70rest-test1.yml @@ -0,0 +1,39 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: rest-test1 + namespace: kafka +spec: + template: + metadata: + name: rest-test1 + spec: + containers: + - name: curl + image: solsson/curl@sha256:8b0927b81d10043e70f3e05e33e36fb9b3b0cbfcbccdb9f04fd53f67a270b874 + env: + - name: REST + value: http://rest.kafka.svc.cluster.local + - name: TOPIC + value: test1 + command: + - /bin/sh + - -c + - > + curl -I $REST; + curl -X POST $REST/consumers/list; + + # http://docs.confluent.io/current/kafka-rest/docs/intro.html + + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v; + + curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v; + + curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v; + + sleep 300 + restartPolicy: Never -- cgit v1.2.3 From 8b1ddad86952bbe70702fb265487c70cbafbf6bd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Thu, 27 Jul 2017 06:14:41 +0200 Subject: Adds more tests, and retries --- test/70rest-test1.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/70rest-test1.yml b/test/70rest-test1.yml index e52da30..ce7b019 100644 --- a/test/70rest-test1.yml +++ b/test/70rest-test1.yml @@ -20,10 +20,13 @@ spec: - /bin/sh - -c - > - curl -I $REST; - curl -X POST $REST/consumers/list; + curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST; - # http://docs.confluent.io/current/kafka-rest/docs/intro.html + curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics; + + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test1; + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date)\"}]}" $REST/topics/$TOPIC -v; + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test2; curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; -- cgit v1.2.3 From 4fc6880c4ef53690a16a63ac1157d0f073ddbb4c Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Jul 2017 15:54:18 +0200 Subject: Starts a test case for rest-proxy --- test/rest-curl.yml | 117 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 test/rest-curl.yml diff --git a/test/rest-curl.yml b/test/rest-curl.yml new file mode 100644 index 0000000..db5758a --- /dev/null +++ b/test/rest-curl.yml @@ -0,0 +1,117 @@ +--- +kind: ConfigMap +metadata: + name: rest-curl + namespace: test-kafka +apiVersion: v1 +data: + + setup.sh: |- + touch /tmp/testlog + + # Don't complete startup until rest proxy is up and running + curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST; + + + curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics; + + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test1; + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date)\"}]}" $REST/topics/$TOPIC -v; + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test2; + + curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v; + + curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v; + + curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v; + + tail -f /tmp/testlog + + continue.sh: |- + exit 0 + + run.sh: |- + exec >> /tmp/testlog + exec 2>&1 + + exit 0 + +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: rest-curl + namespace: test-kafka +spec: + template: + spec: + containers: + - name: topic-create + image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce + command: + - ./bin/kafka-topics.sh + - --zookeeper + - zookeeper.kafka.svc.cluster.local:2181 + - --create + - --if-not-exists + - --topic + - test-rest-curl + - --partitions + - "1" + - --replication-factor + - "1" + restartPolicy: Never +--- +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rest-curl + namespace: test-kafka +spec: + replicas: 1 + template: + metadata: + labels: + test-target: kafka + test-type: readiness + spec: + containers: + - name: testcase + image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf + env: + - name: REST + value: http://rest.kafka.svc.cluster.local + - name: TOPIC + value: test-rest-curl + # Test set up + command: + - /bin/bash + - -e + - /test/setup.sh + # Test run, again and again + readinessProbe: + exec: + command: + - /bin/bash + - -e + - /test/run.sh + # We haven't worked on timing + periodSeconds: 60 + # Test quit on nonzero exit + livenessProbe: + exec: + command: + - /bin/bash + - -e + - /test/continue.sh + volumeMounts: + - name: config + mountPath: /test + volumes: + - name: config + configMap: + name: rest-curl -- cgit v1.2.3 From 713743ec4469dd40b63344fe7b3d81f6dce5a643 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Sat, 29 Jul 2017 16:51:42 +0200 Subject: We should probably wait for CP 3.3.0 because ... I'm not getting the expected behavior. Might also be because of low resources on minikube, but 3.3.0 should only be a few days away. --- test/rest-curl.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/test/rest-curl.yml b/test/rest-curl.yml index db5758a..05fc178 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -9,17 +9,26 @@ data: setup.sh: |- touch /tmp/testlog - # Don't complete startup until rest proxy is up and running - curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST; + # Keep starting up until rest proxy is up and running + curl --retry 10 --retry-delay 30 --retry-connrefused -I -s $REST - curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics; + curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics + echo "" - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test1; - curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date)\"}]}" $REST/topics/$TOPIC -v; - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test2; + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC + echo "" - curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v; + curl -X POST \ + -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ + --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ + $REST/topics/$TOPIC -v --max-time 30 \ + || echo " (timeout might be ok because we only want to send one message)" + # TODO why does the above block? + + + curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions + echo "" curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; -- cgit v1.2.3 From f5876226d96bc355370f76fbbc8437eae1f213e2 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Jul 2017 20:46:25 +0200 Subject: Sample config from v3.3.0 source --- 11confluent-config.yml | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 11confluent-config.yml diff --git a/11confluent-config.yml b/11confluent-config.yml new file mode 100644 index 0000000..8b4087c --- /dev/null +++ b/11confluent-config.yml @@ -0,0 +1,72 @@ +kind: ConfigMap +metadata: + name: confluent-config + namespace: kafka +apiVersion: v1 +data: + schema-registry.properties: |- + # Copyright 2014 Confluent Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + listeners=http://0.0.0.0:8081 + kafkastore.connection.url=localhost:2181 + kafkastore.topic=_schemas + debug=false + + kafka-rest.properties: |- + ## + # Copyright 2015 Confluent Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + ## + + #id=kafka-rest-test-server + #schema.registry.url=http://localhost:8081 + #zookeeper.connect=localhost:2181 + # + # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center + # Make sure that monitoring-interceptors-.jar is on the Java class path + #consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor + #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor + + log4j.properties: |- + log4j.rootLogger=INFO, stdout, file + + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + + log4j.logger.kafka=ERROR, stdout + log4j.logger.org.apache.zookeeper=ERROR, stdout + log4j.logger.org.apache.kafka=ERROR, stdout + log4j.logger.org.I0Itec.zkclient=ERROR, stdout + log4j.additivity.kafka.server=false + log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false + + log4j.appender.file=org.apache.log4j.RollingFileAppender + log4j.appender.file.maxBackupIndex=10 + log4j.appender.file.maxFileSize=100MB + log4j.appender.file.File=${schema-registry.log.dir}/schema-registry.log + log4j.appender.file.layout=org.apache.log4j.PatternLayout + log4j.appender.file.layout.ConversionPattern=[%d] %p %m (%c)%n -- cgit v1.2.3 From 2d4401bc1fd90cc9af82599517d4a1df7a103ebb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Jul 2017 20:47:33 +0200 Subject: Remove file appender as docker implies stdout also the sample is from schema-registry.properties while kafka-rest had only the stdout appender (and no per-package levels) --- 11confluent-config.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index 8b4087c..62d8993 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -51,7 +51,7 @@ data: #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor log4j.properties: |- - log4j.rootLogger=INFO, stdout, file + log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout @@ -63,10 +63,3 @@ data: log4j.logger.org.I0Itec.zkclient=ERROR, stdout log4j.additivity.kafka.server=false log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false - - log4j.appender.file=org.apache.log4j.RollingFileAppender - log4j.appender.file.maxBackupIndex=10 - log4j.appender.file.maxFileSize=100MB - log4j.appender.file.File=${schema-registry.log.dir}/schema-registry.log - log4j.appender.file.layout=org.apache.log4j.PatternLayout - log4j.appender.file.layout.ConversionPattern=[%d] %p %m (%c)%n -- cgit v1.2.3 From 47910682edfadffccbf77b3f0b2722376dc37350 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 07:35:06 +0200 Subject: New image uses config files instead of env --- 11confluent-config.yml | 9 +++++---- 61schemas.yml | 19 +++++++++++++------ 71rest.yml | 18 ++++++++++-------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index 62d8993..df91677 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -19,8 +19,8 @@ data: # See the License for the specific language governing permissions and # limitations under the License. - listeners=http://0.0.0.0:8081 - kafkastore.connection.url=localhost:2181 + listeners=http://0.0.0.0:80 + kafkastore.connection.url=zookeeper:2181 kafkastore.topic=_schemas debug=false @@ -42,8 +42,9 @@ data: ## #id=kafka-rest-test-server - #schema.registry.url=http://localhost:8081 - #zookeeper.connect=localhost:2181 + listeners=http://0.0.0.0:80 + schema.registry.url=http://schemas.kafka.svc.cluster.local:80 + zookeeper.connect=zookeeper:2181 # # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center # Make sure that monitoring-interceptors-.jar is on the Java class path diff --git a/61schemas.yml b/61schemas.yml index 20d1868..8d098dd 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -11,14 +11,21 @@ spec: app: schema-registry spec: containers: - - name: cp-schema-registry - image: confluentinc/cp-schema-registry@sha256:ac1eb34d9a60ce8904eb1bc01fd94bf1f6513924ca507734679d4b513133714c + - name: cp + image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 env: - - name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL - value: zookeeper:2181 + # what does this do? - name: SCHEMA_REGISTRY_HOST_NAME value: schemas - - name: SCHEMA_REGISTRY_LISTENERS - value: http://0.0.0.0:80 + command: + - schema-registry-start + - /etc/schema-registry/schema-registry.properties ports: - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/schema-registry + volumes: + - name: config + configMap: + name: confluent-config diff --git a/71rest.yml b/71rest.yml index 8f52b42..987e013 100644 --- a/71rest.yml +++ b/71rest.yml @@ -11,16 +11,18 @@ spec: app: kafka-rest spec: containers: - - name: cp-kafka-rest - image: confluentinc/cp-kafka-rest@sha256:aa213c1a67eae6ce9836b52a9b5ecee4d6a0b44f2b9cc69f4e4de85131462f1d + - name: cp + image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 env: - - name: KAFKA_REST_ZOOKEEPER_CONNECT - value: zookeeper:2181 + # what does this do? - name: KAFKA_REST_HOST_NAME value: rest - - name: KAFKA_REST_LISTENERS - value: http://0.0.0.0:80 - - name: KAFKA_REST_SCHEMA_REGISTRY_URL - value: http://schemas.kafka.svc.cluster.local:80 ports: - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/kafka-rest + volumes: + - name: config + configMap: + name: confluent-config -- cgit v1.2.3 From 86183d83eb715c97e8647cd16cd963ead542f512 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 10:17:39 +0200 Subject: Runtime conf is logged at start, which documents host.name in case we need it --- 61schemas.yml | 6 +----- 71rest.yml | 9 ++++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/61schemas.yml b/61schemas.yml index 8d098dd..4fe7097 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -12,11 +12,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 - env: - # what does this do? - - name: SCHEMA_REGISTRY_HOST_NAME - value: schemas + image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf command: - schema-registry-start - /etc/schema-registry/schema-registry.properties diff --git a/71rest.yml b/71rest.yml index 987e013..e237357 100644 --- a/71rest.yml +++ b/71rest.yml @@ -12,11 +12,10 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 - env: - # what does this do? - - name: KAFKA_REST_HOST_NAME - value: rest + image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + command: + - kafka-rest-start + - /etc/kafka-rest/kafka-rest.properties ports: - containerPort: 80 volumeMounts: -- cgit v1.2.3 From eb1a1020055ecca572d0dac9a9d5bfb853b3d722 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 12:16:53 +0200 Subject: Once again be explicit (=unsurprising) about log config path --- 61schemas.yml | 3 +++ 71rest.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/61schemas.yml b/61schemas.yml index 4fe7097..f85d74c 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -13,6 +13,9 @@ spec: containers: - name: cp image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + env: + - name: SCHEMA_REGISTRY_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties command: - schema-registry-start - /etc/schema-registry/schema-registry.properties diff --git a/71rest.yml b/71rest.yml index e237357..f03f5b5 100644 --- a/71rest.yml +++ b/71rest.yml @@ -13,6 +13,9 @@ spec: containers: - name: cp image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + env: + - name: KAFKAREST_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties command: - kafka-rest-start - /etc/kafka-rest/kafka-rest.properties -- cgit v1.2.3 From f29e6eb8a20edebce65a3ecca792ce28ba602492 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 13:03:55 +0200 Subject: Logs, unlike docs, revealed that kafka-rest needs ... both zookeeper and bootstrap addresses. Was getting connection errors to localhost:9092 broker id -1, resulting in REST requests never returning. --- 11confluent-config.yml | 3 ++- test/rest-curl.yml | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index df91677..12fa7db 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -43,8 +43,9 @@ data: #id=kafka-rest-test-server listeners=http://0.0.0.0:80 - schema.registry.url=http://schemas.kafka.svc.cluster.local:80 + bootstrap.servers=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 zookeeper.connect=zookeeper:2181 + schema.registry.url=http://schemas.kafka.svc.cluster.local:80 # # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center # Make sure that monitoring-interceptors-.jar is on the Java class path diff --git a/test/rest-curl.yml b/test/rest-curl.yml index 05fc178..7ee719a 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -22,10 +22,7 @@ data: curl -X POST \ -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ - $REST/topics/$TOPIC -v --max-time 30 \ - || echo " (timeout might be ok because we only want to send one message)" - # TODO why does the above block? - + $REST/topics/$TOPIC curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions echo "" -- cgit v1.2.3 From ac5c75a977c796313dc5f84831aa50529457be38 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 14:34:53 +0200 Subject: Tests commands succeed now. Remains to add asserts. --- test/rest-curl.yml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/test/rest-curl.yml b/test/rest-curl.yml index 7ee719a..f8dc8de 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -12,28 +12,43 @@ data: # Keep starting up until rest proxy is up and running curl --retry 10 --retry-delay 30 --retry-connrefused -I -s $REST + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/brokers | egrep '."brokers":.0' - curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics echo "" - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC echo "" curl -X POST \ -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ $REST/topics/$TOPIC + echo "" - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions echo "" - curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; + curl -X POST \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' \ + $REST/consumers/my_json_consumer + echo "" - curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v; + curl -X POST \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + --data "{\"topics\":[\"$TOPIC\"]}" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription \ + -w "%{http_code}" + echo "" - curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v; + curl -X GET \ + -H "Accept: application/vnd.kafka.json.v2+json" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance/records - curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v; + curl -X DELETE \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance tail -f /tmp/testlog -- cgit v1.2.3 From caf56c71bd36be27f1626e1849889a6b893035d4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 14:54:25 +0200 Subject: Looks like Schema Registry works with empty bootstrap servers --- 11confluent-config.yml | 1 + test/rest-curl.yml | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/11confluent-config.yml b/11confluent-config.yml index 12fa7db..b56484b 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -20,6 +20,7 @@ data: # limitations under the License. listeners=http://0.0.0.0:80 + #kafkastore.bootstrap.servers kafkastore.connection.url=zookeeper:2181 kafkastore.topic=_schemas debug=false diff --git a/test/rest-curl.yml b/test/rest-curl.yml index f8dc8de..5c6489d 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -50,6 +50,27 @@ data: -H "Content-Type: application/vnd.kafka.v2+json" \ $REST/consumers/my_json_consumer/instances/my_consumer_instance + # schema-registry + + curl -X GET $SCHEMAS/subjects + echo "" + + curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \ + --data '{"schema": "{\"type\": \"string\"}"}' \ + $SCHEMAS/subjects/$TOPIC-key/versions + echo "" + + curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \ + --data '{"schema": "{\"type\": \"string\"}"}' \ + $SCHEMAS/subjects/$TOPIC-value/versions + echo "" + + curl -X GET $SCHEMAS/schemas/ids/1 + echo "" + + curl -X GET $SCHEMAS/subjects/$TOPIC-value/versions/1 + echo "" + tail -f /tmp/testlog continue.sh: |- @@ -104,6 +125,8 @@ spec: - name: testcase image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf env: + - name: SCHEMAS + value: http://schemas.kafka.svc.cluster.local - name: REST value: http://rest.kafka.svc.cluster.local - name: TOPIC -- cgit v1.2.3 From ff698326390c191a39c95b38d019f7b306fb0063 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 15:05:06 +0200 Subject: Demonstrates that rest+schemas work together --- test/rest-curl.yml | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/test/rest-curl.yml b/test/rest-curl.yml index 5c6489d..998a36f 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -71,6 +71,27 @@ data: curl -X GET $SCHEMAS/subjects/$TOPIC-value/versions/1 echo "" + # rest + schema + # TODO new topic needed because this breaks json consumer above + + curl -X POST -H "Content-Type: application/vnd.kafka.avro.v2+json" \ + -H "Accept: application/vnd.kafka.v2+json" \ + --data '{"value_schema": "{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"name\": \"name\", \"type\": \"string\"}]}", "records": [{"value": {"name": "testUser"}}]}' \ + $REST/topics/$TOPIC + echo "" + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" \ + --data '{"name": "my_consumer_instance", "format": "avro", "auto.offset.reset": "earliest"}' \ + $REST/consumers/my_avro_consumer + echo "" + + curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" \ + --data "{\"topics\":[\"$TOPIC\"]}" \ + $REST/consumers/my_avro_consumer/instances/my_consumer_instance/subscription + + curl -X GET -H "Accept: application/vnd.kafka.avro.v2+json" \ + $REST/consumers/my_avro_consumer/instances/my_consumer_instance/records + tail -f /tmp/testlog continue.sh: |- -- cgit v1.2.3 From edeca3df36e72ddb464a10c580ae33b8ed1d2f17 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Aug 2017 08:22:42 +0200 Subject: Moves to an addon folder, kubectl apply -f addon-cp/ --- 11confluent-config.yml | 68 ------------------------------------------- 60schemas-service.yml | 10 ------- 61schemas.yml | 30 ------------------- 70rest-service.yml | 10 ------- 71rest.yml | 30 ------------------- addon-cp/confluent-config.yml | 68 +++++++++++++++++++++++++++++++++++++++++++ addon-cp/rest-service.yml | 10 +++++++ addon-cp/rest.yml | 30 +++++++++++++++++++ addon-cp/schemas-service.yml | 10 +++++++ addon-cp/schemas.yml | 30 +++++++++++++++++++ 10 files changed, 148 insertions(+), 148 deletions(-) delete mode 100644 11confluent-config.yml delete mode 100644 60schemas-service.yml delete mode 100644 61schemas.yml delete mode 100644 70rest-service.yml delete mode 100644 71rest.yml create mode 100644 addon-cp/confluent-config.yml create mode 100644 addon-cp/rest-service.yml create mode 100644 addon-cp/rest.yml create mode 100644 addon-cp/schemas-service.yml create mode 100644 addon-cp/schemas.yml diff --git a/11confluent-config.yml b/11confluent-config.yml deleted file mode 100644 index b56484b..0000000 --- a/11confluent-config.yml +++ /dev/null @@ -1,68 +0,0 @@ -kind: ConfigMap -metadata: - name: confluent-config - namespace: kafka -apiVersion: v1 -data: - schema-registry.properties: |- - # Copyright 2014 Confluent Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - - listeners=http://0.0.0.0:80 - #kafkastore.bootstrap.servers - kafkastore.connection.url=zookeeper:2181 - kafkastore.topic=_schemas - debug=false - - kafka-rest.properties: |- - ## - # Copyright 2015 Confluent Inc. - # - # Licensed under the Apache License, Version 2.0 (the "License"); - # you may not use this file except in compliance with the License. - # You may obtain a copy of the License at - # - # http://www.apache.org/licenses/LICENSE-2.0 - # - # Unless required by applicable law or agreed to in writing, software - # distributed under the License is distributed on an "AS IS" BASIS, - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - # See the License for the specific language governing permissions and - # limitations under the License. - ## - - #id=kafka-rest-test-server - listeners=http://0.0.0.0:80 - bootstrap.servers=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 - zookeeper.connect=zookeeper:2181 - schema.registry.url=http://schemas.kafka.svc.cluster.local:80 - # - # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center - # Make sure that monitoring-interceptors-.jar is on the Java class path - #consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor - #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor - - log4j.properties: |- - log4j.rootLogger=INFO, stdout - - log4j.appender.stdout=org.apache.log4j.ConsoleAppender - log4j.appender.stdout.layout=org.apache.log4j.PatternLayout - log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n - - log4j.logger.kafka=ERROR, stdout - log4j.logger.org.apache.zookeeper=ERROR, stdout - log4j.logger.org.apache.kafka=ERROR, stdout - log4j.logger.org.I0Itec.zkclient=ERROR, stdout - log4j.additivity.kafka.server=false - log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false diff --git a/60schemas-service.yml b/60schemas-service.yml deleted file mode 100644 index 2c30905..0000000 --- a/60schemas-service.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: schemas - namespace: kafka -spec: - ports: - - port: 80 - selector: - app: schema-registry diff --git a/61schemas.yml b/61schemas.yml deleted file mode 100644 index f85d74c..0000000 --- a/61schemas.yml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: schemas - namespace: kafka -spec: - replicas: 1 - template: - metadata: - labels: - app: schema-registry - spec: - containers: - - name: cp - image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf - env: - - name: SCHEMA_REGISTRY_LOG4J_OPTS - value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties - command: - - schema-registry-start - - /etc/schema-registry/schema-registry.properties - ports: - - containerPort: 80 - volumeMounts: - - name: config - mountPath: /etc/schema-registry - volumes: - - name: config - configMap: - name: confluent-config diff --git a/70rest-service.yml b/70rest-service.yml deleted file mode 100644 index a3ffda6..0000000 --- a/70rest-service.yml +++ /dev/null @@ -1,10 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: rest - namespace: kafka -spec: - ports: - - port: 80 - selector: - app: kafka-rest diff --git a/71rest.yml b/71rest.yml deleted file mode 100644 index f03f5b5..0000000 --- a/71rest.yml +++ /dev/null @@ -1,30 +0,0 @@ -apiVersion: apps/v1beta1 -kind: Deployment -metadata: - name: rest - namespace: kafka -spec: - replicas: 1 - template: - metadata: - labels: - app: kafka-rest - spec: - containers: - - name: cp - image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf - env: - - name: KAFKAREST_LOG4J_OPTS - value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties - command: - - kafka-rest-start - - /etc/kafka-rest/kafka-rest.properties - ports: - - containerPort: 80 - volumeMounts: - - name: config - mountPath: /etc/kafka-rest - volumes: - - name: config - configMap: - name: confluent-config diff --git a/addon-cp/confluent-config.yml b/addon-cp/confluent-config.yml new file mode 100644 index 0000000..b56484b --- /dev/null +++ b/addon-cp/confluent-config.yml @@ -0,0 +1,68 @@ +kind: ConfigMap +metadata: + name: confluent-config + namespace: kafka +apiVersion: v1 +data: + schema-registry.properties: |- + # Copyright 2014 Confluent Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + listeners=http://0.0.0.0:80 + #kafkastore.bootstrap.servers + kafkastore.connection.url=zookeeper:2181 + kafkastore.topic=_schemas + debug=false + + kafka-rest.properties: |- + ## + # Copyright 2015 Confluent Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + ## + + #id=kafka-rest-test-server + listeners=http://0.0.0.0:80 + bootstrap.servers=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 + zookeeper.connect=zookeeper:2181 + schema.registry.url=http://schemas.kafka.svc.cluster.local:80 + # + # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center + # Make sure that monitoring-interceptors-.jar is on the Java class path + #consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor + #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor + + log4j.properties: |- + log4j.rootLogger=INFO, stdout + + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + + log4j.logger.kafka=ERROR, stdout + log4j.logger.org.apache.zookeeper=ERROR, stdout + log4j.logger.org.apache.kafka=ERROR, stdout + log4j.logger.org.I0Itec.zkclient=ERROR, stdout + log4j.additivity.kafka.server=false + log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false diff --git a/addon-cp/rest-service.yml b/addon-cp/rest-service.yml new file mode 100644 index 0000000..a3ffda6 --- /dev/null +++ b/addon-cp/rest-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: rest + namespace: kafka +spec: + ports: + - port: 80 + selector: + app: kafka-rest diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml new file mode 100644 index 0000000..f03f5b5 --- /dev/null +++ b/addon-cp/rest.yml @@ -0,0 +1,30 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: rest + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: kafka-rest + spec: + containers: + - name: cp + image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + env: + - name: KAFKAREST_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties + command: + - kafka-rest-start + - /etc/kafka-rest/kafka-rest.properties + ports: + - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/kafka-rest + volumes: + - name: config + configMap: + name: confluent-config diff --git a/addon-cp/schemas-service.yml b/addon-cp/schemas-service.yml new file mode 100644 index 0000000..2c30905 --- /dev/null +++ b/addon-cp/schemas-service.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: Service +metadata: + name: schemas + namespace: kafka +spec: + ports: + - port: 80 + selector: + app: schema-registry diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml new file mode 100644 index 0000000..f85d74c --- /dev/null +++ b/addon-cp/schemas.yml @@ -0,0 +1,30 @@ +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: schemas + namespace: kafka +spec: + replicas: 1 + template: + metadata: + labels: + app: schema-registry + spec: + containers: + - name: cp + image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + env: + - name: SCHEMA_REGISTRY_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties + command: + - schema-registry-start + - /etc/schema-registry/schema-registry.properties + ports: + - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/schema-registry + volumes: + - name: config + configMap: + name: confluent-config -- cgit v1.2.3 From 4ee79a4e25325aad52ced1c771fe4e45316d0af8 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Aug 2017 08:24:32 +0200 Subject: Stay online at update, even with only one replica --- addon-cp/rest.yml | 5 +++++ addon-cp/schemas.yml | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index f03f5b5..a450eab 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -5,6 +5,11 @@ metadata: namespace: kafka spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 template: metadata: labels: diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml index f85d74c..00bd62f 100644 --- a/addon-cp/schemas.yml +++ b/addon-cp/schemas.yml @@ -5,6 +5,11 @@ metadata: namespace: kafka spec: replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 template: metadata: labels: -- cgit v1.2.3 From 2f946b26f308d2ee6e3eb2b0d307c30b98a462a4 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Aug 2017 08:35:09 +0200 Subject: Adds readiness probes --- addon-cp/rest.yml | 4 ++++ addon-cp/schemas.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index a450eab..a08587d 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -24,6 +24,10 @@ spec: command: - kafka-rest-start - /etc/kafka-rest/kafka-rest.properties + readinessProbe: + httpGet: + path: / + port: 80 ports: - containerPort: 80 volumeMounts: diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml index 00bd62f..8265640 100644 --- a/addon-cp/schemas.yml +++ b/addon-cp/schemas.yml @@ -24,6 +24,10 @@ spec: command: - schema-registry-start - /etc/schema-registry/schema-registry.properties + readinessProbe: + httpGet: + path: / + port: 80 ports: - containerPort: 80 volumeMounts: -- cgit v1.2.3 From c20adc0a6cfc54e91db66271ef7b7481d4ff7f0a Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 7 Aug 2017 08:36:04 +0200 Subject: Adds liveness probes identical to readiness --- addon-cp/rest.yml | 4 ++++ addon-cp/schemas.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index a08587d..60fcc31 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -28,6 +28,10 @@ spec: httpGet: path: / port: 80 + livenessProbe: + httpGet: + path: / + port: 80 ports: - containerPort: 80 volumeMounts: diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml index 8265640..4d99ce0 100644 --- a/addon-cp/schemas.yml +++ b/addon-cp/schemas.yml @@ -28,6 +28,10 @@ spec: httpGet: path: / port: 80 + livenessProbe: + httpGet: + path: / + port: 80 ports: - containerPort: 80 volumeMounts: -- cgit v1.2.3 From 3b6a11ecfd707e738b6e65c4c179048e77336dfd Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 3 Oct 2017 15:29:59 +0200 Subject: Confluent platform built from same java image as kafka:0.11.0.1 --- addon-cp/rest.yml | 2 +- addon-cp/schemas.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml index 60fcc31..c835ebf 100644 --- a/addon-cp/rest.yml +++ b/addon-cp/rest.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + image: solsson/kafka-cp@sha256:89e5b3b326bf8ef888efa9501cd22fb941ca7b5ccef8dfc2f44a1e218f4427e1 env: - name: KAFKAREST_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml index 4d99ce0..c762d63 100644 --- a/addon-cp/schemas.yml +++ b/addon-cp/schemas.yml @@ -17,7 +17,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + image: solsson/kafka-cp@sha256:89e5b3b326bf8ef888efa9501cd22fb941ca7b5ccef8dfc2f44a1e218f4427e1 env: - name: SCHEMA_REGISTRY_LOG4J_OPTS value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties -- cgit v1.2.3