From f5876226d96bc355370f76fbbc8437eae1f213e2 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Jul 2017 20:46:25 +0200 Subject: Sample config from v3.3.0 source --- 11confluent-config.yml | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 11confluent-config.yml diff --git a/11confluent-config.yml b/11confluent-config.yml new file mode 100644 index 0000000..8b4087c --- /dev/null +++ b/11confluent-config.yml @@ -0,0 +1,72 @@ +kind: ConfigMap +metadata: + name: confluent-config + namespace: kafka +apiVersion: v1 +data: + schema-registry.properties: |- + # Copyright 2014 Confluent Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + listeners=http://0.0.0.0:8081 + kafkastore.connection.url=localhost:2181 + kafkastore.topic=_schemas + debug=false + + kafka-rest.properties: |- + ## + # Copyright 2015 Confluent Inc. + # + # Licensed under the Apache License, Version 2.0 (the "License"); + # you may not use this file except in compliance with the License. + # You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + ## + + #id=kafka-rest-test-server + #schema.registry.url=http://localhost:8081 + #zookeeper.connect=localhost:2181 + # + # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center + # Make sure that monitoring-interceptors-.jar is on the Java class path + #consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor + #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor + + log4j.properties: |- + log4j.rootLogger=INFO, stdout, file + + log4j.appender.stdout=org.apache.log4j.ConsoleAppender + log4j.appender.stdout.layout=org.apache.log4j.PatternLayout + log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n + + log4j.logger.kafka=ERROR, stdout + log4j.logger.org.apache.zookeeper=ERROR, stdout + log4j.logger.org.apache.kafka=ERROR, stdout + log4j.logger.org.I0Itec.zkclient=ERROR, stdout + log4j.additivity.kafka.server=false + log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false + + log4j.appender.file=org.apache.log4j.RollingFileAppender + log4j.appender.file.maxBackupIndex=10 + log4j.appender.file.maxFileSize=100MB + log4j.appender.file.File=${schema-registry.log.dir}/schema-registry.log + log4j.appender.file.layout=org.apache.log4j.PatternLayout + log4j.appender.file.layout.ConversionPattern=[%d] %p %m (%c)%n -- cgit v1.2.3 From 2d4401bc1fd90cc9af82599517d4a1df7a103ebb Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Mon, 31 Jul 2017 20:47:33 +0200 Subject: Remove file appender as docker implies stdout also the sample is from schema-registry.properties while kafka-rest had only the stdout appender (and no per-package levels) --- 11confluent-config.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index 8b4087c..62d8993 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -51,7 +51,7 @@ data: #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor log4j.properties: |- - log4j.rootLogger=INFO, stdout, file + log4j.rootLogger=INFO, stdout log4j.appender.stdout=org.apache.log4j.ConsoleAppender log4j.appender.stdout.layout=org.apache.log4j.PatternLayout @@ -63,10 +63,3 @@ data: log4j.logger.org.I0Itec.zkclient=ERROR, stdout log4j.additivity.kafka.server=false log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false - - log4j.appender.file=org.apache.log4j.RollingFileAppender - log4j.appender.file.maxBackupIndex=10 - log4j.appender.file.maxFileSize=100MB - log4j.appender.file.File=${schema-registry.log.dir}/schema-registry.log - log4j.appender.file.layout=org.apache.log4j.PatternLayout - log4j.appender.file.layout.ConversionPattern=[%d] %p %m (%c)%n -- cgit v1.2.3 From 47910682edfadffccbf77b3f0b2722376dc37350 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 07:35:06 +0200 Subject: New image uses config files instead of env --- 11confluent-config.yml | 9 +++++---- 61schemas.yml | 19 +++++++++++++------ 71rest.yml | 18 ++++++++++-------- 3 files changed, 28 insertions(+), 18 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index 62d8993..df91677 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -19,8 +19,8 @@ data: # See the License for the specific language governing permissions and # limitations under the License. - listeners=http://0.0.0.0:8081 - kafkastore.connection.url=localhost:2181 + listeners=http://0.0.0.0:80 + kafkastore.connection.url=zookeeper:2181 kafkastore.topic=_schemas debug=false @@ -42,8 +42,9 @@ data: ## #id=kafka-rest-test-server - #schema.registry.url=http://localhost:8081 - #zookeeper.connect=localhost:2181 + listeners=http://0.0.0.0:80 + schema.registry.url=http://schemas.kafka.svc.cluster.local:80 + zookeeper.connect=zookeeper:2181 # # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center # Make sure that monitoring-interceptors-.jar is on the Java class path diff --git a/61schemas.yml b/61schemas.yml index 20d1868..8d098dd 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -11,14 +11,21 @@ spec: app: schema-registry spec: containers: - - name: cp-schema-registry - image: confluentinc/cp-schema-registry@sha256:ac1eb34d9a60ce8904eb1bc01fd94bf1f6513924ca507734679d4b513133714c + - name: cp + image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 env: - - name: SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL - value: zookeeper:2181 + # what does this do? - name: SCHEMA_REGISTRY_HOST_NAME value: schemas - - name: SCHEMA_REGISTRY_LISTENERS - value: http://0.0.0.0:80 + command: + - schema-registry-start + - /etc/schema-registry/schema-registry.properties ports: - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/schema-registry + volumes: + - name: config + configMap: + name: confluent-config diff --git a/71rest.yml b/71rest.yml index 8f52b42..987e013 100644 --- a/71rest.yml +++ b/71rest.yml @@ -11,16 +11,18 @@ spec: app: kafka-rest spec: containers: - - name: cp-kafka-rest - image: confluentinc/cp-kafka-rest@sha256:aa213c1a67eae6ce9836b52a9b5ecee4d6a0b44f2b9cc69f4e4de85131462f1d + - name: cp + image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 env: - - name: KAFKA_REST_ZOOKEEPER_CONNECT - value: zookeeper:2181 + # what does this do? - name: KAFKA_REST_HOST_NAME value: rest - - name: KAFKA_REST_LISTENERS - value: http://0.0.0.0:80 - - name: KAFKA_REST_SCHEMA_REGISTRY_URL - value: http://schemas.kafka.svc.cluster.local:80 ports: - containerPort: 80 + volumeMounts: + - name: config + mountPath: /etc/kafka-rest + volumes: + - name: config + configMap: + name: confluent-config -- cgit v1.2.3 From 86183d83eb715c97e8647cd16cd963ead542f512 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 10:17:39 +0200 Subject: Runtime conf is logged at start, which documents host.name in case we need it --- 61schemas.yml | 6 +----- 71rest.yml | 9 ++++----- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/61schemas.yml b/61schemas.yml index 8d098dd..4fe7097 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -12,11 +12,7 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 - env: - # what does this do? - - name: SCHEMA_REGISTRY_HOST_NAME - value: schemas + image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf command: - schema-registry-start - /etc/schema-registry/schema-registry.properties diff --git a/71rest.yml b/71rest.yml index 987e013..e237357 100644 --- a/71rest.yml +++ b/71rest.yml @@ -12,11 +12,10 @@ spec: spec: containers: - name: cp - image: solsson/kafka-cp@sha256:53075cf5877c1311516d5c78a4d52fd46d3d369514681e150f96bc06997fa0e4 - env: - # what does this do? - - name: KAFKA_REST_HOST_NAME - value: rest + image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + command: + - kafka-rest-start + - /etc/kafka-rest/kafka-rest.properties ports: - containerPort: 80 volumeMounts: -- cgit v1.2.3 From eb1a1020055ecca572d0dac9a9d5bfb853b3d722 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 12:16:53 +0200 Subject: Once again be explicit (=unsurprising) about log config path --- 61schemas.yml | 3 +++ 71rest.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/61schemas.yml b/61schemas.yml index 4fe7097..f85d74c 100644 --- a/61schemas.yml +++ b/61schemas.yml @@ -13,6 +13,9 @@ spec: containers: - name: cp image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + env: + - name: SCHEMA_REGISTRY_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties command: - schema-registry-start - /etc/schema-registry/schema-registry.properties diff --git a/71rest.yml b/71rest.yml index e237357..f03f5b5 100644 --- a/71rest.yml +++ b/71rest.yml @@ -13,6 +13,9 @@ spec: containers: - name: cp image: solsson/kafka-cp@sha256:a22047b9e8bf4b8badfd2fbba47f2d1acdcbb84dfb03c61a15e1ac203036cedf + env: + - name: KAFKAREST_LOG4J_OPTS + value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties command: - kafka-rest-start - /etc/kafka-rest/kafka-rest.properties -- cgit v1.2.3 From f29e6eb8a20edebce65a3ecca792ce28ba602492 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 13:03:55 +0200 Subject: Logs, unlike docs, revealed that kafka-rest needs ... both zookeeper and bootstrap addresses. Was getting connection errors to localhost:9092 broker id -1, resulting in REST requests never returning. --- 11confluent-config.yml | 3 ++- test/rest-curl.yml | 5 +---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/11confluent-config.yml b/11confluent-config.yml index df91677..12fa7db 100644 --- a/11confluent-config.yml +++ b/11confluent-config.yml @@ -43,8 +43,9 @@ data: #id=kafka-rest-test-server listeners=http://0.0.0.0:80 - schema.registry.url=http://schemas.kafka.svc.cluster.local:80 + bootstrap.servers=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092 zookeeper.connect=zookeeper:2181 + schema.registry.url=http://schemas.kafka.svc.cluster.local:80 # # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center # Make sure that monitoring-interceptors-.jar is on the Java class path diff --git a/test/rest-curl.yml b/test/rest-curl.yml index 05fc178..7ee719a 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -22,10 +22,7 @@ data: curl -X POST \ -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ - $REST/topics/$TOPIC -v --max-time 30 \ - || echo " (timeout might be ok because we only want to send one message)" - # TODO why does the above block? - + $REST/topics/$TOPIC curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions echo "" -- cgit v1.2.3 From ac5c75a977c796313dc5f84831aa50529457be38 Mon Sep 17 00:00:00 2001 From: Staffan Olsson Date: Tue, 1 Aug 2017 14:34:53 +0200 Subject: Tests commands succeed now. Remains to add asserts. --- test/rest-curl.yml | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/test/rest-curl.yml b/test/rest-curl.yml index 7ee719a..f8dc8de 100644 --- a/test/rest-curl.yml +++ b/test/rest-curl.yml @@ -12,28 +12,43 @@ data: # Keep starting up until rest proxy is up and running curl --retry 10 --retry-delay 30 --retry-connrefused -I -s $REST + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/brokers | egrep '."brokers":.0' - curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics echo "" - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC echo "" curl -X POST \ -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \ --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \ $REST/topics/$TOPIC + echo "" - curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions + curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions echo "" - curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v; + curl -X POST \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' \ + $REST/consumers/my_json_consumer + echo "" - curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v; + curl -X POST \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + --data "{\"topics\":[\"$TOPIC\"]}" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription \ + -w "%{http_code}" + echo "" - curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v; + curl -X GET \ + -H "Accept: application/vnd.kafka.json.v2+json" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance/records - curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v; + curl -X DELETE \ + -H "Content-Type: application/vnd.kafka.v2+json" \ + $REST/consumers/my_json_consumer/instances/my_consumer_instance tail -f /tmp/testlog -- cgit v1.2.3