aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorStaffan Olsson <staffan@repos.se>2017-11-29 06:52:48 +0100
committerStaffan Olsson <staffan@repos.se>2017-11-29 06:52:48 +0100
commit59a2afa9fa4a216a7896ecb05565bf83d8a9166f (patch)
treeab9caf3d85c4c544b3a7519b39861fde851a7398
parent99d22ad8a96a847928264f80c340931f37ee7376 (diff)
parent3b6a11ecfd707e738b6e65c4c179048e77336dfd (diff)
downloadkubernetes-kafka-59a2afa9fa4a216a7896ecb05565bf83d8a9166f.tar.gz
kubernetes-kafka-59a2afa9fa4a216a7896ecb05565bf83d8a9166f.tar.bz2
kubernetes-kafka-59a2afa9fa4a216a7896ecb05565bf83d8a9166f.zip
Merge remote-tracking branch 'origin/addon-rest' into 1.8-confluent-rest
-rw-r--r--addon-cp/confluent-config.yml68
-rw-r--r--addon-cp/rest-service.yml10
-rw-r--r--addon-cp/rest.yml43
-rw-r--r--addon-cp/schemas-service.yml10
-rw-r--r--addon-cp/schemas.yml43
-rw-r--r--test/70rest-test1.yml42
-rw-r--r--test/rest-curl.yml182
7 files changed, 398 insertions, 0 deletions
diff --git a/addon-cp/confluent-config.yml b/addon-cp/confluent-config.yml
new file mode 100644
index 0000000..b56484b
--- /dev/null
+++ b/addon-cp/confluent-config.yml
@@ -0,0 +1,68 @@
+kind: ConfigMap
+metadata:
+ name: confluent-config
+ namespace: kafka
+apiVersion: v1
+data:
+ schema-registry.properties: |-
+ # Copyright 2014 Confluent Inc.
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+
+ listeners=http://0.0.0.0:80
+ #kafkastore.bootstrap.servers
+ kafkastore.connection.url=zookeeper:2181
+ kafkastore.topic=_schemas
+ debug=false
+
+ kafka-rest.properties: |-
+ ##
+ # Copyright 2015 Confluent Inc.
+ #
+ # Licensed under the Apache License, Version 2.0 (the "License");
+ # you may not use this file except in compliance with the License.
+ # You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing, software
+ # distributed under the License is distributed on an "AS IS" BASIS,
+ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ # See the License for the specific language governing permissions and
+ # limitations under the License.
+ ##
+
+ #id=kafka-rest-test-server
+ listeners=http://0.0.0.0:80
+ bootstrap.servers=kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
+ zookeeper.connect=zookeeper:2181
+ schema.registry.url=http://schemas.kafka.svc.cluster.local:80
+ #
+ # Configure interceptor classes for sending consumer and producer metrics to Confluent Control Center
+ # Make sure that monitoring-interceptors-<version>.jar is on the Java class path
+ #consumer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor
+ #producer.interceptor.classes=io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor
+
+ log4j.properties: |-
+ log4j.rootLogger=INFO, stdout
+
+ log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+ log4j.appender.stdout.layout.ConversionPattern=[%d] %p %m (%c:%L)%n
+
+ log4j.logger.kafka=ERROR, stdout
+ log4j.logger.org.apache.zookeeper=ERROR, stdout
+ log4j.logger.org.apache.kafka=ERROR, stdout
+ log4j.logger.org.I0Itec.zkclient=ERROR, stdout
+ log4j.additivity.kafka.server=false
+ log4j.additivity.kafka.consumer.ZookeeperConsumerConnector=false
diff --git a/addon-cp/rest-service.yml b/addon-cp/rest-service.yml
new file mode 100644
index 0000000..a3ffda6
--- /dev/null
+++ b/addon-cp/rest-service.yml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: rest
+ namespace: kafka
+spec:
+ ports:
+ - port: 80
+ selector:
+ app: kafka-rest
diff --git a/addon-cp/rest.yml b/addon-cp/rest.yml
new file mode 100644
index 0000000..c835ebf
--- /dev/null
+++ b/addon-cp/rest.yml
@@ -0,0 +1,43 @@
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: rest
+ namespace: kafka
+spec:
+ replicas: 1
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: kafka-rest
+ spec:
+ containers:
+ - name: cp
+ image: solsson/kafka-cp@sha256:89e5b3b326bf8ef888efa9501cd22fb941ca7b5ccef8dfc2f44a1e218f4427e1
+ env:
+ - name: KAFKAREST_LOG4J_OPTS
+ value: -Dlog4j.configuration=file:/etc/kafka-rest/log4j.properties
+ command:
+ - kafka-rest-start
+ - /etc/kafka-rest/kafka-rest.properties
+ readinessProbe:
+ httpGet:
+ path: /
+ port: 80
+ livenessProbe:
+ httpGet:
+ path: /
+ port: 80
+ ports:
+ - containerPort: 80
+ volumeMounts:
+ - name: config
+ mountPath: /etc/kafka-rest
+ volumes:
+ - name: config
+ configMap:
+ name: confluent-config
diff --git a/addon-cp/schemas-service.yml b/addon-cp/schemas-service.yml
new file mode 100644
index 0000000..2c30905
--- /dev/null
+++ b/addon-cp/schemas-service.yml
@@ -0,0 +1,10 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: schemas
+ namespace: kafka
+spec:
+ ports:
+ - port: 80
+ selector:
+ app: schema-registry
diff --git a/addon-cp/schemas.yml b/addon-cp/schemas.yml
new file mode 100644
index 0000000..c762d63
--- /dev/null
+++ b/addon-cp/schemas.yml
@@ -0,0 +1,43 @@
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: schemas
+ namespace: kafka
+spec:
+ replicas: 1
+ strategy:
+ type: RollingUpdate
+ rollingUpdate:
+ maxUnavailable: 0
+ maxSurge: 1
+ template:
+ metadata:
+ labels:
+ app: schema-registry
+ spec:
+ containers:
+ - name: cp
+ image: solsson/kafka-cp@sha256:89e5b3b326bf8ef888efa9501cd22fb941ca7b5ccef8dfc2f44a1e218f4427e1
+ env:
+ - name: SCHEMA_REGISTRY_LOG4J_OPTS
+ value: -Dlog4j.configuration=file:/etc/schema-registry/log4j.properties
+ command:
+ - schema-registry-start
+ - /etc/schema-registry/schema-registry.properties
+ readinessProbe:
+ httpGet:
+ path: /
+ port: 80
+ livenessProbe:
+ httpGet:
+ path: /
+ port: 80
+ ports:
+ - containerPort: 80
+ volumeMounts:
+ - name: config
+ mountPath: /etc/schema-registry
+ volumes:
+ - name: config
+ configMap:
+ name: confluent-config
diff --git a/test/70rest-test1.yml b/test/70rest-test1.yml
new file mode 100644
index 0000000..ce7b019
--- /dev/null
+++ b/test/70rest-test1.yml
@@ -0,0 +1,42 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: rest-test1
+ namespace: kafka
+spec:
+ template:
+ metadata:
+ name: rest-test1
+ spec:
+ containers:
+ - name: curl
+ image: solsson/curl@sha256:8b0927b81d10043e70f3e05e33e36fb9b3b0cbfcbccdb9f04fd53f67a270b874
+ env:
+ - name: REST
+ value: http://rest.kafka.svc.cluster.local
+ - name: TOPIC
+ value: test1
+ command:
+ - /bin/sh
+ - -c
+ - >
+ curl --retry 10 --retry-delay 30 --retry-connrefused -I $REST;
+
+ curl -H 'Accept: application/vnd.kafka.v2+json' $REST/topics;
+
+ curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test1;
+ curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date)\"}]}" $REST/topics/$TOPIC -v;
+ curl --retry 10 -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/test2;
+
+ curl -X POST -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" --data '{"records":[{"value":{"foo":"bar"}}]}' $REST/topics/$TOPIC -v;
+
+ curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' $REST/consumers/my_json_consumer -v;
+
+ curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" --data "{\"topics\":[\"$TOPIC\"]}" $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription -v;
+
+ curl -X GET -H "Accept: application/vnd.kafka.json.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance/records -v;
+
+ curl -X DELETE -H "Content-Type: application/vnd.kafka.v2+json" $REST/consumers/my_json_consumer/instances/my_consumer_instance -v;
+
+ sleep 300
+ restartPolicy: Never
diff --git a/test/rest-curl.yml b/test/rest-curl.yml
new file mode 100644
index 0000000..998a36f
--- /dev/null
+++ b/test/rest-curl.yml
@@ -0,0 +1,182 @@
+---
+kind: ConfigMap
+metadata:
+ name: rest-curl
+ namespace: test-kafka
+apiVersion: v1
+data:
+
+ setup.sh: |-
+ touch /tmp/testlog
+
+ # Keep starting up until rest proxy is up and running
+ curl --retry 10 --retry-delay 30 --retry-connrefused -I -s $REST
+
+ curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/brokers | egrep '."brokers":.0'
+
+ curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics
+ echo ""
+
+ curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC
+ echo ""
+
+ curl -X POST \
+ -H "Content-Type: application/vnd.kafka.json.v2+json" -H "Accept: application/vnd.kafka.v2+json" \
+ --data "{\"records\":[{\"value\":\"Test from $HOSTNAME at $(date -u -Iseconds)\"}]}" \
+ $REST/topics/$TOPIC
+ echo ""
+
+ curl -s -H 'Accept: application/vnd.kafka.v2+json' $REST/topics/$TOPIC/partitions
+ echo ""
+
+ curl -X POST \
+ -H "Content-Type: application/vnd.kafka.v2+json" \
+ --data '{"name": "my_consumer_instance", "format": "json", "auto.offset.reset": "earliest"}' \
+ $REST/consumers/my_json_consumer
+ echo ""
+
+ curl -X POST \
+ -H "Content-Type: application/vnd.kafka.v2+json" \
+ --data "{\"topics\":[\"$TOPIC\"]}" \
+ $REST/consumers/my_json_consumer/instances/my_consumer_instance/subscription \
+ -w "%{http_code}"
+ echo ""
+
+ curl -X GET \
+ -H "Accept: application/vnd.kafka.json.v2+json" \
+ $REST/consumers/my_json_consumer/instances/my_consumer_instance/records
+
+ curl -X DELETE \
+ -H "Content-Type: application/vnd.kafka.v2+json" \
+ $REST/consumers/my_json_consumer/instances/my_consumer_instance
+
+ # schema-registry
+
+ curl -X GET $SCHEMAS/subjects
+ echo ""
+
+ curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \
+ --data '{"schema": "{\"type\": \"string\"}"}' \
+ $SCHEMAS/subjects/$TOPIC-key/versions
+ echo ""
+
+ curl -X POST -H "Content-Type: application/vnd.schemaregistry.v1+json" \
+ --data '{"schema": "{\"type\": \"string\"}"}' \
+ $SCHEMAS/subjects/$TOPIC-value/versions
+ echo ""
+
+ curl -X GET $SCHEMAS/schemas/ids/1
+ echo ""
+
+ curl -X GET $SCHEMAS/subjects/$TOPIC-value/versions/1
+ echo ""
+
+ # rest + schema
+ # TODO new topic needed because this breaks json consumer above
+
+ curl -X POST -H "Content-Type: application/vnd.kafka.avro.v2+json" \
+ -H "Accept: application/vnd.kafka.v2+json" \
+ --data '{"value_schema": "{\"type\": \"record\", \"name\": \"User\", \"fields\": [{\"name\": \"name\", \"type\": \"string\"}]}", "records": [{"value": {"name": "testUser"}}]}' \
+ $REST/topics/$TOPIC
+ echo ""
+
+ curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" \
+ --data '{"name": "my_consumer_instance", "format": "avro", "auto.offset.reset": "earliest"}' \
+ $REST/consumers/my_avro_consumer
+ echo ""
+
+ curl -X POST -H "Content-Type: application/vnd.kafka.v2+json" \
+ --data "{\"topics\":[\"$TOPIC\"]}" \
+ $REST/consumers/my_avro_consumer/instances/my_consumer_instance/subscription
+
+ curl -X GET -H "Accept: application/vnd.kafka.avro.v2+json" \
+ $REST/consumers/my_avro_consumer/instances/my_consumer_instance/records
+
+ tail -f /tmp/testlog
+
+ continue.sh: |-
+ exit 0
+
+ run.sh: |-
+ exec >> /tmp/testlog
+ exec 2>&1
+
+ exit 0
+
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+ name: rest-curl
+ namespace: test-kafka
+spec:
+ template:
+ spec:
+ containers:
+ - name: topic-create
+ image: solsson/kafka:0.11.0.0@sha256:b27560de08d30ebf96d12e74f80afcaca503ad4ca3103e63b1fd43a2e4c976ce
+ command:
+ - ./bin/kafka-topics.sh
+ - --zookeeper
+ - zookeeper.kafka.svc.cluster.local:2181
+ - --create
+ - --if-not-exists
+ - --topic
+ - test-rest-curl
+ - --partitions
+ - "1"
+ - --replication-factor
+ - "1"
+ restartPolicy: Never
+---
+apiVersion: apps/v1beta1
+kind: Deployment
+metadata:
+ name: rest-curl
+ namespace: test-kafka
+spec:
+ replicas: 1
+ template:
+ metadata:
+ labels:
+ test-target: kafka
+ test-type: readiness
+ spec:
+ containers:
+ - name: testcase
+ image: solsson/curl@sha256:8c0c5d669b3dd67932da934024252af59fb9d0fa0e5118b5a737b35c5e1487bf
+ env:
+ - name: SCHEMAS
+ value: http://schemas.kafka.svc.cluster.local
+ - name: REST
+ value: http://rest.kafka.svc.cluster.local
+ - name: TOPIC
+ value: test-rest-curl
+ # Test set up
+ command:
+ - /bin/bash
+ - -e
+ - /test/setup.sh
+ # Test run, again and again
+ readinessProbe:
+ exec:
+ command:
+ - /bin/bash
+ - -e
+ - /test/run.sh
+ # We haven't worked on timing
+ periodSeconds: 60
+ # Test quit on nonzero exit
+ livenessProbe:
+ exec:
+ command:
+ - /bin/bash
+ - -e
+ - /test/continue.sh
+ volumeMounts:
+ - name: config
+ mountPath: /test
+ volumes:
+ - name: config
+ configMap:
+ name: rest-curl