Converts (console-) produce-consume test to online ...

style like kafkacat, but without the timestamp diff feature
because I couldn't get message timestamp out of the console consumer.

Also I had to increase PC_WAIT.

2 files changed, 168 insertions, 113 deletions

diff --git a/test/basic-produce-consume.yml b/test/basic-produce-consume.yml
deleted file mode 100644
index 32ec4ec..0000000
--- a/test/basic-produce-consume.yml
+++ /dev/null
@@ -1,113 +0,0 @@
----
-kind: ConfigMap
-metadata:
-  name: basic-produce-consume
-  namespace: test-kafka
-apiVersion: v1
-data:
-
-  setup.sh: |-
-    touch /tmp/testlog
-
-    ./bin/kafka-topics.sh --zookeeper $ZOOKEEPER \
-      --create --if-not-exists --topic test-basic-produce-consume \
-      --partitions 1 --replication-factor 1
-
-    # Despite the deprecation warning --zookeeper nothing is consumed when using --bootstrap-server
-    ./bin/kafka-console-consumer.sh --zookeeper $ZOOKEEPER --topic test-basic-produce-consume > /tmp/testconsumed &
-
-    tail -f /tmp/testlog
-
-  continue.sh: |-
-    exit 0
-
-  run.sh: |-
-    exec >> /tmp/testlog
-    exec 2>&1
-
-    unique=$(date -Ins)
-
-    echo "Test $unique" | ./bin/kafka-console-producer.sh --broker-list $BOOTSTRAP --topic test-basic-produce-consume
-    echo ""
-    tail -n 1 /tmp/testconsumed | grep $unique
-
-    # How to make this test fail:
-    #apt-get update && apt-get install -y --no-install-recommends procps
-    #pkill java
-
-    exit 0
----
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: basic-produce-consume
-  namespace: test-kafka
-spec:
-  template:
-    spec:
-      containers:
-      - name: topic-create
-        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
-        command:
-        - ./bin/kafka-topics.sh
-        - --zookeeper
-        -   zookeeper.kafka.svc.cluster.local:2181
-        - --create
-        - --if-not-exists
-        - --topic
-        -   test-basic-produce-consume
-        - --partitions
-        -   "1"
-        - --replication-factor
-        -   "2"
-      restartPolicy: Never
----
-apiVersion: apps/v1beta1
-kind: Deployment
-metadata:
-  name: basic-produce-consume
-  namespace: test-kafka
-spec:
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        test-target: kafka
-        test-type: readiness
-    spec:
-      containers:
-      - name: testcase
-        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
-        env:
-        - name: BOOTSTRAP
-          value: kafka-0.broker.kafka.svc.cluster.local:9092,kafka-1.broker.kafka.svc.cluster.local:9092,kafka-2.broker.kafka.svc.cluster.local:9092
-        - name: ZOOKEEPER
-          value: zookeeper.kafka.svc.cluster.local:2181
-        # Test set up
-        command:
-        - /bin/bash
-        - -e
-        - /test/setup.sh
-        # Test run, again and again
-        readinessProbe:
-          exec:
-            command:
-            - /bin/bash
-            - -e
-            - /test/run.sh
-          # JVM start is slow, can we keep producer started and restore the default preriod 10s?
-          periodSeconds: 30
-        # Test quit on nonzero exit
-        livenessProbe:
-          exec:
-            command:
-            - /bin/bash
-            - -e
-            - /test/continue.sh
-        volumeMounts:
-        - name: config
-          mountPath: /test
-      volumes:
-      - name: config
-        configMap:
-          name: basic-produce-consume
diff --git a/test/produce-consume.yml b/test/produce-consume.yml
new file mode 100644
index 0000000..f568c56
--- /dev/null
+++ b/test/produce-consume.yml
@@ -0,0 +1,168 @@
+---
+kind: ConfigMap
+metadata:
+  name: produce-consume
+  namespace: test-kafka
+apiVersion: v1
+data:
+
+  setup.sh: |-
+    touch /tmp/testlog
+
+    tail -f /tmp/testlog
+
+  test.sh: |-
+    exec >> /tmp/testlog
+    exec 2>&1
+
+    # As low as in kafkacat based test didn't work, but this value can likely be squeezed
+    PC_WAIT=2.0
+
+    UNIQUE="${HOSTNAME}@$(date -u -Ins)"
+
+    echo "Test $UNIQUE" >> /shared/produce.tmp
+    sleep $PC_WAIT
+    LAST=$(tail -n 1 /shared/consumed.tmp)
+    [ -z "$LAST" ] && echo "Nothing consumed yet" && exit 1
+
+    # Haven't found how to get message timestamp in console-consumer, see kafkacat based test instead
+    LAST_MSG=$LAST
+
+    if [[ "$LAST_MSG" != *"$UNIQUE" ]]; then
+      echo "Last message (at $(date +%FT%T)) isn't from this test run ($UNIQUE):"
+      echo "$LAST_MSG"
+      exit 11
+    fi
+
+    echo "OK ($LAST_MSG at $(date +%FT%T))"
+    # We haven't asserted that the consumer works, so we'll just have to assume that it will exit if it fails
+
+    exit 0
+
+  quit-on-nonzero-exit.sh: |-
+    exec >> /tmp/testlog
+    exec 2>&1
+
+    exit 0
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: produce-consume
+  namespace: test-kafka
+spec:
+  template:
+    spec:
+      containers:
+      - name: topic-create
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        command:
+        - ./bin/kafka-topics.sh
+        - --zookeeper
+        -   zookeeper.kafka.svc.cluster.local:2181
+        - --create
+        - --if-not-exists
+        - --topic
+        -   test-produce-consume
+        - --partitions
+        -   "1"
+        - --replication-factor
+        -   "2"
+      restartPolicy: Never
+---
+apiVersion: apps/v1beta2
+kind: Deployment
+metadata:
+  name: produce-consume
+  namespace: test-kafka
+spec:
+  replicas: 1
+  strategy:
+    type: Recreate
+  selector:
+    matchLabels:
+      test-target: kafka
+      test-type: readiness
+  template:
+    metadata:
+      labels:
+        test-target: kafka
+        test-type: readiness
+        # for example:
+        # readonly - can be used in production
+        # isolated - read/write but in a manner that does not affect other services
+        # load - unsuitable for production because it uses significant resources
+        # chaos - unsuitable for production because it injects failure modes
+        #test-use:
+    spec:
+      containers:
+      - name: producer
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        env:
+        - name: BOOTSTRAP
+          value: kafka-0.broker.kafka.svc.cluster.local:9092
+        command:
+        - /bin/bash
+        - -cex
+        - >
+          echo "--- start $HOSTNAME $(date --iso-8601='ns' -u) ---" >> /shared/produce.tmp
+          ;
+          tail -f /shared/produce.tmp |
+          ./bin/kafka-console-producer.sh --broker-list $BOOTSTRAP --topic test-produce-consume
+          ;
+        volumeMounts:
+        - name: config
+          mountPath: /test
+        - name: shared
+          mountPath: /shared
+      - name: consumer
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        env:
+        - name: BOOTSTRAP
+          value: kafka-0.broker.kafka.svc.cluster.local:9092
+        command:
+        - /bin/bash
+        - -cex
+        - >
+          ./bin/kafka-console-consumer.sh --bootstrap-server $BOOTSTRAP --topic test-produce-consume |
+          tee /shared/consumed.tmp
+          ;
+        volumeMounts:
+        - name: config
+          mountPath: /test
+        - name: shared
+          mountPath: /shared
+      - name: testcase
+        image: solsson/kafkacat@sha256:ebebf47061300b14a4b4c2e1e4303ab29f65e4b95d34af1b14bb8f7ec6da7cef
+        env:
+        - name: BOOTSTRAP
+          value: kafka-0.broker.kafka.svc.cluster.local:9092
+        command:
+        - /bin/bash
+        - -e
+        - /test/setup.sh
+        readinessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/test.sh
+          initialDelaySeconds: 10
+          periodSeconds: 10
+        livenessProbe:
+          exec:
+            command:
+            - /bin/bash
+            - -e
+            - /test/quit-on-nonzero-exit.sh
+        volumeMounts:
+        - name: config
+          mountPath: /test
+        - name: shared
+          mountPath: /shared
+      volumes:
+      - name: config
+        configMap:
+          name: produce-consume
+      - name: shared
+        emptyDir: {}