From f0d06583d51b144e87f06eb425b763d1f202bc25 Mon Sep 17 00:00:00 2001
From: Staffan Olsson <staffan@repos.se>
Date: Fri, 2 Feb 2018 18:16:00 +0100
Subject: Adds the necessary json mod to require more replicas,

which seems to be equivalent to setting replication.factor 3 at topic create.
---
 maintenance/increase-replication-factor.yml     | 51 --------------------
 maintenance/replication-factor-increase-job.yml | 63 +++++++++++++++++++++++++
 2 files changed, 63 insertions(+), 51 deletions(-)
 delete mode 100644 maintenance/increase-replication-factor.yml
 create mode 100644 maintenance/replication-factor-increase-job.yml

(limited to 'maintenance')

diff --git a/maintenance/increase-replication-factor.yml b/maintenance/increase-replication-factor.yml
deleted file mode 100644
index e9e184e..0000000
--- a/maintenance/increase-replication-factor.yml
+++ /dev/null
@@ -1,51 +0,0 @@
-apiVersion: batch/v1
-kind: Job
-metadata:
-  name: reassign-partitions
-  namespace: kafka
-spec:
-  template:
-    metadata:
-      name: reassign-partitions
-    spec:
-      containers:
-      - name: kafka
-        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
-        env:
-        - name: ZOOKEEPER
-          value: zookeeper.kafka:2181
-        # the following must be edited per job
-        - name: TOPICS
-          value: test-produce-consume,test-kafkacat
-        - name: BROKERS
-          value: 0,2
-        command:
-        - /bin/bash
-        - -ce
-        - >
-          echo '{"topics":[' > /tmp/reassign-topics.json;
-          echo -n ' {"topic":"' >> /tmp/reassign-topics.json;
-          echo -n $TOPICS | sed 's/,/"},\n {"topic":"/g' >> /tmp/reassign-topics.json;
-          echo '"}' >> /tmp/reassign-topics.json;
-          echo ']}' >> /tmp/reassign-topics.json;
-
-          echo "# reassign-topics.json";
-          cat /tmp/reassign-topics.json;
-
-          ./bin/kafka-reassign-partitions.sh 
-          --zookeeper=$ZOOKEEPER
-          --generate
-          --topics-to-move-json-file=/tmp/reassign-topics.json
-          --broker-list=$BROKERS > /tmp/generated.txt;
-
-          tail -n 1 /tmp/generated.txt > /tmp/proposed-reassignment.json;
-
-          echo "# proposed-reassignment.json";
-          cat /tmp/proposed-reassignment.json;
-
-          ./bin/kafka-reassign-partitions.sh
-          --zookeeper=$ZOOKEEPER
-          --execute
-          --reassignment-json-file=/tmp/proposed-reassignment.json;
-      restartPolicy: Never
-  backoffLimit: 3
diff --git a/maintenance/replication-factor-increase-job.yml b/maintenance/replication-factor-increase-job.yml
new file mode 100644
index 0000000..19147d0
--- /dev/null
+++ b/maintenance/replication-factor-increase-job.yml
@@ -0,0 +1,63 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: replication-factor-increase
+  namespace: kafka
+spec:
+  template:
+    metadata:
+      name: replication-factor-increase
+    spec:
+      containers:
+      - name: kafka
+        image: solsson/kafka:1.0.0@sha256:17fdf1637426f45c93c65826670542e36b9f3394ede1cb61885c6a4befa8f72d
+        env:
+        - name: ZOOKEEPER
+          value: zookeeper.kafka:2181
+        # the following must be edited per job
+        - name: TOPICS
+          value: ""
+        - name: BROKERS
+          value: 0,1,2
+        command:
+        - /bin/bash
+        - -ce
+        - >
+          if [ -z "$TOPICS" ];  then
+            echo "Please set the TOPICS env (comma-separated) and re-create the job"
+            tail -f /dev/null
+          fi
+
+          echo '{"topics":[' > /tmp/reassign-topics.json;
+          echo -n ' {"topic":"' >> /tmp/reassign-topics.json;
+          echo -n $TOPICS | sed 's/,/"},\n {"topic":"/g' >> /tmp/reassign-topics.json;
+          echo '"}' >> /tmp/reassign-topics.json;
+          echo ']}' >> /tmp/reassign-topics.json;
+
+          echo "# reassign-topics.json";
+          cat /tmp/reassign-topics.json;
+
+          ./bin/kafka-reassign-partitions.sh
+          --zookeeper=$ZOOKEEPER
+          --generate
+          --topics-to-move-json-file=/tmp/reassign-topics.json
+          --broker-list=$BROKERS > /tmp/generated.txt;
+
+          tail -n 1 /tmp/generated.txt > /tmp/proposed-reassignment.json;
+
+          echo "# proposed-reassignment.json";
+          cat /tmp/proposed-reassignment.json;
+
+          sed -i 's/"replicas":\[.\]/"replicas":[0,1,2]/g' /tmp/proposed-reassignment.json;
+          sed -i 's/,"log_dirs":\["any"\]//g' /tmp/proposed-reassignment.json;
+          echo "# proposed-reassignment.json";
+          cat /tmp/proposed-reassignment.json;
+
+          ./bin/kafka-reassign-partitions.sh
+          --zookeeper=$ZOOKEEPER
+          --execute
+          --reassignment-json-file=/tmp/proposed-reassignment.json;
+
+          echo "# Reassignment exited. Upon success you may want to run preferred-replica-election."
+      restartPolicy: Never
+  backoffLimit: 3
-- 
cgit v1.2.3