Add kafka cluster

kafka/README.md

@@ -8,5 +8,53 @@ Currently we support Kakfa 0.11.*.
 
 ```bash
 $ docker-compose up -d
+```
+
+## Test
+
+Open a new shell to the cli container:
+
+```bash
 $ docker exec -it cli bash
 ```
+
+In the container shell, create a new topic and wait for new msg  (dir default to KAFKA_HOME=/opt/kafka/).
+
+```bash
+# pwd
+/opt/kafka
+
+# bash /tmp/topic_create.sh
+Create a topic test by connecting to zookeeper
+Created topic test
+bin/kafka-topics.sh --create --zookeeper zookeeper:2181 --replication-factor 1 --partitions 1 --topic test
+Created topic test.
+
+# bash /tmp/topic_list.sh
+List all topics at zookeeper
+bin/kafka-topics.sh --list --zookeeper zookeeper:2181
+test
+
+# export KAFKA_HOST=kafka1
+# bash /tmp/msg_recv.sh
+Recving msg to topic test by connecting to kafka1
+bin/kafka-console-consumer.sh --bootstrap-server kafka1:9092 --topic test --from-beginning
+```
+
+Now open a new shell to the cli container.
+
+```bash
+# export KAFKA_HOST=kafka2
+# bash /tmp/msg_send.sh
+Send msg to topic test by connecting to kafka2
+bin/kafka-console-producer.sh --broker-list kafka2:9092 --topic test
+>msg_hello
+```
+
+Check the recving msg shell to get that msg.
+
+## Stop
+
+```bash
+$ docker-compose down
+```

kafka/docker-compose.yaml

@@ -9,27 +9,54 @@ version: '2'
 
 services:
   zookeeper:
-    image: yeasy/kafka:0.11.0.1
+    image: yeasy/kafka:latest
     container_name: zookeeper
     hostname: zookeeper
     ports:
       - "2181:2181"  # zk connection port
     command: bash -c 'bash /tmp/start_zk.sh'
 
-  kafka: # kafka service
+  kafka0: # kafka service
-    image: yeasy/kafka:0.11.0.1
+    image: yeasy/kafka:latest
-    container_name: kafka
+    container_name: kafka0
-    hostname: kafka
+    hostname: kafka0
     ports:
       - "9092:9092"
+    environment:
+      - KAFKA_BROKER_ID=0 # when bootup, will read this and update the server.properties
+    depends_on:
+      - zookeeper
+    command: bash -c 'bash /tmp/start_kafka.sh'
+
+  kafka1: # kafka service
+    image: yeasy/kafka:latest
+    container_name: kafka1
+    hostname: kafka1
+    ports:
+      - "9093:9092"
+    environment:
+      - KAFKA_BROKER_ID=1 # when bootup, will read this and update the server.properties
+    depends_on:
+      - zookeeper
+    command: bash -c 'bash /tmp/start_kafka.sh'
+
+  kafka2: # kafka service
+    image: yeasy/kafka:latest
+    container_name: kafka2
+    hostname: kafka2
+    ports:
+      - "9094:9092"
+    environment:
+      - KAFKA_BROKER_ID=2 # when bootup, will read this and update the server.properties
     depends_on:
       - zookeeper
     command: bash -c 'bash /tmp/start_kafka.sh'
 
   cli:  # used for testing kafka cmds, see /tmp/*.sh
-    image: yeasy/kafka:0.11.0.1
+    image: yeasy/kafka:latest
     container_name: cli
     hostname: cli
     environment:
       - ZK_HOST=zookeeper # zk host to connect to
-    command: bash -c 'while true; do sleep 20170915; done'
+      - KAFKA_HOST=kafka1 # customized the kafka broker to send
+    command: bash -c 'while true; do sleep 20200116; done'