Add mongo elasticsearch sync using mongo-connector

README.md

@@ -22,6 +22,9 @@ Using consul to make a service-discoverable architecture.
 ## mongo_cluster
 Start 3 mongo instance to make a replica set.
 
+## mongo-elasticsearch
+Start mongo (as cluster) and elasticsearch, use a mongo-connector to sync the data from mongo to elasticsearch.
+
 ## mongo_webui
 Start 1 mongo instance and a mongo-express web tool to watch it.
 

mongo-elasticsearch/docker-compose.yml

@@ -0,0 +1,53 @@
+# This will start 5 nodes:
+# mongo will server as the only member of the cluster
+# mongosetup will init mongo as the master node
+# elasticsearch will server as the index search db
+# kibana will server as the web ui of elasticsearch
+# mongo-connector will pipeline mongodb content to the elasticsearch
+
+mongo:
+  image: mongo:3.0
+  hostname: mongo
+  mem_limit: 1024m
+  environment:
+    - TZ=Asia/Shanghai
+  expose:
+    - "27017"
+    - "28017"
+  restart: always
+  entrypoint: [ "/usr/bin/mongod", "--replSet", "rs", "--smallfiles" ]
+
+mongosetup:
+  image: mongo:3.0
+  mem_limit: 1024m
+  environment:
+    - TZ=Asia/Shanghai
+  links:
+    - mongo:mongo
+  volumes:
+    - ./scripts:/scripts
+  entrypoint: [ "bash", "/scripts/mongosetup.sh" ]
+
+elasticsearch:
+  image: elasticsearch:1.7
+  mem_limit: 1024m
+  environment:
+    - TZ=Asia/Shanghai
+  #volumes:
+    #- /opt/data/elasticsearch:/usr/share/elasticsearch/data
+  expose:
+    - "9200"
+    - "9300"
+
+kibana:
+  image: kibana:4.1
+  ports:
+    - "5601:5601"
+  links:
+    - elasticsearch:elasticsearch
+
+mongoconnector:
+  image: yeasy/mongo-connector
+  links:
+    - elasticsearch:elasticsearch
+    - mongo:mongo

mongo-elasticsearch/scripts/mongosetup.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+
+MONGODB1=`ping -c 1 mongo | head -1  | cut -d "(" -f 2 | cut -d ")" -f 1`
+
+echo "Waiting for startup.."
+until curl http://${MONGODB1}:28017/serverStatus\?text\=1 2>&1 | grep uptime | head -1; do
+  printf '.'
+  sleep 2
+done
+
+echo curl http://${MONGODB1}:28017/serverStatus\?text\=1 2>&1 | grep uptime | head -1
+echo "Started.."
+
+sleep 15
+
+echo SETUP time now: `date +"%T" `
+mongo --host ${MONGODB1}:27017 <<EOF
+   var cfg = {
+        "_id": "rs",
+        "version": 1,
+        "members": [
+            {
+                "_id": 0,
+                "host": "${MONGODB1}:27017",
+                "priority": 1
+            }
+        ]
+    };
+    rs.initiate(cfg, { force: true });
+    rs.reconfig(cfg, { force: true });
+    db.getMongo().setReadPref('nearest');
+EOF
+
+echo "rs.isMaster()" > is_master_check
+is_master_result=`mongo --host ${MONGODB1} < is_master_check`
+
+expected_result="\"ismaster\" : true"
+
+while true;
+do
+  if [ "${is_master_result/$expected_result}" = "$is_master_result" ] ; then
+    echo "Waiting for Mongod node to assume primary status..."
+    sleep 3
+    is_master_result=`mongo --host ${MONGODB1} < is_master_check`
+    echo ${is_master_result}
+  else
+    echo "Mongod node is now primary"
+    break;
+  fi
+done
+
+ping 127.0.0.1 > /dev/null