Add mongo elasticsearch sync using mongo-connector

pull/1/head
Baohua Yang 2015-08-21 15:37:35 +08:00
parent b9ae9fe9f8
commit da80fd7da4
3 changed files with 108 additions and 0 deletions

View File

@ -22,6 +22,9 @@ Using consul to make a service-discoverable architecture.
## mongo_cluster
Start 3 mongo instance to make a replica set.
## mongo-elasticsearch
Start mongo (as cluster) and elasticsearch, use a mongo-connector to sync the data from mongo to elasticsearch.
## mongo_webui
Start 1 mongo instance and a mongo-express web tool to watch it.

View File

@ -0,0 +1,53 @@
# This will start 5 nodes:
# mongo will server as the only member of the cluster
# mongosetup will init mongo as the master node
# elasticsearch will server as the index search db
# kibana will server as the web ui of elasticsearch
# mongo-connector will pipeline mongodb content to the elasticsearch
mongo:
image: mongo:3.0
hostname: mongo
mem_limit: 1024m
environment:
- TZ=Asia/Shanghai
expose:
- "27017"
- "28017"
restart: always
entrypoint: [ "/usr/bin/mongod", "--replSet", "rs", "--smallfiles" ]
mongosetup:
image: mongo:3.0
mem_limit: 1024m
environment:
- TZ=Asia/Shanghai
links:
- mongo:mongo
volumes:
- ./scripts:/scripts
entrypoint: [ "bash", "/scripts/mongosetup.sh" ]
elasticsearch:
image: elasticsearch:1.7
mem_limit: 1024m
environment:
- TZ=Asia/Shanghai
#volumes:
#- /opt/data/elasticsearch:/usr/share/elasticsearch/data
expose:
- "9200"
- "9300"
kibana:
image: kibana:4.1
ports:
- "5601:5601"
links:
- elasticsearch:elasticsearch
mongoconnector:
image: yeasy/mongo-connector
links:
- elasticsearch:elasticsearch
- mongo:mongo

View File

@ -0,0 +1,52 @@
#!/bin/bash
MONGODB1=`ping -c 1 mongo | head -1 | cut -d "(" -f 2 | cut -d ")" -f 1`
echo "Waiting for startup.."
until curl http://${MONGODB1}:28017/serverStatus\?text\=1 2>&1 | grep uptime | head -1; do
printf '.'
sleep 2
done
echo curl http://${MONGODB1}:28017/serverStatus\?text\=1 2>&1 | grep uptime | head -1
echo "Started.."
sleep 15
echo SETUP time now: `date +"%T" `
mongo --host ${MONGODB1}:27017 <<EOF
var cfg = {
"_id": "rs",
"version": 1,
"members": [
{
"_id": 0,
"host": "${MONGODB1}:27017",
"priority": 1
}
]
};
rs.initiate(cfg, { force: true });
rs.reconfig(cfg, { force: true });
db.getMongo().setReadPref('nearest');
EOF
echo "rs.isMaster()" > is_master_check
is_master_result=`mongo --host ${MONGODB1} < is_master_check`
expected_result="\"ismaster\" : true"
while true;
do
if [ "${is_master_result/$expected_result}" = "$is_master_result" ] ; then
echo "Waiting for Mongod node to assume primary status..."
sleep 3
is_master_result=`mongo --host ${MONGODB1} < is_master_check`
echo ${is_master_result}
else
echo "Mongod node is now primary"
break;
fi
done
ping 127.0.0.1 > /dev/null