2015-09-22 15:22:12 +08:00
|
|
|
# http://github.com/yeasy/docker-compose-files
|
|
|
|
|
# This compose file will start spark master node and the worker node.
|
|
|
|
|
# All nodes will become a cluster automatically.
|
|
|
|
|
# You can run: docker-compose scale worker=2
|
2015-09-24 10:43:24 +08:00
|
|
|
# After startup, try submit a pi calculation application.
|
2018-09-10 20:00:47 +08:00
|
|
|
# /usr/local/spark/bin/spark-submit --master spark://master:7077 --class org.apache.spark.examples.SparkPi /usr/local/spark/lib/spark-examples-1.4.0-hadoop2.6.0.jar 1000
|
2015-09-22 15:22:12 +08:00
|
|
|
|
2023-03-03 11:41:39 +08:00
|
|
|
version: "3.9"
|
|
|
|
|
services:
|
|
|
|
|
master:
|
|
|
|
|
image: sequenceiq/spark:1.6.0
|
|
|
|
|
hostname: master
|
|
|
|
|
ports:
|
|
|
|
|
- "4040:4040"
|
|
|
|
|
- "8042:8042"
|
|
|
|
|
- "7077:7077"
|
|
|
|
|
- "8088:8088"
|
|
|
|
|
- "8080:8080"
|
|
|
|
|
restart: always
|
|
|
|
|
#mem_limit: 1024m
|
|
|
|
|
command: bash /usr/local/spark/sbin/start-master.sh && ping localhost > /dev/null
|
2015-09-22 15:22:12 +08:00
|
|
|
|
2023-03-03 11:41:39 +08:00
|
|
|
worker:
|
|
|
|
|
image: sequenceiq/spark:1.6.0
|
|
|
|
|
links:
|
|
|
|
|
- master:master
|
|
|
|
|
expose:
|
|
|
|
|
- "8081"
|
|
|
|
|
restart: always
|
|
|
|
|
command: bash /usr/local/spark/sbin/start-slave.sh spark://master:7077 && ping localhost >/dev/null
|
