spark_public_dns

spark_public_dns from Pastebin

    
        version: '3'
services:
    spark-master:
        image: gettyimages/spark
        command: bin/spark-class org.apache.spark.deploy.master.Master -h spark-master
        hostname: spark-master
        environment:
          MASTER: spark://spark-master:7077
          SPARK_CONF_DIR: /conf
          SPARK_PUBLIC_DNS:  #For cloud here ip or dns of the cloud server will come.
          #PYSPARK_PYTHON: python3.5.3 bin/pyspark # For same python version in workers and master
        expose:
          - 7001
          - 7002
          - 7003
          - 7004
          - 7005
          - 7006
          - 7077
          - 6066
        networks:
          - spark-network
        ports:
          - 6066:6066
          - 7077:7077
          - 8080:8080
        volumes:
          - v-spark-master:/conf
     
    spark-worker-1:
        image: gettyimages/spark
        command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
        hostname: spark-worker-1
        environment:
          SPARK_CONF_DIR: /conf
          SPARK_PUBLIC_DNS:  #For cloud here ip or dns of the cloud server will come.
          SPARK_WORKER_CORES: 1
          SPARK_WORKER_MEMORY: 3g
          SPARK_WORKER_PORT: 8881
          SPARK_WORKER_WEBUI_PORT: 8081
          #PYSPARK_PYTHON: python3.5.3 bin/pyspark # For same python version in workers and master
        links:
          - spark-master
        expose:
          - 7012
          - 7013
          - 7014
          - 7015
          - 7016
          - 8881
        networks:
          - spark-network
        ports:
          - 8081:8081
        volumes:
          - v-spark-worker-1:/conf

    spark-worker-2:
        image: gettyimages/spark
        command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
        hostname: spark-worker-2
        environment:
          SPARK_CONF_DIR: /conf
          SPARK_PUBLIC_DNS:  #For cloud here ip or dns of the cloud server will come.
          SPARK_WORKER_CORES: 1
          SPARK_WORKER_MEMORY: 3g
          SPARK_WORKER_PORT: 8881
          SPARK_WORKER_WEBUI_PORT: 8081
          #PYSPARK_PYTHON: python3.5.3 bin/pyspark # For same python version in workers and master
        links:
          - spark-master
        expose:
          - 7012
          - 7013
          - 7014
          - 7015
          - 7016
          - 8881
        networks:
          - spark-network
        ports:
          - 8082:8081
        volumes:
          - v-spark-worker-2:/conf


volumes:
    v-spark-master:
    v-spark-worker-1:
    v-spark-worker-2:



networks:
    spark-network: