This repository has been archived by the owner on Oct 4, 2023. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 2
/
Copy pathdocker-cloud.yml
98 lines (95 loc) · 2.51 KB
/
docker-cloud.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
spark-yarn-master:
image: 'jpizarrom/spark-yarn:1.6.0-master'
command: '-d'
mem_limit: 1024m
environment:
- SPARK_PUBLIC_DNS=$DOCKERCLOUD_CONTAINER_FQDN
- SPARK_MASTER_PORT=7077
- SPARK_MASTER_WEBUI_PORT=8080
- SPARK_WORKER_PORT=8888
- SPARK_WORKER_WEBUI_PORT=8081
- YARN_RESOURCEMANAGER_HEAPSIZE=256
- YARN_PROXYSERVER_HEAPSIZE=128
- YARN_NODEMANAGER_HEAPSIZE=256
- HADOOP_JOB_HISTORYSERVER_HEAPSIZE=256
- HADOOP_NAMENODE_HEAPSIZE=256
- HADOOP_DATANODE_HEAPSIZE=128
hostname: sandbox
ports:
- '4040'
- '8042:8042'
- '8080'
- '8088:8088'
- '8081'
- '50070:50070'
spark-yarn-nodemanager:
image: 'jpizarrom/spark-yarn:1.6.0-nodemanager'
command: '-d'
mem_limit: 1024m
environment:
- SPARK_PUBLIC_DNS=$DOCKERCLOUD_CONTAINER_FQDN
- SPARK_MASTER_PORT=7077
- SPARK_MASTER_WEBUI_PORT=8080
- SPARK_WORKER_PORT=8888
- SPARK_WORKER_WEBUI_PORT=8081
- YARN_RESOURCEMANAGER_HEAPSIZE=256
- YARN_PROXYSERVER_HEAPSIZE=128
- YARN_NODEMANAGER_HEAPSIZE=256
- HADOOP_JOB_HISTORYSERVER_HEAPSIZE=256
- HADOOP_NAMENODE_HEAPSIZE=256
- HADOOP_DATANODE_HEAPSIZE=128
extra_hosts:
- 'sandbox:172.17.0.6'
ports:
- '4040'
- '8042'
- '8080'
- '8081'
target_num_containers: 2
spark-yarn-client:
image: 'jpizarrom/spark-yarn:1.6.0-client'
command: '-d'
mem_limit: 1024m
environment:
- SPARK_PUBLIC_DNS=$DOCKERCLOUD_CONTAINER_FQDN
- SPARK_MASTER_PORT=7077
- SPARK_MASTER_WEBUI_PORT=8080
- SPARK_WORKER_PORT=8888
- SPARK_WORKER_WEBUI_PORT=8081
- YARN_RESOURCEMANAGER_HEAPSIZE=256
- YARN_PROXYSERVER_HEAPSIZE=128
- YARN_NODEMANAGER_HEAPSIZE=256
- HADOOP_JOB_HISTORYSERVER_HEAPSIZE=256
- HADOOP_NAMENODE_HEAPSIZE=256
- HADOOP_DATANODE_HEAPSIZE=128
extra_hosts:
- 'sandbox:172.17.0.6'
ports:
- '4040'
- '8042'
- '8080'
- '8081'
spark-yarn-zeppelin:
image: 'jpizarrom/spark-yarn:1.6.0-zeppelin'
# command: '-d'
mem_limit: 3072m
environment:
- SPARK_PUBLIC_DNS=$DOCKERCLOUD_CONTAINER_FQDN
- SPARK_MASTER_PORT=7077
- SPARK_MASTER_WEBUI_PORT=8080
- SPARK_WORKER_PORT=8888
- SPARK_WORKER_WEBUI_PORT=8081
- YARN_RESOURCEMANAGER_HEAPSIZE=256
- YARN_PROXYSERVER_HEAPSIZE=128
- YARN_NODEMANAGER_HEAPSIZE=256
- HADOOP_JOB_HISTORYSERVER_HEAPSIZE=256
- HADOOP_NAMENODE_HEAPSIZE=256
- HADOOP_DATANODE_HEAPSIZE=128
extra_hosts:
- 'sandbox:172.17.0.6'
ports:
- '4040'
- '8042'
- '8080'
- '8081'
- '8082'