diff --git a/docker/ResourceManager.dockerfile b/docker/ResourceManager.dockerfile
index a364847..8bb1d5a 100644
--- a/docker/ResourceManager.dockerfile
+++ b/docker/ResourceManager.dockerfile
@@ -21,7 +21,7 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh
RUN sh /install-yarn.sh
# Copy over myriad libraries
-ADD /libs/myriad-executor-0.0.1.jar /usr/local/libexec/mesos/
+ADD /libs/myriad-executor-runnable-0.0.1.jar /usr/local/libexec/mesos/
ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/
diff --git a/docker/build-myriad.sh b/docker/build-myriad.sh
index 39f012f..ca89d01 100755
--- a/docker/build-myriad.sh
+++ b/docker/build-myriad.sh
@@ -8,7 +8,6 @@
if [ ! "$1" == "--skipBuild" ]; then
../gradlew build -b ../build.gradle
- ../gradlew build capsuleExecutor -b ../build.gradle
fi
# Create the temp directories
@@ -21,6 +20,7 @@ if [ ! -d "libs" ]; then
fi
# Copy over the Java Libraries
-cp -rf ../build/libs/* libs/
+cp -rf ../myriad-scheduler/build/libs/* libs/
+cp -rf ../myriad-executor/build/libs/myriad-executor-runnable-0.0.1.jar libs/
echo "Init complete! " #Modify config/myriad-default-config.yml to your liking before building the docker image"
diff --git a/docs/API.md b/docs/API.md
index d97524f..c8478d4 100644
--- a/docs/API.md
+++ b/docs/API.md
@@ -118,7 +118,7 @@ Sample Response:
"jvmMaxMemoryMB": {
"present": true
},
- "path": "file:///root/myriad-executor-0.0.1.jar"
+ "path": "file:///root/myriad-executor-runnable-0.0.1.jar"
}
}
```
\ No newline at end of file
diff --git a/docs/myriad-configuration.md b/docs/myriad-configuration.md
index d68eac2..6c43c02 100644
--- a/docs/myriad-configuration.md
+++ b/docs/myriad-configuration.md
@@ -66,10 +66,10 @@ nodemanager:
# configures yarn-site.xml to attach YARN's cgroups under Mesos' cgroup hierarchy.
executor:
jvmMaxMemoryMB: 256 # Xmx for myriad's executor that launches Node Manager.
- path: file://localhost/usr/local/libexec/mesos/myriad-executor-0.0.1.jar # Path for the myriad's executor binary.
+ path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar # Path for the myriad's executor binary.
# Also supports, hdfs:// notation.
# These are for remote distribution. Hdfs is assumed, but http, file, and ftp are also possible.
- # path: hdfs://namenode:port/dist/myriad-executor-0.0.1.jar # Path for the myriad's executor binary.
+ # path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar # Path for the myriad's executor binary.
# nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz # the uri to d/l hadoop from # Path to the Hadoop tarball
# Environment variables required to launch Node Manager process. Admin can also pass other environment variables to NodeManager.
diff --git a/docs/myriad-dev.md b/docs/myriad-dev.md
index 7cffcaf..b839fce 100644
--- a/docs/myriad-dev.md
+++ b/docs/myriad-dev.md
@@ -10,51 +10,73 @@ Myriad requires two components to be built:
* **Myriad Scheduler** - This component plugs into Resource Manager process and negotiates resources from Mesos. It is responsible to launch Node Manager processes via Mesos.
* **Myriad Executor** - This component implements [Mesos Executor](http://mesos.apache.org/api/latest/java/org/apache/mesos/Executor.html) interface. It is launched by Myriad Scheduler via Mesos and runs as a separate process on each mesos-slave. Myriad Executor is responsible for launching Node Manager process as a Mesos Task.
+The project is a multi-project build using gradle. You can go to the `$PROJECT_HOME` (which is where you cloned the git project to) and type `./gradlew build` which will build all projects or you can go into each individual project and build them separately.
+
+### Build All
+**Before** building you may still want to follow the instructions in **Building Myriad Scheduler** in order to build with the appropriate yml file.
+
+To build all, from the `$PROJECT_HOME`, run:
+
+```
+./gradlew build
+```
+
+At this point the jars will be located in the following directories (relative to $PROJECT_HOME).
+
+```
+# scheduler jars
+./myriad-scheduler/build/libs/
+
+# executor jar
+./myriad-executor/build/libs/
+
+```
+
### Building Myriad Scheduler
-**Before** building Myriad Scheduler, please modify [myriad-config-default.yml](../src/main/resources/myriad-config-default.yml) with appropriate configuration properties (Please read [Myriad Configuration Properties](myriad-configuration.md)). This is needed because currently myriad-config-default.yml will be embedded into Myriad Scheduler jar.
+**Before** building Myriad Scheduler, please modify [myriad-config-default.yml](../myriad-scheduler/src/main/resources/myriad-config-default.yml) with appropriate configuration properties (Please read [Myriad Configuration Properties](myriad-configuration.md)). This is needed because currently myriad-config-default.yml will be embedded into Myriad Scheduler jar.
-To build Myriad Scheduler run:
+To build Myriad Scheduler, from `$PROJECT_HOME/myriad-scheduler` run:
```bash
./gradlew build
```
-This will build myriad-x.x.x.jar and download the runtime jars and place them inside ```PROJECT_HOME/build/libs/``` directory.
+This will build myriad-x.x.x.jar and download the runtime jars and place them inside `./build/libs/` directory (relative to `$PROJECT_HOME/myriad-scheduler`).
### Building Myriad Executor
-To build self-contained executor jar, run:
+To build self-contained executor jar, from `$PROJECT_HOME/myriad-executor` run:
```bash
-./gradlew capsuleExecutor
+./gradlew build
```
-This will build myriad-executor-x.x.x.jar and place it inside ```PROJECT_HOME/build/libs/``` directory.
+This will build myriad-executor-runnable-xxx.jar and place it inside `PROJECT_HOME/myriad-executor/build/libs/` directory.
### Deploying Myriad Scheduler
To deploy Myriad Scheduler, please follow the below steps:
1. Build Myriad Scheduler
-2. Copy all jars under ```PROJECT_HOME/build/libs/``` directory, to YARN ResourceManager's classpath. For ex: Copy all jars to ```$YARN_HOME/share/hadoop/yarn/lib/```
+2. Copy all jars under `PROJECT_HOME/myriad-scheduler/build/libs/` directory, to YARN ResourceManager's classpath. For ex: Copy all jars to `$YARN_HOME/share/hadoop/yarn/lib/`
### Deploying Myriad Executor
To deploy Myriad Executor, please follow the below steps:
1. Build Myriad Executor
-2. Copy myriad-executor-xxx.jar from ```PROJECT_HOME/build/libs/``` to each mesos slave's ```/usr/local/libexec/mesos``` directory.
+2. Copy myriad-executor-runnable-xxx.jar from `PROJECT_HOME/myriad-executor/build/libs/` to each mesos slave's `/usr/local/libexec/mesos` directory.
-Note: For advanced readers one can also copy myriad-executor-xxx.jar to any other directory on slave filesystem, or it can be copied to HDFS as well. In either case, one needs to update the executor's path property in myriad-config-default.yml file, and prepend the path with either ```file://``` or ```hdfs://```, as appropriate.
+Note: For advanced readers one can also copy myriad-executor-runnable-xxx.jar to any other directory on slave filesystem, or it can be copied to HDFS as well. In either case, one needs to update the executor's path property in myriad-config-default.yml file, and prepend the path with either `file://` or `hdfs://`, as appropriate.
### Running Myriad Scheduler
To run Myriad Scheduler, you need to follow following steps:
-* Add ```MESOS_NATIVE_JAVA_LIBRARY``` environment variable to ResourceManager's environment variables, for ex: Add following to ```$YARN_HOME/etc/hadoop/hadoop-env.sh```:
+* Add `MESOS_NATIVE_JAVA_LIBRARY` environment variable to ResourceManager's environment variables, for ex: Add following to `$YARN_HOME/etc/hadoop/hadoop-env.sh`:
```bash
export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so
```
-* Add following to ```$YARN_HOME/etc/hadoop/yarn-site.xml```:
+* Add following to `$YARN_HOME/etc/hadoop/yarn-site.xml`:
```xml
@@ -110,7 +132,7 @@ Optional: If you would like to enable cgroups, please add following to ```yarn-s
```
-And, following to ```$YARN_HOME/etc/hadoop/myriad-config-default.yml```:
+And, following to `$YARN_HOME/etc/hadoop/myriad-config-default.yml`:
```yaml
...
diff --git a/docs/myriad-remote-distribution-configuration.md b/docs/myriad-remote-distribution-configuration.md
index 7c3306a..009d6a1 100644
--- a/docs/myriad-remote-distribution-configuration.md
+++ b/docs/myriad-remote-distribution-configuration.md
@@ -12,29 +12,35 @@ your installation.
First configure the Resource Manager as normal.
-At this point you build myriad with the commands:
-```Shell
+At this point, from the project root you build myriad with the commands:
+
+```
./gradlew build
-./gradlew capsuleExecutor
```
+
and copy the jars and config onto your yarn classpath:
-```Shell
-cp build/libs/*.jar /opt/hadoop-2.5.0/share/hadoop/yarn/lib/
-cp src/main/resources/myriad-config-default.xml
+
```
-You will also need to place `myriad-executor-0.0.1.jar` in hdfs
-```Shell
-hadoop fs -put build/libs/myriad-executor-0.0.1.jar /dist
+cp myriad-scheduler/build/libs/*.jar /opt/hadoop-2.5.0/share/hadoop/yarn/lib/
+cp myriad-scheduler/src/main/resources/myriad-config-default.yml /opt/hadoop-2.5.0/share/hadoop/yarn/lib/
+```
+
+You will also need to place `myriad-executor-runnable-0.0.1.jar` in hdfs
+
```
+hadoop fs -put myriad-executor/build/libs/myriad-executor-runnable-0.0.1.jar /dist
+```
+
Edit `/opt/hadoop/etc/hadoop/myriad-config-default`. For standard configuration see
[myriad-configuration](myriad-configuration.md). To enable remote binary distribution you must set the following options:
+
```YAML
frameworkSuperUser: admin # Must be root or have passwordless sudo on all nodes!
frameworkUser: hduser # Should be the same user running the resource manager.
# Must exist on all nodes and be in the 'hadoop' group
executor:
nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz
- path: hdfs://namenode:port/dist/myriad-executor-0.0.1.jar
+ path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar
yarnEnvironment:
YARN_HOME: hadoop-2.5.0 # This should be relative if nodeManagerUri is set
```
@@ -42,11 +48,13 @@ yarnEnvironment:
Configure `/opt/hadoop-2.5.0/etc/hadoop/yarn-site.xml` as instructed in: [myriad-configuration](myriad-configuration.md)
Create the tarball and place it in hdfs:
-```Shell
+
+```
cd ~
sudo cp -rp /opt/hadoop-2.5.0 .
sudo rm hadoop-2.5.0/etc/hadoop/*.xml
sudo tar -zcpf ~/hadoop-2.5.0.tar.gz hadoop-2.5.0
hadoop fs -put ~/hadoop-2.5.0.tar.gz /dist
```
+
You can now start the resource manager and attempt to flex up the cluster!
diff --git a/docs/vagrant.md b/docs/vagrant.md
index 0dddf9c..1355e3c 100644
--- a/docs/vagrant.md
+++ b/docs/vagrant.md
@@ -7,32 +7,41 @@ You can use following guide to setup a cluster in a virtual machine.
* Vagrant
To start the cluster run following:
-```shell
+
+```
vagrant up
```
+
At this point the VM will have a single node mesos cluster running.
To ssh in the cluster run following:
-```shell
+
+```
vagrant ssh
```
+
The password for vagrant user is 'vagrant'.
To setup YARN/Hadoop inside VM, run following:
-```shell
+
+```
cd /vagrant
./setup-yarn-1.sh
```
+
This will create a user hduser in group hadoop. Remember the password that you provide for this user.
Now, do following:
-```shell
+
+```
sudo su - hduser
cd /vagrant
./setup-yarn-2.sh
```
+
If everything goes fine you'll see following processes running (process ids will be different):
-```shell
+
+```
9844 Jps
6709 NameNode
6393 JobHistoryServer
@@ -40,22 +49,18 @@ If everything goes fine you'll see following processes running (process ids will
```
To build myriad scheduler inside VM, you can do following:
-```shell
+
+```
cd /vagrant
./gradlew build
```
-At this point, myriad's scheduler jar and all the runtime dependencies will be available here: /vagrant/build/libs/*. Please copy these jars to $YARN_HOME/share/hadoop/yarn/lib/.
+At this point, myriad's scheduler jar and all the runtime dependencies will be available here: `/vagrant/myriad-scheduler/build/libs/*`. Please copy these jars to `$YARN_HOME/share/hadoop/yarn/lib/`.
-To build self-contained executable JAR, you can run following:
-```shell
-cd /vagrant
-./gradlew capsuleExecutor
-```
-
-At this point, the self-contained myriad executor jar will be available here: /vagrant/build/libs/myriad-executor-x.y.z.jar. Please copy this jar to /usr/local/libexec/mesos/.
+The self-contained myriad executor jar will be available here: `/vagrant/myriad-executor/build/libs/myriad-executor-runnable-x.y.z.jar`. Please copy this jar to `/usr/local/libexec/mesos/`.
To configure YARN to use Myriad, please update ```$YARN_HOME/etc/hadoop/yarn-site.xml``` with following:
+
```xml
yarn.nodemanager.resource.cpu-vcores
@@ -75,6 +80,7 @@ To configure YARN to use Myriad, please update ```$YARN_HOME/etc/hadoop/yarn-sit
```
To configure Myriad itself, please add following file to ```$YARN_HOME/etc/hadoop/myriad-default-config.yml```:
+
```yml
mesosMaster: 10.141.141.20:5050
checkpoint: false
@@ -101,11 +107,12 @@ nodemanager:
cgroups: false
executor:
jvmMaxMemoryMB: 256
- path: file://localhost/usr/local/libexec/mesos/myriad-executor-0.0.1.jar
+ path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar
```
To launch Myriad, you can run following:
-```shell
+
+```
sudo su hduser
yarn-daemon.sh start resourcemanager
```
diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java b/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java
index 9c788f8..29ae506 100644
--- a/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java
+++ b/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java
@@ -49,7 +49,7 @@
* cgroups: false
* executor:
* jvmMaxMemoryMB: 256
- * path: file://localhost/usr/local/libexec/mesos/myriad-executor-0.0.1.jar
+ * path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar
* yarnEnvironment:
* YARN_HOME: /usr/local/hadoop
*/
diff --git a/myriad-scheduler/src/main/resources/myriad-config-default.yml b/myriad-scheduler/src/main/resources/myriad-config-default.yml
index 7916340..d6f4746 100644
--- a/myriad-scheduler/src/main/resources/myriad-config-default.yml
+++ b/myriad-scheduler/src/main/resources/myriad-config-default.yml
@@ -28,10 +28,10 @@ nodemanager:
cgroups: false
executor:
jvmMaxMemoryMB: 256
- path: file:///usr/local/libexec/mesos/myriad-executor-0.0.1.jar
+ path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar
#The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid.
#nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz
- #path: hdfs://namenode:port/dist/myriad-executor-0.0.1.jar
+ #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar
yarnEnvironment:
YARN_HOME: /usr/local/hadoop
#YARN_HOME: hadoop-2.5.0 #this should be relative if nodeManagerUri is set