diff --git a/docker/ResourceManager.dockerfile b/docker/ResourceManager.dockerfile index a364847..8bb1d5a 100644 --- a/docker/ResourceManager.dockerfile +++ b/docker/ResourceManager.dockerfile @@ -21,7 +21,7 @@ ADD myriad-bin/install-yarn.sh /install-yarn.sh RUN sh /install-yarn.sh # Copy over myriad libraries -ADD /libs/myriad-executor-0.0.1.jar /usr/local/libexec/mesos/ +ADD /libs/myriad-executor-runnable-0.0.1.jar /usr/local/libexec/mesos/ ADD /libs/* /usr/local/hadoop/share/hadoop/yarn/lib/ diff --git a/docker/build-myriad.sh b/docker/build-myriad.sh index 39f012f..ca89d01 100755 --- a/docker/build-myriad.sh +++ b/docker/build-myriad.sh @@ -8,7 +8,6 @@ if [ ! "$1" == "--skipBuild" ]; then ../gradlew build -b ../build.gradle - ../gradlew build capsuleExecutor -b ../build.gradle fi # Create the temp directories @@ -21,6 +20,7 @@ if [ ! -d "libs" ]; then fi # Copy over the Java Libraries -cp -rf ../build/libs/* libs/ +cp -rf ../myriad-scheduler/build/libs/* libs/ +cp -rf ../myriad-executor/build/libs/myriad-executor-runnable-0.0.1.jar libs/ echo "Init complete! " #Modify config/myriad-default-config.yml to your liking before building the docker image" diff --git a/docs/API.md b/docs/API.md index d97524f..c8478d4 100644 --- a/docs/API.md +++ b/docs/API.md @@ -118,7 +118,7 @@ Sample Response: "jvmMaxMemoryMB": { "present": true }, - "path": "file:///root/myriad-executor-0.0.1.jar" + "path": "file:///root/myriad-executor-runnable-0.0.1.jar" } } ``` \ No newline at end of file diff --git a/docs/myriad-configuration.md b/docs/myriad-configuration.md index d68eac2..6c43c02 100644 --- a/docs/myriad-configuration.md +++ b/docs/myriad-configuration.md @@ -66,10 +66,10 @@ nodemanager: # configures yarn-site.xml to attach YARN's cgroups under Mesos' cgroup hierarchy. executor: jvmMaxMemoryMB: 256 # Xmx for myriad's executor that launches Node Manager. - path: file://localhost/usr/local/libexec/mesos/myriad-executor-0.0.1.jar # Path for the myriad's executor binary. + path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar # Path for the myriad's executor binary. # Also supports, hdfs:// notation. # These are for remote distribution. Hdfs is assumed, but http, file, and ftp are also possible. - # path: hdfs://namenode:port/dist/myriad-executor-0.0.1.jar # Path for the myriad's executor binary. + # path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar # Path for the myriad's executor binary. # nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz # the uri to d/l hadoop from # Path to the Hadoop tarball # Environment variables required to launch Node Manager process. Admin can also pass other environment variables to NodeManager. diff --git a/docs/myriad-dev.md b/docs/myriad-dev.md index 7cffcaf..b839fce 100644 --- a/docs/myriad-dev.md +++ b/docs/myriad-dev.md @@ -10,51 +10,73 @@ Myriad requires two components to be built: * **Myriad Scheduler** - This component plugs into Resource Manager process and negotiates resources from Mesos. It is responsible to launch Node Manager processes via Mesos. * **Myriad Executor** - This component implements [Mesos Executor](http://mesos.apache.org/api/latest/java/org/apache/mesos/Executor.html) interface. It is launched by Myriad Scheduler via Mesos and runs as a separate process on each mesos-slave. Myriad Executor is responsible for launching Node Manager process as a Mesos Task. +The project is a multi-project build using gradle. You can go to the `$PROJECT_HOME` (which is where you cloned the git project to) and type `./gradlew build` which will build all projects or you can go into each individual project and build them separately. + +### Build All +**Before** building you may still want to follow the instructions in **Building Myriad Scheduler** in order to build with the appropriate yml file. + +To build all, from the `$PROJECT_HOME`, run: + +``` +./gradlew build +``` + +At this point the jars will be located in the following directories (relative to $PROJECT_HOME). + +``` +# scheduler jars +./myriad-scheduler/build/libs/ + +# executor jar +./myriad-executor/build/libs/ + +``` + ### Building Myriad Scheduler -**Before** building Myriad Scheduler, please modify [myriad-config-default.yml](../src/main/resources/myriad-config-default.yml) with appropriate configuration properties (Please read [Myriad Configuration Properties](myriad-configuration.md)). This is needed because currently myriad-config-default.yml will be embedded into Myriad Scheduler jar. +**Before** building Myriad Scheduler, please modify [myriad-config-default.yml](../myriad-scheduler/src/main/resources/myriad-config-default.yml) with appropriate configuration properties (Please read [Myriad Configuration Properties](myriad-configuration.md)). This is needed because currently myriad-config-default.yml will be embedded into Myriad Scheduler jar. -To build Myriad Scheduler run: +To build Myriad Scheduler, from `$PROJECT_HOME/myriad-scheduler` run: ```bash ./gradlew build ``` -This will build myriad-x.x.x.jar and download the runtime jars and place them inside ```PROJECT_HOME/build/libs/``` directory. +This will build myriad-x.x.x.jar and download the runtime jars and place them inside `./build/libs/` directory (relative to `$PROJECT_HOME/myriad-scheduler`). ### Building Myriad Executor -To build self-contained executor jar, run: +To build self-contained executor jar, from `$PROJECT_HOME/myriad-executor` run: ```bash -./gradlew capsuleExecutor +./gradlew build ``` -This will build myriad-executor-x.x.x.jar and place it inside ```PROJECT_HOME/build/libs/``` directory. +This will build myriad-executor-runnable-xxx.jar and place it inside `PROJECT_HOME/myriad-executor/build/libs/` directory. ### Deploying Myriad Scheduler To deploy Myriad Scheduler, please follow the below steps: 1. Build Myriad Scheduler -2. Copy all jars under ```PROJECT_HOME/build/libs/``` directory, to YARN ResourceManager's classpath. For ex: Copy all jars to ```$YARN_HOME/share/hadoop/yarn/lib/``` +2. Copy all jars under `PROJECT_HOME/myriad-scheduler/build/libs/` directory, to YARN ResourceManager's classpath. For ex: Copy all jars to `$YARN_HOME/share/hadoop/yarn/lib/` ### Deploying Myriad Executor To deploy Myriad Executor, please follow the below steps: 1. Build Myriad Executor -2. Copy myriad-executor-xxx.jar from ```PROJECT_HOME/build/libs/``` to each mesos slave's ```/usr/local/libexec/mesos``` directory. +2. Copy myriad-executor-runnable-xxx.jar from `PROJECT_HOME/myriad-executor/build/libs/` to each mesos slave's `/usr/local/libexec/mesos` directory. -Note: For advanced readers one can also copy myriad-executor-xxx.jar to any other directory on slave filesystem, or it can be copied to HDFS as well. In either case, one needs to update the executor's path property in myriad-config-default.yml file, and prepend the path with either ```file://``` or ```hdfs://```, as appropriate. +Note: For advanced readers one can also copy myriad-executor-runnable-xxx.jar to any other directory on slave filesystem, or it can be copied to HDFS as well. In either case, one needs to update the executor's path property in myriad-config-default.yml file, and prepend the path with either `file://` or `hdfs://`, as appropriate. ### Running Myriad Scheduler To run Myriad Scheduler, you need to follow following steps: -* Add ```MESOS_NATIVE_JAVA_LIBRARY``` environment variable to ResourceManager's environment variables, for ex: Add following to ```$YARN_HOME/etc/hadoop/hadoop-env.sh```: +* Add `MESOS_NATIVE_JAVA_LIBRARY` environment variable to ResourceManager's environment variables, for ex: Add following to `$YARN_HOME/etc/hadoop/hadoop-env.sh`: ```bash export MESOS_NATIVE_JAVA_LIBRARY=/usr/local/lib/libmesos.so ``` -* Add following to ```$YARN_HOME/etc/hadoop/yarn-site.xml```: +* Add following to `$YARN_HOME/etc/hadoop/yarn-site.xml`: ```xml @@ -110,7 +132,7 @@ Optional: If you would like to enable cgroups, please add following to ```yarn-s ``` -And, following to ```$YARN_HOME/etc/hadoop/myriad-config-default.yml```: +And, following to `$YARN_HOME/etc/hadoop/myriad-config-default.yml`: ```yaml ... diff --git a/docs/myriad-remote-distribution-configuration.md b/docs/myriad-remote-distribution-configuration.md index 7c3306a..009d6a1 100644 --- a/docs/myriad-remote-distribution-configuration.md +++ b/docs/myriad-remote-distribution-configuration.md @@ -12,29 +12,35 @@ your installation. First configure the Resource Manager as normal. -At this point you build myriad with the commands: -```Shell +At this point, from the project root you build myriad with the commands: + +``` ./gradlew build -./gradlew capsuleExecutor ``` + and copy the jars and config onto your yarn classpath: -```Shell -cp build/libs/*.jar /opt/hadoop-2.5.0/share/hadoop/yarn/lib/ -cp src/main/resources/myriad-config-default.xml + ``` -You will also need to place `myriad-executor-0.0.1.jar` in hdfs -```Shell -hadoop fs -put build/libs/myriad-executor-0.0.1.jar /dist +cp myriad-scheduler/build/libs/*.jar /opt/hadoop-2.5.0/share/hadoop/yarn/lib/ +cp myriad-scheduler/src/main/resources/myriad-config-default.yml /opt/hadoop-2.5.0/share/hadoop/yarn/lib/ +``` + +You will also need to place `myriad-executor-runnable-0.0.1.jar` in hdfs + ``` +hadoop fs -put myriad-executor/build/libs/myriad-executor-runnable-0.0.1.jar /dist +``` + Edit `/opt/hadoop/etc/hadoop/myriad-config-default`. For standard configuration see [myriad-configuration](myriad-configuration.md). To enable remote binary distribution you must set the following options: + ```YAML frameworkSuperUser: admin # Must be root or have passwordless sudo on all nodes! frameworkUser: hduser # Should be the same user running the resource manager. # Must exist on all nodes and be in the 'hadoop' group executor: nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz - path: hdfs://namenode:port/dist/myriad-executor-0.0.1.jar + path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar yarnEnvironment: YARN_HOME: hadoop-2.5.0 # This should be relative if nodeManagerUri is set ``` @@ -42,11 +48,13 @@ yarnEnvironment: Configure `/opt/hadoop-2.5.0/etc/hadoop/yarn-site.xml` as instructed in: [myriad-configuration](myriad-configuration.md) Create the tarball and place it in hdfs: -```Shell + +``` cd ~ sudo cp -rp /opt/hadoop-2.5.0 . sudo rm hadoop-2.5.0/etc/hadoop/*.xml sudo tar -zcpf ~/hadoop-2.5.0.tar.gz hadoop-2.5.0 hadoop fs -put ~/hadoop-2.5.0.tar.gz /dist ``` + You can now start the resource manager and attempt to flex up the cluster! diff --git a/docs/vagrant.md b/docs/vagrant.md index 0dddf9c..1355e3c 100644 --- a/docs/vagrant.md +++ b/docs/vagrant.md @@ -7,32 +7,41 @@ You can use following guide to setup a cluster in a virtual machine. * Vagrant To start the cluster run following: -```shell + +``` vagrant up ``` + At this point the VM will have a single node mesos cluster running. To ssh in the cluster run following: -```shell + +``` vagrant ssh ``` + The password for vagrant user is 'vagrant'. To setup YARN/Hadoop inside VM, run following: -```shell + +``` cd /vagrant ./setup-yarn-1.sh ``` + This will create a user hduser in group hadoop. Remember the password that you provide for this user. Now, do following: -```shell + +``` sudo su - hduser cd /vagrant ./setup-yarn-2.sh ``` + If everything goes fine you'll see following processes running (process ids will be different): -```shell + +``` 9844 Jps 6709 NameNode 6393 JobHistoryServer @@ -40,22 +49,18 @@ If everything goes fine you'll see following processes running (process ids will ``` To build myriad scheduler inside VM, you can do following: -```shell + +``` cd /vagrant ./gradlew build ``` -At this point, myriad's scheduler jar and all the runtime dependencies will be available here: /vagrant/build/libs/*. Please copy these jars to $YARN_HOME/share/hadoop/yarn/lib/. +At this point, myriad's scheduler jar and all the runtime dependencies will be available here: `/vagrant/myriad-scheduler/build/libs/*`. Please copy these jars to `$YARN_HOME/share/hadoop/yarn/lib/`. -To build self-contained executable JAR, you can run following: -```shell -cd /vagrant -./gradlew capsuleExecutor -``` - -At this point, the self-contained myriad executor jar will be available here: /vagrant/build/libs/myriad-executor-x.y.z.jar. Please copy this jar to /usr/local/libexec/mesos/. +The self-contained myriad executor jar will be available here: `/vagrant/myriad-executor/build/libs/myriad-executor-runnable-x.y.z.jar`. Please copy this jar to `/usr/local/libexec/mesos/`. To configure YARN to use Myriad, please update ```$YARN_HOME/etc/hadoop/yarn-site.xml``` with following: + ```xml yarn.nodemanager.resource.cpu-vcores @@ -75,6 +80,7 @@ To configure YARN to use Myriad, please update ```$YARN_HOME/etc/hadoop/yarn-sit ``` To configure Myriad itself, please add following file to ```$YARN_HOME/etc/hadoop/myriad-default-config.yml```: + ```yml mesosMaster: 10.141.141.20:5050 checkpoint: false @@ -101,11 +107,12 @@ nodemanager: cgroups: false executor: jvmMaxMemoryMB: 256 - path: file://localhost/usr/local/libexec/mesos/myriad-executor-0.0.1.jar + path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar ``` To launch Myriad, you can run following: -```shell + +``` sudo su hduser yarn-daemon.sh start resourcemanager ``` diff --git a/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java b/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java index 9c788f8..29ae506 100644 --- a/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java +++ b/myriad-scheduler/src/main/java/com/ebay/myriad/configuration/MyriadConfiguration.java @@ -49,7 +49,7 @@ * cgroups: false * executor: * jvmMaxMemoryMB: 256 - * path: file://localhost/usr/local/libexec/mesos/myriad-executor-0.0.1.jar + * path: file://localhost/usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar * yarnEnvironment: * YARN_HOME: /usr/local/hadoop */ diff --git a/myriad-scheduler/src/main/resources/myriad-config-default.yml b/myriad-scheduler/src/main/resources/myriad-config-default.yml index 7916340..d6f4746 100644 --- a/myriad-scheduler/src/main/resources/myriad-config-default.yml +++ b/myriad-scheduler/src/main/resources/myriad-config-default.yml @@ -28,10 +28,10 @@ nodemanager: cgroups: false executor: jvmMaxMemoryMB: 256 - path: file:///usr/local/libexec/mesos/myriad-executor-0.0.1.jar + path: file:///usr/local/libexec/mesos/myriad-executor-runnable-0.0.1.jar #The following should be used for a remotely distributed URI, hdfs assumed but other URI types valid. #nodeManagerUri: hdfs://namenode:port/dist/hadoop-2.5.0.tar.gz - #path: hdfs://namenode:port/dist/myriad-executor-0.0.1.jar + #path: hdfs://namenode:port/dist/myriad-executor-runnable-0.0.1.jar yarnEnvironment: YARN_HOME: /usr/local/hadoop #YARN_HOME: hadoop-2.5.0 #this should be relative if nodeManagerUri is set