Merge branch 'modularGradle' into 2.0-client

Conflicts:
	.gitignore
	gradle/check.gradle

.gitignore

@@ -67,5 +67,5 @@ atlassian-ide-plugin.xml
 
 # NetBeans specific files/directories
 .nbattrs
-/.nb-gradle/private/
 .nb-gradle-properties
+.nb-gradle/

deploy/aws/emr_genie_bootstrap.sh

@@ -21,19 +21,21 @@
 set -x
 set -e
 
+TOMCAT_VERSION=6.0.39
+
 # Install Tomcat
-cd $HOME; wget http://mirror.sdunix.com/apache/tomcat/tomcat-6/v6.0.37/bin/apache-tomcat-6.0.37.tar.gz
-tar zxvf apache-tomcat-6.0.37.tar.gz
+cd $HOME; wget http://archive.apache.org/dist/tomcat/tomcat-6/v${TOMCAT_VERSION}/bin/apache-tomcat-${TOMCAT_VERSION}.tar.gz
+tar zxvf apache-tomcat-${TOMCAT_VERSION}.tar.gz
 
 # Change port to 7001 to work out of the box
-perl -p -i.bak -ne "s#8080#7001#g" /home/hadoop/apache-tomcat-6.0.37/conf/server.xml
+perl -p -i.bak -ne "s#8080#7001#g" /home/hadoop/apache-tomcat-${TOMCAT_VERSION}/conf/server.xml
 
-# update /home/hadoop/apache-tomcat-6.0.37/conf/web.xml to enable directory browsing
+# update /home/hadoop/apache-tomcat-${TOMCAT_VERSION}/conf/web.xml to enable directory browsing
 perl -0777 -p -i.bak -ne 's#(<param-name>listings</param-name>\s*<param-value>)false(</param-value>)#\1true\2#;' \
-    /home/hadoop/apache-tomcat-6.0.37/conf/web.xml
+    /home/hadoop/apache-tomcat-${TOMCAT_VERSION}/conf/web.xml
 
 # Set up Genie specific properties
-export CATALINA_HOME=/home/hadoop/apache-tomcat-6.0.37
+export CATALINA_HOME=/home/hadoop/apache-tomcat-${TOMCAT_VERSION}
 export CATALINA_OPTS="-Darchaius.deployment.applicationId=genie -Dnetflix.datacenter=cloud"
 
 # Set up directories needed

genie-server/src/main/java/com/netflix/genie/server/jobmanager/impl/JobMonitor.java

@@ -220,6 +220,9 @@ private int waitForExit() {
     public void run() {
         GenieNodeStatistics stats = GenieNodeStatistics.getInstance();
 
+        // flag to track if the job is killed. Used to determine status of email sent.
+        boolean killed = false;
+
         // wait for process to complete
         int exitCode = waitForExit();
         ji.setExitCode(exitCode);
@@ -269,6 +272,7 @@ public void run() {
                 // if job status is killed, the kill thread will update status
                 logger.debug("Job has been killed - will not update DB: "
                         + ji.getJobID());
+                killed = true;
                 rwl.writeLock().unlock();
             }
         } finally {
@@ -285,7 +289,7 @@ public void run() {
         if (emailTo != null) {
             logger.info("User email address: " + emailTo);
 
-            if (sendEmail(emailTo)) {
+            if (sendEmail(emailTo, killed)) {
                 // Email sent successfully. Update success email counter
                 stats.incrSuccessfulEmailCount();
             } else {
@@ -302,7 +306,7 @@ public void run() {
      * mail properties and try and send email about Job Status.
      * @return 0 for success, -1 for failure
      */
-    private boolean sendEmail(String emailTo) {
+    private boolean sendEmail(String emailTo, boolean killed) {
         logger.debug("called");
 
         if (!config.getBoolean("netflix.genie.server.mail.enable", false)) {
@@ -366,11 +370,19 @@ private boolean sendEmail(String emailTo) {
             message.addRecipient(Message.RecipientType.TO,
                     new InternetAddress(emailTo));
 
+            String jobStatus = new String();
+
+            if (killed) {
+                jobStatus = "KILLED";
+            } else {
+                jobStatus = ji.getStatus();
+            }
+
             // Set Subject: header field
             message.setSubject("Genie Job "
                          + ji.getJobName()
                          + " completed with Status: "
-                         + ji.getStatus());
+                         + jobStatus);
 
             // Now set the actual message
             String body = "Your Genie Job is complete\n\n"

genie-web-int-tests/build.gradle

@@ -0,0 +1,20 @@
+apply plugin: 'java'
+
+// dummy test - since integration tests are run by build task
+task test(overwrite:true)
+
+dependencies {
+    compile project(':genie-web')
+
+    testCompile 'com.sun.jersey:jersey-servlet:1.11'
+    testCompile 'com.netflix.karyon:karyon-extensions-testsuite:1.0.22'
+    testCompile 'org.jboss.arquillian.junit:arquillian-junit-container:1.1.0.Final'
+    testCompile 'org.jboss.shrinkwrap:shrinkwrap-impl-base:1.1.2'
+    testCompile 'org.jboss.shrinkwrap.resolver:shrinkwrap-resolver-impl-maven:2.0.0'
+    testRuntime 'org.jboss.arquillian.container:arquillian-tomcat-embedded-7:1.0.0.CR5'
+    testRuntime 'org.apache.tomcat.embed:tomcat-embed-core:7.0.42'
+    testRuntime 'org.apache.tomcat.embed:tomcat-embed-logging-juli:7.0.42'
+    testRuntime ('org.apache.tomcat.embed:tomcat-embed-jasper:7.0.42') {
+        exclude group: 'org.eclipse.jdt.core.compiler', module: 'ecj'
+    }
+}

genie-web/src/test/python/jobs/hadoop2JobTest.py

@@ -0,0 +1,73 @@
+##
+#
+#  Copyright 2013 Netflix, Inc.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+#
+##
+
+import sys
+sys.path.append('../utils')
+
+import time
+import eureka
+import jobs
+
+import os
+
+# the S3 prefix where the tests are located
+GENIE_TEST_PREFIX = os.getenv("GENIE_TEST_PREFIX")
+
+# get the serviceUrl from the eureka client
+serviceUrl = eureka.EurekaClient().getServiceBaseUrl() + '/genie/v0/jobs'
+
+def testJsonSubmitjob():
+    print "Running testJsonSubmitjob "
+    payload = '''
+    {
+        "jobInfo":
+        {
+            "jobName": "HADOOP-JOB-TEST", 
+            "userName" : "blahgenieamsharma", 
+            "groupName" : "hadoop", 
+            "userAgent" : "laptop",
+            "jobType": "hadoop", 
+            "schedule": "ADHOC",
+            "clusterName": "h24query",
+            "cmdArgs":"jar hadoop-examples.jar pi 50 10", 
+            "fileDependencies":"''' + GENIE_TEST_PREFIX + '''/hadoop-examples.jar"
+        }
+    }
+    '''
+    print payload
+    print "\n"
+    return jobs.submitJob(serviceUrl, payload)
+
+# driver method for all tests                
+if __name__ == "__main__":
+   print "Running unit tests:\n"
+   jobID = testJsonSubmitjob()
+   print "\n"
+   while True:
+       print jobs.getJobInfo(serviceUrl, jobID)
+       print "\n"
+       status = jobs.getJobStatus(serviceUrl, jobID)
+       print status
+       print "\n"
+
+       if (status != 'RUNNING') and (status != 'INIT'):
+           print "Final status: ", status
+           print "Job has terminated - exiting"
+           break
+
+       time.sleep(5)

genie-web/src/test/python/jobs/hive2JobTest.py

@@ -0,0 +1,75 @@
+##
+#
+#  Copyright 2013 Netflix, Inc.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+#
+##
+
+import sys
+sys.path.append('../utils')
+
+import time
+import eureka
+import jobs
+
+import os
+
+# the S3 prefix where the tests are located
+GENIE_TEST_PREFIX = os.getenv("GENIE_TEST_PREFIX")
+
+# get the serviceUrl from the eureka client
+serviceUrl = eureka.EurekaClient().getServiceBaseUrl() + '/genie/v0/jobs'
+
+def testJsonSubmitjob():
+    print "Running testJsonSubmitjob "
+    payload = '''
+    {  
+        "jobInfo":
+        {
+            "jobName": "HIVE-JOB-TEST",
+            "description": "This is a test", 
+            "userName" : "amsharma", 
+            "groupName" : "hadoop", 
+            "jobType": "hive", 
+            "configuration": "prod",
+            "schedule": "ADHOC",
+            "clusterName": "h24query",
+            "cmdArgs": "-f hive.q",
+            "fileDependencies":"''' + GENIE_TEST_PREFIX + '''/hive.q"
+        }
+    }
+    '''
+    print payload
+    print "\n"
+    return jobs.submitJob(serviceUrl, payload)
+
+# driver method for all tests                
+if __name__ == "__main__":
+   print "Running unit tests:\n"
+   jobID = testJsonSubmitjob()
+   print "\n"
+   while True:
+       print jobs.getJobInfo(serviceUrl, jobID)
+       print "\n"
+       status = jobs.getJobStatus(serviceUrl, jobID)
+       print status
+       print "\n"
+
+       if (status != 'RUNNING') and (status != 'INIT'):
+           print "Final status: ", status
+           print "Job has terminated - exiting"
+           break
+
+       time.sleep(5)
+

genie-web/src/test/python/jobs/pig2JobTest.py

@@ -0,0 +1,75 @@
+##
+#
+#  Copyright 2013 Netflix, Inc.
+#
+#     Licensed under the Apache License, Version 2.0 (the "License");
+#     you may not use this file except in compliance with the License.
+#     You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#     Unless required by applicable law or agreed to in writing, software
+#     distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+#
+##
+
+import sys
+sys.path.append('../utils')
+
+import time
+import eureka
+import jobs
+
+import os
+
+# the S3 prefix where the tests are located
+GENIE_TEST_PREFIX = os.getenv("GENIE_TEST_PREFIX")
+
+# get the serviceUrl from the eureka client
+serviceUrl = eureka.EurekaClient().getServiceBaseUrl() + '/genie/v0/jobs'
+
+def testJsonSubmitjob():
+    print "Running testJsonSubmitjob "
+    payload = '''
+    {
+        "jobInfo":
+        {
+            "jobName": "PIG-JOB-TEST", 
+            "userName" : "amsharma", 
+            "groupName" : "hadoop", 
+            "userAgent" : "laptop",
+            "jobType": "pig", 
+            "configuration": "prod", 
+            "schedule": "adHoc",
+            "clusterName": "h24query",
+            "cmdArgs":" -f pig2.q", 
+            "fileDependencies":"''' + GENIE_TEST_PREFIX + '''/pig2.q"
+        }
+    }
+    '''
+    print payload
+    print "\n"
+    return jobs.submitJob(serviceUrl, payload)
+
+# driver method for all tests
+if __name__ == "__main__":    
+    print "Running unit tests:\n"
+    jobID = testJsonSubmitjob()
+    print "\n"
+
+    while True:
+       print jobs.getJobInfo(serviceUrl, jobID)
+       print "\n"
+       status = jobs.getJobStatus(serviceUrl, jobID)
+       print status
+       print "\n"
+
+       if (status != 'RUNNING') and (status != 'INIT'):
+           print "Final status: ", status
+           print "Job has terminated - exiting"
+           break
+
+       time.sleep(5)

gradle/check.gradle

@@ -3,7 +3,7 @@ subprojects {
     apply plugin: 'checkstyle'
     checkstyle {
         ignoreFailures = true 
-        configFile = rootProject.file('codequality/checkstyle.xml')
+        configFile = new File(project.parent.projectDir, 'codequality/checkstyle.xml')
     }
 
     // FindBugs

gradle/convention.gradle

@@ -21,8 +21,8 @@ subprojects { project ->
         extension 'jar'
     }
 
-    configurations.add('sources')
-    configurations.add('javadoc')
+    configurations.create('sources')
+    configurations.create('javadoc')
     configurations.archives {
         extendsFrom configurations.sources
         extendsFrom configurations.javadoc

gradle/license.gradle

@@ -1,10 +1,10 @@
 // Dependency for plugin was set in buildscript.gradle
 
 subprojects {
-apply plugin: 'license' //nl.javadude.gradle.plugins.license.LicensePlugin
-license {
-    header rootProject.file('codequality/HEADER')
-    ext.year = Calendar.getInstance().get(Calendar.YEAR)
-    skipExistingHeaders true
-}
+    apply plugin: 'license' //nl.javadude.gradle.plugins.license.LicensePlugin
+    license {
+        header new File(project.parent.projectDir, '/codequality/HEADER')
+        ext.year = Calendar.getInstance().get(Calendar.YEAR)
+        skipExistingHeaders true
+    }
 }