From c2bed3510cf1ef59746f8b46998817ca35e63b8e Mon Sep 17 00:00:00 2001 From: Amit Sharma Date: Thu, 10 Mar 2016 13:53:41 -0800 Subject: [PATCH] {#101} More integration tests for JobRestController. --- .gitignore | 4 + .../JpaJobPersistenceServiceImpl.java | 1 + .../impl/JobKickoffTaskUnitTests.java | 17 ++ .../SequenceWorkFlowExecutorUnitTests.java | 17 ++ .../web/configs/JobConfigIntegrationTest.java | 7 +- .../JobRestControllerIntegrationTests.java | 249 +++++++++++++++++- gradle/installViaTravis.sh | 2 +- 7 files changed, 283 insertions(+), 14 deletions(-) diff --git a/.gitignore b/.gitignore index f2890e08861..09007c18678 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,7 @@ genie-client/src/main/python/build genie-client/src/main/python/nflx_genie_client.egg-info genie-client/src/main/python/MANIFEST genie-common/src/generated + +# Credential Files # +###################################### +*credentials.properties diff --git a/genie-core/src/main/java/com/netflix/genie/core/jpa/services/JpaJobPersistenceServiceImpl.java b/genie-core/src/main/java/com/netflix/genie/core/jpa/services/JpaJobPersistenceServiceImpl.java index aa6357c5353..c52e09758a8 100644 --- a/genie-core/src/main/java/com/netflix/genie/core/jpa/services/JpaJobPersistenceServiceImpl.java +++ b/genie-core/src/main/java/com/netflix/genie/core/jpa/services/JpaJobPersistenceServiceImpl.java @@ -350,6 +350,7 @@ public void createJobExecution( jobEntity.setExecution(jobExecutionEntity); jobEntity.setStatus(JobStatus.RUNNING); jobEntity.setStatusMsg("Job is Running"); + jobEntity.setStarted(new Date()); } /** diff --git a/genie-core/src/test/java/com/netflix/genie/core/jobs/workflow/impl/JobKickoffTaskUnitTests.java b/genie-core/src/test/java/com/netflix/genie/core/jobs/workflow/impl/JobKickoffTaskUnitTests.java index 4d45a325135..5af68a0ff69 100644 --- a/genie-core/src/test/java/com/netflix/genie/core/jobs/workflow/impl/JobKickoffTaskUnitTests.java +++ b/genie-core/src/test/java/com/netflix/genie/core/jobs/workflow/impl/JobKickoffTaskUnitTests.java @@ -1,3 +1,20 @@ +/* + * + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.genie.core.jobs.workflow.impl; import com.netflix.genie.common.exceptions.GenieException; diff --git a/genie-core/src/test/java/com/netflix/genie/core/jobs/workflow/impl/SequenceWorkFlowExecutorUnitTests.java b/genie-core/src/test/java/com/netflix/genie/core/jobs/workflow/impl/SequenceWorkFlowExecutorUnitTests.java index f8448749e1e..e5e02c1dbcc 100644 --- a/genie-core/src/test/java/com/netflix/genie/core/jobs/workflow/impl/SequenceWorkFlowExecutorUnitTests.java +++ b/genie-core/src/test/java/com/netflix/genie/core/jobs/workflow/impl/SequenceWorkFlowExecutorUnitTests.java @@ -1,3 +1,20 @@ +/* + * + * Copyright 2015 Netflix, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ package com.netflix.genie.core.jobs.workflow.impl; import com.netflix.genie.common.exceptions.GenieException; diff --git a/genie-web/src/test/java/com/netflix/genie/web/configs/JobConfigIntegrationTest.java b/genie-web/src/test/java/com/netflix/genie/web/configs/JobConfigIntegrationTest.java index 21bbaffdeaa..8376d8daa5b 100644 --- a/genie-web/src/test/java/com/netflix/genie/web/configs/JobConfigIntegrationTest.java +++ b/genie-web/src/test/java/com/netflix/genie/web/configs/JobConfigIntegrationTest.java @@ -6,6 +6,7 @@ import org.springframework.core.io.Resource; import org.springframework.core.io.ResourceLoader; +import java.io.File; import java.io.IOException; /** @@ -31,9 +32,9 @@ public class JobConfigIntegrationTest { public Resource jobsDir( final ResourceLoader resourceLoader ) throws IOException { - //final String currentDir = new File("/tmp").getAbsolutePath(); - //final Resource jobsDirResource = resourceLoader.getResource(currentDir + "/build/tmp/"); - final Resource jobsDirResource = resourceLoader.getResource("file:///tmp/"); + final String currentDir = new File(".").getCanonicalPath(); + final Resource jobsDirResource = resourceLoader.getResource(currentDir + "/build/tmp/genie/"); + //final Resource jobsDirResource = resourceLoader.getResource("file:///tmp/"); return jobsDirResource; } } diff --git a/genie-web/src/test/java/com/netflix/genie/web/controllers/JobRestControllerIntegrationTests.java b/genie-web/src/test/java/com/netflix/genie/web/controllers/JobRestControllerIntegrationTests.java index 668a633db46..d87e70e547e 100644 --- a/genie-web/src/test/java/com/netflix/genie/web/controllers/JobRestControllerIntegrationTests.java +++ b/genie-web/src/test/java/com/netflix/genie/web/controllers/JobRestControllerIntegrationTests.java @@ -25,6 +25,7 @@ import com.netflix.genie.common.dto.Command; import com.netflix.genie.common.dto.CommandStatus; import com.netflix.genie.common.dto.JobRequest; +import com.netflix.genie.common.dto.JobStatus; import com.netflix.genie.core.jpa.repositories.JpaApplicationRepository; import com.netflix.genie.core.jpa.repositories.JpaClusterRepository; import com.netflix.genie.core.jpa.repositories.JpaCommandRepository; @@ -34,21 +35,26 @@ import lombok.extern.slf4j.Slf4j; import org.hamcrest.Matchers; import org.junit.After; +import org.junit.Assert; import org.junit.Before; import org.junit.Test; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Value; import org.springframework.core.io.DefaultResourceLoader; import org.springframework.core.io.ResourceLoader; +import org.springframework.hateoas.MediaTypes; import org.springframework.http.HttpHeaders; import org.springframework.http.MediaType; +import org.springframework.test.web.servlet.MvcResult; import org.springframework.test.web.servlet.request.MockMvcRequestBuilders; import org.springframework.test.web.servlet.result.MockMvcResultMatchers; import java.util.ArrayList; +import java.util.Date; import java.util.HashSet; import java.util.List; import java.util.Set; +import java.util.UUID; /** * Integration tests for Jobs REST API. @@ -60,6 +66,13 @@ @Slf4j public class JobRestControllerIntegrationTests extends RestControllerIntegrationTestsBase { + protected static final String STATUS_MESSAGE_PATH = "$.statusMsg"; + protected static final String CLUSTER_NAME_PATH = "$.clusterName"; + protected static final String COMMAND_NAME_PATH = "$.commandName"; + protected static final String ARCHIVE_LOCATION_PATH = "$.archiveLocation"; + protected static final String STARTED_PATH = "$.started"; + protected static final String FINISHED_PATH = "$.finished"; + private static final String BASE_DIR = "com/netflix/genie/web/controllers/JobRestControllerIntegrationTests/"; private static final String FILE_DELIMITER = "/"; @@ -68,6 +81,8 @@ public class JobRestControllerIntegrationTests extends RestControllerIntegration private static final String JOB_NAME = "List Directories bash job"; private static final String JOB_USER = "genie"; private static final String JOB_VERSION = "1.0"; + private static final String JOB_DESCRIPTION = "Genie 3 Test Job"; + private static final String JOB_STATUS_MSG = "Job finished successfully."; private static final String APP1_ID = "app1"; private static final String APP1_NAME = "Application 1"; @@ -226,8 +241,7 @@ private void createAnApplication( .andReturn(); } - private void createAllClusters( - ) throws Exception { + private void createAllClusters() throws Exception { final String setUpFile = this.resourceLoader.getResource( BASE_DIR @@ -279,8 +293,7 @@ private void createAllClusters( .andReturn(); } - private void createAllCommands( - ) throws Exception { + private void createAllCommands() throws Exception { final String setUpFile = this.resourceLoader.getResource( BASE_DIR @@ -314,7 +327,7 @@ private void createAllCommands( CMD1_VERSION, CommandStatus.ACTIVE, "/bin/bash", - 1000 + 1 ) .withId(CMD1_ID) .withSetupFile(setUpFile) @@ -347,8 +360,10 @@ public void cleanup() { this.commandRepository.deleteAll(); this.applicationRepository.deleteAll(); } + + /** - * Test the job submit method. + * Test the job submit method for success. * * @throws Exception If there is a problem. */ @@ -383,7 +398,7 @@ public void testSubmitJobMethod() throws Exception { // + "dep2" // ).getFile().getAbsolutePath(); dependencies.add(depFile1); - //dependencies.add(depFile2); +// dependencies.add(depFile2); final Set commandCriteria = new HashSet<>(); commandCriteria.add("bash"); @@ -398,10 +413,218 @@ public void testSubmitJobMethod() throws Exception { .withDisableLogArchival(true) .withSetupFile(setUpFile) .withFileDependencies(dependencies) + .withDescription(JOB_DESCRIPTION) + .build(); + + final MvcResult result = this.mvc + .perform( + MockMvcRequestBuilders + .post(JOBS_API) + .contentType(MediaType.APPLICATION_JSON) + .content(OBJECT_MAPPER.writeValueAsBytes(jobRequest)) + ) + .andExpect(MockMvcResultMatchers.status().isAccepted()) + .andExpect(MockMvcResultMatchers.header().string(HttpHeaders.LOCATION, Matchers.notNullValue())) + .andReturn(); + + final String jobId = this.getIdFromLocation(result.getResponse().getHeader(HttpHeaders.LOCATION)); + try { + Thread.sleep(1000); + } catch (InterruptedException ie) { + //Handle exception + } + + this.mvc + .perform(MockMvcRequestBuilders.get(JOBS_API + "/" + jobId)) + .andExpect(MockMvcResultMatchers.status().isOk()) + .andExpect(MockMvcResultMatchers.content().contentType(MediaTypes.HAL_JSON)) + .andExpect(MockMvcResultMatchers.jsonPath(ID_PATH, Matchers.is(jobId))) + .andExpect(MockMvcResultMatchers.jsonPath(CREATED_PATH, Matchers.notNullValue())) + .andExpect(MockMvcResultMatchers.jsonPath(UPDATED_PATH, Matchers.notNullValue())) + .andExpect(MockMvcResultMatchers.jsonPath(VERSION_PATH, Matchers.is(JOB_VERSION))) + .andExpect(MockMvcResultMatchers.jsonPath(USER_PATH, Matchers.is(JOB_USER))) + .andExpect(MockMvcResultMatchers.jsonPath(NAME_PATH, Matchers.is(JOB_NAME))) + .andExpect(MockMvcResultMatchers.jsonPath(DESCRIPTION_PATH, Matchers.is(JOB_DESCRIPTION))) + .andExpect(MockMvcResultMatchers.jsonPath(STATUS_PATH, Matchers.is(JobStatus.SUCCEEDED.toString()))) + .andExpect(MockMvcResultMatchers.jsonPath(STATUS_MESSAGE_PATH, Matchers.is(JOB_STATUS_MSG))) + .andExpect(MockMvcResultMatchers.jsonPath(STARTED_PATH, Matchers.not(new Date(0)))) + .andExpect(MockMvcResultMatchers.jsonPath(FINISHED_PATH, Matchers.notNullValue())) + .andExpect(MockMvcResultMatchers.jsonPath(ARCHIVE_LOCATION_PATH, Matchers.isEmptyOrNullString())) + .andExpect(MockMvcResultMatchers.jsonPath(CLUSTER_NAME_PATH, Matchers.is(CLUSTER1_NAME))) + .andExpect(MockMvcResultMatchers.jsonPath(COMMAND_NAME_PATH, Matchers.is(CMD1_NAME))) + .andExpect(MockMvcResultMatchers.jsonPath(LINKS_PATH + ".*", Matchers.hasSize(4))) + .andExpect(MockMvcResultMatchers.jsonPath(LINKS_PATH, Matchers.hasKey(SELF_LINK_KEY))); + + this.mvc + .perform(MockMvcRequestBuilders.get(JOBS_API + "/" + jobId + "/output")) + .andExpect(MockMvcResultMatchers.jsonPath("parent", Matchers.isEmptyOrNullString())); + + Assert.assertThat(this.jobRepository.count(), Matchers.is(1L)); + Assert.assertThat(this.jobRequestRepository.count(), Matchers.is(1L)); + Assert.assertThat(this.jobExecutionRepository.count(), Matchers.is(1L)); + } + + /** + * Test the job submit method for job already exists. + * + * @throws Exception If there is a problem. + */ + @Test + public void testSubmitJobMethodAlreadyExists() throws Exception { + final String commandArgs = "-c 'echo hello world'"; + + final List clusterCriteriaList = new ArrayList<>(); + final Set clusterTags = new HashSet<>(); + clusterTags.add("localhost"); + final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); + clusterCriteriaList.add(clusterCriteria); + + final String jobId = UUID.randomUUID().toString(); + + final Set commandCriteria = new HashSet<>(); + commandCriteria.add("bash"); + final JobRequest jobRequest = new JobRequest.Builder( + JOB_NAME, + JOB_USER, + JOB_VERSION, + commandArgs, + clusterCriteriaList, + commandCriteria + ) + .withId(jobId) + .withDisableLogArchival(true) + .build(); + + this.mvc + .perform( + MockMvcRequestBuilders + .post(JOBS_API) + .contentType(MediaType.APPLICATION_JSON) + .content(OBJECT_MAPPER.writeValueAsBytes(jobRequest)) + ); + + this.mvc + .perform( + MockMvcRequestBuilders + .post(JOBS_API) + .contentType(MediaType.APPLICATION_JSON) + .content(OBJECT_MAPPER.writeValueAsBytes(jobRequest)) + ) + .andExpect(MockMvcResultMatchers.status().isConflict()); + } + + /** + * Test the job submit method for incorrect cluster resolved. + * + * @throws Exception If there is a problem. + */ + @Test + public void testSubmitJobMethodMissingCluster() throws Exception { + final String commandArgs = "-c 'echo hello world'"; + + final List clusterCriteriaList = new ArrayList<>(); + final Set clusterTags = new HashSet<>(); + clusterTags.add("undefined"); + final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); + clusterCriteriaList.add(clusterCriteria); + + final String jobId = UUID.randomUUID().toString(); + + final Set commandCriteria = new HashSet<>(); + commandCriteria.add("bash"); + final JobRequest jobRequest = new JobRequest.Builder( + JOB_NAME, + JOB_USER, + JOB_VERSION, + commandArgs, + clusterCriteriaList, + commandCriteria + ) + .withId(jobId) + .withDisableLogArchival(true) .build(); - //final MvcResult result = this.mvc + .perform( + MockMvcRequestBuilders + .post(JOBS_API) + .contentType(MediaType.APPLICATION_JSON) + .content(OBJECT_MAPPER.writeValueAsBytes(jobRequest)) + .accept(MediaType.APPLICATION_JSON) + ) + .andExpect(MockMvcResultMatchers.status().isPreconditionFailed()); + } + + /** + * Test the job submit method for incorrect command resolved. + * + * @throws Exception If there is a problem. + */ + @Test + public void testSubmitJobMethodMissingCommand() throws Exception { + final String commandArgs = "-c 'echo hello world'"; + + final List clusterCriteriaList = new ArrayList<>(); + final Set clusterTags = new HashSet<>(); + clusterTags.add("localhost"); + final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); + clusterCriteriaList.add(clusterCriteria); + + final String jobId = UUID.randomUUID().toString(); + + final Set commandCriteria = new HashSet<>(); + commandCriteria.add("undefined"); + final JobRequest jobRequest = new JobRequest.Builder( + JOB_NAME, + JOB_USER, + JOB_VERSION, + commandArgs, + clusterCriteriaList, + commandCriteria + ) + .withId(jobId) + .withDisableLogArchival(true) + .build(); + + this.mvc + .perform( + MockMvcRequestBuilders + .post(JOBS_API) + .contentType(MediaType.APPLICATION_JSON) + .content(OBJECT_MAPPER.writeValueAsBytes(jobRequest)) + ) + .andExpect(MockMvcResultMatchers.status().isPreconditionFailed()); + } + + /** + * Test the job submit method for when the job fails. + * + * @throws Exception If there is a problem. + */ + @Test + public void testSubmitJobMethodFailure() throws Exception { + final String commandArgs = "-c 'ls foo'"; + + final List clusterCriteriaList = new ArrayList<>(); + final Set clusterTags = new HashSet<>(); + clusterTags.add("localhost"); + final ClusterCriteria clusterCriteria = new ClusterCriteria(clusterTags); + clusterCriteriaList.add(clusterCriteria); + + final Set commandCriteria = new HashSet<>(); + commandCriteria.add("bash"); + final JobRequest jobRequest = new JobRequest.Builder( + JOB_NAME, + JOB_USER, + JOB_VERSION, + commandArgs, + clusterCriteriaList, + commandCriteria + ) + .withDisableLogArchival(true) + .build(); + + final MvcResult result = this.mvc .perform( MockMvcRequestBuilders .post(JOBS_API) @@ -412,13 +635,19 @@ public void testSubmitJobMethod() throws Exception { .andExpect(MockMvcResultMatchers.header().string(HttpHeaders.LOCATION, Matchers.notNullValue())) .andReturn(); - //final String jobId = this.getIdFromLocation(result.getResponse().getHeader(HttpHeaders.LOCATION)); + final String jobId = this.getIdFromLocation(result.getResponse().getHeader(HttpHeaders.LOCATION)); try { Thread.sleep(1000); } catch (InterruptedException ie) { //Handle exception } - log.info("Done"); + + this.mvc + .perform(MockMvcRequestBuilders.get(JOBS_API + "/" + jobId)) + .andExpect(MockMvcResultMatchers.status().isOk()) + .andExpect(MockMvcResultMatchers.content().contentType(MediaTypes.HAL_JSON)) + .andExpect(MockMvcResultMatchers.jsonPath(ID_PATH, Matchers.is(jobId))) + .andExpect(MockMvcResultMatchers.jsonPath(STATUS_PATH, Matchers.is(JobStatus.FAILED.toString()))); } private String getIdFromLocation(final String location) { diff --git a/gradle/installViaTravis.sh b/gradle/installViaTravis.sh index 65dd246c129..e70c0de22ea 100755 --- a/gradle/installViaTravis.sh +++ b/gradle/installViaTravis.sh @@ -6,7 +6,7 @@ if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then ./gradlew assemble elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" == "" ]; then echo -e 'Assemble Branch with Snapshot => Branch ['$TRAVIS_BRANCH']' - ./gradlew -Prelease.travisci=true assemble + ./gradlew -Prelease.travisci=true assemble --stacktrace --debug elif [ "$TRAVIS_PULL_REQUEST" == "false" ] && [ "$TRAVIS_TAG" != "" ]; then echo -e 'Assemble Branch for Release => Branch ['$TRAVIS_BRANCH'] Tag ['$TRAVIS_TAG']' ./gradlew -Prelease.travisci=true -Prelease.useLastTag=true assemble