diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/GeneExpressionExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/GeneExpressionExecutor.java index ca6baa9fa..57150355b 100644 --- a/src/main/java/org/alliancegenome/curation_api/jobs/executors/GeneExpressionExecutor.java +++ b/src/main/java/org/alliancegenome/curation_api/jobs/executors/GeneExpressionExecutor.java @@ -48,9 +48,6 @@ public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) { } bulkLoadFileDAO.merge(bulkLoadFileHistory.getBulkLoadFile()); - bulkLoadFileHistory.setCount(ANNOTATIONS, geneExpressionIngestFmsDTO.getData().size()); - updateHistory(bulkLoadFileHistory); - List annotationIdsLoaded = new ArrayList<>(); List annotationIdsBefore = geneExpressionAnnotationService.getAnnotationIdsByDataProvider(dataProvider); @@ -83,7 +80,7 @@ public APIResponse runLoadAPI(GeneExpressionAnnotationService service, String da if (dataProviderName != null) { dataProvider = BackendBulkDataProvider.valueOf(dataProviderName); } - boolean success = runLoad(service, history, dataProvider, objectList, idsLoaded, true, "Records"); + boolean success = runLoad(service, history, dataProvider, objectList, idsLoaded, true, ANNOTATIONS); if (success) { loadExperiments(history, dataProvider, new ArrayList<>()); } @@ -114,5 +111,6 @@ private void loadExperiments(BulkLoadFileHistory history, BackendBulkDataProvide ph.progressProcess(); } updateHistory(history); + ph.finishProcess(); } } diff --git a/src/main/java/org/alliancegenome/curation_api/jobs/executors/LoadFileExecutor.java b/src/main/java/org/alliancegenome/curation_api/jobs/executors/LoadFileExecutor.java index 7f619c3d0..ef1034f1d 100644 --- a/src/main/java/org/alliancegenome/curation_api/jobs/executors/LoadFileExecutor.java +++ b/src/main/java/org/alliancegenome/curation_api/jobs/executors/LoadFileExecutor.java @@ -211,6 +211,7 @@ protected boolean runLoad(BaseUpser } ph.startProcess(loadMessage, objectList.size()); + history.setCount(countType, objectList.size()); updateHistory(history); for (T dtoObject : objectList) { try { @@ -230,7 +231,7 @@ protected boolean runLoad(BaseUpser history.incrementFailed(countType); addException(history, new ObjectUpdateExceptionData(dtoObject, e.getMessage(), e.getStackTrace())); } - if (terminateFailing && history.getErrorRate() > 0.25) { + if (terminateFailing && history.getErrorRate(countType) > 0.25) { Log.error("Failure Rate > 25% aborting load"); updateHistory(history); updateExceptions(history); diff --git a/src/test/java/org/alliancegenome/curation_api/ExpressionBulkUploadFmsITCase.java b/src/test/java/org/alliancegenome/curation_api/ExpressionBulkUploadFmsITCase.java index c02becbf1..7b4430edf 100644 --- a/src/test/java/org/alliancegenome/curation_api/ExpressionBulkUploadFmsITCase.java +++ b/src/test/java/org/alliancegenome/curation_api/ExpressionBulkUploadFmsITCase.java @@ -13,6 +13,7 @@ import org.junit.jupiter.api.*; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; import static org.hamcrest.Matchers.hasSize; @@ -68,7 +69,12 @@ public void init() { @Order(1) public void expressionBulkUploadAllFields() throws Exception { loadRequiredEntities(); - checkSuccessfulBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "AF_01_all_fields.json"); + + HashMap> params = new HashMap<>(); + params.put("Annotations", createCountParams(1, 0, 1, 0)); + params.put("Experiments", createCountParams(1, 0, 1, 0)); + + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "AF_01_all_fields.json", params); RestAssured.given().when() .header("Content-Type", "application/json") @@ -123,43 +129,52 @@ public void expressionBulkUploadAllFields() throws Exception { @Test @Order(2) public void expressionBulkUploadMissingRequiredFields() throws Exception { - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_01_no_geneId.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_02_no_dateAssigned.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_03_no_evidence.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_04_no_assay.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_05_no_whenExpressed.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_06_no_whereExpressed.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_07_nowhenExpressedStageName.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_08_nowhereExpressedStatement.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_09_norCellComponentNORanatStructure.json"); + HashMap> params = new HashMap<>(); + params.put("Annotations", createCountParams(1, 1, 0, 0)); + + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_01_no_geneId.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_02_no_dateAssigned.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_03_no_evidence.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_04_no_assay.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_05_no_whenExpressed.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_06_no_whereExpressed.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_07_nowhenExpressedStageName.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_08_nowhereExpressedStatement.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_09_norCellComponentNORanatStructure.json", params); } @Test @Order(3) public void expressionBulkUploadEmptyRequiredFields() throws Exception { - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_01_empty_geneId.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_02_empty_dateAssigned.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_03_empty_assay.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_04_empty_whenExpressedStageName.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_05_empty_whereExpressedStatement.json"); + HashMap> params = new HashMap<>(); + params.put("Annotations", createCountParams(1, 1, 0, 0)); + + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_01_empty_geneId.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_02_empty_dateAssigned.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_03_empty_assay.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_04_empty_whenExpressedStageName.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_05_empty_whereExpressedStatement.json", params); } @Test @Order(4) public void expressionBulkUploadInvalidFields() throws Exception { - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_01_invalid_geneId.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_02_invalid_dateAssigned.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_03_invalid_assay.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_04_invalid_publicationId.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_05_invalid_stageterm.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_06_invalid_anatomical_structure.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_07_invalid_anatomical_substructure.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_08_invalid_cellularcomponent.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_09_invalid_anatomicalstructurequalifier.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_10_invalid_anatomicalsubstructurequalifier.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_11_invalid_cellularcomponentqualifier.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_12_invalid_anatomicalstructureuberonslimterms.json"); - checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_13_invalid_anatomicalsubstructureuberonslimterms.json"); + HashMap> params = new HashMap<>(); + params.put("Annotations", createCountParams(1, 1, 0, 0)); + + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_01_invalid_geneId.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_02_invalid_dateAssigned.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_03_invalid_assay.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_04_invalid_publicationId.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_05_invalid_stageterm.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_06_invalid_anatomical_structure.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_07_invalid_anatomical_substructure.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_08_invalid_cellularcomponent.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_09_invalid_anatomicalstructurequalifier.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_10_invalid_anatomicalsubstructurequalifier.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_11_invalid_cellularcomponentqualifier.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_12_invalid_anatomicalstructureuberonslimterms.json", params); + checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_13_invalid_anatomicalsubstructureuberonslimterms.json", params); } private void loadRequiredEntities() throws Exception { @@ -176,7 +191,7 @@ private void loadRequiredEntities() throws Exception { Vocabulary vocabulary2 = createVocabulary(VocabularyConstants.GENE_EXPRESSION_VOCABULARY, false); createVocabularyTerm(vocabulary2, VocabularyConstants.GENE_EXPRESSION_RELATION_TERM, false); Vocabulary stageUberonTermVocabulary = getVocabulary(VocabularyConstants.STAGE_UBERON_SLIM_TERMS); - Vocabulary spatialExpressionQualififerVocabulary = getVocabulary(VocabularyConstants.SPATIAL_EXPRESSION_QUALIFIERS); + Vocabulary spatialExpressionQualifierVocabulary = getVocabulary(VocabularyConstants.SPATIAL_EXPRESSION_QUALIFIERS); VocabularyTermSet anatatomicalStructureQualifierTermset = getVocabularyTermSet(VocabularyConstants.ANATOMICAL_STRUCTURE_QUALIFIER); VocabularyTermSet anatatomicalSubstructureQualifierTermset = getVocabularyTermSet(VocabularyConstants.ANATOMICAL_SUBSTRUCTURE_QUALIFIER); VocabularyTermSet cellularComponentQualifierTermset = getVocabularyTermSet(VocabularyConstants.CELLULAR_COMPONENT_QUALIFIER);