Skip to content

Commit

Permalink
Merge pull request #1740 from alliance-genome/SCRUM-3953_count_fix
Browse files Browse the repository at this point in the history 
SCRUM-3953 (update) Fix record counts
  • Loading branch information
@markquintontulloch
markquintontulloch authored Nov 25, 2024
2 parents 58c36ea + 1d21b06 commit 7ad5182
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 34 deletions.
6 changes: 2 additions & 4 deletions src/main/java/org/alliancegenome/curation_api/jobs/executors/GeneExpressionExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,6 @@ public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) {
}
bulkLoadFileDAO.merge(bulkLoadFileHistory.getBulkLoadFile());

bulkLoadFileHistory.setCount(ANNOTATIONS, geneExpressionIngestFmsDTO.getData().size());
updateHistory(bulkLoadFileHistory);

List<Long> annotationIdsLoaded = new ArrayList<>();
List<Long> annotationIdsBefore = geneExpressionAnnotationService.getAnnotationIdsByDataProvider(dataProvider);

Expand Down Expand Up @@ -83,7 +80,7 @@ public APIResponse runLoadAPI(GeneExpressionAnnotationService service, String da
if (dataProviderName != null) {
dataProvider = BackendBulkDataProvider.valueOf(dataProviderName);
}
boolean success = runLoad(service, history, dataProvider, objectList, idsLoaded, true, "Records");
boolean success = runLoad(service, history, dataProvider, objectList, idsLoaded, true, ANNOTATIONS);
if (success) {
loadExperiments(history, dataProvider, new ArrayList<>());
}
Expand Down Expand Up @@ -114,5 +111,6 @@ private void loadExperiments(BulkLoadFileHistory history, BackendBulkDataProvide
ph.progressProcess();
}
updateHistory(history);
ph.finishProcess();
}
}
3 changes: 2 additions & 1 deletion src/main/java/org/alliancegenome/curation_api/jobs/executors/LoadFileExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,7 @@ protected <E extends AuditedObject, T extends BaseDTO> boolean runLoad(BaseUpser
}
ph.startProcess(loadMessage, objectList.size());

history.setCount(countType, objectList.size());
updateHistory(history);
for (T dtoObject : objectList) {
try {
Expand All @@ -230,7 +231,7 @@ protected <E extends AuditedObject, T extends BaseDTO> boolean runLoad(BaseUpser
history.incrementFailed(countType);
addException(history, new ObjectUpdateExceptionData(dtoObject, e.getMessage(), e.getStackTrace()));
}
if (terminateFailing && history.getErrorRate() > 0.25) {
if (terminateFailing && history.getErrorRate(countType) > 0.25) {
Log.error("Failure Rate > 25% aborting load");
updateHistory(history);
updateExceptions(history);
Expand Down
73 changes: 44 additions & 29 deletions src/test/java/org/alliancegenome/curation_api/ExpressionBulkUploadFmsITCase.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
import org.junit.jupiter.api.*;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import static org.hamcrest.Matchers.hasSize;
Expand Down Expand Up @@ -68,7 +69,12 @@ public void init() {
@Order(1)
public void expressionBulkUploadAllFields() throws Exception {
loadRequiredEntities();
checkSuccessfulBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "AF_01_all_fields.json");

HashMap<String, HashMap<String, Integer>> params = new HashMap<>();
params.put("Annotations", createCountParams(1, 0, 1, 0));
params.put("Experiments", createCountParams(1, 0, 1, 0));

checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "AF_01_all_fields.json", params);

RestAssured.given().when()
.header("Content-Type", "application/json")
Expand Down Expand Up @@ -123,43 +129,52 @@ public void expressionBulkUploadAllFields() throws Exception {
@Test
@Order(2)
public void expressionBulkUploadMissingRequiredFields() throws Exception {
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_01_no_geneId.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_02_no_dateAssigned.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_03_no_evidence.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_04_no_assay.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_05_no_whenExpressed.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_06_no_whereExpressed.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_07_nowhenExpressedStageName.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_08_nowhereExpressedStatement.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "MR_09_norCellComponentNORanatStructure.json");
HashMap<String, HashMap<String, Integer>> params = new HashMap<>();
params.put("Annotations", createCountParams(1, 1, 0, 0));

checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_01_no_geneId.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_02_no_dateAssigned.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_03_no_evidence.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_04_no_assay.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_05_no_whenExpressed.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_06_no_whereExpressed.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_07_nowhenExpressedStageName.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_08_nowhereExpressedStatement.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "MR_09_norCellComponentNORanatStructure.json", params);
}

@Test
@Order(3)
public void expressionBulkUploadEmptyRequiredFields() throws Exception {
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_01_empty_geneId.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_02_empty_dateAssigned.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_03_empty_assay.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_04_empty_whenExpressedStageName.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "ER_05_empty_whereExpressedStatement.json");
HashMap<String, HashMap<String, Integer>> params = new HashMap<>();
params.put("Annotations", createCountParams(1, 1, 0, 0));

checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_01_empty_geneId.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_02_empty_dateAssigned.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_03_empty_assay.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_04_empty_whenExpressedStageName.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "ER_05_empty_whereExpressedStatement.json", params);
}

@Test
@Order(4)
public void expressionBulkUploadInvalidFields() throws Exception {
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_01_invalid_geneId.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_02_invalid_dateAssigned.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_03_invalid_assay.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_04_invalid_publicationId.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_05_invalid_stageterm.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_06_invalid_anatomical_structure.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_07_invalid_anatomical_substructure.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_08_invalid_cellularcomponent.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_09_invalid_anatomicalstructurequalifier.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_10_invalid_anatomicalsubstructurequalifier.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_11_invalid_cellularcomponentqualifier.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_12_invalid_anatomicalstructureuberonslimterms.json");
checkFailedBulkLoad(expressionBulkPostEndpoint, expressionTestFilePath + "IV_13_invalid_anatomicalsubstructureuberonslimterms.json");
HashMap<String, HashMap<String, Integer>> params = new HashMap<>();
params.put("Annotations", createCountParams(1, 1, 0, 0));

checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_01_invalid_geneId.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_02_invalid_dateAssigned.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_03_invalid_assay.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_04_invalid_publicationId.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_05_invalid_stageterm.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_06_invalid_anatomical_structure.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_07_invalid_anatomical_substructure.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_08_invalid_cellularcomponent.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_09_invalid_anatomicalstructurequalifier.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_10_invalid_anatomicalsubstructurequalifier.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_11_invalid_cellularcomponentqualifier.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_12_invalid_anatomicalstructureuberonslimterms.json", params);
checkBulkLoadRecordCounts(expressionBulkPostEndpoint, expressionTestFilePath + "IV_13_invalid_anatomicalsubstructureuberonslimterms.json", params);
}

private void loadRequiredEntities() throws Exception {
Expand All @@ -176,7 +191,7 @@ private void loadRequiredEntities() throws Exception {
Vocabulary vocabulary2 = createVocabulary(VocabularyConstants.GENE_EXPRESSION_VOCABULARY, false);
createVocabularyTerm(vocabulary2, VocabularyConstants.GENE_EXPRESSION_RELATION_TERM, false);
Vocabulary stageUberonTermVocabulary = getVocabulary(VocabularyConstants.STAGE_UBERON_SLIM_TERMS);
Vocabulary spatialExpressionQualififerVocabulary = getVocabulary(VocabularyConstants.SPATIAL_EXPRESSION_QUALIFIERS);
Vocabulary spatialExpressionQualifierVocabulary = getVocabulary(VocabularyConstants.SPATIAL_EXPRESSION_QUALIFIERS);
VocabularyTermSet anatatomicalStructureQualifierTermset = getVocabularyTermSet(VocabularyConstants.ANATOMICAL_STRUCTURE_QUALIFIER);
VocabularyTermSet anatatomicalSubstructureQualifierTermset = getVocabularyTermSet(VocabularyConstants.ANATOMICAL_SUBSTRUCTURE_QUALIFIER);
VocabularyTermSet cellularComponentQualifierTermset = getVocabularyTermSet(VocabularyConstants.CELLULAR_COMPONENT_QUALIFIER);
Expand Down

0 comments on commit 7ad5182

Please sign in to comment.