Skip to content

Commit

Permalink
Merge pull request #1656 from alliance-genome/remove_dataprovider_dups
Browse files Browse the repository at this point in the history 
Remove and add back FK's
  • Loading branch information
@oblodgett
oblodgett authored Sep 4, 2024
2 parents 16fc009 + a80f4ac commit 6d14d69
Show file tree
Hide file tree
Showing 21 changed files with 398 additions and 375 deletions.
6 changes: 3 additions & 3 deletions ...n/java/org/alliancegenome/curation_api/controllers/crud/CodingSequenceCrudController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
import org.alliancegenome.curation_api.controllers.base.BaseEntityCrudController;
import org.alliancegenome.curation_api.dao.CodingSequenceDAO;
import org.alliancegenome.curation_api.interfaces.crud.CodingSequenceCrudInterface;
import org.alliancegenome.curation_api.jobs.executors.Gff3CDSExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3CDSLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptCDSExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3CDSExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3CDSLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptCDSExecutor;
import org.alliancegenome.curation_api.model.entities.CodingSequence;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory;
import org.alliancegenome.curation_api.model.ingest.dto.fms.Gff3DTO;
Expand Down
6 changes: 3 additions & 3 deletions src/main/java/org/alliancegenome/curation_api/controllers/crud/ExonCrudController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,9 @@
import org.alliancegenome.curation_api.controllers.base.BaseEntityCrudController;
import org.alliancegenome.curation_api.dao.ExonDAO;
import org.alliancegenome.curation_api.interfaces.crud.ExonCrudInterface;
import org.alliancegenome.curation_api.jobs.executors.Gff3ExonExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3ExonLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptExonExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3ExonExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3ExonLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptExonExecutor;
import org.alliancegenome.curation_api.model.entities.Exon;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory;
import org.alliancegenome.curation_api.model.ingest.dto.fms.Gff3DTO;
Expand Down
10 changes: 5 additions & 5 deletions src/main/java/org/alliancegenome/curation_api/controllers/crud/TranscriptCrudController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@
import org.alliancegenome.curation_api.controllers.base.BaseEntityCrudController;
import org.alliancegenome.curation_api.dao.TranscriptDAO;
import org.alliancegenome.curation_api.interfaces.crud.TranscriptCrudInterface;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptCDSExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptExonExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptGeneExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptCDSExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptExonExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptGeneExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptLocationExecutor;
import org.alliancegenome.curation_api.model.entities.Transcript;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory;
import org.alliancegenome.curation_api.model.ingest.dto.fms.Gff3DTO;
Expand Down
9 changes: 9 additions & 0 deletions src/main/java/org/alliancegenome/curation_api/jobs/executors/BulkLoadJobExecutor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,15 @@
import org.alliancegenome.curation_api.enums.BackendBulkLoadType;
import org.alliancegenome.curation_api.jobs.executors.associations.alleleAssociations.AlleleGeneAssociationExecutor;
import org.alliancegenome.curation_api.jobs.executors.associations.constructAssociations.ConstructGenomicEntityAssociationExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3CDSExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3CDSLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3ExonExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3ExonLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptCDSExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptExonExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptGeneExecutor;
import org.alliancegenome.curation_api.jobs.executors.gff.Gff3TranscriptLocationExecutor;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory;

import jakarta.enterprise.context.ApplicationScoped;
Expand Down
74 changes: 0 additions & 74 deletions src/main/java/org/alliancegenome/curation_api/jobs/executors/Gff3Executor.java
View file

This file was deleted.

55 changes: 28 additions & 27 deletions ...n_api/jobs/executors/Gff3CDSExecutor.java → ...i/jobs/executors/gff/Gff3CDSExecutor.java
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.alliancegenome.curation_api.jobs.executors;
package org.alliancegenome.curation_api.jobs.executors.gff;

import java.io.FileInputStream;
import java.util.ArrayList;
Expand All @@ -10,13 +10,16 @@
import org.alliancegenome.curation_api.exceptions.ObjectUpdateException;
import org.alliancegenome.curation_api.exceptions.ObjectUpdateException.ObjectUpdateExceptionData;
import org.alliancegenome.curation_api.jobs.util.CsvSchemaBuilder;
import org.alliancegenome.curation_api.model.entities.CodingSequence;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkFMSLoad;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory;
import org.alliancegenome.curation_api.model.ingest.dto.fms.Gff3DTO;
import org.alliancegenome.curation_api.response.APIResponse;
import org.alliancegenome.curation_api.response.LoadHistoryResponce;
import org.alliancegenome.curation_api.services.CodingSequenceService;
import org.alliancegenome.curation_api.services.Gff3Service;
import org.alliancegenome.curation_api.services.helpers.gff3.Gff3AttributesHelper;
import org.alliancegenome.curation_api.services.validation.dto.Gff3DtoValidator;
import org.alliancegenome.curation_api.util.ProcessDisplayHelper;
import org.apache.commons.lang3.tuple.ImmutablePair;

Expand All @@ -33,6 +36,7 @@ public class Gff3CDSExecutor extends Gff3Executor {

@Inject Gff3Service gff3Service;
@Inject CodingSequenceService cdsService;
@Inject Gff3DtoValidator gff3DtoValidator;

public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) {
try {
Expand All @@ -54,16 +58,16 @@ public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) {
BulkFMSLoad fmsLoad = (BulkFMSLoad) bulkLoadFileHistory.getBulkLoad();
BackendBulkDataProvider dataProvider = BackendBulkDataProvider.valueOf(fmsLoad.getFmsDataSubType());

List<ImmutablePair<Gff3DTO, Map<String, String>>> preProcessedGffData = preProcessGffData(gffData, dataProvider);
List<ImmutablePair<Gff3DTO, Map<String, String>>> preProcessedCDSGffData = Gff3AttributesHelper.getCDSGffData(gffData, dataProvider);

gffData.clear();

List<Long> idsAdded = new ArrayList<>();

bulkLoadFileHistory.setTotalRecords((long) preProcessedGffData.size());
bulkLoadFileHistory.setTotalRecords((long) preProcessedCDSGffData.size());
updateHistory(bulkLoadFileHistory);

boolean success = runLoad(bulkLoadFileHistory, gffHeaderData, preProcessedGffData, idsAdded, dataProvider);
boolean success = runLoad(bulkLoadFileHistory, gffHeaderData, preProcessedCDSGffData, idsAdded, dataProvider);

if (success) {
runCleanup(cdsService, bulkLoadFileHistory, dataProvider.name(), cdsService.getIdsByDataProvider(dataProvider), idsAdded, "GFF coding sequence");
Expand All @@ -82,31 +86,13 @@ private boolean runLoad(BulkLoadFileHistory history, List<String> gffHeaderData,
ph.addDisplayHandler(loadProcessDisplayService);
ph.startProcess("GFF CDS update for " + dataProvider.name(), gffData.size());

loadCDSEntities(history, gffData, idsAdded, dataProvider, ph);

ph.finishProcess();

return true;
}

public APIResponse runLoadApi(String dataProviderName, String assemblyName, List<Gff3DTO> gffData) {
List<Long> idsAdded = new ArrayList<>();
BackendBulkDataProvider dataProvider = BackendBulkDataProvider.valueOf(dataProviderName);
List<ImmutablePair<Gff3DTO, Map<String, String>>> preProcessedGffData = preProcessGffData(gffData, dataProvider);
BulkLoadFileHistory history = new BulkLoadFileHistory(preProcessedGffData.size());

runLoad(history, null, preProcessedGffData, idsAdded, dataProvider);
history.finishLoad();

return new LoadHistoryResponce(history);
}


private void loadCDSEntities(BulkLoadFileHistory history, List<ImmutablePair<Gff3DTO, Map<String, String>>> gffData, List<Long> idsAdded, BackendBulkDataProvider dataProvider, ProcessDisplayHelper ph) {
updateHistory(history);
for (ImmutablePair<Gff3DTO, Map<String, String>> gff3EntryPair : gffData) {
try {
gff3Service.loadCDSEntity(history, gff3EntryPair, idsAdded, dataProvider);
CodingSequence cds = gff3DtoValidator.validateCdsEntry(gff3EntryPair.getKey(), gff3EntryPair.getValue(), dataProvider);
if (cds != null) {
idsAdded.add(cds.getId());
}
history.incrementCompleted();
} catch (ObjectUpdateException e) {
history.incrementFailed();
Expand All @@ -119,7 +105,22 @@ private void loadCDSEntities(BulkLoadFileHistory history, List<ImmutablePair<Gff
ph.progressProcess();
}
updateHistory(history);
}

ph.finishProcess();

return true;
}

public APIResponse runLoadApi(String dataProviderName, String assemblyName, List<Gff3DTO> gffData) {
List<Long> idsAdded = new ArrayList<>();
BackendBulkDataProvider dataProvider = BackendBulkDataProvider.valueOf(dataProviderName);
List<ImmutablePair<Gff3DTO, Map<String, String>>> preProcessedCDSGffData = Gff3AttributesHelper.getCDSGffData(gffData, dataProvider);
BulkLoadFileHistory history = new BulkLoadFileHistory(preProcessedCDSGffData.size());

runLoad(history, null, preProcessedCDSGffData, idsAdded, dataProvider);
history.finishLoad();

return new LoadHistoryResponce(history);
}

}
15 changes: 8 additions & 7 deletions ...bs/executors/Gff3CDSLocationExecutor.java → ...xecutors/gff/Gff3CDSLocationExecutor.java
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package org.alliancegenome.curation_api.jobs.executors;
package org.alliancegenome.curation_api.jobs.executors.gff;

import java.io.FileInputStream;
import java.util.ArrayList;
Expand All @@ -16,6 +16,7 @@
import org.alliancegenome.curation_api.response.APIResponse;
import org.alliancegenome.curation_api.response.LoadHistoryResponce;
import org.alliancegenome.curation_api.services.associations.codingSequenceAssociations.CodingSequenceGenomicLocationAssociationService;
import org.alliancegenome.curation_api.services.helpers.gff3.Gff3AttributesHelper;
import org.alliancegenome.curation_api.util.ProcessDisplayHelper;
import org.apache.commons.lang3.tuple.ImmutablePair;

Expand Down Expand Up @@ -52,16 +53,16 @@ public void execLoad(BulkLoadFileHistory bulkLoadFileHistory) {
BulkFMSLoad fmsLoad = (BulkFMSLoad) bulkLoadFileHistory.getBulkLoad();
BackendBulkDataProvider dataProvider = BackendBulkDataProvider.valueOf(fmsLoad.getFmsDataSubType());

List<ImmutablePair<Gff3DTO, Map<String, String>>> preProcessedGffData = preProcessGffData(gffData, dataProvider);
List<ImmutablePair<Gff3DTO, Map<String, String>>> preProcessedCDSGffData = Gff3AttributesHelper.getCDSGffData(gffData, dataProvider);

gffData.clear();

List<Long> idsAdded = new ArrayList<>();

bulkLoadFileHistory.setTotalRecords((long) preProcessedGffData.size());
bulkLoadFileHistory.setTotalRecords((long) preProcessedCDSGffData.size());
updateHistory(bulkLoadFileHistory);

boolean success = runLoad(bulkLoadFileHistory, gffHeaderData, preProcessedGffData, idsAdded, dataProvider, null);
boolean success = runLoad(bulkLoadFileHistory, gffHeaderData, preProcessedCDSGffData, idsAdded, dataProvider, null);

if (success) {
runCleanup(cdsLocationService, bulkLoadFileHistory, dataProvider.name(), cdsLocationService.getIdsByDataProvider(dataProvider), idsAdded, "GFF coding sequence genomic location association");
Expand Down Expand Up @@ -98,10 +99,10 @@ private boolean runLoad(BulkLoadFileHistory history, List<String> gffHeaderData,
public APIResponse runLoadApi(String dataProviderName, String assemblyName, List<Gff3DTO> gffData) {
List<Long> idsAdded = new ArrayList<>();
BackendBulkDataProvider dataProvider = BackendBulkDataProvider.valueOf(dataProviderName);
List<ImmutablePair<Gff3DTO, Map<String, String>>> preProcessedGffData = preProcessGffData(gffData, dataProvider);
BulkLoadFileHistory history = new BulkLoadFileHistory(preProcessedGffData.size());
List<ImmutablePair<Gff3DTO, Map<String, String>>> preProcessedCDSGffData = Gff3AttributesHelper.getCDSGffData(gffData, dataProvider);
BulkLoadFileHistory history = new BulkLoadFileHistory(preProcessedCDSGffData.size());

runLoad(history, null, preProcessedGffData, idsAdded, dataProvider, assemblyName);
runLoad(history, null, preProcessedCDSGffData, idsAdded, dataProvider, assemblyName);
history.finishLoad();

return new LoadHistoryResponce(history);
Expand Down
40 changes: 40 additions & 0 deletions src/main/java/org/alliancegenome/curation_api/jobs/executors/gff/Gff3Executor.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package org.alliancegenome.curation_api.jobs.executors.gff;

import java.util.List;

import org.alliancegenome.curation_api.enums.BackendBulkDataProvider;
import org.alliancegenome.curation_api.exceptions.ObjectUpdateException;
import org.alliancegenome.curation_api.exceptions.ObjectUpdateException.ObjectUpdateExceptionData;
import org.alliancegenome.curation_api.jobs.executors.LoadFileExecutor;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory;
import org.alliancegenome.curation_api.services.Gff3Service;
import org.alliancegenome.curation_api.util.ProcessDisplayHelper;

import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;

@ApplicationScoped
public class Gff3Executor extends LoadFileExecutor {

@Inject Gff3Service gff3Service;

protected String loadGenomeAssembly(String assemblyName, BulkLoadFileHistory history, List<String> gffHeaderData, BackendBulkDataProvider dataProvider, ProcessDisplayHelper ph) {
try {
assemblyName = gff3Service.loadGenomeAssembly(assemblyName, gffHeaderData, dataProvider);
history.incrementCompleted();
} catch (ObjectUpdateException e) {
//e.printStackTrace();
history.incrementFailed();
addException(history, e.getData());
} catch (Exception e) {
e.printStackTrace();
history.incrementFailed();
addException(history, new ObjectUpdateExceptionData(gffHeaderData, e.getMessage(), e.getStackTrace()));
}
updateHistory(history);
ph.progressProcess();

return assemblyName;
}

}
Loading

0 comments on commit 6d14d69

Please sign in to comment.