Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial Code for removing data provider dups #1652

Merged
merged 21 commits into from
Sep 3, 2024
Merged

Initial Code for removing data provider dups #1652

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
b15c397
Initial Code for removing dups
oblodgett Aug 29, 2024
d670d07
Fixed migration file
oblodgett Aug 29, 2024
b043391
Added missing indexes and updated dao's to use cache
oblodgett Aug 29, 2024
fdefa73
Initial code to restructure bulk load file history
oblodgett Aug 29, 2024
a01b94e
Fix for linking objects together
oblodgett Aug 29, 2024
f187d73
Added more code for better performance
oblodgett Aug 30, 2024
04cd0ee
Merge pull request #1653 from alliance-genome/restructure_filehistory
oblodgett Aug 30, 2024
a1800fe
Updated UI
oblodgett Aug 30, 2024
54f90c5
Merge pull request #1654 from alliance-genome/restructure_filehistory
oblodgett Aug 30, 2024
7b14a71
Pretty
oblodgett Aug 30, 2024
da0e2fa
Fix for missing DP
oblodgett Aug 30, 2024
31e0923
Revert DP DAO
oblodgett Aug 30, 2024
2a78f2e
Another attempt to fix DP's
oblodgett Aug 30, 2024
e234093
Added fix for null cross references
oblodgett Aug 30, 2024
09c7fa9
Cache only when it comes from the DB
oblodgett Aug 30, 2024
ebaf68f
Update IT's
oblodgett Aug 30, 2024
09b8de9
Fix IT's
oblodgett Aug 30, 2024
d9cb049
Fix IT's
oblodgett Aug 30, 2024
0917172
Fix IT's
oblodgett Aug 30, 2024
a34f939
More fixes for GFF
oblodgett Aug 30, 2024
7ad4d9e
Removed exceptions
oblodgett Aug 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 15 additions & 14 deletions src/main/cliapp/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion src/main/cliapp/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@
"react-use-websocket": "^3.0.0",
"react18-json-view": "^0.2.8",
"sass": "^1.32.8",
"update-browserslist-db": "^1.1.0",
"use-immer": "^0.7.0"
},
"scripts": {
Expand All @@ -56,7 +57,7 @@
"eslintConfig": {
"extends": [
"react-app",
"prettier"
"prettier"
]
},
"browserslist": [
Expand Down
39 changes: 26 additions & 13 deletions src/main/cliapp/src/containers/dataLoadsPage/DataLoadsComponent.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ export const DataLoadsComponent = () => {
const [bulkLoadDialog, setBulkLoadDialog] = useState(false);
const [expandedGroupRows, setExpandedGroupRows] = useState(null);
const [expandedLoadRows, setExpandedLoadRows] = useState(null);
const [expandedFileRows, setExpandedFileRows] = useState(null);
//const [expandedFileRows, setExpandedFileRows] = useState(null);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should that be removed? Or is that commenting out only temporary?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ya the tables need to get reworked, in order to support to new structure.

const [expandedErrorLoadRows, setExpandedErrorLoadRows] = useState(null);
const [disableFormFields, setDisableFormFields] = useState(false);
const errorMessage = useRef(null);
Expand Down Expand Up @@ -105,8 +105,8 @@ export const DataLoadsComponent = () => {
if (group.loads) {
for (let load of group.loads) {
load.group = group.id;
if (load.loadFiles) {
let sortedFiles = sortFilesByDate(load.loadFiles);
if (load.history) {
let sortedFiles = sortFilesByDate(load.history);
if (sortedFiles[0].bulkloadStatus === 'FAILED') {
_errorLoads.push(load);
}
Expand Down Expand Up @@ -161,7 +161,7 @@ export const DataLoadsComponent = () => {
};

const urlTemplate = (rowData) => {
return <a href={rowData.s3Url}>Download</a>;
return <a href={rowData.bulkLoadFile.s3Url}>Download</a>;
};

const refresh = () => {
Expand Down Expand Up @@ -190,9 +190,9 @@ export const DataLoadsComponent = () => {
setDisableFormFields(true);
};

const deleteLoadFile = (rowData) => {
const deleteLoadFileHistory = (rowData) => {
getService()
.deleteLoadFile(rowData.id)
.deleteLoadFileHistory(rowData.id)
.then((response) => {
queryClient.invalidateQueries(['bulkloadtable']);
});
Expand Down Expand Up @@ -306,7 +306,7 @@ export const DataLoadsComponent = () => {
key="delete"
icon="pi pi-trash"
className="p-button-rounded p-button-danger mr-2"
onClick={() => deleteLoadFile(rowData)}
onClick={() => deleteLoadFileHistory(rowData)}
/>
);
}
Expand Down Expand Up @@ -354,7 +354,7 @@ export const DataLoadsComponent = () => {
);
}

if (!rowData.loadFiles || rowData.loadFiles.length === 0) {
if (!rowData.history || rowData.history.length === 0) {
ret.push(
<Button
key="delete"
Expand Down Expand Up @@ -450,6 +450,7 @@ export const DataLoadsComponent = () => {
if (rowData.bulkloadStatus === 'FAILED') {
styleClass = 'p-button-danger';
}

if (
rowData.bulkloadStatus &&
(rowData.bulkloadStatus.endsWith('STARTED') ||
Expand All @@ -470,12 +471,12 @@ export const DataLoadsComponent = () => {

const bulkloadStatusTemplate = (rowData) => {
let sortedFiles = [];
if (rowData.loadFiles) {
sortedFiles = sortFilesByDate(rowData.loadFiles);
if (rowData.history) {
sortedFiles = sortFilesByDate(rowData.history);
}
let latestStatus = null;
let latestError = null;
if (rowData.loadFiles) {
if (rowData.history) {
latestStatus = sortedFiles[0].bulkloadStatus;
latestError = sortedFiles[0].errorMessage;
}
Expand Down Expand Up @@ -507,13 +508,23 @@ export const DataLoadsComponent = () => {
<DataTable key="historyTable" value={sortedHistory} responsiveLayout="scroll">
<Column field="loadStarted" header="Load Started" />
<Column field="loadFinished" header="Load Finished" />
<Column field="bulkloadStatus" body={bulkloadFileStatusTemplate} header="Status" />
<Column field="completedRecords" header="Records Completed" />
<Column field="failedRecords" header="Records Failed" />
<Column field="totalRecords" header="Total Records" />
<Column field="deletedRecords" header="Deletes Completed" />
<Column field="deleteFailedRecords" header="Deletes Failed" />
<Column field="totalDeleteRecords" header="Total Deletes" />

<Column field="bulkLoadFile.md5Sum" header="MD5 Sum" />
<Column field="bulkLoadFile.fileSize" header="Compressed File Size" />
<Column field="bulkLoadFile.recordCount" header="Record Count" />
<Column field="bulkLoadFile.s3Url" header="S3 Url (Download)" body={urlTemplate} />
<Column field="bulkLoadFile.linkMLSchemaVersion" header="LinkML Schema Version" />
{showModRelease(file)}

<Column body={historyActionBodyTemplate} exportable={false} style={{ minWidth: '8rem' }}></Column>
<Column body={loadFileActionBodyTemplate} exportable={false} style={{ minWidth: '8rem' }}></Column>
</DataTable>
</div>
);
Expand Down Expand Up @@ -551,6 +562,7 @@ export const DataLoadsComponent = () => {
return sortedFiles;
};

/*
const fileTable = (load) => {
let sortedFiles = [];
if (load.loadFiles) {
Expand Down Expand Up @@ -584,6 +596,7 @@ export const DataLoadsComponent = () => {
</div>
);
};
*/

const loadTable = (group) => {
let sortedLoads = [];
Expand All @@ -599,7 +612,7 @@ export const DataLoadsComponent = () => {
responsiveLayout="scroll"
expandedRows={expandedLoadRows}
onRowToggle={(e) => setExpandedLoadRows(e.data)}
rowExpansionTemplate={fileTable}
rowExpansionTemplate={historyTable}
dataKey="id"
>
<Column expander style={{ width: '3em' }} />
Expand Down Expand Up @@ -795,7 +808,7 @@ export const DataLoadsComponent = () => {
responsiveLayout="scroll"
expandedRows={expandedErrorLoadRows}
onRowToggle={(e) => setExpandedErrorLoadRows(e.data)}
rowExpansionTemplate={fileTable}
rowExpansionTemplate={historyTable}
dataKey="id"
>
<Column expander style={{ width: '3em' }} />
Expand Down
19 changes: 17 additions & 2 deletions src/main/cliapp/src/service/DataLoadService.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,6 @@ export class DataLoadService extends BaseAuthService {
delete newLoad[objectKey];
}
}
console.log('Creating: ');
console.log(newLoad);
return this.api.post(`/${endpoint}`, newLoad);
}

Expand Down Expand Up @@ -77,10 +75,27 @@ export class DataLoadService extends BaseAuthService {
return this.api.delete(`/bulkloadfile/${id}`);
}

deleteLoadFileHistory(id) {
return this.api.delete(`/bulkloadfilehistory/${id}`);
}

getBackendBulkLoadTypes(loadType) {
const bulkLoadTypes = {
BulkFMSLoad: [
'GFF',

'GFF_EXON',
'GFF_CDS',
'GFF_TRANSCRIPT',

'GFF_EXON_LOCATION',
'GFF_CDS_LOCATION',
'GFF_TRANSCRIPT_LOCATION',

'GFF_TRANSCRIPT_CDS',
'GFF_TRANSCRIPT_EXON',
'GFF_TRANSCRIPT_GENE',

'HTPDATASET',
'INTERACTION-GEN',
'INTERACTION-MOL',
Expand Down
30 changes: 24 additions & 6 deletions ...n/java/org/alliancegenome/curation_api/controllers/crud/CodingSequenceCrudController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@
import org.alliancegenome.curation_api.controllers.base.BaseEntityCrudController;
import org.alliancegenome.curation_api.dao.CodingSequenceDAO;
import org.alliancegenome.curation_api.interfaces.crud.CodingSequenceCrudInterface;
import org.alliancegenome.curation_api.jobs.executors.Gff3Executor;
import org.alliancegenome.curation_api.jobs.executors.Gff3CDSExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3CDSLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptCDSExecutor;
import org.alliancegenome.curation_api.model.entities.CodingSequence;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory;
import org.alliancegenome.curation_api.model.ingest.dto.fms.Gff3DTO;
import org.alliancegenome.curation_api.response.APIResponse;
import org.alliancegenome.curation_api.response.LoadHistoryResponce;
import org.alliancegenome.curation_api.response.ObjectResponse;
import org.alliancegenome.curation_api.services.CodingSequenceService;

Expand All @@ -19,19 +23,33 @@
@RequestScoped
public class CodingSequenceCrudController extends BaseEntityCrudController<CodingSequenceService, CodingSequence, CodingSequenceDAO> implements CodingSequenceCrudInterface {

@Inject
CodingSequenceService codingSequenceService;
@Inject
Gff3Executor gff3Executor;
@Inject CodingSequenceService codingSequenceService;
@Inject Gff3CDSExecutor gff3CDSExecutor;
@Inject Gff3CDSLocationExecutor gff3CDSLocationExecutor;
@Inject Gff3TranscriptCDSExecutor gff3TranscriptCDSExecutor;

@Override
@PostConstruct
protected void init() {
setService(codingSequenceService);
}

@Override
public APIResponse updateCodingSequences(String dataProvider, String assembly, List<Gff3DTO> gffData) {
return gff3Executor.runLoadApi(dataProvider, assembly, gffData);
BulkLoadFileHistory history = new BulkLoadFileHistory();
LoadHistoryResponce resp = (LoadHistoryResponce) gff3CDSExecutor.runLoadApi(dataProvider, assembly, gffData);
history.setFailedRecords(history.getFailedRecords() + resp.getHistory().getFailedRecords());
history.setCompletedRecords(history.getCompletedRecords() + resp.getHistory().getCompletedRecords());
history.setTotalRecords(history.getTotalRecords() + resp.getHistory().getTotalRecords());
resp = (LoadHistoryResponce) gff3CDSLocationExecutor.runLoadApi(dataProvider, assembly, gffData);
history.setFailedRecords(history.getFailedRecords() + resp.getHistory().getFailedRecords());
history.setCompletedRecords(history.getCompletedRecords() + resp.getHistory().getCompletedRecords());
history.setTotalRecords(history.getTotalRecords() + resp.getHistory().getTotalRecords());
resp = (LoadHistoryResponce) gff3TranscriptCDSExecutor.runLoadApi(dataProvider, assembly, gffData);
history.setFailedRecords(history.getFailedRecords() + resp.getHistory().getFailedRecords());
history.setCompletedRecords(history.getCompletedRecords() + resp.getHistory().getCompletedRecords());
history.setTotalRecords(history.getTotalRecords() + resp.getHistory().getTotalRecords());
return new LoadHistoryResponce(history);
}

@Override
Expand Down
29 changes: 23 additions & 6 deletions src/main/java/org/alliancegenome/curation_api/controllers/crud/ExonCrudController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,14 @@
import org.alliancegenome.curation_api.controllers.base.BaseEntityCrudController;
import org.alliancegenome.curation_api.dao.ExonDAO;
import org.alliancegenome.curation_api.interfaces.crud.ExonCrudInterface;
import org.alliancegenome.curation_api.jobs.executors.Gff3Executor;
import org.alliancegenome.curation_api.jobs.executors.Gff3ExonExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3ExonLocationExecutor;
import org.alliancegenome.curation_api.jobs.executors.Gff3TranscriptExonExecutor;
import org.alliancegenome.curation_api.model.entities.Exon;
import org.alliancegenome.curation_api.model.entities.bulkloads.BulkLoadFileHistory;
import org.alliancegenome.curation_api.model.ingest.dto.fms.Gff3DTO;
import org.alliancegenome.curation_api.response.APIResponse;
import org.alliancegenome.curation_api.response.LoadHistoryResponce;
import org.alliancegenome.curation_api.response.ObjectResponse;
import org.alliancegenome.curation_api.services.ExonService;

Expand All @@ -19,10 +23,10 @@
@RequestScoped
public class ExonCrudController extends BaseEntityCrudController<ExonService, Exon, ExonDAO> implements ExonCrudInterface {

@Inject
ExonService exonService;
@Inject
Gff3Executor gff3Executor;
@Inject ExonService exonService;
@Inject Gff3ExonExecutor gff3ExonExecutor;
@Inject Gff3ExonLocationExecutor gff3ExonLocationExecutor;
@Inject Gff3TranscriptExonExecutor gff3TranscriptExonExecutor;

@Override
@PostConstruct
Expand All @@ -32,7 +36,20 @@ protected void init() {

@Override
public APIResponse updateExons(String dataProvider, String assembly, List<Gff3DTO> gffData) {
return gff3Executor.runLoadApi(dataProvider, assembly, gffData);
BulkLoadFileHistory history = new BulkLoadFileHistory();
LoadHistoryResponce resp = (LoadHistoryResponce) gff3ExonExecutor.runLoadApi(dataProvider, assembly, gffData);
history.setFailedRecords(history.getFailedRecords() + resp.getHistory().getFailedRecords());
history.setCompletedRecords(history.getCompletedRecords() + resp.getHistory().getCompletedRecords());
history.setTotalRecords(history.getTotalRecords() + resp.getHistory().getTotalRecords());
resp = (LoadHistoryResponce) gff3ExonLocationExecutor.runLoadApi(dataProvider, assembly, gffData);
history.setFailedRecords(history.getFailedRecords() + resp.getHistory().getFailedRecords());
history.setCompletedRecords(history.getCompletedRecords() + resp.getHistory().getCompletedRecords());
history.setTotalRecords(history.getTotalRecords() + resp.getHistory().getTotalRecords());
resp = (LoadHistoryResponce) gff3TranscriptExonExecutor.runLoadApi(dataProvider, assembly, gffData);
history.setFailedRecords(history.getFailedRecords() + resp.getHistory().getFailedRecords());
history.setCompletedRecords(history.getCompletedRecords() + resp.getHistory().getCompletedRecords());
history.setTotalRecords(history.getTotalRecords() + resp.getHistory().getTotalRecords());
return new LoadHistoryResponce(history);
}

@Override
Expand Down
Loading
Loading