diff --git a/doc/release-notes/8525-ingest-optional-skip.md b/doc/release-notes/8525-ingest-optional-skip.md new file mode 100644 index 00000000000..dfec1336ea3 --- /dev/null +++ b/doc/release-notes/8525-ingest-optional-skip.md @@ -0,0 +1 @@ +Tabular ingest can be skipped via API. diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 31b6d777526..816a2ae3988 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1301,6 +1301,7 @@ When adding a file to a dataset, you can optionally specify the following: - A description of the file. - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset. - Whether or not the file is restricted. +- Whether or not the file skips :doc:`tabular ingest `. If the ``tabIngest`` parameter is not specified, it defaults to ``true``. Note that when a Dataverse instance is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. @@ -1315,13 +1316,13 @@ In the curl example below, all of the above are specified but they are optional. export SERVER_URL=https://demo.dataverse.org export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB - curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false"}' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID" + curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB" + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB" You should expect a 201 ("CREATED") response and JSON indicating the database id that has been assigned to your newly uploaded file. diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 29f08d885cd..85b9eeb5543 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3591,7 +3591,7 @@ public String save() { // have been created in the dataset. dataset = datasetService.find(dataset.getId()); - List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles, null); + List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles, null, true); newFiles.clear(); // and another update command: diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 393aa870bc8..a7fc03e216f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1021,7 +1021,7 @@ public String save() { } // Try to save the NEW files permanently: - List filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null); + List filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true); // reset the working list of fileMetadatas, as to only include the ones // that have been added to the version successfully: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java index 025e1c865a4..e50b731ca02 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java @@ -336,7 +336,7 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + violation.getMessage() + " The invalid value was \"" + violation.getInvalidValue() + "\"."); } else { - ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, null); + ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, null, true); } } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 2c0f0f514cd..5e5e49c2186 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -501,9 +501,11 @@ private boolean runAddReplaceFile(Dataset owner, if (!phase1Success){ return false; } - - - return runAddReplacePhase2(); + boolean tabIngest = true; + if (optionalFileParams != null) { + tabIngest = optionalFileParams.getTabIngest(); + } + return runAddReplacePhase2(tabIngest); } @@ -653,7 +655,7 @@ private boolean runAddReplacePhase1(Dataset owner, public boolean runReplaceFromUI_Phase2(){ - return runAddReplacePhase2(); + return runAddReplacePhase2(true); } @@ -744,7 +746,7 @@ public boolean updateLabelDescriptionRestrictedFromUI(String label, String descr * * @return */ - private boolean runAddReplacePhase2(){ + private boolean runAddReplacePhase2(boolean tabIngest){ if (this.hasError()){ return false; // possible to have errors already... @@ -756,7 +758,7 @@ private boolean runAddReplacePhase2(){ } msgt("step_060_addFilesViaIngestService"); - if (!this.step_060_addFilesViaIngestService()){ + if (!this.step_060_addFilesViaIngestService(tabIngest)){ return false; } @@ -1570,7 +1572,7 @@ private boolean step_055_loadOptionalFileParams(OptionalFileParams optionalFileP return true; } - private boolean step_060_addFilesViaIngestService(){ + private boolean step_060_addFilesViaIngestService(boolean tabIngest){ if (this.hasError()){ return false; @@ -1583,7 +1585,7 @@ private boolean step_060_addFilesViaIngestService(){ } int nFiles = finalFileList.size(); - finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, fileToReplace); + finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, fileToReplace, tabIngest); if (nFiles != finalFileList.size()) { if (nFiles == 1) { @@ -2244,4 +2246,4 @@ public String getFileName() 1) Recovery from adding same file and duplicate being found - draft ok - published verion - nope -*/ \ No newline at end of file +*/ diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index cc75375f979..35687151090 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -63,6 +63,9 @@ public class OptionalFileParams { private boolean restrict = false; public static final String RESTRICT_ATTR_NAME = "restrict"; + + private boolean tabIngest = true; + public static final String TAB_INGEST_ATTR_NAME = "tabIngest"; private String storageIdentifier; public static final String STORAGE_IDENTIFIER_ATTR_NAME = "storageIdentifier"; @@ -173,7 +176,15 @@ public void setRestriction(boolean restrict){ public boolean getRestriction(){ return this.restrict; } - + + public void setTabIngest(boolean tabIngest) { + this.tabIngest = tabIngest; + } + + public boolean getTabIngest() { + return this.tabIngest; + } + public boolean hasCategories(){ if ((categories == null)||(this.categories.isEmpty())){ return false; @@ -347,6 +358,14 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{ this.restrict = Boolean.valueOf(jsonObj.get(RESTRICT_ATTR_NAME).getAsString()); } + + // ------------------------------- + // get tabIngest as boolean + // ------------------------------- + if ((jsonObj.has(TAB_INGEST_ATTR_NAME)) && (!jsonObj.get(TAB_INGEST_ATTR_NAME).isJsonNull())){ + + this.tabIngest = Boolean.valueOf(jsonObj.get(TAB_INGEST_ATTR_NAME).getAsString()); + } // ------------------------------- // get storage identifier as string diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index cf3c62ef584..1add8e53ef0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -154,8 +154,11 @@ public class IngestServiceBean { // attached to the Dataset via some cascade path (for example, via // DataFileCategory objects, if any were already assigned to the files). // It must be called before we attempt to permanently save the files in - // the database by calling the Save command on the dataset and/or version. - public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles, DataFile fileToReplace) { + // the database by calling the Save command on the dataset and/or version. + public List saveAndAddFilesToDataset(DatasetVersion version, + List newFiles, + DataFile fileToReplace, + boolean tabIngest) { List ret = new ArrayList<>(); if (newFiles != null && newFiles.size() > 0) { @@ -327,7 +330,7 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List