From e504bfcc134083699d252bc8d33eadb41625209d Mon Sep 17 00:00:00 2001 From: lubitchv Date: Wed, 23 Mar 2022 17:11:58 -0400 Subject: [PATCH 01/20] tab ingest skip option --- .../edu/harvard/iq/dataverse/DatasetPage.java | 2 +- .../iq/dataverse/EditDatafilesPage.java | 2 +- .../harvard/iq/dataverse/api/Datasets.java | 7 +++- .../datadeposit/MediaResourceManagerImpl.java | 2 +- .../datasetutility/AddReplaceFileHelper.java | 30 ++++++------- .../dataverse/ingest/IngestServiceBean.java | 4 +- .../harvard/iq/dataverse/api/DatasetsIT.java | 42 +++++++++++++++++++ .../edu/harvard/iq/dataverse/api/UtilIT.java | 24 ++++++++--- 8 files changed, 88 insertions(+), 25 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java index 29f08d885cd..85b9eeb5543 100644 --- a/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/DatasetPage.java @@ -3591,7 +3591,7 @@ public String save() { // have been created in the dataset. dataset = datasetService.find(dataset.getId()); - List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles, null); + List filesAdded = ingestService.saveAndAddFilesToDataset(dataset.getEditVersion(), newFiles, null, true); newFiles.clear(); // and another update command: diff --git a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java index 393aa870bc8..a7fc03e216f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java +++ b/src/main/java/edu/harvard/iq/dataverse/EditDatafilesPage.java @@ -1021,7 +1021,7 @@ public String save() { } // Try to save the NEW files permanently: - List filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null); + List filesAdded = ingestService.saveAndAddFilesToDataset(workingVersion, newFiles, null, true); // reset the working list of fileMetadatas, as to only include the ones // that have been added to the version successfully: diff --git a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java index e21396dd487..1c4829aa819 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/Datasets.java @@ -2299,6 +2299,7 @@ public Response completeMPUpload(String partETagBody, @QueryParam("globalid") St @Path("{id}/add") @Consumes(MediaType.MULTIPART_FORM_DATA) public Response addFileToDataset(@PathParam("id") String idSupplied, + @QueryParam("tabingest") Boolean tabIngest, @FormDataParam("jsonData") String jsonData, @FormDataParam("file") InputStream fileInputStream, @FormDataParam("file") FormDataContentDisposition contentDispositionHeader, @@ -2308,6 +2309,9 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, if (!systemConfig.isHTTPUpload()) { return error(Response.Status.SERVICE_UNAVAILABLE, BundleUtil.getStringFromBundle("file.api.httpDisabled")); } + if (tabIngest == null) { + tabIngest = true; + } // ------------------------------------- // (1) Get the user from the API key @@ -2427,7 +2431,8 @@ public Response addFileToDataset(@PathParam("id") String idSupplied, newFileContentType, newStorageIdentifier, fileInputStream, - optionalFileParams); + optionalFileParams, + tabIngest); if (addFileHelper.hasError()){ diff --git a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java index 025e1c865a4..e50b731ca02 100644 --- a/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java +++ b/src/main/java/edu/harvard/iq/dataverse/api/datadeposit/MediaResourceManagerImpl.java @@ -336,7 +336,7 @@ DepositReceipt replaceOrAddFiles(String uri, Deposit deposit, AuthCredentials au throw new SwordError(UriRegistry.ERROR_BAD_REQUEST, "Unable to add file(s) to dataset: " + violation.getMessage() + " The invalid value was \"" + violation.getInvalidValue() + "\"."); } else { - ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, null); + ingestService.saveAndAddFilesToDataset(editVersion, dataFiles, null, true); } } else { diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 2c0f0f514cd..0fb7ea03a4d 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -334,8 +334,9 @@ public boolean runAddFileByDataset(Dataset chosenDataset, String newFileContentType, String newStorageIdentifier, InputStream newFileInputStream, - OptionalFileParams optionalFileParams){ - return this.runAddFileByDataset(chosenDataset,newFileName,newFileContentType,newStorageIdentifier,newFileInputStream,optionalFileParams,false); + OptionalFileParams optionalFileParams, + Boolean tabIngest){ + return this.runAddFileByDataset(chosenDataset,newFileName,newFileContentType,newStorageIdentifier,newFileInputStream,optionalFileParams,false, tabIngest); } @@ -345,7 +346,8 @@ public boolean runAddFileByDataset(Dataset chosenDataset, String newStorageIdentifier, InputStream newFileInputStream, OptionalFileParams optionalFileParams, - boolean multipleFiles) { + boolean multipleFiles, + Boolean tabIngest) { msgt(">> runAddFileByDatasetId"); @@ -363,7 +365,7 @@ public boolean runAddFileByDataset(Dataset chosenDataset, } //return this.runAddFile(this.dataset, newFileName, newFileContentType, newFileInputStream, optionalFileParams); - return this.runAddReplaceFile(dataset, newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams); + return this.runAddReplaceFile(dataset, newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams, tabIngest); } @@ -431,7 +433,7 @@ public boolean runForceReplaceFile(Long oldFileId, } - return this.runAddReplaceFile(fileToReplace.getOwner(), newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams); + return this.runAddReplaceFile(fileToReplace.getOwner(), newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams, true); } @@ -458,7 +460,7 @@ public boolean runReplaceFile(Long oldFileId, if (!this.step_005_loadFileToReplaceById(oldFileId)){ return false; } - return this.runAddReplaceFile(fileToReplace.getOwner(), newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams); + return this.runAddReplaceFile(fileToReplace.getOwner(), newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams, true); } @@ -486,7 +488,7 @@ public boolean runReplaceFile(Long oldFileId, private boolean runAddReplaceFile(Dataset owner, String newFileName, String newFileContentType, String newStorageIdentifier, InputStream newFileInputStream, - OptionalFileParams optionalFileParams){ + OptionalFileParams optionalFileParams, Boolean tabInjest){ // Run "Phase 1" - Initial ingest of file + error check // But don't save the dataset version yet @@ -503,7 +505,7 @@ private boolean runAddReplaceFile(Dataset owner, } - return runAddReplacePhase2(); + return runAddReplacePhase2(tabInjest); } @@ -653,7 +655,7 @@ private boolean runAddReplacePhase1(Dataset owner, public boolean runReplaceFromUI_Phase2(){ - return runAddReplacePhase2(); + return runAddReplacePhase2(true); } @@ -744,7 +746,7 @@ public boolean updateLabelDescriptionRestrictedFromUI(String label, String descr * * @return */ - private boolean runAddReplacePhase2(){ + private boolean runAddReplacePhase2(Boolean tabIngest){ if (this.hasError()){ return false; // possible to have errors already... @@ -756,7 +758,7 @@ private boolean runAddReplacePhase2(){ } msgt("step_060_addFilesViaIngestService"); - if (!this.step_060_addFilesViaIngestService()){ + if (!this.step_060_addFilesViaIngestService(tabIngest)){ return false; } @@ -1570,7 +1572,7 @@ private boolean step_055_loadOptionalFileParams(OptionalFileParams optionalFileP return true; } - private boolean step_060_addFilesViaIngestService(){ + private boolean step_060_addFilesViaIngestService(Boolean tabIngest){ if (this.hasError()){ return false; @@ -1583,7 +1585,7 @@ private boolean step_060_addFilesViaIngestService(){ } int nFiles = finalFileList.size(); - finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, fileToReplace); + finalFileList = ingestService.saveAndAddFilesToDataset(workingVersion, finalFileList, fileToReplace, tabIngest); if (nFiles != finalFileList.size()) { if (nFiles == 1) { @@ -2244,4 +2246,4 @@ public String getFileName() 1) Recovery from adding same file and duplicate being found - draft ok - published verion - nope -*/ \ No newline at end of file +*/ diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index cf3c62ef584..72a278a0849 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -155,7 +155,7 @@ public class IngestServiceBean { // DataFileCategory objects, if any were already assigned to the files). // It must be called before we attempt to permanently save the files in // the database by calling the Save command on the dataset and/or version. - public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles, DataFile fileToReplace) { + public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles, DataFile fileToReplace, Boolean tabIngest) { List ret = new ArrayList<>(); if (newFiles != null && newFiles.size() > 0) { @@ -327,7 +327,7 @@ public List saveAndAddFilesToDataset(DatasetVersion version, List> runAddFileByDatasetId"); @@ -365,7 +363,7 @@ public boolean runAddFileByDataset(Dataset chosenDataset, } //return this.runAddFile(this.dataset, newFileName, newFileContentType, newFileInputStream, optionalFileParams); - return this.runAddReplaceFile(dataset, newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams, tabIngest); + return this.runAddReplaceFile(dataset, newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams); } @@ -433,7 +431,7 @@ public boolean runForceReplaceFile(Long oldFileId, } - return this.runAddReplaceFile(fileToReplace.getOwner(), newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams, true); + return this.runAddReplaceFile(fileToReplace.getOwner(), newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams); } @@ -460,7 +458,7 @@ public boolean runReplaceFile(Long oldFileId, if (!this.step_005_loadFileToReplaceById(oldFileId)){ return false; } - return this.runAddReplaceFile(fileToReplace.getOwner(), newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams, true); + return this.runAddReplaceFile(fileToReplace.getOwner(), newFileName, newFileContentType, newStorageIdentifier, newFileInputStream, optionalFileParams); } @@ -488,7 +486,7 @@ public boolean runReplaceFile(Long oldFileId, private boolean runAddReplaceFile(Dataset owner, String newFileName, String newFileContentType, String newStorageIdentifier, InputStream newFileInputStream, - OptionalFileParams optionalFileParams, Boolean tabInjest){ + OptionalFileParams optionalFileParams){ // Run "Phase 1" - Initial ingest of file + error check // But don't save the dataset version yet @@ -505,7 +503,7 @@ private boolean runAddReplaceFile(Dataset owner, } - return runAddReplacePhase2(tabInjest); + return runAddReplacePhase2(optionalFileParams.getTabIngest()); } @@ -1572,7 +1570,7 @@ private boolean step_055_loadOptionalFileParams(OptionalFileParams optionalFileP return true; } - private boolean step_060_addFilesViaIngestService(Boolean tabIngest){ + private boolean step_060_addFilesViaIngestService(boolean tabIngest){ if (this.hasError()){ return false; diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index cc75375f979..50f3bbdc948 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -63,6 +63,9 @@ public class OptionalFileParams { private boolean restrict = false; public static final String RESTRICT_ATTR_NAME = "restrict"; + + private boolean tabIngest = true; + public static final String TAB_INGEST_ATTR_NAME = "tabIngest"; private String storageIdentifier; public static final String STORAGE_IDENTIFIER_ATTR_NAME = "storageIdentifier"; @@ -173,7 +176,12 @@ public void setRestriction(boolean restrict){ public boolean getRestriction(){ return this.restrict; } - + + public void setTabIngest(boolean tabIngest) { this.tabIngest = tabIngest; } + + public boolean getTabIngest() { return this.tabIngest; } + + public boolean hasCategories(){ if ((categories == null)||(this.categories.isEmpty())){ return false; @@ -347,6 +355,14 @@ private void loadParamsFromJson(String jsonData) throws DataFileTagException{ this.restrict = Boolean.valueOf(jsonObj.get(RESTRICT_ATTR_NAME).getAsString()); } + + // ------------------------------- + // get tabIngest as boolean + // ------------------------------- + if ((jsonObj.has(TAB_INGEST_ATTR_NAME)) && (!jsonObj.get(TAB_INGEST_ATTR_NAME).isJsonNull())){ + + this.tabIngest = Boolean.valueOf(jsonObj.get(TAB_INGEST_ATTR_NAME).getAsString()); + } // ------------------------------- // get storage identifier as string From d0c70e7b9e78ab250088f488465d7b8081ead008 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 24 Mar 2022 13:20:53 -0400 Subject: [PATCH 03/20] small fixes --- .../dataverse/ingest/IngestServiceBean.java | 2 +- .../harvard/iq/dataverse/api/DatasetsIT.java | 11 ++++++----- .../edu/harvard/iq/dataverse/api/UtilIT.java | 19 +++---------------- 3 files changed, 10 insertions(+), 22 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 72a278a0849..d3020f4ddac 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -155,7 +155,7 @@ public class IngestServiceBean { // DataFileCategory objects, if any were already assigned to the files). // It must be called before we attempt to permanently save the files in // the database by calling the Save command on the dataset and/or version. - public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles, DataFile fileToReplace, Boolean tabIngest) { + public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles, DataFile fileToReplace, boolean tabIngest) { List ret = new ArrayList<>(); if (newFiles != null && newFiles.size() > 0) { diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index b794069fbaf..50a0875cfd6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2601,19 +2601,20 @@ public void testAddFileToDatasetTabIngest() throws IOException, InterruptedExcep Integer datasetIdInt = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); String pathToFile = "src/test/resources/sav/dct.sav"; - String jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\"}"; - Response r = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken, true); + String jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\", \"tabIngest\":\"false\"}"; + Response r = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); logger.info(r.prettyPrint()); assertEquals(200, r.getStatusCode()); - Thread.sleep(500); + Thread.sleep(300); pathToFile = "src/test/resources/sav/frequency-test.sav"; - Response rTabIngest = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken, false); + jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\" }"; + Response rTabIngest = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); logger.info(rTabIngest.prettyPrint()); assertEquals(200, rTabIngest.getStatusCode()); //cleanup - Thread.sleep(500); + Thread.sleep(1000); Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken); assertEquals(200, destroyDatasetResponse.getStatusCode()); diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 33dd3ebcb35..d8372fff6c6 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -39,7 +39,7 @@ import org.hamcrest.Description; import org.hamcrest.Matcher; -import static com.jayway.restassured.RestAssured.*; +import static com.jayway.restassured.RestAssured.given; import static com.jayway.restassured.path.xml.XmlPath.from; import edu.harvard.iq.dataverse.DatasetField; @@ -637,22 +637,12 @@ static Response uploadFileViaNative(String datasetId, String pathToFile, JsonObj return uploadFileViaNative(datasetId, pathToFile, jsonObject.toString(), apiToken); } - static Response uploadFileViaNative(String datasetId, String pathToFile, String jsonAsString, String mimeType, String apiToken) { - Boolean tabIngest = null; - return uploadFileViaNative(datasetId, pathToFile, jsonAsString, mimeType, apiToken, tabIngest); - } - static Response uploadFileViaNative(String datasetId, String pathToFile, String jsonAsString, String apiToken) { String nullMimeType = null; - Boolean tabIngest = null; - return uploadFileViaNative(datasetId, pathToFile, jsonAsString, nullMimeType, apiToken, tabIngest); - } - static Response uploadFileViaNative(String datasetId, String pathToFile, String jsonAsString, String apiToken, Boolean tabIngest) { - String nullMimeType = null; - return uploadFileViaNative(datasetId, pathToFile, jsonAsString, nullMimeType, apiToken, tabIngest); + return uploadFileViaNative(datasetId, pathToFile, jsonAsString, nullMimeType, apiToken); } - static Response uploadFileViaNative(String datasetId, String pathToFile, String jsonAsString, String mimeType, String apiToken, Boolean tabIngest) { + static Response uploadFileViaNative(String datasetId, String pathToFile, String jsonAsString, String mimeType, String apiToken) { RequestSpecification requestSpecification = given() .header(API_TOKEN_HTTP_HEADER, apiToken) .multiPart("datasetId", datasetId) @@ -661,9 +651,6 @@ static Response uploadFileViaNative(String datasetId, String pathToFile, String requestSpecification.multiPart("jsonData", jsonAsString); } String postString = "/api/datasets/" + datasetId + "/add"; - if (tabIngest != null) - postString = postString + "?tabingest=" + tabIngest.toString(); - logger.info(postString); return requestSpecification.post(postString); } From 46d07ce41c360f200b38048cb8308c1a52f38b7a Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 24 Mar 2022 13:28:15 -0400 Subject: [PATCH 04/20] docs --- doc/sphinx-guides/source/api/native-api.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 31b6d777526..a60b5fadf5f 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1301,6 +1301,7 @@ When adding a file to a dataset, you can optionally specify the following: - A description of the file. - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset. - Whether or not the file is restricted. +- Wheather or not the file skips tabular ingest Note that when a Dataverse instance is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. @@ -1315,13 +1316,13 @@ In the curl example below, all of the above are specified but they are optional. export SERVER_URL=https://demo.dataverse.org export PERSISTENT_ID=doi:10.5072/FK2/J8SJZB - curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false"}' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID" + curl -H X-Dataverse-key:$API_TOKEN -X POST -F "file=@$FILENAME" -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "$SERVER_URL/api/datasets/:persistentId/add?persistentId=$PERSISTENT_ID" The fully expanded example above (without environment variables) looks like this: .. code-block:: bash - curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB" + curl -H X-Dataverse-key:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx -X POST -F file=@data.tsv -F 'jsonData={"description":"My description.","directoryLabel":"data/subdir1","categories":["Data"], "restrict":"false", "tabIngest":"false"}' "https://demo.dataverse.org/api/datasets/:persistentId/add?persistentId=doi:10.5072/FK2/J8SJZB" You should expect a 201 ("CREATED") response and JSON indicating the database id that has been assigned to your newly uploaded file. From 3f92d83a53ef11ba56fa2c9c9a3a1f754f418dd2 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 24 Mar 2022 13:35:48 -0400 Subject: [PATCH 05/20] UtilIT back --- .../edu/harvard/iq/dataverse/api/UtilIT.java | 66 +------------------ 1 file changed, 3 insertions(+), 63 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index d8372fff6c6..5ccccd8ec08 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -39,9 +39,9 @@ import org.hamcrest.Description; import org.hamcrest.Matcher; -import static com.jayway.restassured.RestAssured.given; +import static com.jayway.restassured.RestAssured.put; import static com.jayway.restassured.path.xml.XmlPath.from; - +import static com.jayway.restassured.RestAssured.given; import edu.harvard.iq.dataverse.DatasetField; import edu.harvard.iq.dataverse.DatasetFieldConstant; import edu.harvard.iq.dataverse.DatasetFieldType; @@ -650,8 +650,7 @@ static Response uploadFileViaNative(String datasetId, String pathToFile, String if (jsonAsString != null) { requestSpecification.multiPart("jsonData", jsonAsString); } - String postString = "/api/datasets/" + datasetId + "/add"; - return requestSpecification.post(postString); + return requestSpecification.post("/api/datasets/" + datasetId + "/add"); } static Response uploadAuxFile(Long fileId, String pathToFile, String formatTag, String formatVersion, String mimeType, boolean isPublic, String type, String apiToken) { @@ -2397,34 +2396,6 @@ static Response checkDatasetLocks(String idOrPersistentId, String lockType, Stri return response; } - static Response listAllLocks(String apiToken) { - Response response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("api/datasets/locks"); - return response; - } - - static Response listLocksByType(String lockType, String apiToken) { - Response response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("api/datasets/locks?type="+lockType); - return response; - } - - static Response listLocksByUser(String userIdentifier, String apiToken) { - Response response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("api/datasets/locks?userIdentifier="+userIdentifier); - return response; - } - - static Response listLocksByTypeAndUser(String lockType, String userIdentifier, String apiToken) { - Response response = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .get("api/datasets/locks?type="+lockType+"&userIdentifier="+userIdentifier); - return response; - } - static Response lockDataset(long datasetId, String lockType, String apiToken) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) @@ -2778,7 +2749,6 @@ static Response setDefaultLicenseById(Long id, String apiToken) { static Response setLicenseActiveById(Long id, boolean state, String apiToken) { Response activateLicenseResponse = given() .header(API_TOKEN_HTTP_HEADER, apiToken) - .urlEncodingEnabled(false) .put("/api/licenses/"+id.toString() + "/:active/" + state); return activateLicenseResponse; } @@ -2841,34 +2811,4 @@ private static DatasetField constructPrimitive(String fieldName, String value) { new DatasetFieldValue(field, value))); return field; } - - - static Response importDatasetDDIViaNativeApi(String apiToken, String dataverseAlias, String xml, String pid, String release) { - - String postString = "/api/dataverses/" + dataverseAlias + "/datasets/:importddi"; - if (pid != null || release != null ) { - //postString = postString + "?"; - if (pid != null) { - postString = postString + "?pid=" + pid; - if (release != null && release.compareTo("yes") == 0) { - postString = postString + "&release=" + release.toString(); - } - } else { - if (release != null && release.compareTo("yes") == 0) { - postString = postString + "?release=" + release.toString(); - } - } - } - logger.info("Here importDatasetDDIViaNativeApi"); - logger.info(postString); - - RequestSpecification importDDI = given() - .header(API_TOKEN_HTTP_HEADER, apiToken) - .urlEncodingEnabled(false) - .body(xml) - .contentType("application/xml"); - - - return importDDI.post(postString); - } } From 5cbe6a7a9ed1f61ce3673004f6f214e87a52ef05 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 24 Mar 2022 13:37:33 -0400 Subject: [PATCH 06/20] boolean --- .../iq/dataverse/datasetutility/AddReplaceFileHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 10064d8e08d..320ebfd4c85 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -744,7 +744,7 @@ public boolean updateLabelDescriptionRestrictedFromUI(String label, String descr * * @return */ - private boolean runAddReplacePhase2(Boolean tabIngest){ + private boolean runAddReplacePhase2(boolean tabIngest){ if (this.hasError()){ return false; // possible to have errors already... From f0123ec3afe0139621b15c527cd6dcf8a95bead6 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 24 Mar 2022 13:41:13 -0400 Subject: [PATCH 07/20] src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java --- .../edu/harvard/iq/dataverse/api/UtilIT.java | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 5ccccd8ec08..23672c45916 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -2396,6 +2396,34 @@ static Response checkDatasetLocks(String idOrPersistentId, String lockType, Stri return response; } + static Response listAllLocks(String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("api/datasets/locks"); + return response; + } + + static Response listLocksByType(String lockType, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("api/datasets/locks?type="+lockType); + return response; + } + + static Response listLocksByUser(String userIdentifier, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("api/datasets/locks?userIdentifier="+userIdentifier); + return response; + } + + static Response listLocksByTypeAndUser(String lockType, String userIdentifier, String apiToken) { + Response response = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .get("api/datasets/locks?type="+lockType+"&userIdentifier="+userIdentifier); + return response; + } + static Response lockDataset(long datasetId, String lockType, String apiToken) { Response response = given() .header(API_TOKEN_HTTP_HEADER, apiToken) @@ -2749,6 +2777,7 @@ static Response setDefaultLicenseById(Long id, String apiToken) { static Response setLicenseActiveById(Long id, boolean state, String apiToken) { Response activateLicenseResponse = given() .header(API_TOKEN_HTTP_HEADER, apiToken) + .urlEncodingEnabled(false) .put("/api/licenses/"+id.toString() + "/:active/" + state); return activateLicenseResponse; } @@ -2811,4 +2840,34 @@ private static DatasetField constructPrimitive(String fieldName, String value) { new DatasetFieldValue(field, value))); return field; } + + + static Response importDatasetDDIViaNativeApi(String apiToken, String dataverseAlias, String xml, String pid, String release) { + + String postString = "/api/dataverses/" + dataverseAlias + "/datasets/:importddi"; + if (pid != null || release != null ) { + //postString = postString + "?"; + if (pid != null) { + postString = postString + "?pid=" + pid; + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "&release=" + release.toString(); + } + } else { + if (release != null && release.compareTo("yes") == 0) { + postString = postString + "?release=" + release.toString(); + } + } + } + logger.info("Here importDatasetDDIViaNativeApi"); + logger.info(postString); + + RequestSpecification importDDI = given() + .header(API_TOKEN_HTTP_HEADER, apiToken) + .urlEncodingEnabled(false) + .body(xml) + .contentType("application/xml"); + + + return importDDI.post(postString); + } } From 1a040a662f3c462f71a3303e11d865c0c3f70b65 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 24 Mar 2022 13:57:40 -0400 Subject: [PATCH 08/20] style --- .../iq/dataverse/datasetutility/OptionalFileParams.java | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java index 50f3bbdc948..35687151090 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/OptionalFileParams.java @@ -177,10 +177,13 @@ public boolean getRestriction(){ return this.restrict; } - public void setTabIngest(boolean tabIngest) { this.tabIngest = tabIngest; } - - public boolean getTabIngest() { return this.tabIngest; } + public void setTabIngest(boolean tabIngest) { + this.tabIngest = tabIngest; + } + public boolean getTabIngest() { + return this.tabIngest; + } public boolean hasCategories(){ if ((categories == null)||(this.categories.isEmpty())){ From 8457911aa505c4839309eeaad6371e634113d328 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 24 Mar 2022 14:04:19 -0400 Subject: [PATCH 09/20] style --- .../edu/harvard/iq/dataverse/ingest/IngestServiceBean.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index d3020f4ddac..848b621986f 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -155,7 +155,10 @@ public class IngestServiceBean { // DataFileCategory objects, if any were already assigned to the files). // It must be called before we attempt to permanently save the files in // the database by calling the Save command on the dataset and/or version. - public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles, DataFile fileToReplace, boolean tabIngest) { + public List saveAndAddFilesToDataset(DatasetVersion version, + List newFiles, + DataFile fileToReplace, + boolean tabIngest) { List ret = new ArrayList<>(); if (newFiles != null && newFiles.size() > 0) { From d5090b7ae2cc2d0fb11693d15a536fd341f42bed Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 24 Mar 2022 14:14:52 -0400 Subject: [PATCH 10/20] style --- .../edu/harvard/iq/dataverse/ingest/IngestServiceBean.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 848b621986f..1add8e53ef0 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -154,8 +154,8 @@ public class IngestServiceBean { // attached to the Dataset via some cascade path (for example, via // DataFileCategory objects, if any were already assigned to the files). // It must be called before we attempt to permanently save the files in - // the database by calling the Save command on the dataset and/or version. - public List saveAndAddFilesToDataset(DatasetVersion version, + // the database by calling the Save command on the dataset and/or version. + public List saveAndAddFilesToDataset(DatasetVersion version, List newFiles, DataFile fileToReplace, boolean tabIngest) { From 2da5d2d1a12c3a61a7c23eabb09eb1d9654e241d Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 28 Mar 2022 15:29:28 -0400 Subject: [PATCH 11/20] json null replace --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 23672c45916..e6db386bed5 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -717,7 +717,7 @@ static Response getCrawlableFileAccess(String datasetId, String folderName, Stri } static Response replaceFile(String fileIdOrPersistentId, String pathToFile, String apiToken) { - String jsonAsString = null; + String jsonAsString = "{}"; return replaceFile(fileIdOrPersistentId, pathToFile, jsonAsString, apiToken); } From eea514e8f943960c164635cfc7252544bd0f3bf4 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 28 Mar 2022 15:36:40 -0400 Subject: [PATCH 12/20] Remove hardcoded sleep --- src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 50a0875cfd6..863384ba37a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2606,7 +2606,6 @@ public void testAddFileToDatasetTabIngest() throws IOException, InterruptedExcep logger.info(r.prettyPrint()); assertEquals(200, r.getStatusCode()); - Thread.sleep(300); pathToFile = "src/test/resources/sav/frequency-test.sav"; jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\" }"; Response rTabIngest = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); @@ -2614,7 +2613,8 @@ public void testAddFileToDatasetTabIngest() throws IOException, InterruptedExcep assertEquals(200, rTabIngest.getStatusCode()); //cleanup - Thread.sleep(1000); + assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetIdInt, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); + Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken); assertEquals(200, destroyDatasetResponse.getStatusCode()); From 3170ef220b1184536c0b0c05e4aca552b73d15f3 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch <43550154+lubitchv@users.noreply.github.com> Date: Mon, 28 Mar 2022 15:38:02 -0400 Subject: [PATCH 13/20] Update doc/sphinx-guides/source/api/native-api.rst Co-authored-by: Philip Durbin --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index a60b5fadf5f..1d4210c0485 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1301,7 +1301,7 @@ When adding a file to a dataset, you can optionally specify the following: - A description of the file. - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset. - Whether or not the file is restricted. -- Wheather or not the file skips tabular ingest +- Whether or not the file skips tabular ingest. Note that when a Dataverse instance is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. From 9b04c9c8424dcacc3dceb37c6eb18b9fa1fb524c Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 28 Mar 2022 16:31:18 -0400 Subject: [PATCH 14/20] optionalFileParams canbe null --- .../iq/dataverse/datasetutility/AddReplaceFileHelper.java | 8 +++++--- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 4 ++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java index 320ebfd4c85..5e5e49c2186 100644 --- a/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java +++ b/src/main/java/edu/harvard/iq/dataverse/datasetutility/AddReplaceFileHelper.java @@ -501,9 +501,11 @@ private boolean runAddReplaceFile(Dataset owner, if (!phase1Success){ return false; } - - - return runAddReplacePhase2(optionalFileParams.getTabIngest()); + boolean tabIngest = true; + if (optionalFileParams != null) { + tabIngest = optionalFileParams.getTabIngest(); + } + return runAddReplacePhase2(tabIngest); } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index e6db386bed5..7b2a44f7696 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -717,8 +717,8 @@ static Response getCrawlableFileAccess(String datasetId, String folderName, Stri } static Response replaceFile(String fileIdOrPersistentId, String pathToFile, String apiToken) { - String jsonAsString = "{}"; - return replaceFile(fileIdOrPersistentId, pathToFile, jsonAsString, apiToken); + String jsonString = null; + return replaceFile(fileIdOrPersistentId, pathToFile, jsonString, apiToken); } static Response replaceFile(String fileIdOrPersistentId, String pathToFile, JsonObject jsonObject, String apiToken) { From 4fc0a35c8fe981701b1daaf1a5bc9079e90ddca8 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Mon, 28 Mar 2022 16:33:58 -0400 Subject: [PATCH 15/20] json --- src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java index 7b2a44f7696..23672c45916 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/UtilIT.java @@ -717,8 +717,8 @@ static Response getCrawlableFileAccess(String datasetId, String folderName, Stri } static Response replaceFile(String fileIdOrPersistentId, String pathToFile, String apiToken) { - String jsonString = null; - return replaceFile(fileIdOrPersistentId, pathToFile, jsonString, apiToken); + String jsonAsString = null; + return replaceFile(fileIdOrPersistentId, pathToFile, jsonAsString, apiToken); } static Response replaceFile(String fileIdOrPersistentId, String pathToFile, JsonObject jsonObject, String apiToken) { From 2163fdc29edf57cb62005071c11cbe249c233a41 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch Date: Tue, 29 Mar 2022 16:13:54 -0400 Subject: [PATCH 16/20] test and doc --- doc/sphinx-guides/source/api/native-api.rst | 2 +- .../harvard/iq/dataverse/api/DatasetsIT.java | 41 ------------- .../edu/harvard/iq/dataverse/api/FilesIT.java | 59 +++++++++++++++++-- 3 files changed, 56 insertions(+), 46 deletions(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index 1d4210c0485..c607013cf29 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1301,7 +1301,7 @@ When adding a file to a dataset, you can optionally specify the following: - A description of the file. - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset. - Whether or not the file is restricted. -- Whether or not the file skips tabular ingest. +- Whether or not the file skips tabular ingest. If tabIngest parameter is not specified then it defaults to ``true``. Note that when a Dataverse instance is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. diff --git a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java index 863384ba37a..bedc486657d 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/DatasetsIT.java @@ -2584,47 +2584,6 @@ public void testFilesUnchangedAfterDatasetMetadataUpdate() throws IOException { } - @Test - public void testAddFileToDatasetTabIngest() throws IOException, InterruptedException { - - Response createUser = UtilIT.createRandomUser(); - assertEquals(200, createUser.getStatusCode()); - String username = UtilIT.getUsernameFromResponse(createUser); - String apiToken = UtilIT.getApiTokenFromResponse(createUser); - - Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); - assertEquals(201, createDataverseResponse.getStatusCode()); - String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); - - Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); - assertEquals(201, createDatasetResponse.getStatusCode()); - Integer datasetIdInt = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); - - String pathToFile = "src/test/resources/sav/dct.sav"; - String jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\", \"tabIngest\":\"false\"}"; - Response r = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); - logger.info(r.prettyPrint()); - assertEquals(200, r.getStatusCode()); - - pathToFile = "src/test/resources/sav/frequency-test.sav"; - jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\" }"; - Response rTabIngest = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); - logger.info(rTabIngest.prettyPrint()); - assertEquals(200, rTabIngest.getStatusCode()); - - //cleanup - assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetIdInt, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); - - Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken); - assertEquals(200, destroyDatasetResponse.getStatusCode()); - - Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); - assertEquals(200, deleteDataverseResponse.getStatusCode()); - - Response deleteUserResponse = UtilIT.deleteUser(username); - assertEquals(200, deleteUserResponse.getStatusCode()); - - } } diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index b82514ce083..b5d534fefd2 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -3,8 +3,8 @@ import com.jayway.restassured.RestAssured; import com.jayway.restassured.response.Response; import java.util.logging.Logger; -import org.junit.BeforeClass; -import org.junit.Test; + +import org.junit.*; import com.jayway.restassured.path.json.JsonPath; import com.jayway.restassured.path.xml.XmlPath; import static edu.harvard.iq.dataverse.api.AccessIT.apiToken; @@ -39,11 +39,10 @@ import static org.hamcrest.CoreMatchers.startsWith; import static org.hamcrest.CoreMatchers.nullValue; import org.hamcrest.Matchers; -import org.junit.AfterClass; + import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; -import org.junit.Ignore; public class FilesIT { @@ -1784,4 +1783,56 @@ public void testRange() throws IOException { } + @Test + public void testAddFileToDatasetSkipTabIngest() throws IOException, InterruptedException { + + Response createUser = UtilIT.createRandomUser(); + Assert.assertEquals(200, createUser.getStatusCode()); + String username = UtilIT.getUsernameFromResponse(createUser); + String apiToken = UtilIT.getApiTokenFromResponse(createUser); + + Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); + Assert.assertEquals(201, createDataverseResponse.getStatusCode()); + String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); + + Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); + Assert.assertEquals(201, createDatasetResponse.getStatusCode()); + Integer datasetIdInt = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); + + String pathToFile = "src/test/resources/sav/dct.sav"; + String jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\", \"tabIngest\":\"false\"}"; + Response r = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); + logger.info(r.prettyPrint()); + Assert.assertEquals(200, r.getStatusCode()); + + Long dataFileId = JsonPath.from(r.body().asString()).getLong("data.files[0].dataFile.id"); + Response fileMeta = UtilIT.getDataFileMetadataDraft(dataFileId, apiToken); + String label = JsonPath.from(fileMeta.body().asString()).getString("label"); + Assert.assertEquals("dct.sav", label); + + pathToFile = "src/test/resources/sav/frequency-test.sav"; + jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\" }"; + Response rTabIngest = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); + logger.info(rTabIngest.prettyPrint()); + Assert.assertEquals(200, rTabIngest.getStatusCode()); + + assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetIdInt, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); + + Long ingDataFileId = JsonPath.from(rTabIngest.body().asString()).getLong("data.files[0].dataFile.id"); + Response ingFileMeta = UtilIT.getDataFileMetadataDraft(ingDataFileId, apiToken); + String ingLabel = JsonPath.from(ingFileMeta.body().asString()).getString("label"); + Assert.assertEquals("frequency-test.tab", ingLabel); + + //cleanup + Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken); + Assert.assertEquals(200, destroyDatasetResponse.getStatusCode()); + + Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); + Assert.assertEquals(200, deleteDataverseResponse.getStatusCode()); + + Response deleteUserResponse = UtilIT.deleteUser(username); + Assert.assertEquals(200, deleteUserResponse.getStatusCode()); + + } + } From 3cf56e8f29c0f3e8575a00b3650abec5faf2a728 Mon Sep 17 00:00:00 2001 From: Victoria Lubitch Date: Tue, 29 Mar 2022 16:27:24 -0400 Subject: [PATCH 17/20] assert --- .../edu/harvard/iq/dataverse/api/FilesIT.java | 25 ++++++++++--------- 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index b5d534fefd2..cf29c2ab814 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -4,7 +4,8 @@ import com.jayway.restassured.response.Response; import java.util.logging.Logger; -import org.junit.*; +import org.junit.Test; +import org.junit.BeforeClass; import com.jayway.restassured.path.json.JsonPath; import com.jayway.restassured.path.xml.XmlPath; import static edu.harvard.iq.dataverse.api.AccessIT.apiToken; @@ -1787,51 +1788,51 @@ public void testRange() throws IOException { public void testAddFileToDatasetSkipTabIngest() throws IOException, InterruptedException { Response createUser = UtilIT.createRandomUser(); - Assert.assertEquals(200, createUser.getStatusCode()); + assertEquals(200, createUser.getStatusCode()); String username = UtilIT.getUsernameFromResponse(createUser); String apiToken = UtilIT.getApiTokenFromResponse(createUser); Response createDataverseResponse = UtilIT.createRandomDataverse(apiToken); - Assert.assertEquals(201, createDataverseResponse.getStatusCode()); + assertEquals(201, createDataverseResponse.getStatusCode()); String dataverseAlias = UtilIT.getAliasFromResponse(createDataverseResponse); Response createDatasetResponse = UtilIT.createRandomDatasetViaNativeApi(dataverseAlias, apiToken); - Assert.assertEquals(201, createDatasetResponse.getStatusCode()); + assertEquals(201, createDatasetResponse.getStatusCode()); Integer datasetIdInt = JsonPath.from(createDatasetResponse.body().asString()).getInt("data.id"); String pathToFile = "src/test/resources/sav/dct.sav"; String jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\", \"tabIngest\":\"false\"}"; Response r = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); logger.info(r.prettyPrint()); - Assert.assertEquals(200, r.getStatusCode()); - + assertEquals(200, r.getStatusCode()); + Long dataFileId = JsonPath.from(r.body().asString()).getLong("data.files[0].dataFile.id"); Response fileMeta = UtilIT.getDataFileMetadataDraft(dataFileId, apiToken); String label = JsonPath.from(fileMeta.body().asString()).getString("label"); - Assert.assertEquals("dct.sav", label); + assertEquals("dct.sav", label); pathToFile = "src/test/resources/sav/frequency-test.sav"; jsonAsString = "{\"description\":\"My description.\",\"directoryLabel\":\"data/subdir1\",\"categories\":[\"Data\"], \"restrict\":\"false\" }"; Response rTabIngest = UtilIT.uploadFileViaNative(datasetIdInt.toString(), pathToFile, jsonAsString, apiToken); logger.info(rTabIngest.prettyPrint()); - Assert.assertEquals(200, rTabIngest.getStatusCode()); + assertEquals(200, rTabIngest.getStatusCode()); assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetIdInt, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); Long ingDataFileId = JsonPath.from(rTabIngest.body().asString()).getLong("data.files[0].dataFile.id"); Response ingFileMeta = UtilIT.getDataFileMetadataDraft(ingDataFileId, apiToken); String ingLabel = JsonPath.from(ingFileMeta.body().asString()).getString("label"); - Assert.assertEquals("frequency-test.tab", ingLabel); + assertEquals("frequency-test.tab", ingLabel); //cleanup Response destroyDatasetResponse = UtilIT.destroyDataset(datasetIdInt, apiToken); - Assert.assertEquals(200, destroyDatasetResponse.getStatusCode()); + assertEquals(200, destroyDatasetResponse.getStatusCode()); Response deleteDataverseResponse = UtilIT.deleteDataverse(dataverseAlias, apiToken); - Assert.assertEquals(200, deleteDataverseResponse.getStatusCode()); + assertEquals(200, deleteDataverseResponse.getStatusCode()); Response deleteUserResponse = UtilIT.deleteUser(username); - Assert.assertEquals(200, deleteUserResponse.getStatusCode()); + assertEquals(200, deleteUserResponse.getStatusCode()); } From fe9e7e660cf51e169eed15dd9eb16e21662ad3d5 Mon Sep 17 00:00:00 2001 From: lubitchv Date: Thu, 31 Mar 2022 17:14:01 -0400 Subject: [PATCH 18/20] add waitForLock --- src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java index cf29c2ab814..60cb608a50a 100644 --- a/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java +++ b/src/test/java/edu/harvard/iq/dataverse/api/FilesIT.java @@ -1806,6 +1806,8 @@ public void testAddFileToDatasetSkipTabIngest() throws IOException, InterruptedE logger.info(r.prettyPrint()); assertEquals(200, r.getStatusCode()); + assertTrue("Failed test if Ingest Lock exceeds max duration " + pathToFile, UtilIT.sleepForLock(datasetIdInt, "Ingest", apiToken, UtilIT.MAXIMUM_INGEST_LOCK_DURATION)); + Long dataFileId = JsonPath.from(r.body().asString()).getLong("data.files[0].dataFile.id"); Response fileMeta = UtilIT.getDataFileMetadataDraft(dataFileId, apiToken); String label = JsonPath.from(fileMeta.body().asString()).getString("label"); From 47c1413f9290088dc6d0dfd11096ae8a57b599a6 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 12 Apr 2022 10:00:52 -0400 Subject: [PATCH 19/20] link to tabular ingest docs #8525 --- doc/sphinx-guides/source/api/native-api.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/sphinx-guides/source/api/native-api.rst b/doc/sphinx-guides/source/api/native-api.rst index c607013cf29..816a2ae3988 100644 --- a/doc/sphinx-guides/source/api/native-api.rst +++ b/doc/sphinx-guides/source/api/native-api.rst @@ -1301,7 +1301,7 @@ When adding a file to a dataset, you can optionally specify the following: - A description of the file. - The "File Path" of the file, indicating which folder the file should be uploaded to within the dataset. - Whether or not the file is restricted. -- Whether or not the file skips tabular ingest. If tabIngest parameter is not specified then it defaults to ``true``. +- Whether or not the file skips :doc:`tabular ingest `. If the ``tabIngest`` parameter is not specified, it defaults to ``true``. Note that when a Dataverse instance is configured to use S3 storage with direct upload enabled, there is API support to send a file directly to S3. This is more complex and is described in the :doc:`/developers/s3-direct-upload-api` guide. From 66b774bc2d16308be16158b0da883344131f7db5 Mon Sep 17 00:00:00 2001 From: Philip Durbin Date: Tue, 12 Apr 2022 10:01:36 -0400 Subject: [PATCH 20/20] add release note #8525 --- doc/release-notes/8525-ingest-optional-skip.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 doc/release-notes/8525-ingest-optional-skip.md diff --git a/doc/release-notes/8525-ingest-optional-skip.md b/doc/release-notes/8525-ingest-optional-skip.md new file mode 100644 index 00000000000..dfec1336ea3 --- /dev/null +++ b/doc/release-notes/8525-ingest-optional-skip.md @@ -0,0 +1 @@ +Tabular ingest can be skipped via API.