Iqss/7493 batch archiving api call

doc/sphinx-guides/source/installation/config.rst

@@ -937,10 +937,10 @@ For example:
 
 ``cp <your key file> /usr/local/payara5/glassfish/domains/domain1/files/googlecloudkey.json``
 
-.. _Archiving API Call:
+.. _Archiving API Calls:
 
-API Call
-++++++++
+API Calls
++++++++++
 
 Once this configuration is complete, you, as a user with the *PublishDataset* permission, should be able to use the API call to manually submit a DatasetVersion for processing:
 
@@ -952,6 +952,18 @@ where:
 
 ``{version}`` is the friendly version number, e.g. "1.2".
 
+A batch API call is also available that will attempt to archive any currently unarchived dataset versions:
+
+``curl -H "X-Dataverse-key: <key>" http://localhost:8080/api/admin/archiveAllUnarchivedDataVersions``
+
+The call supports three optional query parameters that can be used in combination:
+
+``listonly={true/false}`` default is false. Using true retrieves the list of unarchived versions but does not attempt to archive any.
+
+``latestonly={true/false}`` default is false. Using true only lists/processes the most recently published version of a given dataset (instead of all published versions).
+
+``limit={n}`` default is no limit/process all unarchived versions (subject to other parameters). Defines a maximum number of versions to attempt to archive in response to one invocation of the API call.
+
 The submitDataVersionToArchive API (and the workflow discussed below) attempt to archive the dataset version via an archive specific method. For Chronopolis, a DuraCloud space named for the dataset (it's DOI with ':' and '.' replaced with '-') is created and two files are uploaded to it: a version-specific datacite.xml metadata file and a BagIt bag containing the data and an OAI-ORE map file. (The datacite.xml file, stored outside the Bag as well as inside is intended to aid in discovery while the ORE map file is 'complete', containing all user-entered metadata and is intended as an archival record.)
 
 In the Chronopolis case, since the transfer from the DuraCloud front-end to archival storage in Chronopolis can take significant time, it is currently up to the admin/curator to submit a 'snap-shot' of the space within DuraCloud and to monitor its successful transfer. Once transfer is complete the space should be deleted, at which point the Dataverse Software API call can be used to submit a Bag for other versions of the same Dataset. (The space is reused, so that archival copies of different Dataset versions correspond to different snapshots of the same DuraCloud space.).

src/main/java/edu/harvard/iq/dataverse/DatasetVersionServiceBean.java

@@ -1008,7 +1008,7 @@ public List<HashMap<String, Object>> getBasicDatasetVersionInfo(Dataset dataset)
     } // end getBasicDatasetVersionInfo
 
 
-
+    //Not used?
     public HashMap getFileMetadataHistory(DataFile df){
 
         if (df == null){
@@ -1187,4 +1187,28 @@ private DatasetVersion getPreviousVersionWithUnf(DatasetVersion datasetVersion)
         return null;
     }
 
+    /**
+     * Execute a query to return DatasetVersion
+     * 
+     * @param queryString
+     * @return 
+     */
+    public List<DatasetVersion> getUnarchivedDatasetVersions(){
+
+        String queryString = "SELECT OBJECT(o) FROM DatasetVersion AS o WHERE o.releaseTime IS NOT NULL and o.archivalCopyLocation IS NULL";
+
+        try {
+            TypedQuery<DatasetVersion> query = em.createQuery(queryString, DatasetVersion.class);
+            List<DatasetVersion> dsl = query.getResultList();
+            return dsl;
+
+        } catch (javax.persistence.NoResultException e) {
+            logger.log(Level.FINE, "No unarchived DatasetVersions found: {0}", queryString);
+            return null;
+        } catch (EJBException e) {
+            logger.log(Level.WARNING, "EJBException exception: {0}", e.getMessage());
+            return null;
+        }
+    } // end getUnarchivedDatasetVersions
+
 } // end class

src/main/java/edu/harvard/iq/dataverse/api/Admin.java

@@ -45,6 +45,7 @@
 import javax.json.JsonArrayBuilder;
 import javax.json.JsonObjectBuilder;
 import javax.ws.rs.DELETE;
+import javax.ws.rs.DefaultValue;
 import javax.ws.rs.GET;
 import javax.ws.rs.POST;
 import javax.ws.rs.PUT;
@@ -1773,6 +1774,93 @@ public void run() {
         }
     }
 
+
+    /**
+     * Iteratively archives all unarchived dataset versions
+     * @param
+     * listonly - don't archive, just list unarchived versions
+     * limit - max number to process
+     * lastestonly - only archive the latest versions
+     * @return
+     */
+    @GET
+    @Path("/archiveAllUnarchivedDataVersions")
+    public Response archiveAllUnarchivedDatasetVersions(@QueryParam("listonly") boolean listonly, @QueryParam("limit") Integer limit, @QueryParam("latestonly") boolean latestonly) {
+
+        try {
+            AuthenticatedUser au = findAuthenticatedUserOrDie();
+            // Note - the user is being set in the session so it becomes part of the
+            // DataverseRequest and is sent to the back-end command where it is used to get
+            // the API Token which is then used to retrieve files (e.g. via S3 direct
+            // downloads) to create the Bag
+            session.setUser(au);
+            List<DatasetVersion> dsl = datasetversionService.getUnarchivedDatasetVersions();
+            if (dsl != null) {
+                if (listonly) {
+                    JsonArrayBuilder jab = Json.createArrayBuilder();
+                    logger.info("Unarchived versions found: ");
+                    int current = 0;
+                    for (DatasetVersion dv : dsl) {
+                        if (limit != null && current >= limit) {
+                            break;
+                        }
+                        if (!latestonly || dv.equals(dv.getDataset().getLatestVersionForCopy())) {
+                            jab.add(dv.getDataset().getGlobalId().toString() + ", v" + dv.getFriendlyVersionNumber());
+                            logger.info("    " + dv.getDataset().getGlobalId().toString() + ", v" + dv.getFriendlyVersionNumber());
+                            current++;
+                        }
+                    }
+                    return ok(jab); 
+                }
+                String className = settingsService.getValueForKey(SettingsServiceBean.Key.ArchiverClassName);
+                AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dsl.get(0));
+
+                if (cmd != null) {
+                    new Thread(new Runnable() {
+                        public void run() {
+                            int total = dsl.size();
+                            int successes = 0;
+                            int failures = 0;
+                            for (DatasetVersion dv : dsl) {
+                                if (limit != null && (successes + failures) >= limit) {
+                                    break;
+                                }
+                                if (!latestonly || dv.equals(dv.getDataset().getLatestVersionForCopy())) {
+                                    try {
+                                        AbstractSubmitToArchiveCommand cmd = ArchiverUtil.createSubmitToArchiveCommand(className, dvRequestService.getDataverseRequest(), dv);
+
+                                        dv = commandEngine.submit(cmd);
+                                        if (dv.getArchivalCopyLocation() != null) {
+                                            successes++;
+                                            logger.info("DatasetVersion id=" + dv.getDataset().getGlobalId().toString() + " v" + dv.getFriendlyVersionNumber() + " submitted to Archive at: "
+                                                    + dv.getArchivalCopyLocation());
+                                        } else {
+                                            failures++;
+                                            logger.severe("Error submitting version due to conflict/error at Archive for " + dv.getDataset().getGlobalId().toString() + " v" + dv.getFriendlyVersionNumber());
+                                        }
+                                    } catch (CommandException ex) {
+                                        failures++;
+                                        logger.log(Level.SEVERE, "Unexpected Exception calling  submit archive command", ex);
+                                    }
+                                }
+                                logger.fine(successes + failures + " of " + total + " archive submissions complete");
+                            }
+                            logger.info("Archiving complete: " + successes + " Successes, " + failures + " Failures. See prior log messages for details.");
+                        }
+                    }).start();
+                    return ok("Archiving all unarchived published dataset versions using " + cmd.getClass().getCanonicalName() + ". Processing can take significant time for large datasets/ large numbers of dataset versions. View log and/or check archive for results.");
+                } else {
+                    logger.log(Level.SEVERE, "Could not find Archiver class: " + className);
+                    return error(Status.INTERNAL_SERVER_ERROR, "Could not find Archiver class: " + className);
+                }
+            } else {
+                return error(Status.BAD_REQUEST, "No unarchived published dataset versions found");
+            }
+        } catch (WrappedResponse e1) {
+            return error(Status.UNAUTHORIZED, "api key required");
+        }
+    }
+
 	@DELETE
 	@Path("/clearMetricsCache")
 	public Response clearMetricsCache() {