diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java index 77dbeebdf22..4bc478a8d32 100644 --- a/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/IngestServiceBean.java @@ -1074,6 +1074,104 @@ public String getFilesTempDirectory() { return filesTempDirectory; } + + + /** + * Based on the systemConfig, return the ingest size limit for this file type + * + * @param dataFile + * @return long + */ + private long getIngestSizeLimit(DataFile dataFile){ + + if (dataFile == null){ + throw new NullPointerException("dataFile should not be null!"); + } + + long ingestSizeLimit = -1; + try { + ingestSizeLimit = systemConfig.getTabularIngestSizeLimit(getTabDataReaderByMimeType(dataFile.getContentType()).getFormatName()); + } catch (IOException ioex) { + logger.warning("IO Exception trying to retrieve the ingestable format identifier from the plugin for type "+dataFile.getContentType()+" (non-fatal);"); + } + + return ingestSizeLimit; + } + + /** + * Start Ingest of a single file that has NOT been previously ingested + * + * @param dataFile + * @param user + * @return + */ + public SimpleIngestMessage startIngestSingleFile(DataFile dataFile, AuthenticatedUser user){ + + // ---------------------------- + // Check for nulls + // ---------------------------- + if (dataFile==null){ + throw new NullPointerException("dataFile cannot be null"); + } + if (user == null){ + throw new NullPointerException("user cannot be null"); + } + + // ---------------------------- + // Check if the user is a super user + // ---------------------------- + if (!(user.isSuperuser())){ + return SimpleIngestMessage.getInfoFail("Permission denied. Currently this call is restricted to superusers."); + } + + // ---------------------------- + // Check for an existing DataTable + // ---------------------------- + if (dataFile.getDataTable() != null){ + return SimpleIngestMessage.getInfoFail("This file has already been ingested. (A data table exists for this file)."); + } + + // ---------------------------- + // Set the appropriate status on the file + // ---------------------------- + dataFile.SetIngestScheduled(); + dataFile = fileService.save(dataFile); + + // ---------------------------- + // Check size limit + // ---------------------------- + long ingestSizeLimit = this.getIngestSizeLimit(dataFile); + + if (ingestSizeLimit != -1 || dataFile.getFilesize() >= ingestSizeLimit) { + dataFile.setIngestDone(); + dataFile = fileService.save(dataFile); + + String errMsg = "Skipping tabular ingest of the file " + dataFile.getFileMetadata().getLabel() + "(file id: " + dataFile.getId() + "), because of the size limit (set to "+ ingestSizeLimit +" bytes)."; + logger.info(errMsg); + return SimpleIngestMessage.getInfoFail(errMsg); + } + + // ---------------------------- + // Prepare for ingest! + // ---------------------------- + + // Update status + dataFile.SetIngestInProgress(); + dataFile = fileService.save(dataFile); + logger.fine("Attempting to queue the file " + dataFile.getFileMetadata().getLabel() + " for ingest, file id: " + dataFile.getId()); + + // Make an IngestMessage object + IngestMessage ingestMessage = new IngestMessage(IngestMessage.INGEST_MESAGE_LEVEL_INFO); + ingestMessage.addFileId(dataFile.getId()); + + // Actually send the file to the ingest queue + sendIngestMessageToQueue(ingestMessage); + + return SimpleIngestMessage.getInfoSuccess(); + } + + + // TODO: consider creating a version of this method that would take // datasetversion as the argument. // -- L.A. 4.0 post-beta. @@ -1090,12 +1188,8 @@ public void startIngestJobs(Dataset dataset, AuthenticatedUser user) { // (switching to refinding via id resolves that) dataFile = fileService.find(dataFile.getId()); - long ingestSizeLimit = -1; - try { - ingestSizeLimit = systemConfig.getTabularIngestSizeLimit(getTabDataReaderByMimeType(dataFile.getContentType()).getFormatName()); - } catch (IOException ioex) { - logger.warning("IO Exception trying to retrieve the ingestable format identifier from the plugin for type "+dataFile.getContentType()+" (non-fatal);"); - } + long ingestSizeLimit = this.getIngestSizeLimit(dataFile); + if (ingestSizeLimit == -1 || dataFile.getFilesize() < ingestSizeLimit) { dataFile.SetIngestInProgress(); @@ -1145,6 +1239,9 @@ public int compare(DataFile d1, DataFile d2) { logger.fine("Sorted order: "+i+" (size="+scheduledFilesArray[i].getFilesize()+")"); } + this.sendIngestMessageToQueue(ingestMessage); + } + /* QueueConnection conn = null; QueueSession session = null; QueueSender sender = null; @@ -1181,9 +1278,60 @@ public int compare(DataFile d1, DataFile d2) { ex.printStackTrace(); } } - } + }*/ } + + /** + * Shared ingest method for single files and datasets + * + * @param ingestMessage + */ + private void sendIngestMessageToQueue(IngestMessage ingestMessage){ + + if (ingestMessage == null){ + throw new NullPointerException("ingestMessage should not be null"); + } + + + QueueConnection conn = null; + QueueSession session = null; + QueueSender sender = null; + try { + conn = factory.createQueueConnection(); + session = conn.createQueueSession(false, 0); + sender = session.createSender(queue); + + //ingestMessage.addFile(new File(tempFileLocation)); + Message message = session.createObjectMessage(ingestMessage); + + //try { + sender.send(message); + //} catch (JMSException ex) { + // ex.printStackTrace(); + //} + + } catch (JMSException ex) { + ex.printStackTrace(); + //throw new IOException(ex.getMessage()); + } finally { + try { + + if (sender != null) { + sender.close(); + } + if (session != null) { + session.close(); + } + if (conn != null) { + conn.close(); + } + } catch (JMSException ex) { + ex.printStackTrace(); + } + } + } // end: sendIngestMessageToQueue + public void produceSummaryStatistics(DataFile dataFile) throws IOException { /* logger.info("Skipping summary statistics and UNF."); diff --git a/src/main/java/edu/harvard/iq/dataverse/ingest/SimpleIngestMessage.java b/src/main/java/edu/harvard/iq/dataverse/ingest/SimpleIngestMessage.java new file mode 100644 index 00000000000..d2658b377f6 --- /dev/null +++ b/src/main/java/edu/harvard/iq/dataverse/ingest/SimpleIngestMessage.java @@ -0,0 +1,44 @@ +/* + * To change this license header, choose License Headers in Project Properties. + * To change this template file, choose Tools | Templates + * and open the template in the editor. + */ +package edu.harvard.iq.dataverse.ingest; + +/** + * + * @author rmp553 + */ + class SimpleIngestMessage{ + + public boolean success = true; + public String errorMessage = null; + + public SimpleIngestMessage(boolean success, String errorMessage){ + this.success = success; + if (this.success == false){ + if (errorMessage == null){ + throw new NullPointerException("errorMessage cannot be null. If this was unsuccessfuly, you must include an errorMessage."); + } + this.errorMessage = errorMessage; + } + } + + public boolean wasErrorFound(){ + return this.success; + } + + public String getErrorMessage(){ + return this.errorMessage; + } + + public static SimpleIngestMessage getInfoSuccess(){ + return new SimpleIngestMessage(true, null); + } + + public static SimpleIngestMessage getInfoFail(String errorMsg){ + return new SimpleIngestMessage(false, errorMsg); + } + +} // end SimpleIngestMessage +