(#2906) No NRT export within 24h of previous

WebApp/junit/resources/db_migrations/V44__last_nrt_export_2906.sql

@@ -0,0 +1,2 @@
+-- Add field for last NRT export time
+ALTER TABLE instrument ADD COLUMN last_nrt_export BIGINT NULL DEFAULT NULL AFTER `nrt`;

WebApp/junit/resources/sql/data/DataSet/DataReduction/ControsPco2ReducerTest/dataset_both_zeros/V2000000__dataset_both_zeros.sql

@@ -2,7 +2,7 @@
 -- It consists of a sequence of ZERO, FLUSH, MEASUREMENTS, ZERO
 
 -- Instrument
-INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
+INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,NULL,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
 
 -- Instrument Variables
 INSERT INTO `instrument_variables` VALUES (124,6,'{"zero_flush":"0.00","zero_mode":"Continuous"}','2024-02-01 15:01:57','2024-02-01 16:01:57');

WebApp/junit/resources/sql/data/DataSet/DataReduction/ControsPco2ReducerTest/dataset_post_zero/V2000000__dataset_post_zero.sql

@@ -3,7 +3,7 @@
 -- There is no pre-zero
 
 -- Instrument
-INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
+INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,NULL,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
 
 -- Instrument Variables
 INSERT INTO `instrument_variables` VALUES (124,6,'{"zero_flush":"0.00","zero_mode":"Continuous"}','2024-02-01 15:01:57','2024-02-01 16:01:57');

WebApp/junit/resources/sql/data/DataSet/DataReduction/ControsPco2ReducerTest/dataset_pre_zero/V2000000__dataset_pre_zero.sql

@@ -3,7 +3,7 @@
 -- There is no post-zerp
 
 -- Instrument
-INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
+INSERT INTO `instrument` VALUES (124,1,'Contros','IT-FOS-PALOMA','48MB',0,NULL,'{"depth":"0","latitude":"0.0","postFlushingTime":"0","preFlushingTime":"0","longitude":"0.0","sensorGroups":[{"name":"Default","members":["p_NDIR","p_in","Zero","Flush","Runtime","Signal_raw","Signal_ref","T_gas","TEMP","SAL"]}],"diagnosticQC":{}}','2024-02-01 15:01:57','2024-02-01 16:01:57');
 
 -- Instrument Variables
 INSERT INTO `instrument_variables` VALUES (124,6,'{"zero_flush":"0.00","zero_mode":"Continuous"}','2024-02-01 15:01:57','2024-02-01 16:01:57');

WebApp/junit/resources/sql/data/Instrument/SensorDefinition/VariableTest/flagCascade/V2000000__flagCascade.sql

@@ -68,7 +68,7 @@ INSERT INTO variable_sensors
 
 
 -- Now an instrument using the variable
-INSERT INTO instrument VALUES (1,1,'Test Instrument','Test Instrument','TEST',0,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
+INSERT INTO instrument VALUES (1,1,'Test Instrument','Test Instrument','TEST',0,NULL,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
 
 INSERT INTO instrument_variables (instrument_id, variable_id)
   VALUES (1, (SELECT id FROM variables WHERE name = 'testVar'));

WebApp/junit/resources/sql/data/Instrument/SensorDefinition/VariableTest/flagCascadeDepends/V2000000__flagCascadeDepends.sql

@@ -26,7 +26,7 @@ INSERT INTO variable_sensors
 
 
 -- Now an instrument using the variable
-INSERT INTO instrument VALUES (1,1,'Test Instrument','Test Instrument','TEST',0,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
+INSERT INTO instrument VALUES (1,1,'Test Instrument','Test Instrument','TEST',0,NULL,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
 
 INSERT INTO instrument_variables (instrument_id, variable_id)
   VALUES (1, (SELECT id FROM variables WHERE name = 'testVar'));

WebApp/junit/resources/sql/testbase/instrument/V1000001__instruments.sql

@@ -1,7 +1,7 @@
 -- Benguela Stream instrument definition
 
 -- Instrument
-INSERT INTO instrument VALUES (1,1,'Benguela Stream','Benguela Stream','BSBS',0,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
+INSERT INTO instrument VALUES (1,1,'Benguela Stream','Benguela Stream','BSBS',0,NULL,'','2019-01-28 13:31:21','2019-01-28 14:31:21');
 
 -- Instrument uses the basic marine pCO2 variable
 INSERT INTO instrument_variables (instrument_id, variable_id)

WebApp/src/uk/ac/exeter/QuinCe/api/export/CompleteExport.java

@@ -21,6 +21,6 @@ protected int getNewStatus() {
 
   @Override
   protected void additionalAction(Connection conn, long id) throws Exception {
-    DataSetDB.setDatasetExported(conn, id);
+    DataSetDB.setDatasetExported(conn, id, true);
   }
 }

WebApp/src/uk/ac/exeter/QuinCe/api/export/ExportList.java

@@ -1,6 +1,7 @@
 package uk.ac.exeter.QuinCe.api.export;
 
 import java.sql.Connection;
+import java.time.LocalDateTime;
 import java.util.Collection;
 
 import javax.ws.rs.GET;
@@ -16,6 +17,7 @@
 import uk.ac.exeter.QuinCe.data.Instrument.Instrument;
 import uk.ac.exeter.QuinCe.data.Instrument.InstrumentDB;
 import uk.ac.exeter.QuinCe.utils.DatabaseUtils;
+import uk.ac.exeter.QuinCe.utils.DateTimeUtils;
 import uk.ac.exeter.QuinCe.utils.ExceptionUtils;
 import uk.ac.exeter.QuinCe.web.system.ResourceManager;
 
@@ -44,6 +46,8 @@
 @Path("/export/exportList")
 public class ExportList {
 
+  private static final long MIN_NRT_EXPORT_INTERVAL = 86400L;
+
   /**
    * The main processing method for the API call.
    *
@@ -71,22 +75,40 @@ public String getExportList() throws Exception {
       JsonArray json = new JsonArray();
 
       for (DataSet dataset : datasets) {
-        JsonObject datasetJson = new JsonObject();
-
-        datasetJson.addProperty("id", dataset.getId());
-        datasetJson.addProperty("name", dataset.getName());
-
         Instrument instrument = InstrumentDB.getInstrument(conn,
           dataset.getInstrumentId());
-        JsonObject instrumentJson = new JsonObject();
-        instrumentJson.addProperty("name", instrument.getName());
-        instrumentJson.addProperty("user", instrument.getOwner().getFullName());
-        datasetJson.add("instrument", instrumentJson);
 
-        json.add(datasetJson);
+        boolean canExport = true;
+
+        if (dataset.isNrt()) {
+          if (null != instrument.getLastNrtExport()) {
+            long timeSinceLastNrtExport = DateTimeUtils.secondsBetween(
+              instrument.getLastNrtExport(), LocalDateTime.now());
+            if (timeSinceLastNrtExport < MIN_NRT_EXPORT_INTERVAL) {
+              canExport = false;
+            }
+          }
+        }
+
+        if (canExport) {
+          JsonObject datasetJson = new JsonObject();
+
+          datasetJson.addProperty("id", dataset.getId());
+          datasetJson.addProperty("name", dataset.getName());
+
+          JsonObject instrumentJson = new JsonObject();
+          instrumentJson.addProperty("name", instrument.getName());
+          instrumentJson.addProperty("user",
+            instrument.getOwner().getFullName());
+          datasetJson.add("instrument", instrumentJson);
+
+          json.add(datasetJson);
+        }
       }
       result = json.toString();
-    } catch (Exception e) {
+    } catch (
+
+    Exception e) {
       ExceptionUtils.printStackTrace(e);
       throw e;
     } finally {

WebApp/src/uk/ac/exeter/QuinCe/data/Dataset/DataSetDB.java

@@ -113,6 +113,9 @@ public class DataSetDB {
   private static final String DATASET_EXPORTED_STATEMENT = "UPDATE dataset "
     + "SET exported = 1 WHERE id = ?";
 
+  private static final String LAST_NRT_STATEMENT = "UPDATE instrument SET "
+    + "last_nrt_export = ? WHERE id = ?";
+
   private static final String SENSOR_OFFSETS_PROPERTY = "__SENSOR_OFFSETS";
 
   /**
@@ -1261,18 +1264,62 @@ public static Map<Long, Integer> getDataSetCounts(DataSource dataSource,
     return result;
   }
 
-  public static void setDatasetExported(Connection conn, long datasetId)
-    throws DatabaseException {
+  /**
+   * Mark a {@link DataSet} as having been exported.
+   *
+   * @param conn
+   *          A database connection.
+   * @param datasetId
+   *          The {@link DataSet}'s ID
+   * @param recordNrt
+   *          If this is an NRT dataset, record the last NRT export time for the
+   *          instrument.
+   * @throws DatabaseException
+   * @throws RecordNotFoundException
+   * @throws MissingParamException
+   */
+  public static void setDatasetExported(Connection conn, long datasetId,
+    boolean recordNrt)
+    throws DatabaseException, MissingParamException, RecordNotFoundException {
 
     MissingParam.checkMissing(conn, "conn");
     MissingParam.checkPositive(datasetId, "datasetId");
 
-    try (PreparedStatement stmt = conn
-      .prepareStatement(DATASET_EXPORTED_STATEMENT)) {
-      stmt.setLong(1, datasetId);
-      stmt.execute();
+    PreparedStatement setExportedStmt = null;
+    PreparedStatement setLastNrtStmt = null;
+
+    try {
+      conn.setAutoCommit(false);
+
+      setExportedStmt = conn.prepareStatement(DATASET_EXPORTED_STATEMENT);
+      setExportedStmt.setLong(1, datasetId);
+      setExportedStmt.execute();
+
+      if (recordNrt) {
+        DataSet dataset = getDataSet(conn, datasetId);
+        if (dataset.isNrt()) {
+          setLastNrtStmt = conn.prepareStatement(LAST_NRT_STATEMENT);
+          setLastNrtStmt.setLong(1,
+            DateTimeUtils.dateToLong(LocalDateTime.now()));
+          setLastNrtStmt.setLong(2, dataset.getInstrumentId());
+
+          setLastNrtStmt.execute();
+        }
+      }
+
+      conn.commit();
     } catch (SQLException e) {
       throw new DatabaseException("Error setting dataset export status", e);
+    } finally {
+      DatabaseUtils.closeStatements(setExportedStmt, setLastNrtStmt);
+
+      try {
+        conn.setAutoCommit(true);
+      } catch (SQLException e) {
+        // Close the connection so it can't be reused
+        DatabaseUtils.closeConnection(conn);
+        throw new DatabaseException("Error setting dataset export status", e);
+      }
     }
   }
 }

WebApp/src/uk/ac/exeter/QuinCe/data/Instrument/Instrument.java

@@ -1,5 +1,6 @@
 package uk.ac.exeter.QuinCe.data.Instrument;
 
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
@@ -136,6 +137,12 @@ public class Instrument {
    */
   private boolean nrt = false;
 
+  /**
+   * The time at which the most recent automatic export of an NRT dataset was
+   * performed for this Instrument.
+   */
+  private LocalDateTime lastNrtExport = null;
+
   /**
    * The configuration for the behaviour of diagnostic sensors QC.
    */
@@ -169,8 +176,8 @@ public Instrument(User owner, long databaseId, String name,
     InstrumentFileSet fileDefinitions, List<Variable> variables,
     Map<Variable, Properties> variableProperties,
     SensorAssignments sensorAssignments, String platformName,
-    String platformCode, boolean nrt, String propertiesJson)
-    throws SensorGroupsException {
+    String platformCode, boolean nrt, LocalDateTime lastNrtExport,
+    String propertiesJson) throws SensorGroupsException {
 
     this.owner = owner;
     this.id = databaseId;
@@ -182,6 +189,7 @@ public Instrument(User owner, long databaseId, String name,
     this.platformName = platformName;
     this.platformCode = platformCode;
     this.nrt = nrt;
+    this.lastNrtExport = lastNrtExport;
     parsePropertiesJson(propertiesJson);
   }
 
@@ -210,8 +218,8 @@ public Instrument(User owner, long databaseId, String name,
   public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
     List<Variable> variables, Map<Variable, Properties> variableProperties,
     SensorAssignments sensorAssignments, String platformName,
-    String platformCode, boolean nrt, String propertiesJson)
-    throws SensorGroupsException {
+    String platformCode, boolean nrt, LocalDateTime lastNrtExport,
+    String propertiesJson) throws SensorGroupsException {
 
     this.owner = owner;
     this.name = name;
@@ -222,6 +230,7 @@ public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
     this.platformName = platformName;
     this.platformCode = platformCode;
     this.nrt = nrt;
+    this.lastNrtExport = lastNrtExport;
     parsePropertiesJson(propertiesJson);
   }
 
@@ -247,7 +256,8 @@ public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
   public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
     List<Variable> variables, Map<Variable, Properties> variableProperties,
     SensorAssignments sensorAssignments, SensorGroups sensorGroups,
-    String platformName, String platformCode, boolean nrt) {
+    String platformName, String platformCode, boolean nrt,
+    LocalDateTime lastNrtExport) {
 
     this.owner = owner;
     this.name = name;
@@ -259,6 +269,7 @@ public Instrument(User owner, String name, InstrumentFileSet fileDefinitions,
     this.platformName = platformName;
     this.platformCode = platformCode;
     this.nrt = nrt;
+    this.lastNrtExport = lastNrtExport;
     this.properties = new Properties();
   }
 
@@ -1061,4 +1072,8 @@ public List<Variable> getVariables(Collection<Long> ids)
 
     return result;
   }
+
+  public LocalDateTime getLastNrtExport() {
+    return lastNrtExport;
+  }
 }