Improvements related to multidimNcFiles

WEB-INF/classes/com/cohort/util/Calendar2.java

@@ -6,17 +6,12 @@
 
 import com.cohort.array.DoubleArray;
 import com.cohort.array.PrimitiveArray;
-import com.cohort.array.StringArray;
 
-import java.text.SimpleDateFormat;
 import java.util.Arrays;
 import java.util.BitSet;
 import java.util.Calendar;
-import java.util.Date;
 import java.util.GregorianCalendar;
 import java.util.HashMap;
-import java.util.Locale;
-import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 import java.util.TimeZone;
 

WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromMultidimNcFiles.java

@@ -17,6 +17,7 @@
 
 import gov.noaa.pfel.coastwatch.griddata.NcHelper;
 import gov.noaa.pfel.coastwatch.pointdata.Table;
+import gov.noaa.pfel.coastwatch.pointdata.TableFromMultidimNcFile;
 import gov.noaa.pfel.coastwatch.util.FileVisitorDNLS;
 
 import gov.noaa.pfel.erddap.util.EDStatic;
@@ -116,7 +117,9 @@ public Table lowGetSourceDataFromFile(String tFileDir, String tFileName,
             tFileDir + tFileName, fileDir, decompressedDirectory(), 
             EDStatic.decompressedCacheMaxGB, true); //reuseExisting
         if (mustGetData) {
-            table.readMultidimNc(decompFullName, sourceDataNames, null,
+
+            TableFromMultidimNcFile reader = new TableFromMultidimNcFile(table);
+            reader.readMultidimNc(decompFullName, sourceDataNames, null,
                 treatDimensionsAs,
                 getMetadata, standardizeWhat, removeMVRows,  
                 sourceConVars, sourceConOps, sourceConValues);
@@ -231,7 +234,8 @@ public static String generateDatasetsXml(
         //read the sample file
         tStandardizeWhat = tStandardizeWhat < 0 || tStandardizeWhat == Integer.MAX_VALUE?
             DEFAULT_STANDARDIZEWHAT : tStandardizeWhat;
-        dataSourceTable.readMultidimNc(sampleFileName, null, useDimensions,  
+        TableFromMultidimNcFile reader = new TableFromMultidimNcFile(dataSourceTable);
+        reader.readMultidimNc(sampleFileName, null, useDimensions,  
             tDimAs, //treatDimensionsAs
             true, tStandardizeWhat, tRemoveMVRows, //getMetadata, standardizeWhat, removeMVRows
             null, null, null); //conVars, conOps, conVals

download/setupDatasetsXml.html

@@ -172,6 +172,7 @@ <h2><a class="selfLink" id="TableOfContents" href="#TableOfContents" rel="bookma
       <li><a rel="help" href="#standard_name">standard_name</a>
       <li><a rel="help" href="#units">units</a>
       </ul -->
+      <li><a rel="help" href="#removeMVRows"><kbd>&lt;removeMVRows></removeMVRows&gt;</kbd></a>
       <br>&nbsp;
     </ul>
   </ul>
@@ -15755,7 +15756,20 @@ <h2><a class="selfLink" id="details" href="#details" rel="bookmark">Details</a><
         </ul>
 
       </ul> <!-- end of variable attributes -->
-  </ul>
+      <li><a class="selfLink" id="removeMVRows" href="#removeMVRows" rel="bookmark"><kbd><strong>&lt;removeMVRows&gt;</strong></kbd></a> 
+        is an OPTIONAL tag within a <dataset> tag in datasets.xml for EDDTableFromFiles (including all subclasses) datasets, though it is only used for EDDTableFromMultidimNcFiles. It can have a value of true or false. For example,
+          <removeMVRows>true</removeMVRows>
+          <br>
+          This removes any block of rows at the end of a group where all the values are missing_value, _FillValue, or the CoHort ...Array native missing value (or char=#32 for CharArrays).
+          This is for the CF DSG Multidimensional Array file type and similar files.
+          If true, this does the proper test and so always loads all the max dim variables, so it may take extra time. 
+          <br>
+          The default value of <removeMVRows> is false.
+            <br>
+          Recommendation -- If possible for your dataset, we recommend setting removeMVRows to false. Setting removeMVRows to true can significantly slow down requests, though may be needed for some datasets.
+          <br>&nbsp;
+        </ul>
+    </ul>
 </ul>
 
 

pom.xml

@@ -46,7 +46,7 @@
         <erddapcontent.download.version>v2.23</erddapcontent.download.version>
         <erddapreffiles.download.version>1.0.0</erddapreffiles.download.version>
         <netcdfJavaVersion>5.5.3</netcdfJavaVersion>
-        <test.resources.version>test1.01</test.resources.version>
+        <test.resources.version>test1.02</test.resources.version>
         <jettyVersion>12.0.10</jettyVersion>
     </properties>
 

src/test/java/jetty/JettyTests.java

@@ -236,7 +236,7 @@ void testMetadataIso19115() throws Exception {
                         "            <gco:Integer>NUMBER</gco:Integer>\n" + //
                         "          </gmd:dimensionSize>\n" + //
                         "          <gmd:resolution>\n" + //
-                        "            <gco:Measure uom=\"s\">2.176416E8</gco:Measure>\n" + //
+                        "            <gco:Measure uom=\"s\">VALUE</gco:Measure>\n" + //
                         "          </gmd:resolution>\n" + //
                         "        </gmd:MD_Dimension>\n" + //
                         "      </gmd:axisDimensionProperties>\n" + //
@@ -1232,6 +1232,8 @@ void testMetadataIso19115() throws Exception {
         results = results.replaceAll("<gco:Date>....-..-..</gco:Date>", "<gco:Date>YYYY-MM-DD</gco:Date>");
         results = results.replaceAll("<gco:Measure uom=\\\"s\\\">[0-9]+.[0-9]+</gco:Measure>",
                 "<gco:Measure uom=\"s\">VALUE</gco:Measure>");
+        results = results.replaceAll("<gco:Measure uom=\\\"s\\\">.*</gco:Measure>",
+                        "<gco:Measure uom=\"s\">VALUE</gco:Measure>");
         results = results.replaceAll("<gml:endPosition>....-..-..T..:00:00Z</gml:endPosition>",
                 "<gml:endPosition>YYYY-MM-DDThh:00:00Z</gml:endPosition>");
         results = results.replaceAll("<gco:Integer>[0-9]+</gco:Integer>", "<gco:Integer>NUMBER</gco:Integer>");
@@ -2889,7 +2891,7 @@ void testErddap1FilesWAF2() throws Throwable {
         // String2.log("\n*** FileVisitorDNLS.testErddapFilesWAF2()\n");
 
         // *** test localhost
-        String2.log("\nThis test requires erdMH1chla1day in localhost erddap.");
+        // String2.log("\nThis test requires erdMH1chla1day in localhost erddap.");
         String url = "http://localhost:" + PORT + "/erddap/files/erdMH1chla1day/";
         String tFileNameRegex = "A20.*\\.nc(|\\.gz)";
         boolean tRecursive = true;
@@ -4704,7 +4706,7 @@ void testJsonld() throws Throwable {
                         "  \"name\": \"Chlorophyll-a, Aqua MODIS, NPP, L3SMI, Global, 4km, Science Quality, 2003-present (1 Day Composite)\",\n" + //
                         "  \"headline\": \"erdMH1chla1day\",\n" + //
                         "  \"description\": \"This dataset has Level 3, Standard Mapped Image, 4km, chlorophyll-a concentration data from NASA's Aqua Spacecraft.  Measurements are gathered by the Moderate Resolution Imaging Spectroradiometer (MODIS) carried aboard the spacecraft.  This is Science Quality data.  This is the August 2015 version of this dataset.\\n" + //
-                        "_lastModified=2016-10-18T06:45:00.000Z\\n" + //
+                        "_lastModified=YYYY-MM-DDThh:mm:ss.000Z\\n" + //
                         "cdm_data_type=Grid\\n" + //
                         "Conventions=CF-1.6, COARDS, ACDD-1.3\\n" + //
                         "Easternmost_Easting=179.9792\\n" + //
@@ -4731,7 +4733,7 @@ void testJsonld() throws Throwable {
                         "Northernmost_Northing=89.97916\\n" + //
                         "platform=Aqua\\n" + //
                         "processing_level=L3 Mapped\\n" + //
-                        "processing_version=2014.0.1QL\\n" + //
+                        "processing_version=VERSION\\n" + //
                         "product_name=A2016291.L3m_DAY_CHL_chlor_a_4km.nc\\n" + //
                         "project=Ocean Biology Processing Group (NASA/GSFC/OBPG)\\n" + //
                         "sourceUrl=(local files)\\n" + //
@@ -5050,6 +5052,8 @@ void testJsonld() throws Throwable {
         results = results.replaceAll("dateCreated\\\": \\\"....-..-..T..:..:..Z",
                 "dateCreated\\\": \\\"yyyy-mm-ddThh:mm:ssZ");
         results = results.replaceAll("100_multi_........1200", "100_multi_yyyymmdd1200");
+        results = results.replaceAll("_lastModified=....-..-..T..:..:...000Z", "_lastModified=YYYY-MM-DDThh:mm:ss.000Z");
+        results = results.replaceAll("processing_version=[0-9]+.[0-9].?.?.?.?\\\\n", "processing_version=VERSION\\\\n");
         po = Math.max(0, results.indexOf(expected.substring(0, 80)));
         Test.ensureEqual(results.substring(po, Math.min(results.length(), po + expected.length())),
                 expected, "results=\n" + results);