From c0c3db0ee101ba1b98c9a71d9a178fd02bce9dad Mon Sep 17 00:00:00 2001 From: Bob Simons Date: Fri, 21 Aug 2015 10:10:10 -0700 Subject: [PATCH] v1.64 --- WEB-INF/DasDds.bat | 2 +- WEB-INF/DasDds.sh | 6 +- WEB-INF/GenerateDatasetsXml.bat | 2 +- WEB-INF/GenerateDatasetsXml.sh | 6 +- .../classes/com/cohort/array/Attributes.java | 14 +- .../classes/com/cohort/array/ByteArray.java | 17 - .../classes/com/cohort/array/CharArray.java | 16 - .../classes/com/cohort/array/DoubleArray.java | 16 - .../classes/com/cohort/array/FloatArray.java | 17 - .../classes/com/cohort/array/IntArray.java | 17 - .../classes/com/cohort/array/LongArray.java | 16 - .../com/cohort/array/PrimitiveArray.java | 125 +- .../classes/com/cohort/array/ShortArray.java | 17 - .../classes/com/cohort/array/StringArray.java | 17 - .../classes/com/cohort/ema/EmaStringBox.java | 2 +- .../classes/com/cohort/util/Calendar2.java | 2 + WEB-INF/classes/com/cohort/util/File2.java | 52 +- WEB-INF/classes/com/cohort/util/MustBe.java | 24 +- WEB-INF/classes/com/cohort/util/String2.java | 53 +- WEB-INF/classes/com/cohort/util/Test.java | 8 +- WEB-INF/classes/com/cohort/util/TestUtil.java | 4 +- WEB-INF/classes/com/cohort/util/XML.java | 2 +- .../gov/noaa/pfel/coastwatch/MakeEmaWar.java | 2 +- .../gov/noaa/pfel/coastwatch/TestAll.java | 109 +- .../noaa/pfel/coastwatch/TestBrowsers.java | 14 +- .../griddata/DoubleCenterGrids.java | 2 +- .../coastwatch/griddata/FileNameUtility.java | 6 +- .../noaa/pfel/coastwatch/griddata/Grid.java | 2 +- .../griddata/GridDataSetOpendap.java | 2 +- .../griddata/GridDataSetThredds.java | 2 +- ...Nux10S1day_20050712_x-135_X-105_y22_Y50.nc | Bin 62168 -> 62168 bytes .../coastwatch/griddata/OpendapHelper.java | 26 +- .../coastwatch/pointdata/MakeErdJavaZip.java | 4 +- .../coastwatch/pointdata/NdbcMetStation.java | 81 +- .../noaa/pfel/coastwatch/pointdata/Table.java | 52 +- .../pfel/coastwatch/util/FileVisitorDNLS.java | 1128 ++++++++++- .../coastwatch/util/FileVisitorSubdir.java | 92 +- .../gov/noaa/pfel/coastwatch/util/SSR.java | 66 +- .../gov/noaa/pfel/coastwatch/util/Tally.java | 32 +- .../pfel/coastwatch/util/WatchDirectory.java | 6 +- .../classes/gov/noaa/pfel/erddap/Erddap.java | 1780 ++++++++++++++++- .../noaa/pfel/erddap/GenerateDatasetsXml.java | 8 +- .../gov/noaa/pfel/erddap/dataset/EDD.java | 436 +++- .../gov/noaa/pfel/erddap/dataset/EDDGrid.java | 109 +- .../EDDGridAggregateExistingDimension.java | 85 +- .../noaa/pfel/erddap/dataset/EDDGridCopy.java | 40 +- .../erddap/dataset/EDDGridFromBinaryFile.java | 5 +- .../pfel/erddap/dataset/EDDGridFromDap.java | 287 +-- .../erddap/dataset/EDDGridFromEDDTable.java | 8 +- .../erddap/dataset/EDDGridFromErddap.java | 16 +- .../pfel/erddap/dataset/EDDGridFromEtopo.java | 6 +- .../pfel/erddap/dataset/EDDGridFromFiles.java | 161 +- .../erddap/dataset/EDDGridFromMatFiles.java | 9 +- .../dataset/EDDGridFromMergeIRFiles.java | 8 +- .../erddap/dataset/EDDGridFromNcFiles.java | 1291 +++++++++++- .../pfel/erddap/dataset/EDDGridLonPM180.java | 629 ++++++ .../erddap/dataset/EDDGridSideBySide.java | 26 +- .../noaa/pfel/erddap/dataset/EDDTable.java | 53 +- .../pfel/erddap/dataset/EDDTableCopy.java | 11 + .../dataset/EDDTableFromAsciiFiles.java | 14 +- .../dataset/EDDTableFromAwsXmlFiles.java | 13 +- .../pfel/erddap/dataset/EDDTableFromBMDE.java | 2 +- .../erddap/dataset/EDDTableFromCassandra.java | 10 +- .../EDDTableFromColumnarAsciiFiles.java | 10 +- .../dataset/EDDTableFromDapSequence.java | 15 +- .../erddap/dataset/EDDTableFromDatabase.java | 4 +- .../erddap/dataset/EDDTableFromEDDGrid.java | 4 +- .../erddap/dataset/EDDTableFromErddap.java | 20 +- .../erddap/dataset/EDDTableFromFileNames.java | 456 ++++- .../erddap/dataset/EDDTableFromFiles.java | 143 +- .../dataset/EDDTableFromHyraxFiles.java | 55 +- .../pfel/erddap/dataset/EDDTableFromMWFS.java | 4 +- .../pfel/erddap/dataset/EDDTableFromNOS.java | 2 +- .../erddap/dataset/EDDTableFromNcCFFiles.java | 8 +- .../erddap/dataset/EDDTableFromNcFiles.java | 186 +- .../pfel/erddap/dataset/EDDTableFromOBIS.java | 6 +- .../dataset/EDDTableFromPostDatabase.java | 4 +- .../pfel/erddap/dataset/EDDTableFromSOS.java | 51 +- .../dataset/EDDTableFromThreddsFiles.java | 16 +- .../erddap/dataset/EDDTableFromWFSFiles.java | 26 +- .../gov/noaa/pfel/erddap/util/CfToGcmd.txt | 3 + .../gov/noaa/pfel/erddap/util/EDStatic.java | 24 +- .../gov/noaa/pfel/erddap/util/FishBase.java | 2 +- .../noaa/pfel/erddap/util/HtmlWidgets.java | 115 +- .../util/OceanicAtmosphericAcronyms.tsv | 2 + .../gov/noaa/pfel/erddap/util/Projects2.java | 6 +- .../gov/noaa/pfel/erddap/util/cfStdNames.txt | 1 + .../gov/noaa/pfel/erddap/util/messages.xml | 3 + download/changes.html | 138 ++ download/grids.html | 411 ++-- download/setup.html | 219 +- download/setupDatasetsXml.html | 595 ++++-- 92 files changed, 7896 insertions(+), 1701 deletions(-) create mode 100644 WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridLonPM180.java diff --git a/WEB-INF/DasDds.bat b/WEB-INF/DasDds.bat index bec9d5648..a73970aa4 100644 --- a/WEB-INF/DasDds.bat +++ b/WEB-INF/DasDds.bat @@ -1,4 +1,4 @@ rem This is the Windows batch file to run DasDds. rem See http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html#Tools -java -cp ./classes;../../../lib/servlet-api.jar;lib/activation.jar;lib/axis.jar;lib/cassandra-driver-core.jar;lib/netty-all.jar;lib/guava.jar;lib/metrics-core.jar;lib/lz4.jar;lib/snappy-java.jar;lib/commons-compress.jar;lib/commons-discovery.jar;lib/itext-1.3.1.jar;lib/jaxrpc.jar;lib/joda-time.jar;lib/joid.jar;lib/lucene-core.jar;lib/mail.jar;lib/netcdfAll-latest.jar;lib/postgresql.jdbc.jar;lib/saaj.jar;lib/tsik.jar;lib/wsdl4j.jar -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.DasDds %1 %2 %3 %4 %5 %6 %7 %8 %9 \ No newline at end of file +java -cp ./classes;../../../lib/servlet-api.jar;lib/activation.jar;lib/axis.jar;lib/cassandra-driver-core.jar;lib/netty-all.jar;lib/guava.jar;lib/metrics-core.jar;lib/lz4.jar;lib/snappy-java.jar;lib/commons-compress.jar;lib/commons-discovery.jar;lib/itext-1.3.1.jar;lib/jaxrpc.jar;lib/joda-time.jar;lib/joid.jar;lib/lucene-core.jar;lib/mail.jar;lib/netcdfAll-latest.jar;lib/postgresql.jdbc.jar;lib/saaj.jar;lib/tsik.jar;lib/wsdl4j.jar;lib/aws-java-sdk.jar;lib/commons-codec.jar;lib/commons-logging.jar;lib/fluent-hc.jar;lib/httpclient.jar;lib/httpclient-cache.jar;lib/httpcore.jar;lib/httpmime.jar;lib/jna.jar;lib/jna-platform.jar;lib/jackson-annotations.jar;lib/jackson-core.jar;lib/jackson-databind.jar -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.DasDds %1 %2 %3 %4 %5 %6 %7 %8 %9 \ No newline at end of file diff --git a/WEB-INF/DasDds.sh b/WEB-INF/DasDds.sh index 732b55be6..0917a0efe 100644 --- a/WEB-INF/DasDds.sh +++ b/WEB-INF/DasDds.sh @@ -8,6 +8,10 @@ cp3=":lib/commons-compress.jar:lib/commons-discovery.jar:lib/itext-1.3.1.jar" cp4=":lib/jaxrpc.jar:lib/joda-time.jar:lib/joid.jar:lib/lucene-core.jar" cp5=":lib/mail.jar:lib/netcdfAll-latest.jar:lib/postgresql.jdbc.jar" cp6=":lib/saaj.jar:lib/tsik.jar:lib/wsdl4j.jar" -cp0="$cp1$cp2$cp3$cp4$cp5$cp6" +cp7=":lib/aws-java-sdk.jar:lib/commons-codec.jar:lib/commons-logging.jar" +cp8=":lib/fluent-hc.jar:lib/httpclient.jar:lib/httpclient-cache.jar:lib/httpcore.jar" +cp9=":lib/httpmime.jar:lib/jna.jar:lib/jna-platform.jar:lib/jackson-annotations.jar" +cp10=":lib/jackson-core.jar:lib/jackson-databind.jar" +cp0="$cp1$cp2$cp3$cp4$cp5$cp6$cp7$cp8$cp9$cp10" java -cp $cp0 -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.DasDds $1 $2 $3 $4 $5 $6 $7 $8 $9 diff --git a/WEB-INF/GenerateDatasetsXml.bat b/WEB-INF/GenerateDatasetsXml.bat index d41f4e2e9..c5f4014e0 100644 --- a/WEB-INF/GenerateDatasetsXml.bat +++ b/WEB-INF/GenerateDatasetsXml.bat @@ -1,4 +1,4 @@ rem This is the Windows batch file to run GenerateDatasetsXml. rem See http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html#Tools -java -cp ./classes;../../../lib/servlet-api.jar;lib/activation.jar;lib/axis.jar;lib/cassandra-driver-core.jar;lib/netty-all.jar;lib/guava.jar;lib/metrics-core.jar;lib/lz4.jar;lib/snappy-java.jar;lib/commons-compress.jar;lib/commons-discovery.jar;lib/itext-1.3.1.jar;lib/jaxrpc.jar;lib/joda-time.jar;lib/joid.jar;lib/lucene-core.jar;lib/mail.jar;lib/netcdfAll-latest.jar;lib/postgresql.jdbc.jar;lib/saaj.jar;lib/tsik.jar;lib/wsdl4j.jar -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.GenerateDatasetsXml %1 %2 %3 %4 %5 %6 %7 %8 %9 \ No newline at end of file +java -cp ./classes;../../../lib/servlet-api.jar;lib/activation.jar;lib/axis.jar;lib/cassandra-driver-core.jar;lib/netty-all.jar;lib/guava.jar;lib/metrics-core.jar;lib/lz4.jar;lib/snappy-java.jar;lib/commons-compress.jar;lib/commons-discovery.jar;lib/itext-1.3.1.jar;lib/jaxrpc.jar;lib/joda-time.jar;lib/joid.jar;lib/lucene-core.jar;lib/mail.jar;lib/netcdfAll-latest.jar;lib/postgresql.jdbc.jar;lib/saaj.jar;lib/tsik.jar;lib/wsdl4j.jar;lib/aws-java-sdk.jar;lib/commons-codec.jar;lib/commons-logging.jar;lib/fluent-hc.jar;lib/httpclient.jar;lib/httpclient-cache.jar;lib/httpcore.jar;lib/httpmime.jar;lib/jna.jar;lib/jna-platform.jar;lib/jackson-annotations.jar;lib/jackson-core.jar;lib/jackson-databind.jar -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.GenerateDatasetsXml %1 %2 %3 %4 %5 %6 %7 %8 %9 \ No newline at end of file diff --git a/WEB-INF/GenerateDatasetsXml.sh b/WEB-INF/GenerateDatasetsXml.sh index bd5d076b1..a83021676 100644 --- a/WEB-INF/GenerateDatasetsXml.sh +++ b/WEB-INF/GenerateDatasetsXml.sh @@ -8,6 +8,10 @@ cp3=":lib/commons-compress.jar:lib/commons-discovery.jar:lib/itext-1.3.1.jar" cp4=":lib/jaxrpc.jar:lib/joda-time.jar:lib/joid.jar:lib/lucene-core.jar" cp5=":lib/mail.jar:lib/netcdfAll-latest.jar:lib/postgresql.jdbc.jar" cp6=":lib/saaj.jar:lib/tsik.jar:lib/wsdl4j.jar" -cp0="$cp1$cp2$cp3$cp4$cp5$cp6" +cp7=":lib/aws-java-sdk.jar:lib/commons-codec.jar:lib/commons-logging.jar" +cp8=":lib/fluent-hc.jar:lib/httpclient.jar:lib/httpclient-cache.jar:lib/httpcore.jar" +cp9=":lib/httpmime.jar:lib/jna.jar:lib/jna-platform.jar:lib/jackson-annotations.jar" +cp10=":lib/jackson-core.jar:lib/jackson-databind.jar" +cp0="$cp1$cp2$cp3$cp4$cp5$cp6$cp7$cp8$cp9$cp10" java -cp $cp0 -Xms1000M -Xmx1000M gov.noaa.pfel.erddap.GenerateDatasetsXml $1 $2 $3 $4 $5 $6 $7 $8 $9 diff --git a/WEB-INF/classes/com/cohort/array/Attributes.java b/WEB-INF/classes/com/cohort/array/Attributes.java index 6b0bf2f70..f6d8bd845 100644 --- a/WEB-INF/classes/com/cohort/array/Attributes.java +++ b/WEB-INF/classes/com/cohort/array/Attributes.java @@ -117,7 +117,7 @@ public String[] getNames() { * A convenience method which returns the first element of the attribute's * value PrimitiveArray as a String, regardless of the type used to store it. * - * @param name + * @param name the name of an attribute * @return the String attribute or null if trouble (e.g., not found) */ public String getString(String name) { @@ -135,7 +135,7 @@ public String getString(String name) { * A convenience method which assumes the first element of the attribute's * value PrimitiveArray is a CSV String and which splits the string into parts. * - * @param name + * @param name the name of an attribute * @return a String[] or null if trouble (e.g., not found) */ public String[] getStringsFromCSV(String name) { @@ -154,7 +154,7 @@ public String[] getStringsFromCSV(String name) { * A convenience method which returns the first element of the attribute's * value PrimitiveArray as a double, regardless of the type used to store it. * - * @param name + * @param name the name of an attribute * @return the attribute as a double or Double.NaN if trouble (e.g., not found) */ public double getDouble(String name) { @@ -173,7 +173,7 @@ public double getDouble(String name) { * value PrimitiveArray as a double, regardless of the type used to store it. * If the type was float, this returns Math2.floatToDouble(value). * - * @param name + * @param name the name of an attribute * @return the attribute as a nice double or Double.NaN if trouble (e.g., not found) */ public double getNiceDouble(String name) { @@ -191,7 +191,7 @@ public double getNiceDouble(String name) { * A convenience method which returns the first element of the attribute's * value PrimitiveArray as a float, regardless of the type used to store it. * - * @param name + * @param name the name of an attribute * @return the attribute as a float or Float.NaN if trouble (e.g., not found) */ public float getFloat(String name) { @@ -209,7 +209,7 @@ public float getFloat(String name) { * A convenience method which returns the first element of the attribute's * value PrimitiveArray as a long, regardless of the type used to store it. * - * @param name + * @param name the name of an attribute * @return the attribute as a long or Long.MAX_VALUE if trouble (e.g., not found) */ public long getLong(String name) { @@ -227,7 +227,7 @@ public long getLong(String name) { * A convenience method which returns the first element of the attribute's * value PrimitiveArray as an int, regardless of the type used to store it. * - * @param name + * @param name the name of an attribute * @return the attribute as an int or Integer.MAX_VALUE if trouble (e.g., not found) */ public int getInt(String name) { diff --git a/WEB-INF/classes/com/cohort/array/ByteArray.java b/WEB-INF/classes/com/cohort/array/ByteArray.java index 5b885e6bd..e72583bda 100644 --- a/WEB-INF/classes/com/cohort/array/ByteArray.java +++ b/WEB-INF/classes/com/cohort/array/ByteArray.java @@ -60,7 +60,6 @@ public ByteArray(PrimitiveArray primitiveArray) { * @param capacity creates an ByteArray with the specified initial capacity. * @param active if true, size will be set to capacity and all elements * will equal 0; else size = 0. - * @throws Exception if trouble. */ public ByteArray(int capacity, boolean active) { Math2.ensureMemoryAvailable(1L * capacity, "ByteArray"); @@ -85,7 +84,6 @@ public ByteArray(byte[] anArray) { * * @param first the value of the first element. * @param last the value of the last element (inclusive!). - * @throws Exception if trouble. */ public ByteArray(int first, int last) { size = last - first + 1; @@ -414,7 +412,6 @@ public void setFromPA(int index, PrimitiveArray otherPA, int otherIndex) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ public void remove(int index) { if (index >= size) @@ -431,7 +428,6 @@ public void remove(int index) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ public void removeRange(int from, int to) { if (to > size) @@ -456,7 +452,6 @@ public void removeRange(int from, int to) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ public void move(int first, int last, int destination) { String errorIn = String2.ERROR + " in ByteArray.move:\n"; @@ -602,7 +597,6 @@ public String[] toStringArray() { * This gets a specified element. * * @param index 0 ... size-1 - * @throws Exception if trouble. */ public byte get(int index) { if (index >= size) @@ -616,7 +610,6 @@ public byte get(int index) { * * @param index 0 ... size-1 * @param value the value for that element - * @throws Exception if trouble. */ public void set(int index, byte value) { if (index >= size) @@ -632,7 +625,6 @@ public void set(int index, byte value) { * @param index the index number 0 ... size-1 * @return the value as an int. * Byte.MAX_VALUE is returned as Integer.MAX_VALUE. - * @throws Exception if trouble. */ public int getInt(int index) { int i = get(index); @@ -645,7 +637,6 @@ public int getInt(int index) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToByte(i). - * @throws Exception if trouble. */ public void setInt(int index, int i) { set(index, Math2.narrowToByte(i)); @@ -657,7 +648,6 @@ public void setInt(int index, int i) { * @param index the index number 0 ... size-1 * @return the value as a long. * Byte.MAX_VALUE is returned as Long.MAX_VALUE. - * @throws Exception if trouble. */ public long getLong(int index) { int i = get(index); @@ -670,7 +660,6 @@ public long getLong(int index) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToByte(long). - * @throws Exception if trouble. */ public void setLong(int index, long i) { set(index, Math2.narrowToByte(i)); @@ -683,7 +672,6 @@ public void setLong(int index, long i) { * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. * Byte.MAX_VALUE is returned as Float.NaN. - * @throws Exception if trouble. */ public float getFloat(int index) { byte b = get(index); @@ -696,7 +684,6 @@ public float getFloat(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToByte(d). - * @throws Exception if trouble. */ public void setFloat(int index, float d) { set(index, Math2.roundToByte(d)); @@ -709,7 +696,6 @@ public void setFloat(int index, float d) { * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. * Byte.MAX_VALUE is returned as Double.NaN. - * @throws Exception if trouble. */ public double getDouble(int index) { byte b = get(index); @@ -722,7 +708,6 @@ public double getDouble(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToByte(d). - * @throws Exception if trouble. */ public void setDouble(int index, double d) { set(index, Math2.roundToByte(d)); @@ -733,7 +718,6 @@ public void setDouble(int index, double d) { * * @param index the index number 0 .. * @return For numeric types, this returns (String.valueOf(ar[index])), or "" for NaN or infinity. - * @throws Exception if trouble. */ public String getString(int index) { byte b = get(index); @@ -746,7 +730,6 @@ public String getString(int index) { * @param index the index number 0 .. * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parseInt and narrowed by Math2.narrowToByte(i). - * @throws Exception if trouble. */ public void setString(int index, String s) { set(index, Math2.narrowToByte(String2.parseInt(s))); diff --git a/WEB-INF/classes/com/cohort/array/CharArray.java b/WEB-INF/classes/com/cohort/array/CharArray.java index aee2d16b0..35fc0a5f0 100644 --- a/WEB-INF/classes/com/cohort/array/CharArray.java +++ b/WEB-INF/classes/com/cohort/array/CharArray.java @@ -59,7 +59,6 @@ public CharArray(PrimitiveArray primitiveArray) { * @param capacity creates an CharArray with the specified initial capacity. * @param active if true, size will be set to capacity and all elements * will equal 0; else size = 0. - * @throws Exception if trouble. */ public CharArray(int capacity, boolean active) { Math2.ensureMemoryAvailable(2L * capacity, "CharArray"); @@ -353,7 +352,6 @@ public void setFromPA(int index, PrimitiveArray otherPA, int otherIndex) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ public void remove(int index) { if (index >= size) @@ -370,7 +368,6 @@ public void remove(int index) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ public void removeRange(int from, int to) { if (to > size) @@ -395,7 +392,6 @@ public void removeRange(int from, int to) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ public void move(int first, int last, int destination) { String errorIn = String2.ERROR + " in CharArray.move:\n"; @@ -542,7 +538,6 @@ public String[] toStringArray() { * This gets a specified element. * * @param index 0 ... size-1 - * @throws Exception if trouble. */ public char get(int index) { if (index >= size) @@ -556,7 +551,6 @@ public char get(int index) { * * @param index 0 ... size-1 * @param value the value for that element - * @throws Exception if trouble. */ public void set(int index, char value) { if (index >= size) @@ -572,7 +566,6 @@ public void set(int index, char value) { * @param index the index number 0 ... size-1 * @return the value as an int. * Character.MAX_VALUE is returned as Integer.MAX_VALUE. - * @throws Exception if trouble. */ public int getInt(int index) { int i = get(index); @@ -585,7 +578,6 @@ public int getInt(int index) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToChar(i). - * @throws Exception if trouble. */ public void setInt(int index, int i) { set(index, Math2.narrowToChar(i)); @@ -597,7 +589,6 @@ public void setInt(int index, int i) { * @param index the index number 0 ... size-1 * @return the value as a long. * Character.MAX_VALUE is returned as Long.MAX_VALUE. - * @throws Exception if trouble. */ public long getLong(int index) { int i = get(index); @@ -610,7 +601,6 @@ public long getLong(int index) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToChar(long). - * @throws Exception if trouble. */ public void setLong(int index, long i) { set(index, Math2.narrowToChar(i)); @@ -623,7 +613,6 @@ public void setLong(int index, long i) { * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. * Character.MAX_VALUE is returned as Float.NaN. - * @throws Exception if trouble. */ public float getFloat(int index) { char c = get(index); @@ -636,7 +625,6 @@ public float getFloat(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToChar(d). - * @throws Exception if trouble. */ public void setFloat(int index, float d) { set(index, Math2.roundToChar(d)); @@ -649,7 +637,6 @@ public void setFloat(int index, float d) { * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. * Character.MAX_VALUE is returned as Double.NaN. - * @throws Exception if trouble. */ public double getDouble(int index) { char c = get(index); @@ -662,7 +649,6 @@ public double getDouble(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToChar(d). - * @throws Exception if trouble. */ public void setDouble(int index, double d) { set(index, Math2.roundToChar(d)); @@ -673,7 +659,6 @@ public void setDouble(int index, double d) { * * @param index the index number 0 .. * @return This returns (int)(ar[index]), or "" for NaN or infinity. - * @throws Exception if trouble. */ public String getString(int index) { char b = get(index); @@ -686,7 +671,6 @@ public String getString(int index) { * @param index the index number 0 .. * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parseInt and narrowed by Math2.narrowToChar(i). - * @throws Exception if trouble. */ public void setString(int index, String s) { set(index, Math2.narrowToChar(String2.parseInt(s))); diff --git a/WEB-INF/classes/com/cohort/array/DoubleArray.java b/WEB-INF/classes/com/cohort/array/DoubleArray.java index 0242dd8a5..5a87886c0 100644 --- a/WEB-INF/classes/com/cohort/array/DoubleArray.java +++ b/WEB-INF/classes/com/cohort/array/DoubleArray.java @@ -58,7 +58,6 @@ public DoubleArray(PrimitiveArray primitiveArray) { * @param capacity creates an DoubleArray with the specified initial capacity. * @param active if true, size will be set to capacity and all elements * will equal 0; else size = 0. - * @throws Exception if trouble. */ public DoubleArray(int capacity, boolean active) { Math2.ensureMemoryAvailable(8L * capacity, "DoubleArray"); @@ -330,7 +329,6 @@ public void setFromPA(int index, PrimitiveArray otherPA, int otherIndex) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ public void remove(int index) { if (index >= size) @@ -347,7 +345,6 @@ public void remove(int index) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ public void removeRange(int from, int to) { if (to > size) @@ -372,7 +369,6 @@ public void removeRange(int from, int to) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ public void move(int first, int last, int destination) { String errorIn = String2.ERROR + " in DoubleArray.move:\n"; @@ -509,7 +505,6 @@ public String[] toStringArray() { * This gets a specified element. * * @param index 0 ... size-1 - * @throws Exception if trouble. */ public double get(int index) { if (index >= size) @@ -523,7 +518,6 @@ public double get(int index) { * * @param index 0 ... size-1 * @param value the value for that element - * @throws Exception if trouble. */ public void set(int index, double value) { if (index >= size) @@ -539,7 +533,6 @@ public void set(int index, double value) { * * @param index the index number 0 ... size-1 * @return the value as an int. This may return Integer.MAX_VALUE. - * @throws Exception if trouble. */ public int getInt(int index) { return Math2.roundToInt(get(index)); @@ -551,7 +544,6 @@ public int getInt(int index) { * @param index the index number 0 .. size-1 * @param i the value. Integer.MAX_VALUE is converted * to this Double.NaN. - * @throws Exception if trouble. */ public void setInt(int index, int i) { set(index, i == Integer.MAX_VALUE? Double.NaN : i); @@ -563,7 +555,6 @@ public void setInt(int index, int i) { * @param index the index number 0 ... size-1 * @return the value as a long. * This may return Long.MAX_VALUE. - * @throws Exception if trouble. */ public long getLong(int index) { return Math2.roundToLong(get(index)); @@ -575,7 +566,6 @@ public long getLong(int index) { * @param index the index number 0 .. size-1 * @param i the value. Long.MAX_VALUE is converted * to Double.NaN. - * @throws Exception if trouble. */ public void setLong(int index, long i) { set(index, i == Long.MAX_VALUE? Double.NaN : i); @@ -588,7 +578,6 @@ public void setLong(int index, long i) { * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. * Large values like 1e100 are returned as Float.NaN, not Float.POSITIVE_INFINITY. - * @throws Exception if trouble. */ public float getFloat(int index) { return Math2.doubleToFloatNaN(get(index)); @@ -600,7 +589,6 @@ public float getFloat(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToFloat(d). - * @throws Exception if trouble. */ public void setFloat(int index, float d) { set(index, (double)d); @@ -612,7 +600,6 @@ public void setFloat(int index, float d) { * @param index the index number 0 .. size-1 * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. - * @throws Exception if trouble. */ public double getDouble(int index) { return get(index); @@ -624,7 +611,6 @@ public double getDouble(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToDouble(d). - * @throws Exception if trouble. */ public void setDouble(int index, double d) { set(index, d); @@ -635,7 +621,6 @@ public void setDouble(int index, double d) { * * @param index the index number 0 .. * @return For numeric types, this returns (String.valueOf(ar[index])), or "" for NaN or infinity. - * @throws Exception if trouble. */ public String getString(int index) { double b = get(index); @@ -648,7 +633,6 @@ public String getString(int index) { * @param index the index number 0 .. * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parseDouble. - * @throws Exception if trouble. */ public void setString(int index, String s) { set(index, String2.parseDouble(s)); diff --git a/WEB-INF/classes/com/cohort/array/FloatArray.java b/WEB-INF/classes/com/cohort/array/FloatArray.java index 9e078c576..07e09ec15 100644 --- a/WEB-INF/classes/com/cohort/array/FloatArray.java +++ b/WEB-INF/classes/com/cohort/array/FloatArray.java @@ -57,7 +57,6 @@ public FloatArray(PrimitiveArray primitiveArray) { * @param capacity creates an FloatArray with the specified initial capacity. * @param active if true, size will be set to capacity and all elements * will equal 0; else size = 0. - * @throws Exception if trouble. */ public FloatArray(int capacity, boolean active) { Math2.ensureMemoryAvailable(4L * capacity, "FloatArray"); @@ -326,7 +325,6 @@ public void setFromPA(int index, PrimitiveArray otherPA, int otherIndex) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ public void remove(int index) { if (index >= size) @@ -343,7 +341,6 @@ public void remove(int index) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ public void removeRange(int from, int to) { if (to > size) @@ -368,7 +365,6 @@ public void removeRange(int from, int to) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ public void move(int first, int last, int destination) { String errorIn = String2.ERROR + " in FloatArray.move:\n"; @@ -512,7 +508,6 @@ public String[] toStringArray() { * This gets a specified element. * * @param index 0 ... size-1 - * @throws Exception if trouble. */ public float get(int index) { if (index >= size) @@ -526,7 +521,6 @@ public float get(int index) { * * @param index 0 ... size-1 * @param value the value for that element - * @throws Exception if trouble. */ public void set(int index, float value) { if (index >= size) @@ -542,7 +536,6 @@ public void set(int index, float value) { * * @param index the index number 0 ... size-1 * @return the value as an int. This may return Integer.MAX_VALUE. - * @throws Exception if trouble. */ public int getInt(int index) { return Math2.roundToInt(get(index)); @@ -554,7 +547,6 @@ public int getInt(int index) { * @param index the index number 0 .. size-1 * @param i the value. Integer.MAX_VALUE is converted * to this type's missing value. - * @throws Exception if trouble. */ public void setInt(int index, int i) { set(index, i == Integer.MAX_VALUE? Float.NaN : i); @@ -566,7 +558,6 @@ public void setInt(int index, int i) { * @param index the index number 0 ... size-1 * @return the value as a long. * This may return Long.MAX_VALUE. - * @throws Exception if trouble. */ public long getLong(int index) { return Math2.roundToLong(get(index)); @@ -578,7 +569,6 @@ public long getLong(int index) { * @param index the index number 0 .. size-1 * @param i the value. Long.MAX_VALUE is converted * to Float.NaN. - * @throws Exception if trouble. */ public void setLong(int index, long i) { set(index, i == Long.MAX_VALUE? Float.NaN : i); @@ -589,7 +579,6 @@ public void setLong(int index, long i) { * @param index the index number 0 .. size-1 * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. - * @throws Exception if trouble. */ public float getFloat(int index) { return get(index); @@ -600,7 +589,6 @@ public float getFloat(int index) { * * @param index the index number 0 .. size-1 * @param d the value. - * @throws Exception if trouble. */ public void setFloat(int index, float d) { set(index, d); @@ -613,7 +601,6 @@ public void setFloat(int index, float d) { * @param index the index number 0 .. size-1 * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. - * @throws Exception if trouble. */ public double getDouble(int index) { return get(index); @@ -628,7 +615,6 @@ public double getDouble(int index) { * @param index the index number 0 .. size-1 * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. - * @throws Exception if trouble. */ public double getNiceDouble(int index) { return Math2.floatToDouble(get(index)); @@ -640,7 +626,6 @@ public double getNiceDouble(int index) { * @param index the index number 0 .. size-1 * @param d the value. It is narrowed by Math2.doubleToFloatNaN(d), * which converts INFINITY and large values to NaN. - * @throws Exception if trouble. */ public void setDouble(int index, double d) { set(index, Math2.doubleToFloatNaN(d)); @@ -651,7 +636,6 @@ public void setDouble(int index, double d) { * * @param index the index number 0 .. * @return For numeric types, this returns (String.valueOf(ar[index])), or "" for NaN or infinity. - * @throws Exception if trouble. */ public String getString(int index) { float b = get(index); @@ -665,7 +649,6 @@ public String getString(int index) { * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parse and narrowed if needed by methods like * Math2.roundToFloat(d). - * @throws Exception if trouble. */ public void setString(int index, String s) { set(index, String2.parseFloat(s)); diff --git a/WEB-INF/classes/com/cohort/array/IntArray.java b/WEB-INF/classes/com/cohort/array/IntArray.java index 0727cfcb7..075881f6b 100644 --- a/WEB-INF/classes/com/cohort/array/IntArray.java +++ b/WEB-INF/classes/com/cohort/array/IntArray.java @@ -59,7 +59,6 @@ public IntArray(PrimitiveArray primitiveArray) { * @param capacity creates an IntArray with the specified initial capacity. * @param active if true, size will be set to capacity and all elements * will equal 0; else size = 0. - * @throws Exception if trouble. */ public IntArray(int capacity, boolean active) { Math2.ensureMemoryAvailable(4L * capacity, "IntArray"); @@ -73,7 +72,6 @@ public IntArray(int capacity, boolean active) { * * @param first the value of the first element. * @param last the value of the last element (inclusive!). - * @throws Exception if trouble. */ public IntArray(int first, int last) { size = last - first + 1; @@ -343,7 +341,6 @@ public void setFromPA(int index, PrimitiveArray otherPA, int otherIndex) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ public void remove(int index) { if (index >= size) @@ -360,7 +357,6 @@ public void remove(int index) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ public void removeRange(int from, int to) { if (to > size) @@ -385,7 +381,6 @@ public void removeRange(int from, int to) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ public void move(int first, int last, int destination) { String errorIn = String2.ERROR + " in IntArray.move:\n"; @@ -552,7 +547,6 @@ public String[] toStringArray() { * This gets a specified element. * * @param index 0 ... size-1 - * @throws Exception if trouble. */ public int get(int index) { if (index >= size) @@ -566,7 +560,6 @@ public int get(int index) { * * @param index 0 ... size-1 * @param value the value for that element - * @throws Exception if trouble. */ public void set(int index, int value) { if (index >= size) @@ -581,7 +574,6 @@ public void set(int index, int value) { * * @param index the index number 0 ... size-1 * @return the value as an int. - * @throws Exception if trouble. */ public int getInt(int index) { return get(index); @@ -592,7 +584,6 @@ public int getInt(int index) { * * @param index the index number 0 .. size-1 * @param i the value. - * @throws Exception if trouble. */ public void setInt(int index, int i) { set(index, i); @@ -604,7 +595,6 @@ public void setInt(int index, int i) { * @param index the index number 0 ... size-1 * @return the value as a long. * Integer.MAX_VALUE is returned as Long.MAX_VALUE. - * @throws Exception if trouble. */ public long getLong(int index) { int i = get(index); @@ -617,7 +607,6 @@ public long getLong(int index) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToInt(long). - * @throws Exception if trouble. */ public void setLong(int index, long i) { set(index, Math2.narrowToInt(i)); @@ -631,7 +620,6 @@ public void setLong(int index, long i) { * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. * Int.MAX_VALUE is returned as Float.NaN. - * @throws Exception if trouble. */ public float getFloat(int index) { int i = get(index); @@ -644,7 +632,6 @@ public float getFloat(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToInt(d). - * @throws Exception if trouble. */ public void setFloat(int index, float d) { set(index, Math2.roundToInt(d)); @@ -657,7 +644,6 @@ public void setFloat(int index, float d) { * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. * Int.MAX_VALUE is returned as Double.NaN. - * @throws Exception if trouble. */ public double getDouble(int index) { int i = get(index); @@ -670,7 +656,6 @@ public double getDouble(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToInt(d). - * @throws Exception if trouble. */ public void setDouble(int index, double d) { set(index, Math2.roundToInt(d)); @@ -681,7 +666,6 @@ public void setDouble(int index, double d) { * * @param index the index number 0 .. * @return For numeric types, this returns (String.valueOf(ar[index])), or "" for NaN or infinity. - * @throws Exception if trouble. */ public String getString(int index) { int b = get(index); @@ -694,7 +678,6 @@ public String getString(int index) { * @param index the index number 0 .. * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parseInt. - * @throws Exception if trouble. */ public void setString(int index, String s) { set(index, String2.parseInt(s)); diff --git a/WEB-INF/classes/com/cohort/array/LongArray.java b/WEB-INF/classes/com/cohort/array/LongArray.java index 498c12b35..052204350 100644 --- a/WEB-INF/classes/com/cohort/array/LongArray.java +++ b/WEB-INF/classes/com/cohort/array/LongArray.java @@ -59,7 +59,6 @@ public LongArray(PrimitiveArray primitiveArray) { * @param capacity creates an LongArray with the specified initial capacity. * @param active if true, size will be set to capacity and all elements * will equal 0; else size = 0. - * @throws Exception if trouble. */ public LongArray(int capacity, boolean active) { Math2.ensureMemoryAvailable(8L * capacity, "LongArray"); @@ -329,7 +328,6 @@ public void setFromPA(int index, PrimitiveArray otherPA, int otherIndex) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ public void remove(int index) { if (index >= size) @@ -346,7 +344,6 @@ public void remove(int index) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ public void removeRange(int from, int to) { if (to > size) @@ -371,7 +368,6 @@ public void removeRange(int from, int to) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ public void move(int first, int last, int destination) { String errorIn = String2.ERROR + " in LongArray.move:\n"; @@ -517,7 +513,6 @@ public String[] toStringArray() { * This gets a specified element. * * @param index 0 ... size-1 - * @throws Exception if trouble. */ public long get(int index) { if (index >= size) @@ -531,7 +526,6 @@ public long get(int index) { * * @param index 0 ... size-1 * @param value the value for that element - * @throws Exception if trouble. */ public void set(int index, long value) { if (index >= size) @@ -546,7 +540,6 @@ public void set(int index, long value) { * * @param index the index number 0 ... size-1 * @return the value as an int. This may return Integer.MAX_VALUE. - * @throws Exception if trouble. */ public int getInt(int index) { return Math2.roundToInt(get(index)); @@ -558,7 +551,6 @@ public int getInt(int index) { * @param index the index number 0 .. size-1 * @param i the value. Integer.MAX_VALUE is converted * to this type's missing value. - * @throws Exception if trouble. */ public void setInt(int index, int i) { set(index, i == Integer.MAX_VALUE? Long.MAX_VALUE : i); @@ -569,7 +561,6 @@ public void setInt(int index, int i) { * * @param index the index number 0 ... size-1 * @return the value as a long. - * @throws Exception if trouble. */ public long getLong(int index) { return get(index); @@ -580,7 +571,6 @@ public long getLong(int index) { * * @param index the index number 0 .. size-1 * @param i the value. - * @throws Exception if trouble. */ public void setLong(int index, long i) { set(index, i); @@ -594,7 +584,6 @@ public void setLong(int index, long i) { * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. * Long.MAX_VALUE is returned as Float.NaN. - * @throws Exception if trouble. */ public float getFloat(int index) { long tl = get(index); @@ -607,7 +596,6 @@ public float getFloat(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToLong(d). - * @throws Exception if trouble. */ public void setFloat(int index, float d) { set(index, Math2.roundToLong(d)); @@ -620,7 +608,6 @@ public void setFloat(int index, float d) { * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. * Long.MAX_VALUE is returned as Double.NaN. - * @throws Exception if trouble. */ public double getDouble(int index) { long tl = get(index); @@ -633,7 +620,6 @@ public double getDouble(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToLong(d). - * @throws Exception if trouble. */ public void setDouble(int index, double d) { set(index, Math2.roundToLong(d)); @@ -644,7 +630,6 @@ public void setDouble(int index, double d) { * * @param index the index number 0 .. * @return For numeric types, this returns (String.valueOf(ar[index])), or "" for NaN or infinity. - * @throws Exception if trouble. */ public String getString(int index) { long b = get(index); @@ -657,7 +642,6 @@ public String getString(int index) { * @param index the index number 0 .. * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parseLong. - * @throws Exception if trouble. */ public void setString(int index, String s) { set(index, String2.parseLong(s)); diff --git a/WEB-INF/classes/com/cohort/array/PrimitiveArray.java b/WEB-INF/classes/com/cohort/array/PrimitiveArray.java index 957ab45d0..42dba0924 100644 --- a/WEB-INF/classes/com/cohort/array/PrimitiveArray.java +++ b/WEB-INF/classes/com/cohort/array/PrimitiveArray.java @@ -373,7 +373,6 @@ public String elementClassString() { * * @param type an element type (e.g., float.class) * @return the string representation of the element type (e.g., float) - * @throws exception if not one of the PrimitiveArray types */ public static String elementClassToString(Class type) { if (type == double.class) return "double"; @@ -393,7 +392,6 @@ public static String elementClassToString(Class type) { * * @param type an element type string (e.g., "float") * @return the corresponding element type (e.g., float.class) - * @throws exception if not one of the PrimitiveArray types */ public static Class elementStringToClass(String type) { if (type.equals("double")) return double.class; @@ -414,7 +412,6 @@ public static Class elementStringToClass(String type) { * * @param type an element type string (e.g., "float") * @return the corresponding number of bytes - * @throws exception if not one of the PrimitiveArray types */ public static int elementSize(String type) { if (type.equals("double")) return 8; @@ -435,7 +432,6 @@ public static int elementSize(String type) { * * @param type an element type (e.g., float.class) * @return the corresponding number of bytes - * @throws exception if not one of the PrimitiveArray types */ public static int elementSize(Class type) { return elementSize(elementClassToString(type)); @@ -491,7 +487,7 @@ public static String classToEsriPixelType(Class tClass) { * See http://www.techonthenet.com/sql/datatypes.php . *
  • For safety, ByteArray returns SMALLINT (not TINYINT, * which isn't universally supported, e.g., postgresql). - * * * @param stringLengthFactor for StringArrays, this is the factor (typically 1.5) * to be multiplied by the current max string length (then rounded up to @@ -702,7 +698,6 @@ public static double getMissingValue(Class type) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ abstract public void remove(int index); @@ -711,7 +706,6 @@ public static double getMissingValue(Class type) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ abstract public void removeRange(int from, int to); @@ -722,7 +716,6 @@ public static double getMissingValue(Class type) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ abstract public void move(int first, int last, int destination); @@ -773,7 +766,6 @@ public static double getMissingValue(Class type) { * @param index the index number 0 ... size-1 * @return the value as an int. String values are parsed * with String2.parseInt and so may return Integer.MAX_VALUE. - * @throws Exception if trouble. */ abstract public int getInt(int index); @@ -783,7 +775,6 @@ public static double getMissingValue(Class type) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToByte(i). - * @throws Exception if trouble. */ abstract public void setInt(int index, int i); @@ -794,7 +785,6 @@ public static double getMissingValue(Class type) { * @param index the index number 0 ... size-1 * @return the value as a long. String values are parsed * with String2.parseLong and so may return Long.MAX_VALUE. - * @throws Exception if trouble. */ abstract public long getLong(int index); @@ -804,7 +794,6 @@ public static double getMissingValue(Class type) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToByte(i). - * @throws Exception if trouble. */ abstract public void setLong(int index, long i); @@ -815,7 +804,6 @@ public static double getMissingValue(Class type) { * @param index the index number 0 ... size-1 * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. - * @throws Exception if trouble. */ abstract public float getFloat(int index); @@ -825,7 +813,6 @@ public static double getMissingValue(Class type) { * @param index the index number 0 ... size-1 * @param d the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.roundToInt(d). - * @throws Exception if trouble. */ abstract public void setFloat(int index, float d); @@ -836,7 +823,6 @@ public static double getMissingValue(Class type) { * @param index the index number 0 ... size-1 * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. - * @throws Exception if trouble. */ abstract public double getDouble(int index); @@ -847,7 +833,6 @@ public static double getMissingValue(Class type) { * @param index the index number 0 ... size-1 * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. - * @throws Exception if trouble. */ public double getNiceDouble(int index) { return getDouble(index); @@ -859,7 +844,6 @@ public double getNiceDouble(int index) { * @param index the index number 0 ... size-1 * @param d the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.roundToInt(d). - * @throws Exception if trouble. */ abstract public void setDouble(int index, double d); @@ -868,7 +852,6 @@ public double getNiceDouble(int index) { * * @param index the index number 0 ... size-1 * @return For numeric types, this returns ("" + ar[index]), or "" if NaN or infinity. - * @throws Exception if trouble. */ abstract public String getString(int index); @@ -879,7 +862,6 @@ public double getNiceDouble(int index) { * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parse and narrowed if needed by methods like * Math2.roundToInt(d). - * @throws Exception if trouble. */ abstract public void setString(int index, String s); @@ -1372,14 +1354,33 @@ public static void rafWriteDouble(RandomAccessFile raf, Class type, /** * This tests if the other PrimitiveArray has almost equal values. * If both are integer types or String types, this is an exact test (and says null==null is true). - * If either are double types, this tests almostEqual9() (and says NaN==NaN is true). * If either are float types, this tests almostEqual5() (and says NaN==NaN is true). + * If either are double types, this tests almostEqual9() (and says NaN==NaN is true). * * @param other * @return "" if almost equal, or message if not. * other=null throws an exception. */ public String almostEqual(PrimitiveArray other) { + return almostEqual(other, 9); + } + + /** + * This tests if the other PrimitiveArray has almost equal values. + * If both are integer types or String types, this is an exact test (and says null==null is true). + * + * @param other + * @param matchNDigits This is used if this or other is DoubleArray or FloatArray. + * Otherwise, this is ignored. + * (<=)0=no testing. + * 1 to 18 tests hidiv(nDigits,2) digits if either is FloatArray, + * or nDigits if either is DoubleArray. + * (Integer.MAX_VALUE is interpreted as 9.) + * >18 says to test exact equality. + * @return "" if almost equal, or message if not. + * other=null throws an exception. + */ + public String almostEqual(PrimitiveArray other, int matchNDigits) { if (size != other.size()) return MessageFormat.format(ArrayDifferentSize, "" + size, "" + other.size()); @@ -1398,32 +1399,78 @@ public String almostEqual(PrimitiveArray other) { return ""; } - if (this instanceof LongArray && other instanceof LongArray) { - for (int i = 0; i < size; i++) - if (!Test.equal(getLong(i), other.getLong(i))) //this says NaN==NaN is true - return MessageFormat.format(ArrayDifferentValue, "" + i, - "" + getLong(i), "" + other.getLong(i)); + if (this instanceof FloatArray || other instanceof FloatArray) { + matchNDigits = matchNDigits == Integer.MAX_VALUE? 5 : matchNDigits; + if (matchNDigits > 18) { + for (int i = 0; i < size; i++) { + float f1 = getFloat(i); + float f2 = other.getFloat(i); + if (Math2.isFinite(f1)) { + if (Math2.isFinite(f2)) { + if (f1 != f2) //exact + return MessageFormat.format(ArrayDifferentValue, "" + i, + "" + f1, "" + f2); + } else { + return MessageFormat.format(ArrayDifferentValue, "" + i, + "" + f1, "" + f2); + } + } else if (Math2.isFinite(f2)) { + return MessageFormat.format(ArrayDifferentValue, "" + i, + "" + f1, "" + f2); + } + } + } else if (matchNDigits <= 0) { + return ""; + } else { + int tMatchNDigits = Math2.hiDiv(matchNDigits, 2); + for (int i = 0; i < size; i++) + if (!Math2.almostEqual(tMatchNDigits, //this says NaN==NaN is true + getFloat(i), other.getFloat(i))) + return MessageFormat.format(ArrayDifferentValue, "" + i, + "" + getFloat(i), "" + other.getFloat(i)); + } return ""; } - if (this instanceof DoubleArray || this instanceof LongArray || - other instanceof DoubleArray || other instanceof LongArray) { - for (int i = 0; i < size; i++) - if (!Test.equal(getDouble(i), other.getDouble(i))) //this says NaN==NaN is true - return MessageFormat.format(ArrayDifferentValue, "" + i, - "" + getDouble(i), "" + other.getDouble(i)); + if (this instanceof DoubleArray || other instanceof DoubleArray) { + matchNDigits = matchNDigits == Integer.MAX_VALUE? 9 : matchNDigits; + if (matchNDigits > 18) { + for (int i = 0; i < size; i++) { + double d1 = getDouble(i); + double d2 = other.getDouble(i); + if (Math2.isFinite(d1)) { + if (Math2.isFinite(d2)) { + if (d1 != d2) //exact + return MessageFormat.format(ArrayDifferentValue, "" + i, + "" + d1, "" + d2); + } else { + return MessageFormat.format(ArrayDifferentValue, "" + i, + "" + d1, "" + d2); + } + } else if (Math2.isFinite(d2)) { + return MessageFormat.format(ArrayDifferentValue, "" + i, + "" + d1, "" + d2); + } + } + } else if (matchNDigits <= 0) { + return ""; + } else { + for (int i = 0; i < size; i++) + if (!Math2.almostEqual(matchNDigits, getDouble(i), other.getDouble(i))) //this says NaN==NaN is true + return MessageFormat.format(ArrayDifferentValue, "" + i, + "" + getDouble(i), "" + other.getDouble(i)); + } return ""; } - if (this instanceof FloatArray || - other instanceof FloatArray) { + if (this instanceof LongArray || other instanceof LongArray) { for (int i = 0; i < size; i++) - if (!Test.equal(getFloat(i), other.getFloat(i))) //this says NaN==NaN is true + if (getLong(i) != other.getLong(i)) return MessageFormat.format(ArrayDifferentValue, "" + i, - "" + getFloat(i), "" + other.getFloat(i)); - return ""; + "" + getLong(i), "" + other.getLong(i)); } + //test via int's for (int i = 0; i < size; i++) if (getInt(i) != other.getInt(i)) return MessageFormat.format(ArrayDifferentValue, "" + i, @@ -1508,7 +1555,7 @@ public static long rafBinarySearch(RandomAccessFile raf, Class type, * @param highPo the high index to start with, usually * (originalPrimitiveArray.size() - 1) * @param value the value you are searching for - * @return the index of the first element @gt;= value + * @return the index of the first element >= value * (or highPo + 1, if there are none) * @throws Exception if trouble */ @@ -1668,7 +1715,7 @@ public static long rafLastLAE(RandomAccessFile raf, Class type, * (or -index-1 where it should be inserted, with extremes of * -lowPo-1 and -(highPo+1)-1). * [So insert at -response-1.] - * @throws Exception if lowPo > highPo. + * @throws Exception if lowPo > highPo. */ public int binarySearch(int lowPo, int highPo, double value) { @@ -2711,7 +2758,7 @@ public String smallestBiggestSpacing() { * Given nHave values and stride, this returns the actual number of points that will be found. * * @param nHave the size of the array (or end-start+1) - * @param stride (must be >= 1) + * @param stride (must be >= 1) * @return the actual number of points that will be found. */ public static int strideWillFind(int nHave, int stride) { diff --git a/WEB-INF/classes/com/cohort/array/ShortArray.java b/WEB-INF/classes/com/cohort/array/ShortArray.java index 22979f45f..23dc6d53f 100644 --- a/WEB-INF/classes/com/cohort/array/ShortArray.java +++ b/WEB-INF/classes/com/cohort/array/ShortArray.java @@ -59,7 +59,6 @@ public ShortArray(PrimitiveArray primitiveArray) { * @param capacity creates an ShortArray with the specified initial capacity. * @param active if true, size will be set to capacity and all elements * will equal 0; else size = 0. - * @throws Exception if trouble. */ public ShortArray(int capacity, boolean active) { Math2.ensureMemoryAvailable(2L * capacity, "ShortArray"); @@ -73,7 +72,6 @@ public ShortArray(int capacity, boolean active) { * * @param first the value of the first element. * @param last the value of the last element (inclusive). - * @throws Exception if trouble. */ public ShortArray(int first, int last) { size = last - first + 1; @@ -358,7 +356,6 @@ public void setFromPA(int index, PrimitiveArray otherPA, int otherIndex) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ public void remove(int index) { if (index >= size) @@ -375,7 +372,6 @@ public void remove(int index) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ public void removeRange(int from, int to) { if (to > size) @@ -400,7 +396,6 @@ public void removeRange(int from, int to) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ public void move(int first, int last, int destination) { String errorIn = String2.ERROR + " in ShortArray.move:\n"; @@ -546,7 +541,6 @@ public String[] toStringArray() { * This gets a specified element. * * @param index 0 ... size-1 - * @throws Exception if trouble. */ public short get(int index) { if (index >= size) @@ -560,7 +554,6 @@ public short get(int index) { * * @param index 0 ... size-1 * @param value the value for that element - * @throws Exception if trouble. */ public void set(int index, short value) { if (index >= size) @@ -576,7 +569,6 @@ public void set(int index, short value) { * @param index the index number 0 ... size-1 * @return the value as an int. * Short.MAX_VALUE is returned as Integer.MAX_VALUE. - * @throws Exception if trouble. */ public int getInt(int index) { int i = get(index); @@ -589,7 +581,6 @@ public int getInt(int index) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToByte(i). - * @throws Exception if trouble. */ public void setInt(int index, int i) { set(index, Math2.narrowToShort(i)); @@ -601,7 +592,6 @@ public void setInt(int index, int i) { * @param index the index number 0 ... size-1 * @return the value as a long. * Short.MAX_VALUE is returned as Long.MAX_VALUE. - * @throws Exception if trouble. */ public long getLong(int index) { int i = get(index); @@ -614,7 +604,6 @@ public long getLong(int index) { * @param index the index number 0 .. size-1 * @param i the value. For numeric PrimitiveArray's, it is narrowed * if needed by methods like Math2.narrowToShort(long). - * @throws Exception if trouble. */ public void setLong(int index, long i) { set(index, Math2.narrowToShort(i)); @@ -627,7 +616,6 @@ public void setLong(int index, long i) { * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. * Short.MAX_VALUE is returned as Float.NaN. - * @throws Exception if trouble. */ public float getFloat(int index) { short s = get(index); @@ -640,7 +628,6 @@ public float getFloat(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToShort(d). - * @throws Exception if trouble. */ public void setFloat(int index, float d) { set(index, Math2.roundToShort(d)); @@ -653,7 +640,6 @@ public void setFloat(int index, float d) { * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. * Short.MAX_VALUE is returned as Double.NaN. - * @throws Exception if trouble. */ public double getDouble(int index) { short s = get(index); @@ -666,7 +652,6 @@ public double getDouble(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToShort(d). - * @throws Exception if trouble. */ public void setDouble(int index, double d) { set(index, Math2.roundToShort(d)); @@ -677,7 +662,6 @@ public void setDouble(int index, double d) { * * @param index the index number 0 .. * @return For numeric types, this returns (String.valueOf(ar[index])), or "" for NaN or infinity. - * @throws Exception if trouble. */ public String getString(int index) { short b = get(index); @@ -690,7 +674,6 @@ public String getString(int index) { * @param index the index number 0 .. * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parseInt and narrowed by Math2.narrowToShort(i). - * @throws Exception if trouble. */ public void setString(int index, String s) { set(index, Math2.narrowToShort(String2.parseInt(s))); diff --git a/WEB-INF/classes/com/cohort/array/StringArray.java b/WEB-INF/classes/com/cohort/array/StringArray.java index c8cc70bca..abe7b08b7 100644 --- a/WEB-INF/classes/com/cohort/array/StringArray.java +++ b/WEB-INF/classes/com/cohort/array/StringArray.java @@ -74,7 +74,6 @@ public StringArray(PrimitiveArray primitiveArray) { * @param capacity creates an StringArray with the specified initial capacity. * @param active if true, size will be set to capacity and all elements * will equal "", else size = 0. - * @throws Exception if trouble. */ public StringArray(int capacity, boolean active) { Math2.ensureMemoryAvailable(16L * capacity, "StringArray"); //16 is lame estimate of space needed per String @@ -149,7 +148,6 @@ public StringArray(Map map) { /** * This reads the text contents of the specified file using this computer's default charset. * - * @throws Exception if trouble (e.g., file not found) */ public static StringArray fromFile(String fileName) throws Exception { @@ -518,7 +516,6 @@ public void setFromPA(int index, PrimitiveArray otherPA, int otherIndex) { * This removes the specified element. * * @param index the element to be removed, 0 ... size-1 - * @throws Exception if trouble. */ public void remove(int index) { if (index >= size) @@ -536,7 +533,6 @@ public void remove(int index) { * * @param from the first element to be removed, 0 ... size * @param to one after the last element to be removed, from ... size - * @throws Exception if trouble. */ public void removeRange(int from, int to) { if (to > size) @@ -562,7 +558,6 @@ public void removeRange(int from, int to) { * @param first the first to be move * @param last (exclusive) * @param destination the destination, can't be in the range 'first+1..last-1'. - * @throws Exception if trouble */ public void move(int first, int last, int destination) { String errorIn = String2.ERROR + " in StringArray.move:\n"; @@ -699,7 +694,6 @@ public String[] toStringArray() { * This gets a specified element. * * @param index 0 ... size-1 - * @throws Exception if trouble. */ public String get(int index) { if (index >= size) @@ -713,7 +707,6 @@ public String get(int index) { * * @param index 0 ... size-1 * @param value the value for that element - * @throws Exception if trouble. */ public void set(int index, String value) { if (index >= size) @@ -728,7 +721,6 @@ public void set(int index, String value) { * * @param index the index number 0 ... size-1 * @return the value as an int. This uses String2.parseInt. - * @throws Exception if trouble. */ public int getInt(int index) { return String2.parseInt(get(index)); @@ -739,7 +731,6 @@ public int getInt(int index) { * * @param index the index number 0 .. size-1 * @param i the value. - * @throws Exception if trouble. */ public void setInt(int index, int i) { set(index, i == Integer.MAX_VALUE? "" : String.valueOf(i)); @@ -750,7 +741,6 @@ public void setInt(int index, int i) { * * @param index the index number 0 ... size-1 * @return the value as a long. This uses String2.parseLong. - * @throws Exception if trouble. */ public long getLong(int index) { return String2.parseLong(get(index)); @@ -761,7 +751,6 @@ public long getLong(int index) { * * @param index the index number 0 .. size-1 * @param i the value. - * @throws Exception if trouble. */ public void setLong(int index, long i) { set(index, i == Long.MAX_VALUE? "" : String.valueOf(i)); @@ -774,7 +763,6 @@ public void setLong(int index, long i) { * @param index the index number 0 .. size-1 * @return the value as a float. String values are parsed * with String2.parseFloat and so may return Float.NaN. - * @throws Exception if trouble. */ public float getFloat(int index) { return String2.parseFloat(get(index)); @@ -786,7 +774,6 @@ public float getFloat(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToString(d). - * @throws Exception if trouble. */ public void setFloat(int index, float d) { set(index, Math2.isFinite(d)? String.valueOf(d) : ""); @@ -798,7 +785,6 @@ public void setFloat(int index, float d) { * @param index the index number 0 .. size-1 * @return the value as a double. String values are parsed * with String2.parseDouble and so may return Double.NaN. - * @throws Exception if trouble. */ public double getDouble(int index) { return String2.parseDouble(get(index)); @@ -810,7 +796,6 @@ public double getDouble(int index) { * @param index the index number 0 .. size-1 * @param d the value. For numeric PrimitiveArray, it is narrowed * if needed by methods like Math2.roundToString(d). - * @throws Exception if trouble. */ public void setDouble(int index, double d) { set(index, Math2.isFinite(d)? String.valueOf(d) : ""); @@ -821,7 +806,6 @@ public void setDouble(int index, double d) { * * @param index the index number 0 .. * @return For numeric types, this returns array[index]. - * @throws Exception if trouble. */ public String getString(int index) { return get(index); @@ -834,7 +818,6 @@ public String getString(int index) { * @param s the value. For numeric PrimitiveArray's, it is parsed * with String2.parse and narrowed if needed by methods like * Math2.roundToString(d). - * @throws Exception if trouble. */ public void setString(int index, String s) { set(index, s); diff --git a/WEB-INF/classes/com/cohort/ema/EmaStringBox.java b/WEB-INF/classes/com/cohort/ema/EmaStringBox.java index 77b19fdd4..130bd8786 100644 --- a/WEB-INF/classes/com/cohort/ema/EmaStringBox.java +++ b/WEB-INF/classes/com/cohort/ema/EmaStringBox.java @@ -81,7 +81,7 @@ public String getControl(String value) { "\" rows=\"" + rows + "\" cols=\"" + cols + //'wrap' is non-standard. see http://www.idocs.com/tags/forms/_TEXTAREA_WRAP.html - "\" wrap=\"SOFT\""); //was "virtual" + "\" maxlength=\"" + (rows * cols) + "\" wrap=\"SOFT\""); //was "virtual" if (title.length() > 0) sb.append("\n title=\"" + XML.encodeAsHTML(title) + "\""); //style not used here diff --git a/WEB-INF/classes/com/cohort/util/Calendar2.java b/WEB-INF/classes/com/cohort/util/Calendar2.java index 4f6f06474..13e8af445 100644 --- a/WEB-INF/classes/com/cohort/util/Calendar2.java +++ b/WEB-INF/classes/com/cohort/util/Calendar2.java @@ -345,6 +345,8 @@ public static double isoStringToEpochSeconds(String isoZuluString) { * This is like isoStringToEpochSeconds, but returns NaN if trouble. */ public static double safeIsoStringToEpochSeconds(String isoZuluString) { + if (isoZuluString == null || isoZuluString.length() < 4) + return Double.NaN; try { return isoZuluStringToMillis(isoZuluString) / 1000.0; } catch (Exception e) { diff --git a/WEB-INF/classes/com/cohort/util/File2.java b/WEB-INF/classes/com/cohort/util/File2.java index 0c4609744..fa6fbb697 100644 --- a/WEB-INF/classes/com/cohort/util/File2.java +++ b/WEB-INF/classes/com/cohort/util/File2.java @@ -718,21 +718,34 @@ public static boolean copy(String source, String destination) { */ public static boolean copy(String source, OutputStream out) { - FileInputStream in = null; - int bufferSize = 8192; - byte buffer[] = new byte[bufferSize]; - boolean success = false; try { File file = new File(source); if (!file.isFile()) return false; - long remain = file.length(); - in = new FileInputStream(file); - while (remain > 0) { - int read = in.read(buffer); - out.write(buffer, 0, read); - remain -= read; - } + FileInputStream in = new FileInputStream(file); + return copy(in, out); + } catch (Exception e) { + String2.log(MustBe.throwable(String2.ERROR + " in File2.copy: ", e)); + return false; + } + } + + /** + * This copies from an inputStream to an outputStream. + * + * @param in When finished, successful or not, this closes 'in'. + * @param out which is flushed, but not closed, at the end + * @return true if successful. + */ + public static boolean copy(InputStream in, OutputStream out) { + + int bufferSize = 32768; + byte buffer[] = new byte[bufferSize]; + boolean success = false; + try { + int nRead; + while ((nRead = in.read(buffer)) >= 0) //0 shouldn't happen. -1=end of file + out.write(buffer, 0, nRead); out.flush(); success = true; } catch (Exception e) { @@ -838,5 +851,22 @@ public static String addSlash(String dir) { return dir + slash; } + /** + * This returns true if the dir starts with http://, https://, ftp://, sftp://, + * or smb://. + * If dir is null or "", this returns false. + */ + public static boolean isRemote(String dir) { + if (dir == null) + return false; + return + dir.startsWith("http://") || + dir.startsWith("https://") || + dir.startsWith("ftp://") || + dir.startsWith("sftp://") || + dir.startsWith("smb://"); + } + + } diff --git a/WEB-INF/classes/com/cohort/util/MustBe.java b/WEB-INF/classes/com/cohort/util/MustBe.java index 7083a4bab..ef0ee7cca 100644 --- a/WEB-INF/classes/com/cohort/util/MustBe.java +++ b/WEB-INF/classes/com/cohort/util/MustBe.java @@ -356,19 +356,25 @@ public static String allStackTraces(boolean hideThisThread, boolean hideTomcatWa Map.Entry me = (Map.Entry)oar[i]; Thread t = (Thread)me.getKey(); StackTraceElement ste[] = (StackTraceElement[])me.getValue(); - String ste0 = ste.length == 0? "" : ste[0].toString(); + String ste0 = ste.length < 1? "" : ste[0].toString(); + String ste1 = ste.length < 2? "" : ste[1].toString(); + String ste2 = ste.length < 3? "" : ste[2].toString(); if (hideThisThread && ste0.startsWith("java.lang.Thread.dumpThreads(Native Method)")) continue; - //linux if (hideTomcatWaitingThreads && - ste.length == 4 && ste0.startsWith("java.lang.Object.wait(Native Method)") && - ste[2].toString().startsWith("org.apache.tomcat.util.threads.ThreadPool")) - continue; - //Mac OS/X + ste.length >= 3 && + ste0.startsWith("java.lang.Object.wait(Native Method)")) { + if (ste2.startsWith("org.apache.tomcat.util.threads.ThreadPool") || //linux + ste2.startsWith("org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:")) //Mac + continue; + } + //inotify thredd if (hideTomcatWaitingThreads && - ste.length == 5 && ste0.startsWith("java.lang.Object.wait(Native Method)") && - ste[2].toString().startsWith("org.apache.tomcat.util.net.JIoEndpoint$Worker.await(JIoEndpoint.java:")) + ste.length >= 1 && + ste0.startsWith("sun.nio.fs.LinuxWatchService.poll(") || //linux + ste0.startsWith("org.apache.tomcat.jni.Poll.poll(")) //windows continue; + sar[count] = t.toString() + " " + t.getState().toString() + (t.isDaemon()? " daemon\n" : "\n") + String2.toNewlineString(ste) + "\n"; @@ -384,7 +390,7 @@ public static String allStackTraces(boolean hideThisThread, boolean hideTomcatWa //write to StringBuilder StringBuilder sb = new StringBuilder(); - sb.append("Number of " + (hideTomcatWaitingThreads? "non-Tomcat-waiting " : "") + + sb.append("Number of " + (hideTomcatWaitingThreads? "non-Tomcat-waiting non-inotify " : "") + "threads in this JVM = " + count + "\n" + "(format: #threadNumber Thread[threadName,threadPriority,threadGroup] threadStatus)\n\n"); for (int i = 0; i < count; i++) diff --git a/WEB-INF/classes/com/cohort/util/String2.java b/WEB-INF/classes/com/cohort/util/String2.java index 2da2b907e..0e8358900 100644 --- a/WEB-INF/classes/com/cohort/util/String2.java +++ b/WEB-INF/classes/com/cohort/util/String2.java @@ -92,6 +92,8 @@ public class String2 { https://developer.apple.com/library/mac/technotes/tn2002/tn2110.html */ public static boolean OSIsMacOSX = OSName.contains("OS X"); + public final static String AWS_S3_REGEX = "http(s|)://(\\w*)\\.s3\\.amazonaws\\.com/(.*)"; + /** These are NOT thread-safe. Always use them in synchronized blocks ("synchronized(gen....) {}").*/ private static DecimalFormat genStdFormat6 = new DecimalFormat("0.######"); private static DecimalFormat genEngFormat6 = new DecimalFormat("##0.#####E0"); @@ -217,7 +219,8 @@ public static String center(String s, int length) { * * @param s * @param max - * @return s (if it is short) or the first max characters of s + * @return s (if it is short) or the first max characters of s. + * If s==null, this returns "". */ public static String noLongerThan(String s, int max) { if (s == null) @@ -5205,5 +5208,53 @@ public static String periodSpaceConcat(String a, String b) { } + /** + * Given an Amazon AWS S3 URL, this returns the bucketName. + * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html + * If files have file-system-like names, e.g., + * http(s|)://bucketName.s3.amazonaws.com/prefix + * where the prefix is usually in the form dir1/dir2/fileName.ext + * http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_NorESM1-M_209601-209912.nc + * + * @param url + * @return the bucketName or null if not an s3 URL + */ + public static String getAwsS3BucketName(String url) { + if (url == null) + return null; + Pattern pattern = Pattern.compile(String2.AWS_S3_REGEX); + if (url.endsWith(".s3.amazonaws.com")) + url = File2.addSlash(url); + Matcher matcher = pattern.matcher(url); + if (matcher.matches()) + return matcher.group(2); //bucketName + return null; + } + + /** + * Given an Amazon AWS S3 URL, this returns the objectName or prefix. + * http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html + * If files have file-system-like names, e.g., + * http(s|)://bucketName.s3.amazonaws.com/prefix + * where a prefix is usually in the form dir1/dir2/ + * where an objectName is usually in the form dir1/dir2/fileName.ext + * http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_NorESM1-M_209601-209912.nc + * + * @param url If the url is supposed to be for a prefix, use + * getAwsS3Prefix(File2.addSlash(url)) + * @return the prefix or null if not an s3 URL + */ + public static String getAwsS3Prefix(String url) { + if (url == null) + return null; + Pattern pattern = Pattern.compile(String2.AWS_S3_REGEX); + if (url.endsWith(".s3.amazonaws.com")) + url = File2.addSlash(url); + Matcher matcher = pattern.matcher(url); + if (matcher.matches()) + return matcher.group(3); //prefix + return null; + } + } //End of String2 class. diff --git a/WEB-INF/classes/com/cohort/util/Test.java b/WEB-INF/classes/com/cohort/util/Test.java index b2b8f7454..8f7484508 100644 --- a/WEB-INF/classes/com/cohort/util/Test.java +++ b/WEB-INF/classes/com/cohort/util/Test.java @@ -114,8 +114,8 @@ public static void ensureNotEqual(long i1, long i2, String message) } /** - * This returns true if the two float values are almost equal (or both NaN - * or both infinite). + * This returns true if the two float values are almost equal (5 digits) + * or both NaN or both infinite. * * @param f1 * @param f2 @@ -130,8 +130,8 @@ public static boolean equal(float f1, float f2) { } /** - * This returns true if the two double values are almost equal (or both NaN - * or both infinite). + * This returns true if the two double values are almost equal (9 digits) + * or both NaN or both infinite. * * @param d1 * @param d2 diff --git a/WEB-INF/classes/com/cohort/util/TestUtil.java b/WEB-INF/classes/com/cohort/util/TestUtil.java index e109025f5..f5d0eb79d 100644 --- a/WEB-INF/classes/com/cohort/util/TestUtil.java +++ b/WEB-INF/classes/com/cohort/util/TestUtil.java @@ -3929,8 +3929,8 @@ public static void testFile2() throws Exception { Math2.sleep(20); //make the file a little older long fileTime = File2.getLastModified(utilDir + "temp.txt"); long time1 = System.currentTimeMillis(); - Test.ensureEqual(time1 >= fileTime + 10, true, "a1"); - Test.ensureEqual(time1 <= fileTime + 100, true, "a2"); + Test.ensureEqual(time1 >= fileTime + 10, true, "a1 " + time1 + " " + fileTime); + Test.ensureEqual(time1 <= fileTime + 100, true, "a2 " + time1 + " " + fileTime); Test.ensureEqual(File2.touch(utilDir + "temp.txt"), true, "a"); //touch the file long time2 = System.currentTimeMillis(); fileTime = File2.getLastModified(utilDir + "temp.txt"); diff --git a/WEB-INF/classes/com/cohort/util/XML.java b/WEB-INF/classes/com/cohort/util/XML.java index 58e150f3e..bd1c4d7aa 100644 --- a/WEB-INF/classes/com/cohort/util/XML.java +++ b/WEB-INF/classes/com/cohort/util/XML.java @@ -113,7 +113,7 @@ public static String removeHTMLTags(String htmlString) { * @return the encoded string */ public static String encodeAsHTML(String plainText) { -//future should it:* Pairs of spaces are converted to sp +  . +//future: should it convert pairs of spaces to sp +   ? int size = plainText.length(); StringBuilder output = new StringBuilder(size * 2); diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/MakeEmaWar.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/MakeEmaWar.java index d358246d2..f71fe8561 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/MakeEmaWar.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/MakeEmaWar.java @@ -46,7 +46,7 @@ public static void main(String args[]) throws Exception { String coastWatchDir = baseDir + "WEB-INF/classes/gov/noaa/pfel/coastwatch/"; //make the javadoc commands - String commandLine0 = "C:\\Progra~1\\Java\\jdk1.7.0_67\\bin\\javadoc" + + String commandLine0 = "C:\\Progra~1\\Java\\jdk1.8.0_51\\bin\\javadoc" + " -sourcepath " + classPath + //root directory of the classes " -d "; //directory to hold results String commandLine2 = " -subpackages com.cohort"; //the packages to be doc'd diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java index 331a20af1..8add7677e 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java @@ -134,8 +134,22 @@ public static void main(String args[]) throws Throwable { // // Table.testReadNcCFASAProfile(false); -// Table.debugMode = true; //DasDds.main(new String[]{"LRHB2015AISyGRecords", "-verbose"}); -// String2.log(NcHelper.dumpString("c:/data/rutgers/NOAA_COOPS_WIND_STATIONS_SUBSET.nc", false)); +// Table.debugMode = true; DasDds.main(new String[]{"swfscTrinCTD", "-verbose"}); + +/* if (false) { //one time fixup of scrippsGliders + String dir = "/u00/data/points/scrippsGliders/batch2/"; + File file = new File(dir); + String tList[] = file.list(); + for (int i = 0; i < tList.length; i++) { + SSR.dosOrCShell( + "c:\\programs\\nco\\ncatted -O -a standard_name,salinity,o,c,sea_water_practical_salinity " + dir + tList[i], 60).toArray(new String[0]); + SSR.dosOrCShell( + "c:\\programs\\nco\\ncatted -O -a units,salinity,o,c,1 " + dir + tList[i], 60).toArray(new String[0]); + } + } /* */ + +// String2.log(NcHelper.dumpString("/u00/data/points/eb/CoralSea_CS140116.nc", true)); + // Table table = new Table(); //// table.readNDNc("c:/data/rutgers/NOAA_COOPS_WIND_STATIONS_SUBSET.nc", //// new String[]{"longitude","latitude","station","time"}, //loadVars @@ -160,6 +174,7 @@ public static void main(String args[]) throws Throwable { // (EDDTable)EDD.oneFromDatasetXml("pmelTao")).getEmpiricalMinMax("2008-10-05", "2008-10-10", false, false); // String2.log(((EDDTable)EDD.oneFromDatasetXml("nwioosAdcp2003")).toString()); // String2.log(EDD.testDasDds("thierry")); +// String2.log(EDD.generateDatasetsXmlFromFiles("/u00/data/points/tao")); ///u00/data/points/tao // EDDGrid.verbose = true; // EDDGrid.reallyVerbose = true; // EDDGrid.suggestGraphMinMax(); @@ -211,7 +226,7 @@ public static void main(String args[]) throws Throwable { // "http://oceanwatch.pfeg.noaa.gov/thredds/catalog/catalog.xml", // "http://thredds.jpl.nasa.gov/thredds/podaac_catalogs/AQUARIUS_L3_SMI_V20_catalog.xml", // "http://opendap-uat.jpl.nasa.gov/thredds/catalog.xml", -// "http://ferret.pmel.noaa.gov/geoide/CleanCatalogs/ecowatch.ncddc.noaa.gov/thredds/catalog/ncom/ncom_reg1_agg/catalog.xml", +// "http://www.esrl.noaa.gov/psd/thredds/dodsC/Datasets/nodc.woa94/salt.mnltm.nc", // ".*", -1); // String2.log(String2.readFromFile(ftcName)[1]); // @@ -223,9 +238,9 @@ public static void main(String args[]) throws Throwable { // // EDD.debugMode = true; // String2.log("\n" + EDDGridFromDap.generateDatasetsXml(false, //directions -// "http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/PPMW/3day", -//// String2.readLinesFromFile("/temp/urlsFromUAF.txt", "", 1)[2342], //one URL -//// String2.readLinesFromFile("/temp/urlsFromPodaac.txt", "", 1)[161], //one URL +// "http://data1.gfdl.noaa.gov:8380/thredds3/dodsC/ipcc_ar4_CM2.0_R2_20C3M-0_annual_ocean_interp_18610101-20001231", +// String2.readLinesFromFile("/temp/urlsFromUAF.txt", "", 1)[2342], //one URL +// String2.readLinesFromFile("/temp/urlsFromPodaac.txt", "", 1)[161], //one URL // null, null, null, -1, null)); // // crawl UAF clean catalog @@ -248,7 +263,6 @@ public static void main(String args[]) throws Throwable { // Arrays.sort(ar); // String2.log(String2.toNewlineString(ar)); -// EDDGridFromDap.testGetUrlsFromHyraxCatalog(); // EDDGridFromDap.testValidMinMax(); // EDDGridFromErddap.testDataVarOrder(); // String2.log(EDDGridFromErddap.generateDatasetsXml("http://coastwatch.pfeg.noaa.gov/erddap", @@ -256,11 +270,51 @@ public static void main(String args[]) throws Throwable { // String2.log(EDDGridFromErddap.generateDatasetsXml("http://oceanview.pfeg.noaa.gov/erddap", true)); // String2.log(EDDGridFromErddap.generateDatasetsXml("http://upwell.pfeg.noaa.gov/erddap")); // String2.log(EDDGridFromErddap.generateDatasetsXml("http://oos.soest.hawaii.edu/erddap", true)); -// EDDGridFromNcFiles.testUpdate(); -// String2.log(EDDGridFromNcFiles.generateDatasetsXml( -// "/data/VH2/", "V.{7}\\.L3m_DAY_NPP_CHL_chlor_a_4km\\.nc", -// "/data/VH2/V2013100.L3m_DAY_NPP_CHL_chlor_a_4km.nc", -// 90, null)); +// EDDGridFromNcFiles.testAwsS3(false); +// String opt[] = { +// "_BNU-ESM_","_CCSM4_","_CESM1-CAM5_","_CSIRO-Mk3-6-0_","_CanESM2_","_FGOALS-g2_", +// "_FIO-ESM_","_GFDL-CM3_","_GFDL-ESM2G_","_GFDL-ESM2M_","_GISS-E2-R_","_HadGEM2-AO_","_IPSL-CM5A-LR_", +// "_IPSL-CM5A-MR_","_MIROC-ESM-CHEM_","_MIROC-ESM_","_MIROC5_","_MPI-ESM-LR_","_MPI-ESM-MR_", +// "_MRI-CGCM3_","_NorESM1-M_","_bcc-csm1-1_"}; +// //File2.delete("/Temp/AWSDatasets.txt"); +// for (int opti = 0; opti < opt.length; opti++) { +// try { +// String2.appendFile("/Temp/AWSDatasets.txt", +// EDDGridFromNcFiles.generateDatasetsXml( +// "http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/", +// ".*" + opt[opti] + ".*\\.nc", +// "http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS" + opt[opti] + "200601-201012.nc", +// 1000000, null)); +// } catch (Throwable t) { +// String2.appendFile("/Temp/AWSDatasets.txt", MustBe.throwableToString(t)); +// } +// } + +/* + //Move CoastWatch ERDDAP datasets from THREDDS to new RAID files. + String ds = "QS"; //2 letter dataset code (exceptions: MPIC, MPOC) + String vn = "uy10"; //4 letter var name, e.g., ssta, but "" for MPIC MPOC + String cl = "m"; //composite length, e.g., h, 1, 3, 8, 14, m + String tf = "QS1999213_1999243_uy10.nc"; //test .nc (or .ncml) fileName + + EDDGridFromNcFiles.generateDatasetsXmlCoastwatchErdMode = true; + String tdir = "/u00/satellite/" + ds + "/" + + (vn.length() == 0? "" : vn + "/") + cl + "day/"; + String gx = EDDGridFromNcFiles.generateDatasetsXml( + tdir, ".*_" + (vn.length() == 0? "4km\\.ncml" : vn + "\\.nc"), + tdir + tf, 1440, null); + String2.log(gx); + String2.setClipboardString(gx); + String2.pressEnterToContinue("Paste results into datasets2.xml, change destName and long_name."); + + File2.delete("/u00/cwatch/erddap2/dataset/ay/erd" + ds + vn + cl + "day/fileTable.nc"); + String2.log(EDD.testDasDds("erd" + ds + vn + cl + "day")); +/* */ + //while (ds.length() > 0) { + // ds = String2.getStringFromSystemIn("datasetID?"); + // String2.log(EDD.testDasDds(ds)); + //} + /* StringBuilder sb = new StringBuilder(); //for (int i1 = 0; i1 < 4320; i1++) @@ -274,7 +328,7 @@ public static void main(String args[]) throws Throwable { // Projects.viirsLatLon(true); //create // String2.log(NcHelper.dumpString("c:/data/rutgers/NOAA_COOPS_WIND_STATIONS_SUBSET.nc", false)); -// String2.log(NcHelper.dumpString("/u00/data/points/eb/TrinidadHeadLine_CoralSea_CS120921.nc", false)); +// String2.log(NcHelper.dumpString("/u00/data/points/eb/TrinidadHeadLine_CoralSea_CS140116.nc", false)); // String2.log(String2.noLongLines(NcHelper.dumpString("/data/tao/sst0n147e_dy.cdf", "time"), 80, "")); //Table table = new Table(); //table.readNDNc("/u00/data/points/taoOriginal/realtime/airt0n110w_dy.cdf", @@ -288,7 +342,7 @@ public static void main(String args[]) throws Throwable { // EDDGridFromNcFiles.testNcml(); //make e.g., // Projects.makeNcmlCoordValues("V*.L3m_DAY_NPP_CHL_chlor_a_4km", "2012-01-02", "2013-12-31", 1, Calendar.DAY_OF_YEAR); - +// EDDGridFromNcFiles.testGenerateDatasetsXmlWithRemoteThreddsFiles(); // EDDGridFromNcFiles.testSpeed(-1); //-1 for all // EDDGridSideBySide.testTransparentPng(); @@ -476,6 +530,10 @@ public static void main(String args[]) throws Throwable { // EDDTableFromErddap.testApostrophe(); // EDDTableFromEDDGrid.testTableFromGriddap(); // EDDTableFromFiles.testIsOK(); +// String2.log(EDDTableFromFileNames.generateDatasetsXml( +// "http://nasanex.s3.amazonaws.com/", ".*", true, +// 10080, +// "","","","",null)); // String2.log(EDDTableFromHyraxFiles.generateDatasetsXml( // "http://data.nodc.noaa.gov/opendap/wod/monthly/APB/201103-201103/", // "wod_01345934.O\\.nc", @@ -485,11 +543,11 @@ public static void main(String args[]) throws Throwable { // "time", //String tSortedColumnSourceName, // "time", //tSortFilesBySourceNames, // null)); //externalAddAttributes) -// EDDTableFromHyraxFiles.testJpl(false); +// EDDTableFromHyraxFiles.testJpl(true); //deleteCachedInfoAndOneFile // String2.log(EDDTableFromNcCFFiles.generateDatasetsXml( // "/u00/data/points/eb/", "TrinidadHeadLine_CoralSea_.*\\.nc", -// "/u00/data/points/eb/TrinidadHeadLine_CoralSea_CS0610.nc", 1440, +// "/u00/data/points/eb/TrinidadHeadLine_CoralSea_CS140116.nc", 1440, // "", "", "", // "", "", // "", "", "", "", new Attributes())); @@ -504,7 +562,7 @@ public static void main(String args[]) throws Throwable { //String2.log(tTable.toCSVString()); // -// EDDTableFromNcFiles.testBigRequest(); +// EDDTableFromNcFiles.testNewTime(); // NOT FINISHED EDDTableFromNcFiles.bobConsolidateWOD("APB", "1960-01-01"); // EDDTableFromNcFiles.getAllSourceVariableNames( // "c:/data/wod/monthly/APB/", ".*\\.nc"); //201103-201103/ @@ -520,7 +578,7 @@ public static void main(String args[]) throws Throwable { // "long_name"); // String2.log(EDDTableFromNcFiles.generateDatasetsXml( // "/u00/data/points/eb/", "TrinidadHeadLine_CoralSea.*\\.nc", -// "/u00/data/points/eb/TrinidadHeadLine_CoralSea_CS0610.nc", +// "/u00/data/points/eb/TrinidadHeadLine_CoralSea_CS140116.nc", // "", 1440, // "", "", "", // "", "", @@ -547,7 +605,7 @@ public static void main(String args[]) throws Throwable { // Temporarily switching off parts of McAfee : Virus Scan Console (2X speedup!) // On Access Scanner : All Processes // Scan Items: check: specified file types only (instead of usual All Files) -// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2015, 2, 2015, 4, false); //first/last year(1990..)/month(1..), testMode +// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2012, 12, 2015, 7, false); //first/last year(1990..)/month(1..), testMode // log file is c:/data/gtspp/log.txt // 2b) Email the "good" but "impossible" stations to Charles Sun // [was Melanie Hamilton, now retired] @@ -570,7 +628,7 @@ public static void main(String args[]) throws Throwable { // EDDTableFromNcFiles.testErdGtsppBest("erdGtsppBestNc"); // 6) Create ncCF files with the same date range as 2a) above: // !!!! HIDE THE WINDOW !!! IT WILL RUN MUCH FASTER!!! takes ~2 minutes per month processed -// EDDTableFromNcFiles.bobCreateGtsppNcCFFiles(2015, 2, 2015, 4); //e.g., first/last year(1990..)/month(1..) +// EDDTableFromNcFiles.bobCreateGtsppNcCFFiles(2012, 12, 2015, 7); //e.g., first/last year(1990..)/month(1..) // String2.log(NcHelper.dumpString("/u00/data/points/gtsppNcCF/201406a.nc", false)); // 7) * Load erdGtsppBest in localHost ERDDAP. (long time if lots of files changed) // * Generate .json file from @@ -643,6 +701,7 @@ public static void main(String args[]) throws Throwable { EDDGridFromMergeIRFiles.testGenerateDatasetsXml(); EDDGridFromNcFiles.testGenerateDatasetsXml(); EDDGridFromNcFiles.testGenerateDatasetsXml2(); + EDDGridFromNcFiles.testGenerateDatasetsXmlAwsS3(); EDDTableFromAsciiFiles.testGenerateDatasetsXml(); EDDTableFromAwsXmlFiles.testGenerateDatasetsXml(); EDDTableFromCassandra.testGenerateDatasetsXml(); @@ -651,6 +710,7 @@ public static void main(String args[]) throws Throwable { EDDTableFromDatabase.testGenerateDatasetsXml(); EDDTableFromErddap.testGenerateDatasetsXml(); EDDTableFromFileNames.testGenerateDatasetsXml(); + EDDTableFromFileNames.testGenerateDatasetsXmlAwsS3(); EDDTableFromHyraxFiles.testGenerateDatasetsXml(); //EDDTableFromHyraxFiles.testGenerateDatasetsXml2(); //not yet working EDDTableFromNcCFFiles.testGenerateDatasetsXml(); @@ -679,6 +739,9 @@ public static void main(String args[]) throws Throwable { // Erddap.makeErddapContentZip("c:/programs/tomcat/samples/", "c:/backup/"); // Erddap.testHammerGetDatasets(); // File2.touch("c:/u00/cwatch/erddap2/copy/nmspWcosTemp/ANO001/2005/ANO001_021MTBD020R00_20051105.nc"); +// FileVisitorDNLS.testHyrax(); +// FileVisitorDNLS.testThredds(); +// FileVisitorSubdir.testAWSS3(); //FishBase datasets // FishBase.convertHtmlToNc("ABNORM"); @@ -958,7 +1021,7 @@ public static void main(String args[]) throws Throwable { dods.dap.DConnect dConnect; DataHelper dh; DigirHelper dh2; -dods.dap.DSequence ds; +dods.dap.DSequence dseq; DoubleArray doublea; EDDTableFromAllDatasets etfad; EmaAttribute ea; @@ -1308,7 +1371,7 @@ public static void main(String args[]) throws Throwable { EDUnits.test(); Table.testXml(); Subscriptions.test(); - FileVisitorDNLS.test(); + FileVisitorDNLS.test(false); //doBigTest FileVisitorSubdir.test(); WatchDirectory.test(true); //doInteractiveTest boolean doGraphicsTests = true; @@ -1321,7 +1384,7 @@ public static void main(String args[]) throws Throwable { EDDGridFromEtopo.test(true); //EDDGridAggregateExistingDimension.test(); //don't usually run...very slow EDDGridAggregateExistingDimension.testGenerateDatasetsXml(); - EDDGridFromNcFiles.test(true); + EDDGridFromNcFiles.test(true); //deleteCachedInfo EDDGridFromMergeIRFiles.test(); EDDGridFromEDDTable.test(); EDDGridCopy.test(); diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestBrowsers.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestBrowsers.java index 8b325895e..485d5e252 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestBrowsers.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestBrowsers.java @@ -785,7 +785,7 @@ public static void testGridScreen(String url) throws Exception { Test.ensureTrue(String2.indexOf(responseArray, " :sensor = \"AVHRR HRPT\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :source = \"satellite observation: POES, AVHRR HRPT\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :Southernmost_Northing = 22.0; // double") > 0, error); - Test.ensureTrue(String2.indexOf(responseArray, " :standard_name_vocabulary = \"CF Standard Name Table v27\";") > 0, error); + Test.ensureTrue(String2.indexOf(responseArray, " :standard_name_vocabulary = \"CF Standard Name Table v29\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :start_time = 0.0; // double") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :summary = \"NOAA CoastWatch provides sea surface temperature (SST) products derived from NOAA's Polar Operational Environmental Satellites (POES). This data is provided at high resolution (0.0125 degrees) for the North Pacific Ocean. Measurements are gathered by the Advanced Very High Resolution Radiometer (AVHRR) instrument, a multiband radiance sensor carried aboard the NOAA POES satellites.\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :time_coverage_end = \"2006-11-22T00:00:00Z\";") > 0, error); @@ -963,7 +963,7 @@ public static void testGridScreen(String url) throws Exception { Test.ensureTrue(response.indexOf(" :sensor = \"SeaWinds\";") > 0, error); Test.ensureTrue(response.indexOf(" :source = \"satellite observation: QuikSCAT, SeaWinds\";") > 0, error); Test.ensureTrue(response.indexOf(" :Southernmost_Northing = 46.75; // double") > 0, error); - Test.ensureTrue(response.indexOf(" :standard_name_vocabulary = \"CF Standard Name Table v27\";") > 0, error); + Test.ensureTrue(response.indexOf(" :standard_name_vocabulary = \"CF Standard Name Table v29\";") > 0, error); Test.ensureTrue(response.indexOf(" :summary = \"NASA's Jet Propulsion Laboratory (JPL) distributes near real time wind velocity data from the SeaWinds instrument onboard NASA's QuikSCAT satellite. SeaWinds is a microwave scatterometer designed to measure surface winds over the global ocean. Wind velocity fields are provided in zonal, meriodonal, and modulus sets. The reference height for all wind velocities is 10 meters.\";") > 0, error); Test.ensureTrue(response.indexOf(" :time_coverage_end = \"2006-11-21T12:00:00Z\";") > 0, error); Test.ensureTrue(response.indexOf(" :time_coverage_start = \"2006-10-21T12:00:00Z\";") > 0, error); @@ -1177,7 +1177,7 @@ public static void testVectorScreen(String url) throws Exception { Test.ensureTrue(response.indexOf(" :sensor = \"SeaWinds\";\n") > 0, error); Test.ensureTrue(response.indexOf(" :source = \"satellite observation: QuikSCAT, SeaWinds\";\n") > 0, error); Test.ensureTrue(response.indexOf(" :Southernmost_Northing = 22.0; // double\n") > 0, error); - Test.ensureTrue(response.indexOf(" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n") > 0, error); + Test.ensureTrue(response.indexOf(" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n") > 0, error); Test.ensureTrue(response.indexOf(" :start_time = 0.0; // double\n") > 0, error); Test.ensureTrue(response.indexOf(" :summary = \"NASA's Jet Propulsion Laboratory (JPL) distributes near real time wind velocity data from the SeaWinds instrument onboard NASA's QuikSCAT satellite. SeaWinds is a microwave scatterometer designed to measure surface winds over the global ocean. Wind velocity fields are provided in zonal, meriodonal, and modulus sets. The reference height for all wind velocities is 10 meters.\";\n") > 0, error); Test.ensureTrue(response.indexOf(" :time_coverage_end = \"2006-11-22T00:00:00Z\";\n") > 0, error); @@ -1326,7 +1326,7 @@ public static void testStationVectorScreen(String url) throws Exception { Test.ensureTrue(String2.indexOf(responseArray, " :quality = \"Automated QC checks with periodic manual QC\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :source = \"station observation\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :Southernmost_Northing = 32.425; // double") > 0, error); - Test.ensureTrue(String2.indexOf(responseArray, " :standard_name_vocabulary = \"CF Standard Name Table v27\";") > 0, error); + Test.ensureTrue(String2.indexOf(responseArray, " :standard_name_vocabulary = \"CF Standard Name Table v29\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :time_coverage_end = \"2006-11-21T12:00:00Z\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :time_coverage_start = \"2006-11-21T12:00:00Z\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :title = \"Wind Speed, Zonal (NDBC)\";") > 0, error); @@ -1487,7 +1487,7 @@ public static void testStationVectorScreen(String url) throws Exception { Test.ensureTrue(response.indexOf(" :quality = \"Automated QC checks with periodic manual QC\";") > 0, error); Test.ensureTrue(response.indexOf(" :source = \"station observation\";") > 0, error); Test.ensureTrue(response.indexOf(" :Southernmost_Northing = 42.58; // double") > 0, error); - Test.ensureTrue(response.indexOf(" :standard_name_vocabulary = \"CF Standard Name Table v27\";") > 0, error); + Test.ensureTrue(response.indexOf(" :standard_name_vocabulary = \"CF Standard Name Table v29\";") > 0, error); Test.ensureTrue(response.indexOf(" :time_coverage_end = \"2006-11-21T12:00:00Z\";") > 0, error); Test.ensureTrue(response.indexOf(" :time_coverage_start = \"2006-10-21T12:00:00Z\";") > 0, error); Test.ensureTrue(response.indexOf(" :title = \"Wind Speed, Zonal (NDBC)\";") > 0, error); @@ -1627,7 +1627,7 @@ public static void testStationScreen(String url) throws Exception { Test.ensureTrue(String2.indexOf(responseArray, " :quality = \"Automated QC checks with periodic manual QC\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :source = \"station observation\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :Southernmost_Northing = 32.425; // double") > 0, error); - Test.ensureTrue(String2.indexOf(responseArray, " :standard_name_vocabulary = \"CF Standard Name Table v27\";") > 0, error); + Test.ensureTrue(String2.indexOf(responseArray, " :standard_name_vocabulary = \"CF Standard Name Table v29\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :time_coverage_end = \"2006-11-21T12:00:00Z\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :time_coverage_start = \"2006-11-21T12:00:00Z\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :title = \"SST (NDBC)\";") > 0, error); @@ -1785,7 +1785,7 @@ public static void testStationScreen(String url) throws Exception { Test.ensureTrue(String2.indexOf(responseArray, " :quality = \"Automated QC checks with periodic manual QC\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :source = \"station observation\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :Southernmost_Northing = 42.58; // double") > 0, error); - Test.ensureTrue(String2.indexOf(responseArray, " :standard_name_vocabulary = \"CF Standard Name Table v27\";") > 0, error); + Test.ensureTrue(String2.indexOf(responseArray, " :standard_name_vocabulary = \"CF Standard Name Table v29\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :time_coverage_end = \"2006-11-21T12:00:00Z\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :time_coverage_start = \"2006-10-21T12:00:00Z\";") > 0, error); Test.ensureTrue(String2.indexOf(responseArray, " :title = \"SST (NDBC)\";") > 0, error); diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/DoubleCenterGrids.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/DoubleCenterGrids.java index da6367df9..cdaa7490a 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/DoubleCenterGrids.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/DoubleCenterGrids.java @@ -368,7 +368,7 @@ public static void test() throws Exception { " :sensor = \"AVHRR GAC\";\n" + " :source = \"satellite observation: POES, AVHRR GAC\";\n" + " :Southernmost_Northing = 33.5; // double\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :start_time = 0.0; // double\n" + " :summary = \"NOAA CoastWatch provides sea surface temperature (SST) products derived from NOAA's Polar Operational Environmental Satellites (POES). This data provides global area coverage at 0.1 degrees resolution. Measurements are gathered by the Advanced Very High Resolution Radiometer (AVHRR) instrument, a multiband radiance sensor carried aboard the NOAA POES satellites.\";\n" + " :time_coverage_end = \"2005-02-10T00:00:00Z\";\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/FileNameUtility.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/FileNameUtility.java index dedfc65dd..db5acff1c 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/FileNameUtility.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/FileNameUtility.java @@ -83,8 +83,8 @@ public static String getDateCreated() { public static String getProcessingLevel() {return "3 (projected)"; } public static String getProject() {return DataHelper.CW_PROJECT; } public static String getStandardNameVocabulary() { - //2015-04-17 new longer v27 name is from ACDD 1.3 - return "CF Standard Name Table v27"; } //was CF-1.0 and CF-11, 2015-04-17 was CF-12 + //2015-04-17 new, longer name format is from ACDD 1.3 + return "CF Standard Name Table v29"; } //was CF-1.0 and CF-11, 2015-04-17 was CF-12, 2015-07-23 was v27 private String categoryLetters; private String[] categoryNames; @@ -1434,7 +1434,7 @@ public static void main(String args[]) throws Throwable { Test.ensureEqual(getAcknowledgement(), "NOAA NESDIS COASTWATCH, NOAA SWFSC ERD", "getAcknowledgement"); Test.ensureEqual(getLatUnits(), "degrees_north", "getLatUnits"); Test.ensureEqual(getLonUnits(), "degrees_east", "getLonUnits"); - Test.ensureEqual(getStandardNameVocabulary(), "CF Standard Name Table v27", "getStandardNameVocabulary"); + Test.ensureEqual(getStandardNameVocabulary(), "CF Standard Name Table v29", "getStandardNameVocabulary"); Test.ensureEqual(getLicense(), "The data may be used and redistributed for free but is not intended for legal use, since it may contain inaccuracies. Neither the data Contributor, CoastWatch, NOAA, nor the United States Government, nor any of their employees or contractors, makes any warranty, express or implied, including warranties of merchantability and fitness for a particular purpose, or assumes any legal liability for the accuracy, completeness, or usefulness, of this information.", "getLicense"); Test.ensureEqual(fnu.getContributorName(names[i]), fnu.getCourtesy(names[i]), "getContributorName"); Test.ensureEqual(getContributorRole(), "Source of level 2 data.", "getContributorRole"); diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/Grid.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/Grid.java index a4a1becbe..e137d059d 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/Grid.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/Grid.java @@ -4422,7 +4422,7 @@ public static void testNetCDF(FileNameUtility fileNameUtility) throws Exception Test.ensureEqual(grid2.globalAttributes().get("time_coverage_start"), new StringArray(new String[]{"2003-03-04T00:00:00Z"}), "time_coverage_start"); Test.ensureEqual(grid2.globalAttributes().get("time_coverage_end"), new StringArray(new String[]{"2003-03-05T00:00:00Z"}), "time_coverage_end"); //Test.ensureEqual(grid2.globalAttributes().get("time_coverage_resolution", new StringArray(new String[]{""}), "time_coverage_resolution"); - Test.ensureEqual(grid2.globalAttributes().get("standard_name_vocabulary"), new StringArray(new String[]{"CF Standard Name Table v27"}), "standard_name_vocabulary"); + Test.ensureEqual(grid2.globalAttributes().get("standard_name_vocabulary"), new StringArray(new String[]{"CF Standard Name Table v29"}), "standard_name_vocabulary"); Test.ensureEqual(grid2.globalAttributes().get("license"), new StringArray(new String[]{"The data may be used and redistributed for free but is not intended for legal use, since it may contain inaccuracies. Neither the data Contributor, CoastWatch, NOAA, nor the United States Government, nor any of their employees or contractors, makes any warranty, express or implied, including warranties of merchantability and fitness for a particular purpose, or assumes any legal liability for the accuracy, completeness, or usefulness, of this information."}), "license"); Test.ensureEqual(grid2.globalAttributes().get("contributor_name"), new StringArray(new String[]{"NOAA NWS Monterey and NOAA CoastWatch"}), "contributor_name"); Test.ensureEqual(grid2.globalAttributes().get("contributor_role"), new StringArray(new String[]{"Source of level 2 data."}), "contributor_role"); diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/GridDataSetOpendap.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/GridDataSetOpendap.java index 69f91e66c..b7ea0f7a2 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/GridDataSetOpendap.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/GridDataSetOpendap.java @@ -995,7 +995,7 @@ public static void test() throws Exception { Test.ensureEqual(grid.globalAttributes().get("time_coverage_start"), new StringArray(new String[]{"2006-06-10T00:00:00Z"}), "time_coverage_start"); Test.ensureEqual(grid.globalAttributes().get("time_coverage_end"), new StringArray(new String[]{"2006-06-11T00:00:00Z"}), "time_coverage_end"); //Test.ensureEqual(grid.globalAttributes().get("time_coverage_resolution", new StringArray(new String[]{""}), "time_coverage_resolution"); - Test.ensureEqual(grid.globalAttributes().get("standard_name_vocabulary"), new StringArray(new String[]{"CF Standard Name Table v27"}), "standard_name_vocabulary"); + Test.ensureEqual(grid.globalAttributes().get("standard_name_vocabulary"), new StringArray(new String[]{"CF Standard Name Table v29"}), "standard_name_vocabulary"); Test.ensureEqual(grid.globalAttributes().get("license"), new StringArray(new String[]{"The data may be used and redistributed for free but is not intended for legal use, since it may contain inaccuracies. Neither the data Contributor, CoastWatch, NOAA, nor the United States Government, nor any of their employees or contractors, makes any warranty, express or implied, including warranties of merchantability and fitness for a particular purpose, or assumes any legal liability for the accuracy, completeness, or usefulness, of this information."}), "license"); Test.ensureEqual(grid.globalAttributes().get("contributor_name"), new StringArray(new String[]{"Remote Sensing Systems, Inc."}), "contributor_name"); Test.ensureEqual(grid.globalAttributes().get("contributor_role"), new StringArray(new String[]{"Source of level 2 data."}), "contributor_role"); diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/GridDataSetThredds.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/GridDataSetThredds.java index a8a31d2bd..0a278234e 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/GridDataSetThredds.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/GridDataSetThredds.java @@ -1130,7 +1130,7 @@ public static void test() throws Exception { Test.ensureEqual(grid.globalAttributes().get("time_coverage_start"), new StringArray(new String[]{"2006-06-10T00:00:00Z"}), "time_coverage_start"); Test.ensureEqual(grid.globalAttributes().get("time_coverage_end"), new StringArray(new String[]{"2006-06-11T00:00:00Z"}), "time_coverage_end"); //Test.ensureEqual(grid.globalAttributes().get("time_coverage_resolution", new StringArray(new String[]{""}), "time_coverage_resolution"); - Test.ensureEqual(grid.globalAttributes().get("standard_name_vocabulary"), new StringArray(new String[]{"CF Standard Name Table v27"}), "standard_name_vocabulary"); + Test.ensureEqual(grid.globalAttributes().get("standard_name_vocabulary"), new StringArray(new String[]{"CF Standard Name Table v29"}), "standard_name_vocabulary"); Test.ensureEqual(grid.globalAttributes().get("license"), new StringArray(new String[]{"The data may be used and redistributed for free but is not intended for legal use, since it may contain inaccuracies. Neither the data Contributor, CoastWatch, NOAA, nor the United States Government, nor any of their employees or contractors, makes any warranty, express or implied, including warranties of merchantability and fitness for a particular purpose, or assumes any legal liability for the accuracy, completeness, or usefulness, of this information."}), "license"); Test.ensureEqual(grid.globalAttributes().get("contributor_name"), new StringArray(new String[]{"Remote Sensing Systems, Inc."}), "contributor_name"); Test.ensureEqual(grid.globalAttributes().get("contributor_role"), new StringArray(new String[]{"Source of level 2 data."}), "contributor_role"); diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc index c2502177de696a34a6caf4bb0eb3536956745a7e..dceda862813c1f8f45815ccfeab3335bdc761b1a 100644 GIT binary patch delta 66 zcmccdl=;R}<_%d)EEc+k7L)Us6nH?4C Atmospheric Winds > Surface Winds,\n" + "Oceans > Ocean Winds > Surface Winds,\n" + "atmosphere, atmospheric, coastwatch, degrees, global, level, monthly, noaa, ocean, oceans, quality, quikscat, science, science quality, seawinds, surface wcn, wind, winds, x_wind, zonal\";\n" + @@ -2555,13 +2555,16 @@ public static void testDapToNcDGrid() throws Throwable { " :project = \"CoastWatch (http://coastwatch.noaa.gov/)\";\n" + " :projection = \"geographic\";\n" + " :projection_type = \"mapped\";\n" + +" :publisher_email = \"erd.data@noaa.gov\";\n" + +" :publisher_name = \"NOAA NMFS SWFSC ERD\";\n" + +" :publisher_url = \"http://www.pfeg.noaa.gov\";\n" + " :references = \"RSS Inc. Winds: http://www.remss.com/ .\";\n" + " :satellite = \"QuikSCAT\";\n" + " :sensor = \"SeaWinds\";\n" + " :source = \"satellite observation: QuikSCAT, SeaWinds\";\n" + " :sourceUrl = \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/QS/ux10/mday\";\n" + " :Southernmost_Northing = -75.0; // double\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"Remote Sensing Inc. distributes science quality wind velocity data from the SeaWinds instrument onboard NASA's QuikSCAT satellite. SeaWinds is a microwave scatterometer designed to measure surface winds over the global ocean. Wind velocity fields are provided in zonal, meridional, and modulus sets. The reference height for all wind velocities is 10 meters.\";\n" + " :time_coverage_end = \"2009-10-16T12:00:00Z\";\n" + " :time_coverage_start = \"1999-08-16T12:00:00Z\";\n" + @@ -2723,9 +2726,9 @@ public static void testDapToNcDGrid() throws Throwable { " :contributor_name = \"Remote Sensing Systems, Inc.\";\n" + " :contributor_role = \"Source of level 2 data.\";\n" + " :Conventions = \"COARDS, CF-1.6, ACDD-1.3\";\n" + -" :creator_email = \"dave.foley@noaa.gov\";\n" + -" :creator_name = \"NOAA CoastWatch, West Coast Node\";\n" + -" :creator_url = \"http://coastwatch.pfel.noaa.gov\";\n" + +" :creator_email = \"erd.data@noaa.gov\";\n" + +" :creator_name = \"NOAA NMFS SWFSC ERD\";\n" + +" :creator_url = \"http://www.pfeg.noaa.gov\";\n" + " :date_created = \"2010-04-18Z\";\n" + " :date_issued = \"2010-04-18Z\";\n" + " :defaultGraphQuery = \"&.draw=vectors\";\n" + @@ -2748,7 +2751,7 @@ public static void testDapToNcDGrid() throws Throwable { //today + time " http://coastwatch.pfeg.noaa.gov/erddap/griddap/erdQSwindmday.das\";\n" + expected2 = " :infoUrl = \"http://coastwatch.pfeg.noaa.gov/infog/QS_ux10_las.html\";\n" + -" :institution = \"NOAA CoastWatch, West Coast Node\";\n" + +" :institution = \"NOAA NMFS SWFSC ERD\";\n" + " :keywords = \"Atmosphere > Atmospheric Winds > Surface Winds,\n" + "Oceans > Ocean Winds > Surface Winds,\n" + "atmosphere, atmospheric, coastwatch, degrees, global, level, monthly, noaa, ocean, oceans, quality, quikscat, science, science quality, seawinds, surface wcn, wind, winds, x_wind, zonal\";\n" + @@ -2767,13 +2770,16 @@ public static void testDapToNcDGrid() throws Throwable { " :project = \"CoastWatch (http://coastwatch.noaa.gov/)\";\n" + " :projection = \"geographic\";\n" + " :projection_type = \"mapped\";\n" + +" :publisher_email = \"erd.data@noaa.gov\";\n" + +" :publisher_name = \"NOAA NMFS SWFSC ERD\";\n" + +" :publisher_url = \"http://www.pfeg.noaa.gov\";\n" + " :references = \"RSS Inc. Winds: http://www.remss.com/ .\";\n" + " :satellite = \"QuikSCAT\";\n" + " :sensor = \"SeaWinds\";\n" + " :source = \"satellite observation: QuikSCAT, SeaWinds\";\n" + " :sourceUrl = \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/QS/ux10/mday\";\n" + " :Southernmost_Northing = -75.0; // double\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"Remote Sensing Inc. distributes science quality wind velocity data from the SeaWinds instrument onboard NASA's QuikSCAT satellite. SeaWinds is a microwave scatterometer designed to measure surface winds over the global ocean. Wind velocity fields are provided in zonal, meridional, and modulus sets. The reference height for all wind velocities is 10 meters.\";\n" + " :time_coverage_end = \"2009-10-16T12:00:00Z\";\n" + " :time_coverage_start = \"1999-08-16T12:00:00Z\";\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/MakeErdJavaZip.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/MakeErdJavaZip.java index dd0a4bee5..6601db398 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/MakeErdJavaZip.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/MakeErdJavaZip.java @@ -48,7 +48,7 @@ public static void main(String args[]) throws Exception { //make the JavaDocs String coastWatchClass = "gov.noaa.pfel.coastwatch."; - String commandLine0 = "C:\\Progra~1\\Java\\jdk1.7.0_67\\bin\\javadoc" + + String commandLine0 = "C:\\Progra~1\\Java\\jdk1.8.0_51\\bin\\javadoc" + //" -source 1.4" + //use 1.4 for the DODS classes that use "enum" //2011-02-22 Bob Simons changed enum to en. " -sourcepath " + classPath + //root directory of the classes " -d "; //+ baseDir + "ConvertTableDoc" + //dir to hold results @@ -237,7 +237,7 @@ public static void makeConvertTableJar(String destinationDir) throws Exception { //accumulate the file names to be zipped String ctName = destinationDir + "converttable.jar"; StringBuilder cmdLine = new StringBuilder(); - cmdLine.append("\\Progra~1\\Java\\jdk1.7.0_67\\bin\\jar cvf " + ctName); + cmdLine.append("\\Progra~1\\Java\\jdk1.8.0_51\\bin\\jar cvf " + ctName); //I thought I could use -C once and have lots of files after it. //But no. I need to use -C for each file. (maybe just if 'file' is a directory) //And can't use *. List files separately. diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/NdbcMetStation.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/NdbcMetStation.java index a8a26c827..b2ace3f7b 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/NdbcMetStation.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/NdbcMetStation.java @@ -211,7 +211,7 @@ public class NdbcMetStation { * and near real time data (less quality controlled). * This changes every month when I get the latest historical data. */ - public static String firstNearRealTimeData = "2015-05-01T00:00:00"; + public static String firstNearRealTimeData = "2015-07-01T00:00:00"; /** Change current year ~Feb 28 when Jan historical files become available. */ public static String HISTORICAL_FILES_CURRENT_YEAR = "2015"; @@ -1043,7 +1043,7 @@ public static void addLastNDaysInfo(String ncDir, int nDays, boolean testMode) t //is this station in list of available nDay files from ndbc String stationID = stationList[station]; String tBaseUrl, tSuffix; - int tNDays = 0; + int tNDays = 0; //figure out which is to be used for this file if (nDays == 5 && n5DayFileNames.indexOf(stationID, 0) >= 0) { //note first test is of nDays tNDays = 5; tBaseUrl = "http://www.ndbc.noaa.gov/data/5day2/"; @@ -1320,10 +1320,15 @@ public static void makeStationNcFile(String ndbcStationHtmlDir, //get the owner int ownerLine = String2.lineContaining(lines, " maintained by"); //Buoy Center or Station Center, or University... + if (ownerLine == -1) + ownerLine = String2.lineContaining(lines, "Maintained by"); if (ownerLine == -1) ownerLine = String2.lineContaining(lines, " operated by"); + if (ownerLine == -1) + ownerLine = String2.lineContaining(lines, "Owned by"); if (ownerLine == -1) ownerLine = String2.lineContaining(lines, "Information submitted by "); + if (ownerLine == -1) Test.ensureNotEqual(ownerLine, -1, errorInMethod + "'maintained by' line not found.\n" + "stationUrl=" + stationUrl); String owner = XML.removeHTMLTags(lines[ownerLine]); @@ -2412,27 +2417,27 @@ public static void test46088(Table table) throws Exception { // http://www.ndbc.noaa.gov/data/realtime2/46088.txt //45 day //top line has precedence //#YY MM DD hh mm WDIR WSPD GST WVHT DPD APD MWD PRES ATMP WTMP DEWP VIS PTDY TIDE //#yr mo dy hr mn degT m/s m/s m sec sec degT hPa degC degC degC mi hPa ft - //2015 05 01 01 20 240 1.0 2.0 0.1 MM 3.5 MM 1022.3 11.0 10.5 8.0 MM MM MM - //2015 05 01 00 50 270 2.0 2.0 0.1 MM 3.3 MM 1022.9 11.5 10.4 6.0 MM -2.4 MM - seconds = Calendar2.isoStringToEpochSeconds("2015-05-01T01"); //50 min rounds to next hour; usually test 01T01 + //2015 07 01 01 20 240 10.0 11.0 0.7 4 3.4 260 1016.4 12.9 12.0 11.7 MM MM MM + //2015 07 01 00 50 240 10.0 11.0 0.6 3 3.3 265 1016.9 12.7 12.1 11.6 MM -1.9 MM + seconds = Calendar2.isoStringToEpochSeconds("2015-07-01T01"); //50 min rounds to next hour; usually test 01T01 row = table.getColumn(timeIndex).indexOf("" + seconds, 0); Test.ensureEqual(table.getStringData(idIndex, row), "46088", ""); Test.ensureEqual(table.getFloatData(latIndex, row), 48.333f, ""); Test.ensureEqual(table.getFloatData(lonIndex, row), -123.167f, ""); Test.ensureEqual(table.getDoubleData(depthIndex, row), 0, ""); Test.ensureEqual(table.getDoubleData(wdIndex, row), 240, ""); - Test.ensureEqual(table.getFloatData(wspdIndex, row), 1f, ""); - Test.ensureEqual(table.getFloatData(gstIndex, row), 2f, ""); - Test.ensureEqual(table.getFloatData(wvhtIndex, row), .1f, ""); - Test.ensureEqual(table.getFloatData(dpdIndex, row), Float.NaN, ""); - Test.ensureEqual(table.getFloatData(apdIndex, row), 3.5f, ""); - Test.ensureEqual(table.getFloatData(mwdIndex, row), Float.NaN, ""); //or getIntData - Test.ensureEqual(table.getFloatData(aprsIndex, row), 1022.3f, ""); - Test.ensureEqual(table.getFloatData(atmpIndex, row), 11f, ""); - Test.ensureEqual(table.getFloatData(wtmpIndex, row), 10.5f, ""); - Test.ensureEqual(table.getFloatData(dewpIndex, row), 8.0f, ""); + Test.ensureEqual(table.getFloatData(wspdIndex, row), 10f, ""); + Test.ensureEqual(table.getFloatData(gstIndex, row), 11f, ""); + Test.ensureEqual(table.getFloatData(wvhtIndex, row), .7f, ""); + Test.ensureEqual(table.getFloatData(dpdIndex, row), 4f, ""); + Test.ensureEqual(table.getFloatData(apdIndex, row), 3.4f, ""); + Test.ensureEqual(table.getFloatData(mwdIndex, row), 260f, ""); //or getIntData + Test.ensureEqual(table.getFloatData(aprsIndex, row), 1016.4f, ""); + Test.ensureEqual(table.getFloatData(atmpIndex, row), 12.9f, ""); + Test.ensureEqual(table.getFloatData(wtmpIndex, row), 12f, ""); + Test.ensureEqual(table.getFloatData(dewpIndex, row), 11.7f, ""); Test.ensureEqual(table.getFloatData(visIndex, row), Float.NaN, ""); //(float)Math2.roundTo(18.5 * Math2.kmPerMile, decimalDigits[visIndex]), ""); - Test.ensureEqual(table.getFloatData(ptdyIndex, row), -2.4f, ""); + Test.ensureEqual(table.getFloatData(ptdyIndex, row), -1.9f, ""); Test.ensureEqual(table.getFloatData(tideIndex, row), Float.NaN, ""); //(float)Math2.roundTo(3.0 * Math2.meterPerFoot, decimalDigits[tideIndex]), ""); String2.log("test46088 was successful"); @@ -2467,26 +2472,26 @@ public static void test46088AddLastNDays(Table table) throws Exception { //top row has precedence, but not if file already had lower row of data //#YY MM DD hh mm WDIR WSPD GST WVHT DPD APD MWD PRES ATMP WTMP DEWP VIS PTDY TIDE //#yr mo dy hr mn degT m/s m/s m sec sec degT hPa degC degC degC mi hPa ft - //2015 05 26 20 50 260 3.0 3.0 0.3 3 3.4 246 1018.0 11.4 10.6 9.8 MM +0.0 MM - //2015 05 26 20 20 270 3.0 4.0 0.3 3 3.5 218 1018.0 11.4 10.7 9.7 MM MM MM - double seconds = Calendar2.isoStringToEpochSeconds("2015-05-26T21"); //rounded + //2015 07 25 14 20 280 9.0 11.0 0.6 4 3.5 247 1016.5 13.5 12.2 11.7 MM MM MM + //2015 07 25 13 50 280 8.0 10.0 0.5 5 3.5 250 1016.4 13.6 12.1 11.6 MM +0.0 MM + double seconds = Calendar2.isoStringToEpochSeconds("2015-07-25T14"); //rounded int row = table.getColumn(timeIndex).indexOf("" + seconds, 0); Test.ensureTrue(row >= 0, "row=" + row); Test.ensureEqual(table.getStringData(idIndex, row), "46088", ""); Test.ensureEqual(table.getFloatData(latIndex, row), 48.333f, ""); Test.ensureEqual(table.getFloatData(lonIndex, row), -123.167f, ""); Test.ensureEqual(table.getDoubleData(depthIndex, row), 0, ""); - Test.ensureEqual(table.getFloatData(wdIndex, row), 260, ""); - Test.ensureEqual(table.getFloatData(wspdIndex, row), 3f, ""); - Test.ensureEqual(table.getFloatData(gstIndex, row), 3f, ""); - Test.ensureEqual(table.getFloatData(wvhtIndex, row), .3f, ""); - Test.ensureEqual(table.getFloatData(dpdIndex, row), 3f, ""); - Test.ensureEqual(table.getFloatData(apdIndex, row), 3.4f, ""); - Test.ensureEqual(table.getFloatData(mwdIndex, row), 246f, ""); - Test.ensureEqual(table.getFloatData(aprsIndex, row), 1018f, ""); - Test.ensureEqual(table.getFloatData(atmpIndex, row), 11.4f, ""); - Test.ensureEqual(table.getFloatData(wtmpIndex, row), 10.6f, ""); - Test.ensureEqual(table.getFloatData(dewpIndex, row), 9.8f, ""); + Test.ensureEqual(table.getFloatData(wdIndex, row), 280, ""); + Test.ensureEqual(table.getFloatData(wspdIndex, row), 9f, ""); + Test.ensureEqual(table.getFloatData(gstIndex, row), 11f, ""); + Test.ensureEqual(table.getFloatData(wvhtIndex, row), .6f, ""); + Test.ensureEqual(table.getFloatData(dpdIndex, row), 4f, ""); + Test.ensureEqual(table.getFloatData(apdIndex, row), 3.5f, ""); + Test.ensureEqual(table.getFloatData(mwdIndex, row), 247f, ""); + Test.ensureEqual(table.getFloatData(aprsIndex, row), 1016.5f, ""); + Test.ensureEqual(table.getFloatData(atmpIndex, row), 13.5f, ""); + Test.ensureEqual(table.getFloatData(wtmpIndex, row), 12.2f, ""); + Test.ensureEqual(table.getFloatData(dewpIndex, row), 11.7f, ""); Test.ensureEqual(table.getFloatData(visIndex, row), Float.NaN, ""); //(float)Math2.roundTo(18.5 * Math2.kmPerMile, decimalDigits[visIndex]), ""); Test.ensureEqual(table.getFloatData(ptdyIndex, row), 0f, ""); Test.ensureEqual(table.getFloatData(tideIndex, row), Float.NaN, "");//(float)Math2.roundTo(3.0 * Math2.meterPerFoot, decimalDigits[tideIndex]), ""); @@ -2756,11 +2761,11 @@ public static void main(String args[]) throws Exception { //historical monthly files are from: http://www.ndbc.noaa.gov/data/stdmet// e.g., Jan //!!!!**** Windows GUI My Computer doesn't show all the files in the directory! // Use DOS window "dir" or Linux ls instead of the GUI. - //downloadNewHistoricalTxtFiles(ndbcHistoricalTxtDir); //time varies, last done 2015-05-26 + //downloadNewHistoricalTxtFiles(ndbcHistoricalTxtDir); //time varies, last done 2015-07-24 // 3) *** get latest 45 day files //DON'T download45DayTextFiles after 45 days after last historicalTxt date. - //download45DayTxtFiles(ndbc45DayTxtDir); //15-30 minutes, last done 2015-05-26 + //download45DayTxtFiles(ndbc45DayTxtDir); //15-30 minutes, last done 2015-07-24 // 4) *** Make the nc files //!!!!**** EACH MONTH, SOME TESTS NEED UPDATING: SEE "UPDATE_EACH_MONTH" @@ -2768,7 +2773,7 @@ public static void main(String args[]) throws Exception { boolean testMode = false; //used to: always run 'true' then run 'false' String ignoreStationsBefore = " "; //use " " to process all stations or lowercase characters to start in middle //makeSeparateNcFiles(ndbcStationHtmlDir, ndbcHistoricalTxtDir, ndbc45DayTxtDir, - // ndbcNcDir, ignoreStationsBefore, testMode); //M4700 ~1 hr, was ~3 hrs //last done 2015-05-26 + // ndbcNcDir, ignoreStationsBefore, testMode); //M4700 ~1 hr, was ~3 hrs //last done 2015-07-24 test31201Nc(ndbcNcDir); test41009Nc(ndbcNcDir); test41015Nc(ndbcNcDir); @@ -2786,8 +2791,8 @@ public static void main(String args[]) throws Exception { //(5 days takes 12 minutes) //but 45 is more likely to get more information (if needed and if available) //(45 days takes 25 minutes) - testMode = false; //always run 'true' then run 'false' - //addLastNDaysInfo(ndbcNcDir, 5, testMode); + testMode = true; //always run 'true' then run 'false' + //addLastNDaysInfo(ndbcNcDir, 5, testMode); //usually 5 //!!!!**** EACH MONTH, THIS TEST NEED UPDATING test46088AddLastNDaysNc(ndbcNcDir); @@ -2797,7 +2802,9 @@ public static void main(String args[]) throws Exception { * ftp ndbcMett.tgz to coastwatch's /u00/data/points cd /u00/data/points tar zxvf ndbcMett.tgz -chmod -R a+rwx ndbcMett +as su + chown -R tomcat:erddap ndbcMett + chmod -R a+rw ndbcMett rename ndbcMet ndbcMetR20150224 ndbcMet rename ndbcMett ndbcMet ndbcMett rm ndbcMett.tgz @@ -2807,7 +2814,7 @@ public static void main(String args[]) throws Exception { // change 'historic' dates in summary attribute for datasetID=cwwcNDBCMet // to reflect new transition date. // * On laptop, use updateDatasetsXml.py - // * Copy datasetsNewCW.xml to coastwatch and rename to put in place + // * Copy datasetsFEDCW.xml to coastwatch and rename to put in place // * Set cwwcNDBCMet flag. // 9) *** test cwwcNDBCMet sometimes: diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java index 64b50b6fa..d8fc99162 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java @@ -20473,14 +20473,16 @@ public void readErddapInfo(String url) throws Exception { *
    It mimics http://www.ngdc.noaa.gov/metadata/published/NOAA/NESDIS/NGDC/MGG/Hazard_Photos/fgdc/xml/ * stored on Bob's computer as c:/programs/apache/listing.html * See also http://www.ndbc.noaa.gov/data/realtime2/?C=N;O=A - *
    ***WARNING*** The URL for that got the user to this page MUST be a + *
    ***WARNING*** The URL that got the user to this page MUST be a * directoryURL ending in '/', or the links don't work (since they are implied * to be relative to the current URL, not explicit)! *
    This just writes the part inside the 'body' tag. *
    If there is a parentDirectory, its link will be at the top of the list. *
    The table need not be sorted initially. This method handles sorting. - *
    The table should have 4 columns: "Name" (String), "Last modified" (long), - * "Size" (long), and "Description" (String) + *
    The table should have 4 columns: "Name" (String, dir names should end in /), + * "Last modified" (long, in milliseconds, directories/unknown should have Long.MAX_VALUE), + * "Size" (long, directories/unknown should have Long.MAX_VALUE), and + * "Description" (String) *
    The displayed Last Modified time will be Zulu timezone. *
    The displayed size will be some number of bytes, or truncated to some * number of K (1024), M (1024^2), G (1024^3), or T (1024^4), @@ -20506,7 +20508,8 @@ public void readErddapInfo(String url) throws Exception { *
    Currently, only C= and O= parameters are supported. * @param iconUrlDir the public URL directory (with slash at end) with the icon files * @param addParentDir if true, this shows a link to the parent directory - * @param dirNames is the list of subdirectories in the directory (without trailing '/'). + * @param dirNames is the list of subdirectories in the directory + * just the individual words (and without trailing '/'), not the whole URL. * It will be sorted within directoryListing. * @param dirDescriptions may be null */ @@ -23325,17 +23328,15 @@ public static void testReadNcCFMATimeSeriesReversed() throws Exception { time = System.currentTimeMillis(); table.readNcCF("/data/hunter/USGS_DISCHARGE_STATIONS_SUBSET.nc", null, //all vars - StringArray.fromCSV("time"), StringArray.fromCSV(">"), StringArray.fromCSV("2874.7")); + StringArray.fromCSV("time"), StringArray.fromCSV(">"), StringArray.fromCSV("3426.69")); String2.log("time=" + (System.currentTimeMillis() - time)); results = table.dataToCSVString(); expected = -"latitude,discharge,longitude,station,time\n" + -"39.01061275,57.76636788,-75.45794828,1484080.0,2874.70000000007\n" + -"39.01061275,63.99607422,-75.45794828,1484080.0,2874.704166666721\n" + -"43.26944444,176.13078833999998,-73.5958333,1327750.0,2874.708333333372\n" + -"42.78527778,273.54074202,-73.7075,1357500.0,2874.708333333372\n" + -"39.01061275,61.16438952,-75.45794828,1484080.0,2874.708333333372\n" + -"39.01061275,60.59805258,-75.45794828,1484080.0,2874.7125000000233\n"; +"discharge,station,time,longitude,latitude\n" + +"80.98618241999999,1327750.0,3426.6979166667443,-73.5958333,43.26944444\n" + +"181.2278208,1357500.0,3426.6979166667443,-73.7075,42.78527778\n" + +"183.77633703,1357500.0,3426.708333333372,-73.7075,42.78527778\n" + +"-56.06735706,1484085.0,3426.691666666651,-75.3976111,39.05830556\n"; Test.ensureEqual(results, expected, "results=\n" + results); // just read station vars (all stations) no constraints @@ -23430,7 +23431,7 @@ public static void testReadNcCFMATimeSeriesReversed() throws Exception { "1463500.0,40.22166667,-74.7780556,638.2708333332557,97.40995368\n"; //stop there results = results.substring(0, expected.length()); Test.ensureEqual(results, expected, "results=\n" + results); - Test.ensureEqual(table.nRows(), 209126, "wrong nRows"); + Test.ensureEqual(table.nRows(), 256610, "wrong nRows"); // read all data String2.log("\n* read all data"); @@ -23440,21 +23441,20 @@ public static void testReadNcCFMATimeSeriesReversed() throws Exception { String2.log("time=" + (System.currentTimeMillis() - time)); results = table.dataToCSVString(); expected = -"latitude,discharge,longitude,station,time\n" + -"40.22166667,92.02975275,-74.7780556,1463500.0,638.1666666666279\n" + -"40.8847222,1.500792891,-74.2261111,1389500.0,638.1666666666279\n" + -"40.5511111,4.190893356,-74.5483333,1403060.0,638.1666666666279\n" + -"39.9678905,11.921392587,-75.1885123,1474500.0,638.1666666666279\n" + -"40.22166667,92.02975275,-74.7780556,1463500.0,638.1770833332557\n" + -"40.8847222,1.500792891,-74.2261111,1389500.0,638.1770833332557\n" + -"40.5511111,4.190893356,-74.5483333,1403060.0,638.1770833332557\n" + -"40.22166667,92.02975275,-74.7780556,1463500.0,638.1875\n" + -"40.8847222,1.500792891,-74.2261111,1389500.0,638.1875\n" + -"40.5511111,4.275843897,-74.5483333,1403060.0,638.1875\n" + -"39.9678905,11.921392587,-75.1885123,1474500.0,638.1875\n"; //stop there +"discharge,station,time,longitude,latitude\n" + +"92.02975275,1463500.0,638.1666666666279,-74.7780556,40.22166667\n" + +"92.02975275,1463500.0,638.1770833332557,-74.7780556,40.22166667\n" + +"92.02975275,1463500.0,638.1875,-74.7780556,40.22166667\n" + +"92.87925815999999,1463500.0,638.1979166666279,-74.7780556,40.22166667\n" + +"93.72876357,1463500.0,638.2083333332557,-74.7780556,40.22166667\n" + +"93.72876357,1463500.0,638.21875,-74.7780556,40.22166667\n" + +"94.86143745,1463500.0,638.2291666666279,-74.7780556,40.22166667\n" + +"95.71094286,1463500.0,638.2395833332557,-74.7780556,40.22166667\n" + +"95.71094286,1463500.0,638.25,-74.7780556,40.22166667\n" + +"95.71094286,1463500.0,638.2604166666279,-74.7780556,40.22166667\n"; //stop there results = results.substring(0, expected.length()); Test.ensureEqual(results, expected, "results=\n" + results); - Test.ensureEqual(table.nRows(), 1742115, "wrong nRows"); + Test.ensureEqual(table.nRows(), 2315617, "wrong nRows"); // read all vars when obs is constrained, diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorDNLS.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorDNLS.java index e4802741e..c5160650f 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorDNLS.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorDNLS.java @@ -4,12 +4,22 @@ */ package gov.noaa.pfel.coastwatch.util; +import com.amazonaws.AmazonClientException; +import com.amazonaws.AmazonServiceException; +import com.amazonaws.auth.profile.ProfileCredentialsProvider; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3ObjectSummary; + import com.cohort.array.Attributes; import com.cohort.array.DoubleArray; import com.cohort.array.LongArray; import com.cohort.array.StringArray; import com.cohort.util.Calendar2; import com.cohort.util.File2; +import com.cohort.util.Math2; import com.cohort.util.MustBe; import com.cohort.util.SimpleException; import com.cohort.util.String2; @@ -23,6 +33,9 @@ import java.nio.file.Path; import java.nio.file.SimpleFileVisitor; import java.nio.file.FileVisitResult; +import java.util.Date; +import java.util.HashSet; +import java.util.Iterator; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -41,6 +54,7 @@ public class FileVisitorDNLS extends SimpleFileVisitor { */ public static boolean verbose = false; public static boolean reallyVerbose = false; + public static boolean debugMode = false; /** The names of the columns in the table. */ public final static String DIRECTORY = "directory"; @@ -57,12 +71,12 @@ public class FileVisitorDNLS extends SimpleFileVisitor { public Pattern pattern; //from regex public boolean recursive, directoriesToo; static boolean OSIsWindows = String2.OSIsWindows; - public Table table = new Table(); + public Table table; /** dirs will have \\ or / like original constructor tDir, and a matching trailing slash. */ - public StringArray directoryPA = new StringArray(); - public StringArray namePA = new StringArray(); - public LongArray lastModifiedPA = new LongArray(); - public LongArray sizePA = new LongArray(); + public StringArray directoryPA; + public StringArray namePA; + public LongArray lastModifiedPA; + public LongArray sizePA; /** @@ -84,10 +98,11 @@ public FileVisitorDNLS(String tDir, String tRegex, boolean tRecursive, regex = tRegex; pattern = Pattern.compile(regex); directoriesToo = tDirectoriesToo; - table.addColumn(DIRECTORY, directoryPA); - table.addColumn(NAME, namePA); - table.addColumn(LASTMODIFIED, lastModifiedPA); - table.addColumn(SIZE, sizePA); + table = makeEmptyTable(); + directoryPA = (StringArray)table.getColumn(DIRECTORY); + namePA = (StringArray)table.getColumn(NAME); + lastModifiedPA = ( LongArray)table.getColumn(LASTMODIFIED); + sizePA = ( LongArray)table.getColumn(SIZE); } /** Invoked before entering a directory. */ @@ -161,18 +176,201 @@ public String resultsToString() { return table.dataToCSVString(); } + /** + * This returns an empty table with columns suitable for the instance table or oneStep. + */ + public static Table makeEmptyTable() { + Table table = new Table(); + table.addColumn(DIRECTORY, new StringArray()); + table.addColumn(NAME, new StringArray()); + table.addColumn(LASTMODIFIED, new LongArray()); + table.addColumn(SIZE, new LongArray()); + return table; + } + /** * This is a convenience method for using this class. + *

    This works with Amazon AWS S3 bucket URLs. Internal /'s in the keys will be + * treated as folder separators. If there aren't any /'s, all the keys will + * be in the root directory. * * @param tDir The starting directory, with \\ or /, with or without trailing slash. * The resulting directoryPA will contain dirs with matching slashes and trailing slash. * @return a table with columns with DIRECTORY, NAME, LASTMODIFIED, and SIZE columns. * If directoriesToo=true, the original dir won't be included and any * directory's name will be "". + * @throws IOException if trouble */ public static Table oneStep(String tDir, String tRegex, boolean tRecursive, boolean tDirectoriesToo) throws IOException { long time = System.currentTimeMillis(); + + //Is it an S3 bucket with "files"? + Pattern pattern = Pattern.compile(String2.AWS_S3_REGEX); + Matcher matcher = pattern.matcher(File2.addSlash(tDir)); //forcing trailing slash avoids problems + if (matcher.matches()) { + //http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html + //If files have file-system-like names, e.g., + // http://bucketname.s3.amazonaws.com/dir1/dir2/fileName.ext) + // http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_NorESM1-M_209601-209912.nc + // you still can't request just dir2 info because they aren't directories. + // They are just object keys with internal slashes. + //So specify prefix in request. + Table table = makeEmptyTable(); + StringArray directoryPA = (StringArray)table.getColumn(DIRECTORY); + StringArray namePA = (StringArray)table.getColumn(NAME); + LongArray lastModifiedPA = ( LongArray)table.getColumn(LASTMODIFIED); + LongArray sizePA = ( LongArray)table.getColumn(SIZE); + + String bucketName = matcher.group(2); + String prefix = matcher.group(3); + String baseURL = tDir.substring(0, matcher.start(3)); + AmazonS3 s3client = new AmazonS3Client(new ProfileCredentialsProvider()); + try { + if (verbose) + String2.log("FileVisitorDNLS.oneStep getting info from AWS S3 at" + + "\nURL=" + tDir); + //"\nbucket=" + bucketName + " prefix=" + prefix); + + //I wanted to generate lastMod for dir based on lastMod of files + //but it would be inconsistent for different requests (recursive, regex). + //so just a set of dir names. + HashSet dirHashSet = new HashSet(); + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketName) + .withPrefix(prefix); + ObjectListing objectListing; + do { + objectListing = s3client.listObjects(listObjectsRequest); + for (S3ObjectSummary objectSummary : + objectListing.getObjectSummaries()) { + String keyFullName = objectSummary.getKey(); + String keyDir = File2.getDirectory(baseURL + keyFullName); + String keyName = File2.getNameAndExtension(keyFullName); + if (debugMode) String2.log("keyFullName=" + keyFullName + + "\nkeyDir=" + keyDir + + "\n tDir=" + tDir); + if (keyDir.startsWith(tDir) && //it should + (tRecursive || keyDir.length() == tDir.length())) { + + //store this dir + if (tDirectoriesToo) { + //S3 only returns object keys. I must infer/collect directories. + //Store this dir and parents back to tDir. + String choppedKeyDir = keyDir; + while (choppedKeyDir.length() >= tDir.length()) { + if (!dirHashSet.add(choppedKeyDir)) + break; //hash set already had this, so will already have parents + + //chop off last subdirectory + choppedKeyDir = File2.getDirectory( + choppedKeyDir.substring(0, choppedKeyDir.length() - 1)); //remove trailing / + } + } + + //store this file's information + //Sometimes directories appear as files are named "" with size=0. + //I don't store those as files. + if (debugMode) String2.log("keyName=" + keyFullName + + "\n tRegex=" + tRegex + " matches=" + keyName.matches(tRegex)); + if (keyName.length() > 0 && keyName.matches(tRegex)) { + directoryPA.add(keyDir); + namePA.add(keyName); + lastModifiedPA.add(objectSummary.getLastModified().getTime()); //epoch millis + sizePA.add(objectSummary.getSize()); //long + } + } + } + listObjectsRequest.setMarker(objectListing.getNextMarker()); + } while (objectListing.isTruncated()); + + //add directories to the table + if (tDirectoriesToo) { + Iterator it = dirHashSet.iterator(); + while (it.hasNext()) { + directoryPA.add(it.next()); + namePA.add(""); + lastModifiedPA.add(Long.MAX_VALUE); + sizePA.add(Long.MAX_VALUE); + } + table.leftToRightSort(2); + } + + return table; + + } catch (AmazonServiceException ase) { + throw new IOException("AmazonServiceException: " + + ase.getErrorType() + " ERROR, HTTP Code=" + ase.getStatusCode() + + ": " + ase.getMessage(), ase); + } catch (AmazonClientException ace) { + throw new IOException(ace.getMessage(), ace); + } + } + + //HYRAX before THREDDS + //http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/ + if (tDir.matches("http://.+/opendap/.+")) { + try { + Table table = makeEmptyTable(); + StringArray directoryPA = (StringArray)table.getColumn(DIRECTORY); + StringArray namePA = (StringArray)table.getColumn(NAME); + LongArray lastModifiedPA = ( LongArray)table.getColumn(LASTMODIFIED); + LongArray sizePA = ( LongArray)table.getColumn(SIZE); + + DoubleArray lastModDA = new DoubleArray(); + addToHyraxUrlList(tDir, tRegex, tRecursive, tDirectoriesToo, + namePA, lastModDA, sizePA); + lastModifiedPA.append(lastModDA); + int n = namePA.size(); + for (int i = 0; i < n; i++) { + String fn = namePA.get(i); + directoryPA.add(File2.getDirectory(fn)); + namePA.set(i, File2.getNameAndExtension(fn)); + } + + return table; + } catch (Throwable t) { + throw new IOException(t.getMessage(), t); + } + } + + //THREDDS + if (tDir.matches("http://.+/thredds/catalog/.+")) { + try { + Table table = makeEmptyTable(); + StringArray directoryPA = (StringArray)table.getColumn(DIRECTORY); + StringArray namePA = (StringArray)table.getColumn(NAME); + LongArray lastModifiedPA = ( LongArray)table.getColumn(LASTMODIFIED); + LongArray sizePA = ( LongArray)table.getColumn(SIZE); + + DoubleArray lastModDA = new DoubleArray(); + addToThreddsUrlList(tDir, tRegex, tRecursive, tDirectoriesToo, + namePA, lastModDA, sizePA); + lastModifiedPA.append(lastModDA); + int n = namePA.size(); + for (int i = 0; i < n; i++) { + String fn = namePA.get(i); + directoryPA.add(File2.getDirectory(fn)); + namePA.set(i, File2.getNameAndExtension(fn)); + } + + return table; + } catch (Throwable t) { + throw new IOException(t.getMessage(), t); + } + } + + //other remote files +//NOT YET DONE +// if (tDir.startsWith("http://") || tDir.startsWith("http://")) { + + //apache-style directory index? parse the html + //Use an ERDDAP files dir as an example. + + //throw error +// } + + //local files FileVisitorDNLS fv = new FileVisitorDNLS(tDir, tRegex, tRecursive, tDirectoriesToo); Files.walkFileTree(FileSystems.getDefault().getPath(tDir), fv); @@ -189,14 +387,25 @@ public static Table oneStep(String tDir, String tRegex, boolean tRecursive, * * @param tDir The starting directory, with \\ or /, with or without trailing slash. * The resulting directoryPA will contain dirs with matching slashes and trailing slash. - * @return a table with columns with DIRECTORY, NAME, LASTMODIFIED, and SIZE columns. + * @return a table with columns with DIRECTORY, NAME, + * LASTMODIFIED (double epochSeconds), and SIZE (doubles) columns. * If directoriesToo=true, the original dir won't be included and any * directory's name will be "". + * @throws IOException if trouble */ public static Table oneStepDouble(String tDir, String tRegex, boolean tRecursive, boolean tDirectoriesToo) throws IOException { - Table tTable = oneStep(tDir, tRegex, tRecursive, tDirectoriesToo); + return oneStepDouble(oneStep(tDir, tRegex, tRecursive, tDirectoriesToo)); + } + + + /** + * This is a variant of oneStepDouble() that uses an existing table from oneStep(). + * + * @throws IOException if trouble + */ + public static Table oneStepDouble(Table tTable) throws IOException { int nCols = tTable.nColumns(); int nRows = tTable.nRows(); @@ -234,20 +443,35 @@ public static Table oneStepDouble(String tDir, String tRegex, boolean tRecursive } /** - * This is a variant of oneStepDouble (a convenience method for using this class) - * that returns a url column instead of a directory column. + * This is like oneStepDouble (a convenience method for using this class) + * but returns a url column instead of a directory column. * * @param tDir The starting directory, with \\ or /, with or without trailing slash. * @param startOfUrl usually EDStatic.erddapUrl(loggedInAs) + "/files/" + datasetID() + "/" - * @return a table with columns with DIRECTORY, NAME, LASTMODIFIED, and SIZE columns. - * If directoriesToo=true, the original dir won't be included and any - * directory's name will be "". + * @return a table with columns with DIRECTORY (always "/"), NAME, + * LASTMODIFIED (double epochSeconds), and SIZE (doubles) columns. */ - public static Table oneStepAccessibleViaFiles(String tDir, String tRegex, boolean tRecursive, + public static Table oneStepDoubleWithUrlsNotDirs(String tDir, String tRegex, boolean tRecursive, String startOfUrl) throws IOException { tDir = File2.addSlash(String2.replaceAll(tDir, "\\", "/")); //ensure forward/ and trailing/ - Table tTable = oneStepDouble(tDir, tRegex, tRecursive, false); //tDirectoriesToo + return oneStepDoubleWithUrlsNotDirs( + oneStepDouble(tDir, tRegex, tRecursive, false), //tDirectoriesToo + tDir, + startOfUrl); + } + + + /** + * This is a variant of oneStepDoubleWithUrlsNotDirs() that uses an existing + * table from oneStepDouble(tDirToo=false). + * + * @param tTable an existing table from oneStepDouble(tDirToo=false) + * @throws IOException if trouble + */ + public static Table oneStepDoubleWithUrlsNotDirs(Table tTable, String tDir, + String startOfUrl) throws IOException { + int nCols = tTable.nColumns(); int nRows = tTable.nRows(); @@ -278,12 +502,688 @@ public static Table oneStepAccessibleViaFiles(String tDir, String tRegex, boolea } + /** + * This gets the file names from Hyrax catalog directory URL. + * This only finds info for DAP URLs. + * This doesn't find other types of files (although they may be listed) + * This is used by EDDGridFromDap and EDDTableFromHyraxFiles. + * + * @param startUrl the url of the current web directory to which "contents.html" + * can be added to see a hyrax catalog) e.g., + http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/ + or + http://podaac-opendap.jpl.nasa.gov/opendap/hyrax/allData/avhrr/L4/reynolds_er/v3b/monthly/netcdf/2014/ + * @param fileNameRegex e.g., + "pentad.*flk\\.nc\\.gz" + * @param recursive + * @returns a String[] with a list of full URLs of the children (may be new String[0]) + * @throws Throwable if unexpected trouble. But url not responding won't throw Throwable. + */ + public static String[] getUrlsFromHyraxCatalog(String startUrl, String fileNameRegex, + boolean recursive) throws Throwable { + if (verbose) String2.log("getUrlsFromHyraxCatalog regex=" + fileNameRegex); + + //call the recursive method + boolean tDirectoriesToo = false; + StringArray childUrls = new StringArray(); + DoubleArray lastModified = new DoubleArray(); + LongArray size = new LongArray(); + addToHyraxUrlList(startUrl, fileNameRegex, recursive, tDirectoriesToo, + childUrls, lastModified, size); + + return childUrls.toArray(); + } + + /** + * This does the work for getUrlsFromHyraxCatalogs. + * This calls itself recursively, adding into to the PrimitiveArrays as info + * for a DAP URL is found. + * This doesn't find other types of files (although they may be listed) + * + * @param url the url of the directory to which contents.html can be added + * to see a hyrax catalog, e.g., + http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/ + * (If url has a file name, it must be "contents.html".) + * @param fileNameRegex e.g., "pentad.*flk\\.nc\\.gz" + * @param recursive + * @param dirsToo if true, directories should be collected also + * @param childUrls new children will be added to this + * @param lastModified the lastModified time (secondsSinceEpoch, NaN if not available). + * Source times are assumed to be Zulu time zone (which is probably incorrect). + * @param size the file's size (bytes, Long.MAX_VALUE if not available) + * @return true if completely successful (no access errors, all URLs found) + * @throws Throwable if unexpected trouble. But url not responding won't throw Throwable. + */ + public static boolean addToHyraxUrlList(String url, String fileNameRegex, + boolean recursive, boolean dirsToo, + StringArray childUrls, DoubleArray lastModified, LongArray size) throws Throwable { + + if (reallyVerbose) String2.log("\naddToHyraxUrlList childUrls.size=" + childUrls.size() + + "\n url=" + url); + boolean completelySuccessful = true; //but any child can set it to false + String response; + try { + if (url.endsWith("/contents.html")) + url = File2.getDirectory(url); + else url = File2.addSlash(url); //otherwise, assume url is missing final slash + response = SSR.getUrlResponseString(url + "contents.html"); + } catch (Throwable t) { + String2.log(MustBe.throwableToString(t)); + return false; + } + String responseLC = response.toLowerCase(); + if (dirsToo) { + childUrls.add(url); + lastModified.add(Double.NaN); + size.add(Long.MAX_VALUE); + } + + //skip header line and parent directory + int po = responseLC.indexOf("parent directory"); //Lower Case + if (po < 0 ) { + if (reallyVerbose) String2.log("ERROR: \"parent directory\" not found in Hyrax response."); + return false; + } + po += 18; + + //endPre + int endPre = responseLC.indexOf("", po); //Lower Case + if (endPre < 0) + endPre = response.length(); + + //go through file,dir listings + boolean diagnosticMode = false; + while (true) { + + //EXAMPLE http://data.nodc.noaa.gov/opendap/wod/monthly/ No longer available + + //EXAMPLE http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/M07 + //(reformatted: look for tags, not formatting + /* + month_19870701_v11l35flk.nc.gz + 2007-04-04T07:00:00 + 4807310 + + + + + +
    ddx dds 
    //will exist if exists + + + //may or may not exist + //may or may not exist + //the next row... +
    viewers
    + */ + + //find beginRow and nextRow + int beginRow = responseLC.indexOf(" endPre) + return completelySuccessful; + int endRow = responseLC.indexOf(" endPre) + endRow = endPre; + + //if in the middle, skip table + int tablePo = responseLC.indexOf(" 0 && tablePo < endRow) { + int endTablePo = responseLC.indexOf(" endPre) + endTablePo = endPre; + + //find + endRow = responseLC.indexOf(" endPre) + endRow = endPre; + } + String thisRow = response.substring(beginRow, endRow); + String thisRowLC = responseLC.substring(beginRow, endRow); + if (diagnosticMode) + String2.log("<<>>"); + + //look for .das href="wod_013459339O.nc.das">das< + int dasPo = thisRowLC.indexOf(".das\">das<"); + if (diagnosticMode) + String2.log(" .das " + (dasPo < 0? "not " : "") + "found"); + if (dasPo > 0) { + int quotePo = thisRow.lastIndexOf('"', dasPo); + if (quotePo < 0) { + String2.log("ERROR: invalid .das reference:\n " + thisRow); + po = endRow; + continue; + } + String fileName = thisRow.substring(quotePo + 1, dasPo); + if (diagnosticMode) + String2.log(" filename=" + fileName + + (fileName.matches(fileNameRegex)? " does" : " doesn't") + + " match " + fileNameRegex); + if (fileName.matches(fileNameRegex)) { + + //get lastModified time >2011-06-30T04:43:09< + String stime = String2.extractRegex(thisRow, + ">\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}<", 0); + double dtime = Calendar2.safeIsoStringToEpochSeconds( + stime == null? "" : stime.substring(1, stime.length() - 1)); + + //get size e.g., 119K + String sSize = String2.extractRegex(thisRow, + ">(\\d|\\.)+(|K|M|G|T|P)", 0); + long lSize = Long.MAX_VALUE; + if (sSize != null) { + sSize = sSize.substring(1, sSize.length() - 5); + char lastCh = sSize.charAt(sSize.length() - 1); + long times = 1; + if (!String2.isDigit(lastCh)) { + //this is done, but my samples just have nBytes + sSize = sSize.substring(0, sSize.length() - 1); + times = lastCh == 'K'? Math2.BytesPerKB : + lastCh == 'M'? Math2.BytesPerMB : + lastCh == 'G'? Math2.BytesPerGB : + lastCh == 'T'? Math2.BytesPerTB : + lastCh == 'P'? Math2.BytesPerPB : 1; + } + lSize = Math2.roundToLong(String2.parseDouble(sSize) * times); + } + + //then add to PrimitiveArrays + childUrls.add(url + fileName); + lastModified.add(dtime); + size.add(lSize); + //String2.log(" file=" + fileName + " " + stime); + po = endRow; + continue; + } + } + + if (recursive) { + //look for href="199703-199705/contents.html" + int conPo = thisRowLC.indexOf("/contents.html\""); + if (conPo > 0) { + int quotePo = thisRow.lastIndexOf('"', conPo); + if (quotePo < 0) { + String2.log("ERROR: invalid contents.html reference:\n " + thisRow); + po = endRow; + continue; + } + boolean tSuccessful = addToHyraxUrlList( + url + thisRow.substring(quotePo + 1, conPo + 1), + fileNameRegex, recursive, dirsToo, childUrls, lastModified, size); + if (!tSuccessful) + completelySuccessful = false; + po = endRow; + continue; + } + } + po = endRow; + } + } + + + + /** + * This tests Hyrax-related methods. + */ + public static void testHyrax() throws Throwable { + String2.log("\n*** FileVisitorDNLS.testHyrax()\n"); + + try { + + String url = "http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/"; //contents.html + String regex = "month_198(8|9).*flk\\.nc\\.gz"; + boolean recursive = true; + boolean dirsToo = true; + StringArray childUrls = new StringArray(); + DoubleArray lastModified = new DoubleArray(); + LongArray size = new LongArray(); + + //test error via addToHyraxUrlList + //(yes, logged message includes directory name) + Test.ensureEqual( + addToHyraxUrlList(url + "testInvalidUrl", regex, recursive, dirsToo, + childUrls, lastModified, size), + false, ""); + + //test addToHyraxUrlList + childUrls = new StringArray(); + lastModified = new DoubleArray(); + size = new LongArray(); + boolean allOk = addToHyraxUrlList(url, regex, recursive, dirsToo, + childUrls, lastModified, size); + Table table = new Table(); + table.addColumn("URL", childUrls); + table.addColumn("lastModified", lastModified); + table.addColumn("size", size); + String results = table.dataToCSVString(); + String expected = +"URL,lastModified,size\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880101_v11l35flk.nc.gz,1.336863115E9,4981045\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880201_v11l35flk.nc.gz,1.336723222E9,5024372\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880301_v11l35flk.nc.gz,1.336546575E9,5006043\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880401_v11l35flk.nc.gz,1.336860015E9,4948285\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880501_v11l35flk.nc.gz,1.336835143E9,4914250\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880601_v11l35flk.nc.gz,1.336484405E9,4841084\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880701_v11l35flk.nc.gz,1.336815079E9,4837417\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880801_v11l35flk.nc.gz,1.336799789E9,4834242\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880901_v11l35flk.nc.gz,1.336676042E9,4801865\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881001_v11l35flk.nc.gz,1.336566352E9,4770289\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881101_v11l35flk.nc.gz,1.336568382E9,4769160\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881201_v11l35flk.nc.gz,1.336838712E9,4866335\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890101_v11l35flk.nc.gz,1.336886548E9,5003981\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890201_v11l35flk.nc.gz,1.336268373E9,5054907\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890301_v11l35flk.nc.gz,1.336605483E9,4979393\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890401_v11l35flk.nc.gz,1.336350339E9,4960865\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890501_v11l35flk.nc.gz,1.336551575E9,4868541\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890601_v11l35flk.nc.gz,1.336177278E9,4790364\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890701_v11l35flk.nc.gz,1.336685187E9,4854943\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890801_v11l35flk.nc.gz,1.336534686E9,4859216\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890901_v11l35flk.nc.gz,1.33622953E9,4838390\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19891001_v11l35flk.nc.gz,1.336853599E9,4820645\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19891101_v11l35flk.nc.gz,1.336882933E9,4748166\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19891201_v11l35flk.nc.gz,1.336748115E9,4922858\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1990/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1991/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1992/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1993/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1994/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1995/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1996/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1997/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1998/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1999/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2000/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2001/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2002/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2003/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2004/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2005/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2006/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2007/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2008/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2009/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2010/,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/2011/,,\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + Test.ensureTrue(allOk, ""); + + //test getUrlsFromHyraxCatalog + String resultsAr[] = getUrlsFromHyraxCatalog(url, regex, recursive); + String expectedAr[] = new String[]{ +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880101_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880201_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880301_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880401_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880501_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880601_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880701_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880801_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880901_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881001_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881101_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881201_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890101_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890201_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890301_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890401_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890501_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890601_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890701_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890801_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19890901_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19891001_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19891101_v11l35flk.nc.gz", +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1989/month_19891201_v11l35flk.nc.gz"}; + Test.ensureEqual(resultsAr, expectedAr, "results=\n" + results); + + //different test of addToHyraxUrlList + childUrls = new StringArray(); + lastModified = new DoubleArray(); + LongArray fSize = new LongArray(); + url = "http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/"; //startUrl, + regex = "month_[0-9]{8}_v11l35flk\\.nc\\.gz"; //fileNameRegex, + recursive = true; + addToHyraxUrlList(url, regex, recursive, dirsToo, + childUrls, lastModified, fSize); + + results = childUrls.toNewlineString(); + expected = +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19870701_v11l35flk.nc.gz\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19870801_v11l35flk.nc.gz\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19870901_v11l35flk.nc.gz\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19871001_v11l35flk.nc.gz\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19871101_v11l35flk.nc.gz\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19871201_v11l35flk.nc.gz\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + results = lastModified.toString(); + expected = "NaN, 1.336609915E9, 1.336785444E9, 1.336673639E9, 1.336196561E9, 1.336881763E9, 1.336705731E9"; + Test.ensureEqual(results, expected, "results=\n" + results); + + //test via oneStep -- dirs + table = oneStep(url, regex, recursive, true); + results = table.dataToCSVString(); + expected = +"directory,name,lastModified,size\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,,,\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19870701_v11l35flk.nc.gz,1336609915,4807310\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19870801_v11l35flk.nc.gz,1336785444,4835774\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19870901_v11l35flk.nc.gz,1336673639,4809582\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19871001_v11l35flk.nc.gz,1336196561,4803285\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19871101_v11l35flk.nc.gz,1336881763,4787239\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19871201_v11l35flk.nc.gz,1336705731,4432696\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + //test via oneStep -- no dirs + table = oneStep(url, regex, recursive, false); + results = table.dataToCSVString(); + expected = +"directory,name,lastModified,size\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19870701_v11l35flk.nc.gz,1336609915,4807310\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19870801_v11l35flk.nc.gz,1336785444,4835774\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19870901_v11l35flk.nc.gz,1336673639,4809582\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19871001_v11l35flk.nc.gz,1336196561,4803285\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19871101_v11l35flk.nc.gz,1336881763,4787239\n" + +"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/,month_19871201_v11l35flk.nc.gz,1336705731,4432696\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + } catch (Throwable t) { + String2.pressEnterToContinue(MustBe.throwableToString(t) + + "\nUnexpected error."); + } + } + + + + /** + * This parses /thredds/catalog/.../catalog.html files to extract file URLs + * (/thredds/fileServer/.../name.ext), lastModified, and size info. + * This calls itself recursively, adding into to the PrimitiveArrays as info + * for a file URL is found. + * This doesn't find other types of files (although they may be listed) + * + * @param url the url of the current Thredds directory + * (which usually includes /thredds/catalog/) + * to which catalog.html will be added, e.g., + *
    http://data.nodc.noaa.gov/thredds/catalog/pathfinder/Version5.1_CloudScreened/5day/FullRes/ + * (If url has a file name, it must be catalog.html or catalog.xml.) + * @param fileNameRegex e.g., ".*\\.hdf" + * @param childUrls new children will be added to this + * @param lastModified the lastModified time (secondsSinceEpoch, NaN if not available). + * Source times are assumed to be Zulu time zone (which is probably incorrect). + * @param size the file's size (bytes, Long.MAX_VALUE if not available) + * @return true if completely successful (no access errors, all URLs found) + * @throws Throwable if unexpected trouble. But url not responding won't throw Throwable. + */ + public static boolean addToThreddsUrlList(String url, String fileNameRegex, + boolean recursive, boolean dirsToo, + StringArray childUrls, DoubleArray lastModified, LongArray size) throws Throwable { + + if (reallyVerbose) String2.log("\naddToThreddsUrlList childUrls.size=" + childUrls.size() + + "\n url=" + url); + boolean completelySuccessful = true; //but any child can set it to false + String response; + try { + if (url.endsWith("/catalog.html") || url.endsWith("/catalog.xml")) + url = File2.getDirectory(url); + else url = File2.addSlash(url); //otherwise, assume url is missing final slash + response = SSR.getUrlResponseString(url + "catalog.html"); + } catch (Throwable t) { + String2.log(MustBe.throwableToString(t)); + return false; + } + String fileServerDir = String2.replaceAll(url, + "/thredds/catalog/", "/thredds/fileServer/"); + if (dirsToo) { + childUrls.add(fileServerDir); + lastModified.add(Double.NaN); + size.add(Long.MAX_VALUE); + } + + //skip header line and parent directory + int po = response.indexOf("", po); //Lower Case + if (endTable < 0) { + if (reallyVerbose) String2.log("WARNING:
    not found!"); + completelySuccessful = false; + endTable = response.length(); + } + + //go through file,dir listings + boolean diagnosticMode = false; + while (true) { + +/* EXAMPLE from TDS 4.2.10 at +http://data.nodc.noaa.gov/thredds/catalog/pathfinder/Version5.1_CloudScreened/5day/FullRes/1981/catalog.html +... + + + + + + + + + + + + + +*/ + //find beginRow and nextRow + int beginRow = response.indexOf(" endTable) + return completelySuccessful; + int nextRow = response.indexOf(" endTable) + nextRow = endTable; + + String thisRow = response.substring(beginRow, nextRow); + if (diagnosticMode) + String2.log("=== thisRow=" + thisRow); + + //look for ."); + completelySuccessful = false; + po = nextRow; + continue; + } + + //ensure fileName matches regex + if (!content1.matches(fileNameRegex)) { + po = nextRow; + if (diagnosticMode) + String2.log("=== skip this row: content1=" + content1 + " doesn't match regex=" + fileNameRegex); + continue; + } + + //extract approximate size, e.g., 70.03 Mbytes + String sSize = String2.extractRegex(content2, + "(\\d|\\.)+ (|K|M|G|T|P)bytes", 0); + long lSize = Long.MAX_VALUE; + if (sSize != null) { + int spacePo = sSize.indexOf(' '); + char ch = sSize.charAt(spacePo + 1); + long times = ch == 'K'? Math2.BytesPerKB : + ch == 'M'? Math2.BytesPerMB : + ch == 'G'? Math2.BytesPerGB : + ch == 'T'? Math2.BytesPerTB : + ch == 'P'? Math2.BytesPerPB : 1; + lSize = Math2.roundToLong( + String2.parseDouble(sSize.substring(0, spacePo)) * times); + } + + //extract lastModified, e.g., 2011-06-30 04:43:09Z + String stime = String2.extractRegex(content3, + "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}", 0); + double dtime = Calendar2.safeIsoStringToEpochSeconds(stime); + + //add info to PrimitiveArrays + childUrls.add(fileServerDir + content1); + lastModified.add(dtime); + size.add(lSize); + //String2.log(" file=" + fileName + " " + stime); + po = nextRow; + } + } /** - * This tests this class. + * This tests THREDDS-related methods. */ - public static void test() throws Throwable { - String2.log("\n*** FileVisitorDNLS.test"); + public static void testThredds() throws Throwable { + String2.log("\n*** testThredds"); + boolean oReallyVerbose = reallyVerbose; + reallyVerbose = true; + + try{ + String url = "http://data.nodc.noaa.gov/thredds/catalog/aquarius/nodc_binned_V3.0/monthly/"; //catalog.html + String regex = "sss_binned_L3_MON_SCI_V3.0_\\d{4}\\.nc"; + boolean recursive = true; + boolean dirsToo = true; + StringArray childUrls = new StringArray(); + DoubleArray lastModified = new DoubleArray(); + LongArray fSize = new LongArray(); + + //test error via addToThreddsUrlList + //(yes, logged message includes directory name) + Test.ensureEqual( + addToThreddsUrlList(url + "testInvalidUrl", regex, recursive, dirsToo, + childUrls, lastModified, fSize), + false, ""); + + //test addToThreddsUrlList + childUrls = new StringArray(); + lastModified = new DoubleArray(); + fSize = new LongArray(); + addToThreddsUrlList(url, regex, recursive, dirsToo, + childUrls, lastModified, fSize); + + String results = childUrls.toNewlineString(); + String expected = +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/sss_binned_L3_MON_SCI_V3.0_2011.nc\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/sss_binned_L3_MON_SCI_V3.0_2012.nc\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/sss_binned_L3_MON_SCI_V3.0_2013.nc\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/sss_binned_L3_MON_SCI_V3.0_2014.nc\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/sss_binned_L3_MON_SCI_V3.0_2015.nc\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + results = lastModified.toString(); + expected = +"NaN, 1.405495932E9, 1.405492834E9, 1.405483892E9, 1.429802008E9, 1.429867829E9"; + Test.ensureEqual(results, expected, "results=\n" + results); + + results = fSize.toString(); + expected = +"9223372036854775807, 2723152, 6528434, 6528434, 6528434, 1635779"; + Test.ensureEqual(results, expected, "results=\n" + results); + + + //test via oneStep -- dirs + Table table = oneStep(url, regex, recursive, true); + results = table.dataToCSVString(); + expected = +"directory,name,lastModified,size\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,,,\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2011.nc,1405495932,2723152\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2012.nc,1405492834,6528434\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2013.nc,1405483892,6528434\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2014.nc,1429802008,6528434\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2015.nc,1429867829,1635779\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + //test via oneStep -- no dirs + table = oneStep(url, regex, recursive, false); + results = table.dataToCSVString(); + expected = +"directory,name,lastModified,size\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2011.nc,1405495932,2723152\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2012.nc,1405492834,6528434\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2013.nc,1405483892,6528434\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2014.nc,1429802008,6528434\n" + +"http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/,sss_binned_L3_MON_SCI_V3.0_2015.nc,1429867829,1635779\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + } catch (Throwable t) { + String2.pressEnterToContinue(MustBe.throwableToString(t) + + "\nUnexpected error."); + } + reallyVerbose = oReallyVerbose; + } + + + /** + * This tests this class with the local file system. + */ + public static void testLocal(boolean doBigTest) throws Throwable { + String2.log("\n*** FileVisitorDNLS.testLocal"); verbose = true; String contextDir = SSR.getContextDirectory(); //with / separator and / at the end Table table; @@ -370,7 +1270,7 @@ public static void test() throws Throwable { //*** //oneStepAccessibleViaFiles - table = oneStepAccessibleViaFiles("c:/erddapTest/fileNames", ".*\\.png", true, + table = oneStepDoubleWithUrlsNotDirs("c:/erddapTest/fileNames", ".*\\.png", true, "http://127.0.0.1:8080/cwexperimental/files/testFileNames/"); results = table.toCSVString(); expected = @@ -408,57 +1308,153 @@ public static void test() throws Throwable { String unexpected = "\nUnexpected FileVisitorDNLS error (but /data/gtspp/temp dir has variable nFiles):\n"; - for (int attempt = 0; attempt < 2; attempt++) { - try { - //forward slash in huge directory - time = System.currentTimeMillis(); - table = oneStep("/data/gtspp/temp", ".*\\.nc", false, false); - time = System.currentTimeMillis() - time; - //2014-11-25 98436 files in 410ms - StringArray directoryPA = (StringArray)table.getColumn(DIRECTORY); - String2.log("forward test: n=" + directoryPA.size() + " time=" + time); - if (directoryPA.size() < 1000) { - String2.log(directoryPA.size() + " files. Not a good test."); - } else { - Test.ensureBetween(time / (double)directoryPA.size(), 2e-3, 8e-3, - "ms/file (4.1e-3 expected)"); - String dir0 = directoryPA.get(0); - String2.log("forward slash test: dir0=" + dir0); - Test.ensureTrue(dir0.indexOf('\\') < 0, ""); - Test.ensureTrue(dir0.endsWith("/"), ""); + if (doBigTest) { + for (int attempt = 0; attempt < 2; attempt++) { + try { + //forward slash in huge directory + time = System.currentTimeMillis(); + table = oneStep("/data/gtspp/temp", ".*\\.nc", false, false); + time = System.currentTimeMillis() - time; + //2014-11-25 98436 files in 410ms + StringArray directoryPA = (StringArray)table.getColumn(DIRECTORY); + String2.log("forward test: n=" + directoryPA.size() + " time=" + time); + if (directoryPA.size() < 1000) { + String2.log(directoryPA.size() + " files. Not a good test."); + } else { + Test.ensureBetween(time / (double)directoryPA.size(), 2e-3, 8e-3, + "ms/file (4.1e-3 expected)"); + String dir0 = directoryPA.get(0); + String2.log("forward slash test: dir0=" + dir0); + Test.ensureTrue(dir0.indexOf('\\') < 0, ""); + Test.ensureTrue(dir0.endsWith("/"), ""); + } + } catch (Throwable t) { + String2.pressEnterToContinue(unexpected + + MustBe.throwableToString(t)); } - } catch (Throwable t) { - String2.pressEnterToContinue(unexpected + - MustBe.throwableToString(t)); } - } - for (int attempt = 0; attempt < 2; attempt++) { - try { - //backward slash in huge directory - time = System.currentTimeMillis(); - table = oneStep("\\data\\gtspp\\temp", ".*\\.nc", false, false); - time = System.currentTimeMillis() - time; - //2014-11-25 98436 files in 300ms - StringArray directoryPA = (StringArray)table.getColumn(DIRECTORY); - String2.log("backward test: n=" + directoryPA.size() + " time=" + time); - if (directoryPA.size() < 1000) { - String2.log(directoryPA.size() + " files. Not a good test."); - } else { - Test.ensureBetween(time / (double)directoryPA.size(), 1e-3, 8e-3, - "ms/file (3e-3 expected)"); - String dir0 = directoryPA.get(0); - String2.log("backward slash test: dir0=" + dir0); - Test.ensureTrue(dir0.indexOf('/') < 0, ""); - Test.ensureTrue(dir0.endsWith("\\"), ""); + for (int attempt = 0; attempt < 2; attempt++) { + try { + //backward slash in huge directory + time = System.currentTimeMillis(); + table = oneStep("\\data\\gtspp\\temp", ".*\\.nc", false, false); + time = System.currentTimeMillis() - time; + //2014-11-25 98436 files in 300ms + StringArray directoryPA = (StringArray)table.getColumn(DIRECTORY); + String2.log("backward test: n=" + directoryPA.size() + " time=" + time); + if (directoryPA.size() < 1000) { + String2.log(directoryPA.size() + " files. Not a good test."); + } else { + Test.ensureBetween(time / (double)directoryPA.size(), 1e-3, 8e-3, + "ms/file (3e-3 expected)"); + String dir0 = directoryPA.get(0); + String2.log("backward slash test: dir0=" + dir0); + Test.ensureTrue(dir0.indexOf('/') < 0, ""); + Test.ensureTrue(dir0.endsWith("\\"), ""); + } + } catch (Throwable t) { + String2.pressEnterToContinue(unexpected + + MustBe.throwableToString(t)); } - } catch (Throwable t) { - String2.pressEnterToContinue(unexpected + - MustBe.throwableToString(t)); } } - String2.log("\n*** FileVisitorDNLS.test finished."); + String2.log("\n*** FileVisitorDNLS.testLocal finished."); } + /** + * This tests this class with Amazon AWS S3 file system. + * Your S3 credentials must be in + *
    ~/.aws/credentials on Linux, OS X, or Unix + *
    C:\Users\USERNAME\.aws\credentials on Windows + * See http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html . + */ + public static void testAWSS3() throws Throwable { + String2.log("\n*** FileVisitorDNLS.testAWSS3"); + try { + + verbose = true; + String contextDir = SSR.getContextDirectory(); //with / separator and / at the end + Table table; + long time; + int n; + String results, expected; + String parent = "http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/"; + String child = "CONUS/"; + + //recursive and dirToo + table = oneStep(parent, ".*\\.nc", true, true); + results = table.dataToCSVString(); + expected = +"directory,name,lastModified,size\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/,,,\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,,,\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_200601-201012.nc,1382457840000,1368327466\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_201101-201512.nc,1382457577000,1369233982\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_201601-202012.nc,1382260226000,1368563375\n"; + if (expected.length() > results.length()) + String2.log("results=\n" + results); + Test.ensureEqual(results.substring(0, expected.length()), expected, "results=\n" + results); + + //recursive and !dirToo + table = oneStep(parent, ".*\\.nc", true, false); + results = table.dataToCSVString(); + expected = +"directory,name,lastModified,size\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_200601-201012.nc,1382457840000,1368327466\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_201101-201512.nc,1382457577000,1369233982\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_201601-202012.nc,1382260226000,1368563375\n"; + if (expected.length() > results.length()) + String2.log("results=\n" + results); + Test.ensureEqual(results.substring(0, expected.length()), expected, "results=\n" + results); + + //!recursive and dirToo + table = oneStep(parent + child, ".*\\.nc", false, true); + results = table.dataToCSVString(); + expected = +"directory,name,lastModified,size\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,,,\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_200601-201012.nc,1382457840000,1368327466\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_201101-201512.nc,1382457577000,1369233982\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_201601-202012.nc,1382260226000,1368563375\n"; + if (expected.length() > results.length()) + String2.log("results=\n" + results); + Test.ensureEqual(results.substring(0, expected.length()), expected, "results=\n" + results); + + //!recursive and !dirToo + table = oneStep(parent + child, ".*\\.nc", false, false); + results = table.dataToCSVString(); + expected = +"directory,name,lastModified,size\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_200601-201012.nc,1382457840000,1368327466\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_201101-201512.nc,1382457577000,1369233982\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_BNU-ESM_201601-202012.nc,1382260226000,1368563375\n"; + if (expected.length() > results.length()) + String2.log("results=\n" + results); + Test.ensureEqual(results.substring(0, expected.length()), expected, "results=\n" + results); + + String2.log("\n*** FileVisitorDNLS.testAWSS3 finished."); + + } catch (Throwable t) { + String2.pressEnterToContinue(MustBe.throwableToString(t) + + "\nUnexpected error. (Did you create your AWS S3 credentials file?)"); + } + } + + + /** + * This tests the methods in this class. + * + * @throws Throwable if trouble + */ + public static void test(boolean doBigTest) throws Throwable { + String2.log("\n****************** FileVisitorDNLS.test() *****************\n"); +/* */ + //always done + testLocal(doBigTest); + testAWSS3(); + testHyrax(); + testThredds(); + } } diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorSubdir.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorSubdir.java index 251fb1d2e..9285373cd 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorSubdir.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/FileVisitorSubdir.java @@ -4,11 +4,14 @@ */ package gov.noaa.pfel.coastwatch.util; +import com.cohort.array.StringArray; import com.cohort.util.File2; import com.cohort.util.MustBe; import com.cohort.util.String2; import com.cohort.util.Test; +import gov.noaa.pfel.coastwatch.pointdata.Table; + import java.io.IOException; import java.nio.file.attribute.BasicFileAttributes; import java.nio.file.Files; @@ -22,6 +25,8 @@ import java.nio.file.WatchKey; import java.nio.file.WatchService; import java.util.ArrayList; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** @@ -82,32 +87,46 @@ public FileVisitResult visitFileFailed(Path file, IOException exc) { * A convenience method for using this class. * * @param tDir The starting directory, with \\ or /, with or without trailing slash. - * @return an ArrayList<Path> with the dir and subdir paths. - * Note that path.toString() returns the full dir name with the OS's slashes - * (\\ for Windows!). + * @return a StringArray with dir and subdir names + * (with the OS's slashes -- \\ for Windows!). */ - public static ArrayList oneStep(String tDir) + public static StringArray oneStep(String tDir) throws IOException { long time = System.currentTimeMillis(); + + //Is it an S3 bucket with "files"? + //http://docs.aws.amazon.com/AmazonS3/latest/API/RESTBucketGET.html + if (File2.addSlash(tDir).matches(String2.AWS_S3_REGEX)) { //forcing trailing slash avoids problems + Table table = FileVisitorDNLS.oneStep(tDir, "/", //regex that won't match any files + true, true); //tRecursive, tDirectoriesToo + StringArray dirNames = (StringArray)table.getColumn(FileVisitorDNLS.DIRECTORY); + return dirNames; + } + + //do local file system FileVisitorSubdir fv = new FileVisitorSubdir(tDir); Files.walkFileTree(FileSystems.getDefault().getPath(tDir), fv); + int n = fv.results.size(); + StringArray dirNames = new StringArray(n, false); + for (int i = 0; i < n; i++) + dirNames.add(fv.results.get(i).toString()); if (verbose) String2.log("FileVisitorSubdir.oneStep finished successfully. n=" + - fv.results.size() + " time=" + (System.currentTimeMillis() - time)); - return fv.results; + n + " time=" + (System.currentTimeMillis() - time)); + return dirNames; } /** - * This tests this class. + * This tests a local file system. */ - public static void test() throws Throwable { - String2.log("\n*** FileVisitorSubdir.test"); + public static void testLocal() throws Throwable { + String2.log("\n*** FileVisitorSubdir.testLocal"); verbose = true; String contextDir = SSR.getContextDirectory(); //with / separator and / at the end - ArrayList alps; + StringArray alps; long time; alps = oneStep(contextDir + "WEB-INF/classes/com/cohort"); - String results = String2.toNewlineString(alps.toArray()); + String results = alps.toNewlineString(); String expected = "C:\\programs\\tomcat\\webapps\\cwexperimental\\WEB-INF\\classes\\com\\cohort\n" + "C:\\programs\\tomcat\\webapps\\cwexperimental\\WEB-INF\\classes\\com\\cohort\\array\n" + @@ -116,10 +135,55 @@ public static void test() throws Throwable { Test.ensureEqual(results, expected, "results=\\n" + results); alps = oneStep(String2.replaceAll(contextDir + "WEB-INF/classes/com/cohort/", '/', '\\')); - results = String2.toNewlineString(alps.toArray()); - Test.ensureEqual(results, expected, "results=\\n" + results); + results = alps.toNewlineString(); + Test.ensureEqual(results, expected, "results=\n" + results); + + String2.log("\n*** FileVisitorSubdir.testLocal finished."); + } + + /** + * This tests an Amazon AWS S3 file system. + * Your S3 credentials must be in + *
    ~/.aws/credentials on Linux, OS X, or Unix + *
    C:\Users\USERNAME\.aws\credentials on Windows + * See http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html . + */ + public static void testAWSS3() throws Throwable { + String2.log("\n*** FileVisitorSubdir.testAWSS3"); + try { - String2.log("\n*** FileVisitorSubdir.test finished."); + verbose = true; + String contextDir = SSR.getContextDirectory(); //with / separator and / at the end + StringArray alps; + long time; + + alps = oneStep( + "http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/"); + String results = alps.toNewlineString(); + String expected = +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/\n" + +"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + String2.log("\n*** FileVisitorSubdir.testAWSS3 finished."); + + } catch (Throwable t) { + String2.pressEnterToContinue(MustBe.throwableToString(t) + + "\nUnexpected error. (Did you create your AWS S3 credentials file?)"); + } + } + + /** + * This tests the methods in this class. + * + * @throws Throwable if trouble + */ + public static void test() throws Throwable { + String2.log("\n****************** FileVisitorSubdir.test() *****************\n"); +/* */ + //always done + testLocal(); + testAWSS3(); } diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/SSR.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/SSR.java index 2da9a1b56..570accec0 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/SSR.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/SSR.java @@ -990,9 +990,9 @@ public static String embedEPSF(double left, double bottom, double angle, * @param emailAddress (eg. bob DOT simons AT noaa DOT gov) * @return the safer form (eg. bob DOT simons AT noaa DOT gov) */ - public static String getSaveEmailAddress(String emailAddress) { - emailAddress = String2.replaceAll(emailAddress, ".", " DOT "); - emailAddress = String2.replaceAll(emailAddress, "@", " AT "); + public static String getSafeEmailAddress(String emailAddress) { + emailAddress = String2.replaceAll(emailAddress, ".", " dot "); + emailAddress = String2.replaceAll(emailAddress, "@", " at "); return emailAddress; } @@ -1463,7 +1463,7 @@ public static void downloadFile(String urlString, getUrlInputStream(urlString) : getUncompressedUrlInputStream(urlString); OutputStream out = new FileOutputStream(fullFileName); - byte buffer[] = new byte[8192]; + byte buffer[] = new byte[32768]; int nBytes = in.read(buffer); while (nBytes > 0) { out.write(buffer, 0, nBytes); @@ -2501,25 +2501,63 @@ public static InputStream getPostInputStream(String urlString, copy(getPostInputStream(url, "text/xml", "UTF-8", content), System.out); }*/ + + /** + * Copy from source to outputStream. + * + * @param source May be local fileName or a URL. + * @param out At the end, out is flushed, but not closed + * @return true if successful + */ + public static boolean copy(String source, OutputStream out) { + if (source.startsWith("http://") || + source.startsWith("https://") || //untested. + source.startsWith("ftp://")) { //untested. presumably anonymous + //URL + InputStream in = null; + try { + in = (InputStream)(getUrlConnInputStream(source, + 120000)[1]); //timeOutMillis. throws Exception + copy(in, out); + return true; + } catch (Exception e) { + String2.log(String2.ERROR + " in SSR.copy(source=" + source + ")\n" + + MustBe.throwableToString(e)); + return false; + } finally { + try { + if (in != null) + in.close(); + } catch (Exception e2) { + } + } + + } else { + //presumably a file + return File2.copy(source, out); + } + } + + /** * A method to copy the info from an inputStream to an outputStream. + * Based on E.R. Harold's book "Java I/O". * - * Copied from E.R. Harold's book "Java I/O". + * @param in At the end, in isn't closed. + * @param out At the end, out is flushed, but not closed + * @throws IOException if trouble */ public static void copy(InputStream in, OutputStream out) throws IOException { // do not allow other threads to read from the - // input or write to the output while copying is - // taking place + // input or write to the output while copying is taking place synchronized (in) { synchronized (out) { - - byte[] buffer = new byte[256]; - while (true) { - int bytesRead = in.read(buffer); - if (bytesRead == -1) break; - out.write(buffer, 0, bytesRead); - } + byte[] buffer = new byte[32768]; + int nRead; + while ((nRead = in.read(buffer)) >= 0) //0 shouldn't happen. -1=end of file + out.write(buffer, 0, nRead); + out.flush(); } } } diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/Tally.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/Tally.java index f6ea47776..9a63e4c8a 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/Tally.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/Tally.java @@ -117,17 +117,17 @@ public String toString(int maxAttributeNames) { } /** - * This returns a String with the information for one category (or "" if not found). + * This returns a table-like ArrayList with 2 items: + * attributeNames (a StringArray) and counts (an IntArray), + * sorted by counts (descending) then attributeNames (ascending). * - * @param categoryName - * @param maxAttributeNames the maximum number of attribute names printed per - * category + * @return null if no items for categoryName */ - public String toString(String categoryName, int maxAttributeNames) { - StringBuilder results = new StringBuilder(); + public ArrayList getSortedNamesAndCounts(String categoryName) { + ConcurrentHashMap hashMap = (ConcurrentHashMap)mainHashMap.get(categoryName); if (hashMap == null) - return ""; + return null; //make a StringArray of attributeNames and IntArray of counts StringArray attributeNames = new StringArray(); @@ -145,7 +145,25 @@ public String toString(String categoryName, int maxAttributeNames) { arrayList.add(counts); PrimitiveArray.sortIgnoreCase(arrayList, new int[]{1,0}, new boolean[]{false, true}); + return arrayList; + } + + /** + * This returns a String with the information for one category (or "" if not found). + * + * @param categoryName + * @param maxAttributeNames the maximum number of attribute names printed per + * category + */ + public String toString(String categoryName, int maxAttributeNames) { + ArrayList arrayList = getSortedNamesAndCounts(categoryName); + if (arrayList == null) + return ""; + StringArray attributeNames = (StringArray)arrayList.get(0); + IntArray counts = (IntArray)arrayList.get(1); + //print + StringBuilder results = new StringBuilder(); results.append(categoryName + "\n"); int countsSize = counts.size(); int nRows = Math.min(maxAttributeNames, countsSize); diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/WatchDirectory.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/WatchDirectory.java index 09e4c4548..bdfb4a62e 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/WatchDirectory.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/util/WatchDirectory.java @@ -86,12 +86,12 @@ public WatchDirectory(String tWatchDir, boolean tRecursive, WatchEvent.Kind even throw new RuntimeException( "The OS doesn't support WatchService for that file system."); if (recursive) { - ArrayList alps = FileVisitorSubdir.oneStep(watchDir); + StringArray alps = FileVisitorSubdir.oneStep(watchDir); int n = alps.size(); for (int i = 0; i < n; i++) { - WatchKey key = alps.get(i).register(watchService, events); + WatchKey key = Paths.get(alps.get(i)).register(watchService, events); keyToDirMap.put(key, String2.canonical(File2.addSlash( - String2.replaceAll(alps.get(i).toString(), fromSlash, toSlash)))); + String2.replaceAll(alps.get(i), fromSlash, toSlash)))); } } else { WatchKey key = watchPath.register(watchService, events); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java index 1d73565d7..64dbbaede 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java @@ -77,16 +77,8 @@ import javax.servlet.http.HttpServletResponse; import javax.servlet.http.HttpSession; -//import org.apache.lucene.analysis.Analyzer; -//import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; -//import org.apache.lucene.document.Field; -//import org.apache.lucene.index.IndexReader; -//import org.apache.lucene.index.IndexWriter; -//import org.apache.lucene.index.IndexWriterConfig; import org.apache.lucene.index.Term; -//import org.apache.lucene.queryParser.ParseException; -//import org.apache.lucene.queryParser.QueryParser; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.search.TopDocs; @@ -96,9 +88,6 @@ import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.TermQuery; import org.apache.lucene.search.WildcardQuery; -//import org.apache.lucene.store.Directory; -//import org.apache.lucene.store.SimpleFSDirectory; -//import org.apache.lucene.util.Version; import org.verisign.joid.consumer.OpenIdFilter; @@ -609,6 +598,24 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) doSlideSorter(request, response, loggedInAs, userQuery); } else if (endOfRequest.equals("status.html")) { doStatus(request, response, loggedInAs); + } else if (endOfRequest.startsWith("dataProviderForm")) { + if (!EDStatic.dataProviderFormActive) + throw new SimpleException(MessageFormat.format(EDStatic.disabled, + "Data Provider From")); + else if (endOfRequest.equals("dataProviderForm.html")) + doDataProviderForm(request, response, loggedInAs); + else if (endOfRequest.equals("dataProviderForm1.html")) + doDataProviderForm1(request, response, loggedInAs, ipAddress); + else if (endOfRequest.equals("dataProviderForm2.html")) + doDataProviderForm2(request, response, loggedInAs, ipAddress); + else if (endOfRequest.equals("dataProviderForm3.html")) + doDataProviderForm3(request, response, loggedInAs, ipAddress); + else if (endOfRequest.equals("dataProviderForm4.html")) + doDataProviderForm4(request, response, loggedInAs, ipAddress); + else if (endOfRequest.equals("dataProviderFormDone.html")) + doDataProviderFormDone(request, response, loggedInAs); + else sendResourceNotFoundError(request, response, "the first subdirectory or file is unknown"); + } else if (protocol.equals("subscriptions")) { doSubscriptions(request, response, loggedInAs, ipAddress, endOfRequest, protocol, protocolEnd + 1, userQuery); @@ -621,8 +628,7 @@ public void doGet(HttpServletRequest request, HttpServletResponse response) } else if (endOfRequest.equals("version")) { doVersion(request, response); } else { - if (verbose) String2.log(EDStatic.resourceNotFound + " end of protocol list"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "the first subdirectory or file is unknown"); } //tally @@ -726,8 +732,7 @@ public void doIndex(HttpServletRequest request, HttpServletResponse response, //only thing left should be erddap/index.html request if (!requestUrl.equals("/" + EDStatic.warName + "/index.html")) { - if (verbose) String2.log(EDStatic.resourceNotFound + " not /index.html"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "index.html expected"); return; } @@ -1522,6 +1527,1536 @@ public void doLogout(HttpServletRequest request, HttpServletResponse response, } } + /** + * This shows the start of the forms for data providers to fill out. + * Note: default URL length (actually, the whole header) is 8KB. + * That should be plenty. For longer, see tomcat settings: + * http://serverfault.com/questions/56691/whats-the-maximum-url-length-in-tomcat + * + */ + public void doDataProviderForm(HttpServletRequest request, HttpServletResponse response, + String tLoggedInAs) throws Throwable { + + String tErddapUrl = EDStatic.erddapUrl(tLoggedInAs); + OutputStream out = getHtmlOutputStream(request, response); + Writer writer = getHtmlWriter(tLoggedInAs, "Data Provider Form", out); + + try { + writer.write(EDStatic.youAreHere(tLoggedInAs, "Data Provider Form")); + +//begin text +//the overall table that constrains the width of the text +writer.write( +"
    DatasetSizeLast Modified
         +Folder  1981 --
             +1981236-1981240.pfv51sst_5day.hdf70.03 Mbytes2009-11-23 17:58:53Z
    + int td1Po = thisRow.indexOf("= 0? thisRow.indexOf("= 0? thisRow.indexOf(" 0) { + //This is a folder row. + //Row with current dir doesn't match regex; row for subdir does. + if (diagnosticMode) + String2.log("=== folder row"); + String content1 = String2.extractRegex(td1, "href='[^']*/catalog.html'>", 0); + if (recursive && content1 != null && content1.length() > 21) { //21 is non-.* stuff + content1 = content1.substring(6, content1.length() - 2); + completelySuccessful = addToThreddsUrlList( + url + content1, fileNameRegex, recursive, dirsToo, + childUrls, lastModified, size); + } + po = nextRow; + continue; + } + + //file info row will have 3 's. So if not, skip this row. + if (td3Po < 0) { + po = nextRow; + if (diagnosticMode) + String2.log("=== td3Po<0 so go to next row"); + continue; + } + + //look for content in the 3 's + String content1 = String2.extractRegex(td1, ".*", 0); + String content2 = String2.extractRegex( + thisRow.substring(td2Po, td3Po), ".*", 0); + String content3 = String2.extractRegex( + thisRow.substring(td3Po, thisRow.length()), ".*", 0); + content1 = content1 == null? "" : content1.substring(4, content1.length() - 5); + content2 = content2 == null? "" : content2.substring(4, content2.length() - 5); + content3 = content3 == null? "" : content3.substring(4, content3.length() - 5); + if (diagnosticMode) + String2.log("=== content #1=" + content1 + " #2=" + content2 + + " #3=" + content3); + if (content1.length() == 0) { + if (reallyVerbose) + String2.log("WARNING: No content in first ...
    \n" + +"\n" + +"\n" + +"
    \n" + +"This Data Provider Form is for people who have data and want it to be served by this ERDDAP.\n" + +"The overview of the process is:\n" + +"
      \n" + +"
    1. You fill out a 4-part form. When you finish a part,\n" + +" the information you just entered is\n"+ +" sent to the administrator of this ERDDAP (" + + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ").\n" + +"
    2. The ERDDAP administrator will contact you to figure out the best way to\n" + +" transfer the data and to work out other details.\n" + +"  \n" + +"
    \n" + +"\n" + +"

    Things You Need To Know

    \n" + +"The Goal\n" + +"
    The goal of this form is to help you create a description of your dataset\n" + +" (\"metadata\"), so that someone who knows nothing about this dataset will be\n" + +" able to find the dataset when they search for this type of dataset,\n" + +" understand the dataset, use the data properly, and give you credit\n" + +" for having created the dataset.\n" + +"

    Don't skim.\n" + +"
    Please work your way down this document, reading everything carefully. Don't skim.\n" + +" If you skip over some important point, it will just be twice as much\n" + +" work for you (and me) later.\n" + +"

    Do your best.\n" + +"
    This is the first pass at creating metadata for this dataset.\n" + +" The ERDDAP administrator will edit this and work with you to improve it.\n" + +" It is even easy to change the metadata after the dataset is in ERDDAP.\n" + +"
    But, it is better to get this right the first time.\n" + +"
    Please focus, read carefully, be patient, be diligent, and do your best.\n" + +"

    Helpful Hint Icons " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Success! As you see, when you move/hover your mouse over" + + "
    these question mark icons, you will see helpful hints" + + "
    and term definitions.") + "\n" + +"
    If you move/hover your mouse over these question mark icons,\n" + +" you will see helpful hints and term definitions.\n" + +"
    Try it now with the icon above.\n" + +"
    There are lots of these icons on this web page and in ERDDAP in general.\n" + +" The information they contain is essential for filling out this form properly.\n" + +"

    How much time will this take?\n" + +"
    It takes an hour to do a really good job with this form,\n" + +" less if you have all the needed information handy.\n" + +"

    Running out of time?\n" + +"
    If you run out of time before you finish all 4 parts, just leave the\n" + +" tab open in your browser so you can come back and finish it later.\n" + +" There is no time limit for finishing all 4 parts.\n" + +"

    Large Number of Datasets?\n" + +"
    If you have structured information (for example, ISO 19115 files) for a large\n" + +" number of datasets, there are probably ways that we can work together to\n" + +" (semi-)automate the process of getting the datasets into ERDDAP.\n" + +" Please email the administrator of this ERDDAP\n" + +" (" + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ")\n" + +" to discuss the options.\n" + +"

    Need help?\n" + +"
    If you have questions or need help while filling out this form,\n" + +" please send an email to the administrator of this ERDDAP\n" + +" (" + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ").\n" + +"
     \n" + +"\n" + + +"

    " + + "Click Here for Part 1 (of 4) of the Data Provider Form\n" + + +//the end of the overall table that constrains the width of the text +"

    \n"); + + + } catch (Throwable t) { + EDStatic.rethrowClientAbortException(t); //first thing in catch{} + writer.write(EDStatic.htmlForException(t)); + } + endHtmlWriter(out, writer, tErddapUrl, false); + return; + } + + /** + * This is part 1 of the Data Provider Form -- The Data. + */ + public void doDataProviderForm1(HttpServletRequest request, HttpServletResponse response, + String tLoggedInAs, String ipAddress) throws Throwable { + + String tErddapUrl = EDStatic.erddapUrl(tLoggedInAs); + OutputStream out = null; + Writer writer = null; + + try { + + //getParameters + //gridded options + String griddedOptions[] = {"(No, I have tabular data.)", + ".bufr files", ".grib files", ".hdf files", ".mat files", + ".nc files", + "(let's talk)"}; + int griddedOption = Math.max(0, String2.indexOf(griddedOptions, + request.getParameter("griddedOption"))); + + //tabular options + String tabularOptions[] = {"(No, I have gridded data.)", + ".csv files", "database", "Excel files", + ".hdf files", ".mat files", ".nc files", + "(let's talk)"}; + int tabularOption = Math.max(0, String2.indexOf(tabularOptions, + request.getParameter("tabularOption"))); + + //frequency options + String frequencyOptions[] = { + "never", + "rarely", + "yearly", + "monthly", + "daily", + "hourly", + "every minute", + "(irregularly)", + "(let's talk)"}; + int frequencyOption = String2.indexOf(frequencyOptions, + request.getParameter("frequencyOption")); + + String + tYourName = request.getParameter("yourName"), + tEmailAddress = request.getParameter("emailAddress"), + tTimestamp = request.getParameter("timestamp"); + + //validate (same order as form) + StringBuilder errorMsgSB = new StringBuilder(); + tYourName = HtmlWidgets.validateIsSomethingNotTooLong( + "Your Name", "", tYourName, 50, errorMsgSB); + tEmailAddress = HtmlWidgets.validateIsSomethingNotTooLong( + "Email Address", "", tEmailAddress, 50, errorMsgSB); + tTimestamp = HtmlWidgets.validateIsSomethingNotTooLong( + "Timestamp", Calendar2.getCurrentISODateTimeStringLocal(), + tTimestamp, 20, errorMsgSB); + if (griddedOption == 0 && tabularOption == 0) + errorMsgSB.append( + "
    • Error: Please specify (below) how your gridded or tabular data is stored.\n"); + frequencyOption = HtmlWidgets.validate0ToMax( + "Frequency", 0, frequencyOption, frequencyOptions.length - 1, errorMsgSB); + if (errorMsgSB.length() > 0) + errorMsgSB.insert(0, + "
    Please fix these problems, then 'Submit' this part of the form again.\n"); + + String fromInfo = tYourName + " <" + tEmailAddress + "> at " + tTimestamp; + + //if this is a submission, + boolean isSubmission = "Submit".equals(request.getParameter("Submit")); + if (isSubmission && errorMsgSB.length() == 0) { + //convert the info into psuedo datasets.xml + String content = + "Data Provider Form - Part 1\n" + //important! Bob's erd.data gmail filter looks for 'Data Provider Form' + " from " + fromInfo + "\n" + + " ipAddress=" + ipAddress + "\n" + + "\n" + + "griddedOption=" + griddedOptions[griddedOption] + "\n" + + "tabularOption=" + tabularOptions[tabularOption] + "\n" + + "frequencyOption=" + frequencyOptions[frequencyOption] + "\n" + + "\n"; + + //log the content to /logs/dataProviderForm.log + String error = String2.appendFile( + EDStatic.fullLogsDirectory + "dataProviderForm.log", + "*** " + content); + if (error.length() > 0) + String2.log(String2.ERROR + + " while writing to logs/dataProviderForm.log:\n" + + error); + //email the content to the admin + EDStatic.email(EDStatic.adminEmail, + "Data Provider Form - Part 1, from " + fromInfo, + content); + + //redirect to part 2 + sendRedirect(response, tErddapUrl + "/dataProviderForm2.html?" + + "yourName=" + SSR.minimalPercentEncode(tYourName) + + "&emailAddress=" + SSR.minimalPercentEncode(tEmailAddress) + + "×tamp=" + SSR.minimalPercentEncode(tTimestamp)); + + return; + } + + //write the HTML + out = getHtmlOutputStream(request, response); + writer = getHtmlWriter(tLoggedInAs, "Data Provider Form - Part 1", out); + writer.write(EDStatic.youAreHere(tLoggedInAs, "Data Provider Form - Part 1")); + + //begin form + String formName = "f1"; + HtmlWidgets widgets = new HtmlWidgets("", false, //style, false=not htmlTooltips + EDStatic.imageDirUrl(tLoggedInAs)); + widgets.enterTextSubmitsForm = false; + writer.write(widgets.beginForm(formName, + //this could be POST to deal with lots of text. + //but better to change tomcat settings (above) and keep ease of use + "GET", tErddapUrl + "/dataProviderForm1.html", "") + "\n"); + + //hidden fields + writer.write( + widgets.hidden("timestamp", XML.encodeAsHTML(tTimestamp)) + + "\n"); + +//begin text +//the overall table that constrains the width of the text +writer.write( +"\n" + +"\n" + +"\n" + +"
    \n" + +"This is part 1 (of 4) of the Data Provider Form.\n" + +"
    Need help? Send an email to the administrator of this ERDDAP (" + + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ").\n" + +"
     \n" + +"\n"); + +//error message? +if (isSubmission && errorMsgSB.length() > 0) +writer.write("" + errorMsgSB.toString() + " " + + "
     \n"); + +//Contact Info +writer.write( +"

    Your Contact Information

    \n" + +"This will be used by the ERDDAP administrator to contact you.\n" + +"This won't go in the dataset's metadata or be made public.\n" + +"

    What is your name? " + +widgets.textField("yourName", "", //tooltip + 30, 50, tYourName, "") + +"
    What is your email address? " + +widgets.textField("emailAddress", "", //tooltip + 30, 50, tEmailAddress, "") + +"
    This dataset submission's timestamp is " + tTimestamp + ".\n" + +"\n"); + +//The Data +writer.write( +"

    The Data

    \n" + +"ERDDAP deals with a dataset in one of two ways: as gridded data or as tabular data.\n" + + +//Gridded Data +"

    Gridded Data\n" + +"
    ERDDAP can serve data from various types of data files\n" + +"(and from OPeNDAP servers like Hyrax, THREDDS, GrADS, ERDDAP) that contain multi-dimensional\n" + +"gridded data, for example, Level 3 sea surface temperature data (from a satellite) with\n" + +"three dimensions: [time][latitude][longitude].\n" + +"

    The data for a gridded dataset can be stored in one file or many files\n" + +"(typically with one time point per file).\n" + +"

    If your dataset is already served via an OPeNDAP server,\n" + +"skip this form and just email the dataset's OPeNDAP URL\n" + +"to the administrator of this ERDDAP \n" + +" (" + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ").\n" + +"

    How is your gridded data stored?\n" + +widgets.select("griddedOption", "", 1, griddedOptions, griddedOption, "") + +"\n" + + +//Tabular Data +"

    Tabular Data\n" + +"
    ERDDAP can also serve data that can be represented as a single, database-like\n" + +"table, where there is a column for each type of data and a row for each observation.\n" + +"This includes:\n" + +"

      \n" + +"
    • All data that is currently stored in a database.\n" + +"
      (See Data in Databases\n" + +" below for more information.)\n" + +"
    • All in situ data.\n" + +"
      Examples: a time series from an instrument or several similar instruments,\n" + +" profile data from a CTD or a group of CTD's,\n" + +" or data collected during a ship's cruise (the similar cuises over several years).\n" + +"
    • Non-geospatial data that can be represented as a table of data,\n" + +"
      Examples: data from a laboratory experiment, genetic sequence data,\n" + +"
      or a list of bibliographic references.\n" + +"
    • Collections of other types of files (for example, image or audio files).\n" + +"
      ERDDAP can present the file names in a table and let users\n" + +" view or download the files.\n" + +"
    \n" + +"The data for a tabular dataset can be stored in one file or many files\n" + +"(typically with data for one station, one glider, one animal, or one cruise per file).\n" + +"We recommend making one dataset with all of the data that is very similar,\n" + +"and not a lot of separate datasets.\n" + +"For example, you might make one dataset with data from a group of moored buoys,\n" + +"a group of gliders, a group of animals, or a group of cruises (for example, annually on one line).\n" + +"

    How is your tabular data stored?\n" + +widgets.select("tabularOption", "", 1, tabularOptions, tabularOption, "") + +"\n" + + +//Frequency Of Changes +"

    Frequency of Changes\n" + +"
    Some datasets get new data frequently. Some datasets will never be changed.\n" + +"
    How often will this data be changed?\n" + +widgets.select("frequencyOption", "", 1, frequencyOptions, frequencyOption, "") + +"
     \n" + +"\n"); + +//Submit +writer.write( +"

    Finished with part 1?

    \n" + +"Click\n" + +widgets.button("submit", "Submit", "", "Submit", "") + +"to send this information to the ERDDAP administrator and move on to part 2 (of 4).\n" + +"\n"); + +//Data in Databases +writer.write( +"
     " + +"
    \n" + +"

    Additional information about
    Data in Databases

    \n" + +"If your data is in a database,\n" + +"you need to make one,\n" + +" denormalized\n" + +"table or\n" + +" view\n" + +"with all of the data that you want to make available as one dataset in ERDDAP.\n" + +"For large, complex databases, it may make sense to separate out several chunks\n" + +"as denormalized tables,\n" + +"each with a different type of data, which will become separate datasets in ERDDAP.\n" + +"Talk this over with the administrator of this ERDDAP (" + + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ").\n" + +"\n" + +"

    Making a denormalized table may sound like a crazy idea to you.\n" + +"Please trust us. The denormalized table solves several problems:\n" + +"

      \n" + +"
    • It's vastly easier for users.\n" + +"
      When ERDDAP presents the dataset as one, simple, denormalized table,\n" + +" it is very easy for anyone to understand the data. Most users have never\n" + +" heard of normalized tables, and very few understand\n" + +" keys, foreign keys, or table joins,\n" + +" and they almost certainly don't know the details of the different types of joins,\n" + +" or how to specify the SQL to do a join (or multiple joins). Using a denormalized\n" + +" table avoids all those problems. This reason alone justifies the use of a\n" + +" denormalized table for the presentation of the data to ERDDAP users.\n" + +"
       \n" + +"
    • You can make changes for ERDDAP without changing your tables.\n" + +"
      ERDDAP has a few requirements that may be different from how you have set\n" + +" up your database.\n" + +"
      For example, ERDDAP requires that timestamp data be stored in 'timestamp\n" + +" with timezone' fields.\n" + +"
      By making a separate table/view for ERDDAP, you can make these changes\n" + +" when you make the denormalized table for ERDDAP.\n" + +" Thus, you don't have to make any changes to your tables.\n" + +"
       \n" + +"
    • ERDDAP will recreate some of the structure of the normalized tables.\n" + +"
      You can specify which columns of data come from the 'outer' tables and therefore\n" + +" have a limited number of distinct values. ERDDAP will collect all of the\n" + +" distinct values in each of these columns\n" + +" and present them to users in drop-down lists.\n" + +"
       \n" + +"
    • A denormalized table makes the data hand-off from you to the ERDDAP\n" + +" administrator easy.\n" + +"
      You're the expert for this dataset,\n" + +" so it makes sense that you make the decisions\n" + +" about which tables and which columns to join and how to join them.\n" + +" So you don't have to hand us several tables and detailed instructions for\n" + +" several joins, you just have to give us access to the denormalized table.\n" + +"
       \n" + +"
    • A denormalized table allows for efficient access to the data.\n" + +"
      The denormalized form is usually faster to access than the normalized form.\n" + +" Joins can be slow. Multiple joins can be very slow.\n" + +"
    \n" + +"In order to get the data from the database into ERDDAP, there are three options:\n" + +"
      \n" + +"
    • Recommended Option:\n" + +"
      You can create a comma- or tab-separated-value file with\n" + +" the data from the denormalized table.\n" + +"
      If the dataset is huge, then it makes sense to create several files,\n" + +" each with a cohesive subset of the denormalized table\n" + +" (for example, data from a smaller time range).\n" + +"

      The big advantage here is that ERDDAP will be able to handle user requests\n" + +" for data without any further effort by your database.\n" + +" So ERDDAP won't be a burden on your database or a security risk.\n" + +" This is the best option under almost all circumstances because ERDDAP can usually\n" + +" get data from files faster than from a database\n" + +" (if we convert the .csv files to .ncCF files). (Part of the reason is that\n" + +" ERDDAP+files is a read-only system and doesn't have to deal with making changes\n" + +" while providing\n" + +" ACID\n" + +" (Atomicity, Consistency, Isolation, Durability).)\n" + +" Also, you probably won't need a separate server since we can store the data\n" + +" on one of our RAIDs and access it with an existing ERDDAP on an existing server.\n" + +"

    • Okay Option:\n" + +"
      You set up a new database on a different computer with just the\n" + +" denormalized table.\n" + +"
      Since that database can be a free and open source database like PostgreSQL,\n" + +" this option needn't cost a lot.\n" + +"

      The big advantage here are that ERDDAP will be able to handle user requests\n" + +" for data without any further effort by your current database.\n" + +" So ERDDAP won't be a burden on your current database.\n" + +" This also eliminates a lot of security concerns since ERDDAP will not have\n" + +" access to your current database.\n" + +"

    • Discouraged Option:\n" + +"
      We can connect ERDDAP to your current database.\n" + +"
      To do this, you need to:\n" + +"
        \n" + +"
      • Create a separate table or view with the denormalized table of data.\n" + +"
      • Create an \"erddap\" user who has read-only access to only the denormalized table(s).\n" + +"
         \n" + +"
      \n" + +" This is an option if the data changes very frequently and you want to give\n" + +" ERDDAP users instant access to those changes; however, even so,\n" + +" it may make sense to use the file option above and periodically\n" + +" (every 30 minutes?) replace the file that has today's data.\n" + +"
      The huge disadvantages of this approach are that ERDDAP user requests will\n" + +" probably place an unbearably large burden on your database and that\n" + +" the ERDDAP connection is a security risk (although we can minimize/manage the risk).\n" + +"
    \n" + +"When you talk with the administrator of this ERDDAP (" + + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + "),\n" + +"you can discuss which of these options\n" + +"to pursue and how to handle the details.\n" + +"\n"); + +//the end of the overall table that constrains the width of the text +writer.write( +"
    \n"); + +//end form +writer.write(widgets.endForm()); + + } catch (Throwable t) { + EDStatic.rethrowClientAbortException(t); //first thing in catch{} + if (out == null || writer == null) + throw t; + else writer.write(EDStatic.htmlForException(t)); + } + endHtmlWriter(out, writer, tErddapUrl, false); + return; + } + + /** + * This is Data Provider Form 2 + */ + public void doDataProviderForm2(HttpServletRequest request, HttpServletResponse response, + String tLoggedInAs, String ipAddress) throws Throwable { + + String tErddapUrl = EDStatic.erddapUrl(tLoggedInAs); + OutputStream out = null; + Writer writer = null; + + try { + + //get parameters + //cdm_data_type options + String cdmDataTypes[] = { + "Grid", + "Point", + "Profile", + "TimeSeries", + "TimeSeriesProfile", + "Trajectory", + "TrajectoryProfile", + "Other"}; + int defaultCdmDataType = String2.indexOf(cdmDataTypes, "Other"); + int tCdmDataType = String2.indexOf(cdmDataTypes, + request.getParameter("cdm_data_type")); + String cdmDataTypeHelp = +"CDM is Unidata's Common Data Model, a way of categorizing datasets" + +"
    based on the geometry of the dataset. Pick the cdm_data_type which" + +"
    is most appropriate:" + +"
    • Use Grid for all gridded datasets." + +"
    • Use Point for a dataset with unrelated points." + +"
      Example: whale sightings, or stranded marine mammal sightings." + +"
    • Use Profile for data from multiple depths at one or more longitude," + +"
      latitude locations." + +"
      Example: CTD's if not associated with a TimeSeries or Trajectory." + +"
    • Use TimeSeries for data from a set of stations with fixed longitude," + +"
      latitude(,altitude)." + +"
      Examples: moored buoys, or stations." + +"
    • Use TimeSeriesProfile for profiles from a set of stations." + +"
      Examples: stations with CTD's." + +"
    • Use Trajectory for data from a set of longitude,latitude(,altitude)" + +"
      paths called trajectories." + +"
      Examples: ships, surface gliders, or tagged animals." + +"
    • Use TrajectoryProfile for profiles along trajectories." + +"
      Examples: ships + CTD's, or profiling gliders." + +"
    • Use Other if the dataset doesn't have latitude,longitude data or if" + +"
      no other type is appropriate." + +"
      Examples: laboratory analyses, or fish landings by port name (if no lat,lon)."; + + String creatorTypes[] = { + "person", + "group", + "institution", + "position"}; + int tCreatorType = String2.indexOf(creatorTypes, + request.getParameter("creator_type")); + + String + //required + tYourName = request.getParameter("yourName"), + tEmailAddress = request.getParameter("emailAddress"), + tTimestamp = request.getParameter("timestamp"), + tTitle = request.getParameter("title"), + tSummary = request.getParameter("summary"), + tCreatorName = request.getParameter("creator_name"), + tCreatorEmail = request.getParameter("creator_email"), + tInstitution = request.getParameter("institution"), + tInfoUrl = request.getParameter("infoUrl"), + tLicense = request.getParameter("license"), + //optional + tHistory = request.getParameter("history"), + tAcknowledgement = request.getParameter("acknowledgement"), + tID = request.getParameter("id"), + tNamingAuthority = request.getParameter("naming_authority"), + tProductVersion = request.getParameter("product_version"), + tReferences = request.getParameter("references"), + tComment = request.getParameter("comment"); + + //validate (same order as form) + StringBuilder errorMsgSB = new StringBuilder(); + tYourName = HtmlWidgets.validateIsSomethingNotTooLong( + "Your Name", "?", tYourName, 50, null); + tEmailAddress = HtmlWidgets.validateIsSomethingNotTooLong( + "Email Address", "?", tEmailAddress, 50, null); + tTimestamp = HtmlWidgets.validateIsSomethingNotTooLong( + "Timestamp", Calendar2.getCurrentISODateTimeStringLocal() + "?", + tTimestamp, 20, null); + + //required + tTitle = HtmlWidgets.validateIsSomethingNotTooLong("title", "", tTitle, 80, errorMsgSB); + tSummary = HtmlWidgets.validateIsSomethingNotTooLong("summary", "", tSummary, 500, errorMsgSB); + tCreatorName = HtmlWidgets.validateIsSomethingNotTooLong("creator_name", "", tCreatorName, 80, errorMsgSB); + tCreatorType = HtmlWidgets.validate0ToMax("creator_type", 0, + tCreatorType, creatorTypes.length - 1, errorMsgSB); + tCreatorEmail = HtmlWidgets.validateIsSomethingNotTooLong("creator_email", "", tCreatorEmail, 80, errorMsgSB); + tInstitution = HtmlWidgets.validateIsSomethingNotTooLong("institution", "", tInstitution, 80, errorMsgSB); + tInfoUrl = HtmlWidgets.validateIsSomethingNotTooLong("infoUrl", "", tInfoUrl, 120, errorMsgSB); + tLicense = HtmlWidgets.validateIsSomethingNotTooLong("license", "[standard]", tLicense, 500, errorMsgSB); + tCdmDataType = HtmlWidgets.validate0ToMax("cdm_data_type", defaultCdmDataType, + tCdmDataType, cdmDataTypes.length - 1, errorMsgSB); + //optional + tAcknowledgement= HtmlWidgets.validateNotTooLong("acknowledgement", "", tAcknowledgement, 350, errorMsgSB); + tHistory = HtmlWidgets.validateNotTooLong("history", "", tHistory, 500, errorMsgSB); + tID = HtmlWidgets.validateNotTooLong("id", "", tID, 80, errorMsgSB); + tNamingAuthority= HtmlWidgets.validateNotTooLong("naming_authority", "", tNamingAuthority, 160, errorMsgSB); + tProductVersion = HtmlWidgets.validateNotTooLong("product_version", "", tProductVersion, 40, errorMsgSB); + tReferences = HtmlWidgets.validateNotTooLong("references", "", tReferences, 500, errorMsgSB); + tComment = HtmlWidgets.validateNotTooLong("comment", "", tComment, 350, errorMsgSB); + if (errorMsgSB.length() > 0) + errorMsgSB.insert(0, + "
    Please fix these problems, then 'Submit' this part of the form again.\n"); + + String fromInfo = tYourName + " <" + tEmailAddress + "> at " + tTimestamp; + + //if this is a submission, + boolean isSubmission = "Submit".equals(request.getParameter("Submit")); + if (isSubmission && errorMsgSB.length() == 0) { + //convert the info into psuedo datasets.xml + String content = +"Data Provider Form - Part 2\n" + //important! Bob's erd.data gmail filter looks for this +" from " + fromInfo + "\n" + +" ipAddress=" + ipAddress + "\n" + +"\n" + +" \n" + +" " + XML.encodeAsXML(tAcknowledgement) + "\n" + +" " + XML.encodeAsXML(cdmDataTypes[tCdmDataType]) + "\n" + +" ???\n" + +" ???\n" + +" " + XML.encodeAsXML(tComment) + "\n" + +" ACDD-1.3, COARDS, CF-1.6\n" + +" " + XML.encodeAsXML(tCreatorName) + "\n" + +" " + XML.encodeAsXML(creatorTypes[tCreatorType]) + "\n" + +" " + XML.encodeAsXML(tCreatorEmail) + "\n" + +" " + XML.encodeAsXML(tInfoUrl) /*yes, infoUrl*/ + "\n" + +" " + XML.encodeAsXML(tHistory) + "\n" + +" " + XML.encodeAsXML(tID) + "\n" + +" " + XML.encodeAsXML(tInfoUrl) + "\n" + +" " + XML.encodeAsXML(tInstitution) + "\n" + +" \n" + +" " + XML.encodeAsXML(tLicense) + "\n" + +" "+ XML.encodeAsXML(tNamingAuthority) + "\n" + +" " + XML.encodeAsXML(tProductVersion) + "\n" + +" " + XML.encodeAsXML(tReferences) + "\n" + +" (local files)\n" + +" CF Standard Name Table v29\n" + +" " + XML.encodeAsXML(tSummary) + "\n" + +" " + XML.encodeAsXML(tTitle) + "\n" + +" \n" + +"\n"; + + //log the content to /logs/dataProviderForm.log + String error = String2.appendFile( + EDStatic.fullLogsDirectory + "dataProviderForm.log", + "*** " + content); + if (error.length() > 0) + String2.log(String2.ERROR + + " while writing to logs/dataProviderForm.log:\n" + + error); + //email the content to the admin + EDStatic.email(EDStatic.adminEmail, + "Data Provider Form - Part 2, from " + fromInfo, + content); + + //redirect to part 3 + sendRedirect(response, tErddapUrl + "/dataProviderForm3.html?" + + "yourName=" + SSR.minimalPercentEncode(tYourName) + + "&emailAddress=" + SSR.minimalPercentEncode(tEmailAddress) + + "×tamp=" + SSR.minimalPercentEncode(tTimestamp)); + + return; + } + + //write the HTML + out = getHtmlOutputStream(request, response); + writer = getHtmlWriter(tLoggedInAs, "Data Provider Form - Part 2", out); + writer.write(EDStatic.youAreHere(tLoggedInAs, "Data Provider Form - Part 2")); + + //begin form + String formName = "f1"; + HtmlWidgets widgets = new HtmlWidgets("", false, //style, false=not htmlTooltips + EDStatic.imageDirUrl(tLoggedInAs)); + widgets.enterTextSubmitsForm = false; + writer.write(widgets.beginForm(formName, + //this could be POST to deal with lots of text. + //but better to change tomcat settings (above) and keep ease of use + "GET", tErddapUrl + "/dataProviderForm2.html", "") + "\n"); + + //hidden fields + writer.write( + widgets.hidden("yourName", XML.encodeAsHTML(tYourName)) + + widgets.hidden("emailAddress", XML.encodeAsHTML(tEmailAddress)) + + widgets.hidden("timestamp", XML.encodeAsHTML(tTimestamp)) + + "\n"); + +//begin text +//the overall table that constrains the width of the text +writer.write( +"\n" + +"\n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +//Optional +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +"
    \n" + +"This is part 2 (of 4) of the Data Provider Form\n" + +"
    from " + XML.encodeAsHTML(fromInfo) + ".\n" + +"
    Need help? Send an email to the administrator of this ERDDAP (" + + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ").\n" + +"
     \n" + +"\n"); + +//error message? +if (isSubmission && errorMsgSB.length() > 0) +writer.write("" + errorMsgSB.toString() + " " + + "
     \n"); + +//Global Metadata +writer.write( +"

    Global Metadata

    \n" + +"Global metadata is information about the entire dataset. It is a set of\n" + +"attribute=value pairs, for example,\n" + +"
    title=Spray Gliders, Scripps Institution of Oceanography\n" + +"

    .nc files — If your data is in .nc files that already have some metadata,\n" + +"just provide the information below for attributes that aren't in your files\n" + +"or where you want to change the attribute's value.\n" + +"\n" + +"
    " + widgets.beginTable(0, 0, "") + +//Required +"

    Required\n" + +"
    title\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is a short (<=80 characters) description of the dataset. For example," + +"
    Spray Gliders, Scripps Institution of Oceanography") + " \n" + +"     		\n" + +widgets.textField("title", "", 80, 80, tTitle, "") + +"
    summary\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is a paragraph describing the dataset. (<=500 characters)" + + "
    The summary should answer these questions:" + + "
    • Who created the dataset?" + + "
    • What information was collected?" + + "
    • When was the data collected?" + + "
    • Where was it collected?" + + "
    • Why was it collected?" + + "
    • How was it collected?") + " \n" + +"     		\n" + +"
    creator_name\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is the name of the primary person, group, institution," + + "
    or position that created the data. For example," + + "
    John Smith") + " " + +"     		\n" + +widgets.textField("creator_name", "", 40, 40, tCreatorName, "") + +"
    creator_type\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This identifies the creator_name (above) as a person," + + "
    group, institution, or position.") + " " + +"     		\n" + +widgets.select("creator_type", "", 1, creatorTypes, tCreatorType, "") + +"
    creator_email\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is the best contact email address for the creator of this data." + + "
    Use your judgment — the creator_email might be for a" + + "
    different entity than the creator_name." + + "
    For example, your.name@yourOrganization.org") + " \n" + +"     		\n" + +widgets.textField("creator_email", "", 40, 40, tCreatorEmail, "") + +"
    institution\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is the short/abbreviated form of the name of the primary" + + "
    organization that created the data. For example," + + "
    NOAA NMFS SWFSC") + " \n" + +"     		\n" + +widgets.textField("institution", "", 40, 40, tInstitution, "") + +"
    infoUrl\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is a URL with information about this dataset." + + "
    For example, http://spray.ucsd.edu" + + "
    If there is no URL related to the dataset, provide" + + "
    a URL for the group or organization.") + " \n" + +"     		\n" + +widgets.textField("infoUrl", "", 80, 120, tInfoUrl, "") + +"
    license\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is the license and disclaimer for use of this data." + + "
    ERDDAP has a standard license, which you can use via [standard]" + + "
    You can either add to that or replace it. (<=500 characters)" + + "
    The text of the standard license is:" + + "
    " + String2.replaceAll(EDStatic.standardLicense, "\n", "
    ") + + "    ") + " \n" + +"     		\n" + +"
    cdm_data_type\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, cdmDataTypeHelp) + " \n" + +" \n" + +widgets.select("cdm_data_type", "", 1, cdmDataTypes, tCdmDataType, "") + +"
     \n" + +"
    Optional\n" + +"  \n" + +" (Please provide the information if it is available for your dataset.)\n" + +"
    acknowledgement\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is the place to acknowledge various types of support for" + + "
    the project that produced this data. (<=350 characters) For example," + + "
    This project received additional funding from the NOAA" + + "
    Climate and Global Change Program.    ") + " \n" + +"     		\n" + +"
    history\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is a list of the actions (one per line) which led to the creation of this data." + + "
    Ideally, each line includes a timestamp and a description of the action. (<=500 characters) For example," + + "
    Datafiles are downloaded ASAP from http://oceandata.sci.gsfc.nasa.gov/MODISA/L3SMI/ to NOAA NMFS SWFSC ERD." + + "
    NOAA NMFS SWFSC ERD (erd.data@noaa.gov) uses NCML to add the time dimension and slightly modify the metadata.    ") + " \n" + +"     		\n" + +"
    id\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is an identifier for the dataset, as provided by" + + "
    its naming authority. The combination of \"naming authority\"" + + "
    and the \"id\" should be globally unique, but the id can be" + + "
    globally unique by itself also. IDs can be URLs, URNs, DOIs," + + "
    meaningful text strings, a local key, or any other unique" + + "
    string of characters. The id should not include white space" + + "
    characters." + + "
    For example, CMC0.2deg-CMC-L4-GLOB-v2.0") + " \n" + +"     		\n" + +widgets.textField("id", "", 40, 80, tID, "") + +"
    naming_authority\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is the organization that provided the id (above) for the dataset." + + "
    The naming authority should be uniquely specified by this attribute." + + "
    We recommend using reverse-DNS naming for the naming authority;" + + "
    URIs are also acceptable." + + "
    For example, org.ghrsst") + " \n" + +"     		\n" + +widgets.textField("naming_authority", "", 80, 160, tNamingAuthority, "") + +"
    product_version\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is the version identifier of this data. For example, if you" + + "
    plan to add new data yearly, you might use the year as the version identifier." + + "
    For example, 2014") + " \n" + +"     		\n" + +widgets.textField("product_version", "", 20, 40, tProductVersion, "") + +"
    references\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is one or more published or web-based references" + + "
    that describe the data or methods used to produce it. URL's and" + + "
    DOI's are recommend. (<=500 characters) For example,\n" + + "
    Hu, C., Lee Z., and Franz, B.A. (2012). Chlorophyll-a" + + "
    algorithms for oligotrophic oceans: A novel approach" + + "
    based on three-band reflectance difference, J. Geophys." + + "
    Res., 117, C01011, doi:10.1029/2011JC007395.    ") + " \n" + +"     		\n" + +"
    comment\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is miscellaneous information about the data, not" + + "
    captured elsewhere. (<=350 characters) For example," + + "
    No animals were harmed during the collection of this data.") + " \n" + +"     		\n" + +"
     \n" + +"
    \n" + +"\n"); + +//Submit +writer.write( +"

    Finished with part 2?

    \n" + +"Click\n" + +widgets.button("submit", "Submit", "", "Submit", "") + +"to send this information to the ERDDAP administrator and move on to part 3 (of 4).\n" + +"\n"); + +//the end of the overall table that constrains the width of the text +writer.write( +"\n" + +"\n"); + +//end form +writer.write(widgets.endForm()); + + } catch (Throwable t) { + EDStatic.rethrowClientAbortException(t); //first thing in catch{} + if (out == null || writer == null) + throw t; + else writer.write(EDStatic.htmlForException(t)); + } + endHtmlWriter(out, writer, tErddapUrl, false); + return; + } + + /** + * This shows the form for data providers to fill out. + * Note: default URL length (actually, the whole header) is 8KB. + * That should be plenty. For longer, see tomcat settings: + * http://serverfault.com/questions/56691/whats-the-maximum-url-length-in-tomcat + * + */ + public void doDataProviderForm3(HttpServletRequest request, HttpServletResponse response, + String tLoggedInAs, String ipAddress) throws Throwable { + + String tErddapUrl = EDStatic.erddapUrl(tLoggedInAs); + OutputStream out = null; + Writer writer = null; + + try { + + //get parameters + //dataType (pretend char doesn't exist?) + String groupOptions[] = { + "first", "second", "third", "fourth", "fifth"}; + int tGroup = Math.max(0, String2.indexOf(groupOptions, + request.getParameter("group"))); + + String dataTypeOptions[] = {"(unknown)", + "String", "boolean", "byte", "short", "int", "long", "float", "double"}; + String dataTypeHelp = + "This is the data type and precision of this variable." + + "
    If the data file uses a specific type (for example, in a .nc file)," + + "
      specify that type here." + + "
    If the data file doesn't use a specific type (for example, in a .csv file)," + + "
      specify the type that should be used in ERDDAP." + + "
    • Use (unknown) if you don't know." + + "
    • String is a series of characters." + + "
      (For databases, ERDDAP treats all non-numeric data types as Strings.)" + + "
    • boolean is either true or false. ERDDAP will convert these to bytes, 1 or 0." + + "
    • byte is an 8 bit signed integer, +/-127" + + "
    • short is a 16 bit signed integer, +/-32,767" + + "
    • int is a 32 bit signed integer, +/-2,147,483,647" + + "
    • long is a 64 bit signed integer, +/- ~1e19" + + "
    • float is a 32 bit floating point number (up to 7 significant digits)" + + "
    • double is a 64 bit floating point number (up to 17 significant digits)"; + + String ioosCategoryOptions[] = { + "Bathymetry", "Biology", "Bottom Character", "Colored Dissolved Organic Matter", + "Contaminants", "Currents", "Dissolved Nutrients", "Dissolved O2", "Ecology", + "Fish Abundance", "Fish Species", "Heat Flux", "Hydrology", "Ice Distribution", + "Identifier", "Location", "Meteorology", "Ocean Color", "Optical Properties", + "Other", "Pathogens", "pCO2", "Phytoplankton Species", "Pressure", + "Productivity", "Quality", "Salinity", "Sea Level", "Statistics", + "Stream Flow", "Surface Waves", "Taxonomy", "Temperature", "Time", + "Total Suspended Matter", "Unknown", "Wind", "Zooplankton Species", + "Zooplankton Abundance"}; + int ioosUnknown = String2.indexOf(ioosCategoryOptions, "Unknown"); + String ioosCategoryHelp = + "Pick the ioos_category which is most appropriate for this variable." + + "
    • Use Location for place names and for longitude, latitude," + + "
      altitude, and depth." + + "
    • Use Time for date/time." + + "
    • Use Taxonomy for species names." + + "
    • Use Identifier for cruise names, ship names, line names," + + "
      station names, equipment types, serial numbers, etc." + + "
    • Use Ocean Color for chlorophyll." + + "
    • Use Other if no other category in the list is close." + + "
    • Use Unknown if you really don't know."; + + String + tYourName = request.getParameter("yourName"), + tEmailAddress = request.getParameter("emailAddress"), + tTimestamp = request.getParameter("timestamp"); + + //variable attributes + int nVars = 10; + String + tSourceName[] = new String[nVars+1], + tDestinationName[] = new String[nVars+1], + tLongName[] = new String[nVars+1], + tUnits[] = new String[nVars+1], + tRangeMin[] = new String[nVars+1], + tRangeMax[] = new String[nVars+1], + tStandardName[] = new String[nVars+1], + tFillValue[] = new String[nVars+1], + tComment[] = new String[nVars+1]; + int tDataType[] = new int[ nVars+1], + tIoosCategory[] = new int[ nVars+1]; + + for (int var = 1; var <= nVars; var++) { + //String + tSourceName[var] = request.getParameter("sourceName" + var); + tDestinationName[var] = request.getParameter("destinationName" + var); + tLongName[var] = request.getParameter("long_name" + var); + tUnits[var] = request.getParameter("units" + var); + tRangeMin[var] = request.getParameter("rangeMin" + var); + tRangeMax[var] = request.getParameter("rangeMax" + var); + tStandardName[var] = request.getParameter("standard_name" + var); + tFillValue[var] = request.getParameter("FillValue" + var); //note that field name lacks leading '_' + tComment[var] = request.getParameter("comment" + var); //note that field name lacks leading '_' + + //int + tDataType[var] = Math.max(0, String2.indexOf(dataTypeOptions, + request.getParameter("dataType" + var))); + tIoosCategory[var] = String2.indexOf(ioosCategoryOptions, + request.getParameter("ioos_category" + var)); + if (tIoosCategory[var] < 0) + tIoosCategory[var] = ioosUnknown; + } + + //validate (same order as form) + StringBuilder errorMsgSB = new StringBuilder(); + tYourName = HtmlWidgets.validateIsSomethingNotTooLong( + "Your Name", "?", tYourName, 50, null); + tEmailAddress = HtmlWidgets.validateIsSomethingNotTooLong( + "Email Address", "?", tEmailAddress, 50, null); + tTimestamp = HtmlWidgets.validateIsSomethingNotTooLong( + "Timestamp", Calendar2.getCurrentISODateTimeStringLocal() + "?", + tTimestamp, 20, null); + + for (int var = 1; var <= nVars; var++) { + tSourceName[var] = var == 1? + HtmlWidgets.validateIsSomethingNotTooLong( + "sourceName #" + var, "", tSourceName[var], 50, errorMsgSB) : + HtmlWidgets.validateNotTooLong( + "sourceName #" + var, "", tSourceName[var], 50, errorMsgSB); + tDestinationName[var] = HtmlWidgets.validateNotTooLong( + "destinationName #" + var, "", tDestinationName[var], 50, errorMsgSB); + tLongName[var] = HtmlWidgets.validateNotTooLong( + "long_name #" + var, "", tLongName[var], 80, errorMsgSB); + tStandardName[var] = HtmlWidgets.validateNotTooLong( + "standard_name #" + var, "", tStandardName[var], 80, errorMsgSB); + tFillValue[var] = HtmlWidgets.validateNotTooLong( + "_FillValue #" + var, "", tFillValue[var], 20, errorMsgSB); + tUnits[var] = HtmlWidgets.validateNotTooLong( + "units #" + var, "", tUnits[var], 40, errorMsgSB); + tRangeMin[var] = HtmlWidgets.validateNotTooLong( + "range min #" + var, "", tRangeMin[var], 15, errorMsgSB); + tRangeMax[var] = HtmlWidgets.validateNotTooLong( + "range max #" + var, "", tRangeMax[var], 15, errorMsgSB); + tComment[var] = HtmlWidgets.validateNotTooLong( + "comment #" + var, "", tComment[var], 160, errorMsgSB); + } + if (errorMsgSB.length() > 0) + errorMsgSB.insert(0, + "
    Please fix these problems, then 'Submit' this part of the form again.\n"); + + String fromInfo = tYourName + " <" + tEmailAddress + "> at " + tTimestamp; + + //if this is a submission, + boolean isSubmission = "Submit".equals(request.getParameter("Submit")); + if (isSubmission && errorMsgSB.length() == 0) { + //convert the info into psuedo datasets.xml + StringBuilder content = new StringBuilder(); + content.append( + "Data Provider Form - Part 3\n" + //important! Bob's erd.data gmail filter looks for this + " from " + fromInfo + "\n" + + " ipAddress=" + ipAddress + "\n" + + "\n" + + "groupOf10=" + groupOptions[tGroup] + "\n" + + "\n"); + + for (int var = 1; var <= nVars; var++) + content.append( +" \n" + +" " + XML.encodeAsXML(tSourceName[var]) + "\n" + +" " + XML.encodeAsXML(tDestinationName[var]) + "\n" + +" " + XML.encodeAsXML(dataTypeOptions[tDataType[var]]) + "\n" + +" \n" + +" " + + XML.encodeAsXML(tRangeMin[var]) + "\n" + +" " + + XML.encodeAsXML(tRangeMax[var]) + "\n" + +" " + XML.encodeAsXML(tComment[var]) + "\n" + +" " + XML.encodeAsXML(tFillValue[var]) + "\n" + +" "+ XML.encodeAsXML(ioosCategoryOptions[tIoosCategory[var]]) + "\n" + +" " + XML.encodeAsXML(tLongName[var]) + "\n" + +" "+ XML.encodeAsXML(tStandardName[var])+ "\n" + +" " + XML.encodeAsXML(tUnits[var]) + "\n" + +" \n" + +" \n"); + content.append( + "\n" + + "\n"); + + //log the content to /logs/dataProviderForm.log + String error = String2.appendFile( + EDStatic.fullLogsDirectory + "dataProviderForm.log", + "*** " + content.toString()); + if (error.length() > 0) + String2.log(String2.ERROR + + " while writing to logs/dataProviderForm.log:\n" + + error); + //email the content to the admin + EDStatic.email(EDStatic.adminEmail, + "Data Provider Form - Part 3, from " + fromInfo, + content.toString()); + + //redirect to part 4 + sendRedirect(response, tErddapUrl + "/dataProviderForm4.html?" + + "yourName=" + SSR.minimalPercentEncode(tYourName) + + "&emailAddress=" + SSR.minimalPercentEncode(tEmailAddress) + + "×tamp=" + SSR.minimalPercentEncode(tTimestamp)); + + return; + } + + //write the HTML + out = getHtmlOutputStream(request, response); + writer = getHtmlWriter(tLoggedInAs, "Data Provider Form - Part 3", out); + writer.write(EDStatic.youAreHere(tLoggedInAs, "Data Provider Form - Part 3")); + + //begin form + String formName = "f1"; + HtmlWidgets widgets = new HtmlWidgets("", false, //style, false=not htmlTooltips + EDStatic.imageDirUrl(tLoggedInAs)); + widgets.enterTextSubmitsForm = false; + writer.write(widgets.beginForm(formName, + //this could be POST to deal with lots of text. + //but better to change tomcat settings (above) and keep ease of use + "GET", tErddapUrl + "/dataProviderForm3.html", "") + "\n"); + + //hidden fields + writer.write( + widgets.hidden("yourName", XML.encodeAsHTML(tYourName)) + + widgets.hidden("emailAddress", XML.encodeAsHTML(tEmailAddress)) + + widgets.hidden("timestamp", XML.encodeAsHTML(tTimestamp)) + + "\n"); + +//begin text +//the overall table that constrains the width of the text +writer.write( +"\n" + +"\n" + +"\n" + +" \n" + +"\n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +" \n" + +"\n" + +"
    \n" + +"This is part 3 (of 4) of the Data Provider Form\n" + +"
    from " + XML.encodeAsHTML(fromInfo) + ".\n" + +"
    Need help? Send an email to the administrator of this ERDDAP (" + + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ").\n" + +"
     \n" + +"\n"); + +//error message? +if (isSubmission && errorMsgSB.length() > 0) +writer.write("" + errorMsgSB.toString() + " " + + "
     \n"); + +//Variable Metadata +writer.write( +"

    Variable Metadata

    \n" + +"Variable metadata is information that is specific to a given variable within\n" + +"the dataset. It is a set of attribute=value pairs, for example,\n" + +"units=degree_C .\n" + +"\n" + +"

    Fewer Or More Than 10 Variables\n" + //n variables +"
    There are slots on this form for 10 variables.\n" + +"
    If your dataset has 10 or fewer variables, just use the slots that you need.\n" + +"
    If your dataset has more than 10 variables, then for each group of 10 variables:\n" + +"

      \n" + +"
    1. Identify the group: This is the\n" + +widgets.select("group", "", 1, groupOptions, tGroup, "") + +" group of 10 variables.\n" + +"
    2. Fill out the form below for that group of 10 variables.\n" + +"
    3. Click \"I'm finished!\" below to submit the information for that group of 10 variables.\n" + +"
    4. If this isn't the last group, then on the next web page (for Part 4 of this form),\n" + +" press your browser's Back button so that you can fill out this part of the form\n" + +" (Part 3) for the next group of 10 variables.\n" + +"
    \n" + +"\n" + +"

    .nc Files\n" + +"
    If your data is in .nc files that already have some metadata,\n" + +"just provide the information below for attributes that aren't in your files\n" + +"or where you want to change the attribute's value.\n" + +"\n"); + +for (int var = 1; var <= nVars; var++) +writer.write( +widgets.beginTable(0, 0, "") + +"

     
    Variable #" + var + "
    sourceName\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is the name of this variable currently used by the data source." + + "
    For example, wt" + + "
    This is case-sensitive.") + " \n" + +"     		\n" + +widgets.textField("sourceName" + var, "", 20, 50, tSourceName[var], "") + +"
    destinationName\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: You can specify a new, different name for this variable." + + "
    This new name is the one that will be shown to users in ERDDAP." + + "
    For example, waterTemp" + + "
    This is case-sensitive." + + "
    This MUST start with a letter (A-Z, a-z) and MUST be followed by\n" + + "
    0 or more characters (A-Z, a-z, 0-9, and _)." + + "
    • Use latitude for the main latitude variable." + + "
    • Use longitude for the main longitude variable." + + "
    • Use altitude if the variable measures height above sea level." + + "
    • Use depth if the variable measures distance below sea level." + + "
    • Use time for the main date/time variable." + + "
    • Otherwise, it is up to you. If you want to use the sourceName\n" + + "
      as the destinationName, leave this blank.") + " \n" + +"     		\n" + +widgets.textField("destinationName" + var, "", 20, 50, tDestinationName[var], "") + +"
    long_name\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "This is a longer, written-out version of the destinationName." + + "
    For example, Water Temperature" + + "
    Among other uses, it will be used as an axis title on graphs." + + "
    Capitalize each word in the long_name." + + "
    Don't include the units. (ERDDAP will add units when creating" + + "
      an axis title.)") + " \n" + +"     		\n" + +widgets.textField("long_name" + var, "", 40, 80, tLongName[var], "") + +"
    standard_name\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is the name from the CF Standard Name Table" + + "
      which is most appropriate for this variable.\n" + + "
    For example, sea_water_temperature." + + "
    If you don't already know, or if no CF Standard Name is" + + "
      appropriate, just leave this blank. We'll fill it in.") + " \n" + +"     		\n" + +widgets.textField("standard_name" + var, "", 60, 80, tStandardName[var], "") + +"
    dataType\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, dataTypeHelp) + " \n" + +" \n" + +widgets.select("dataType" + var, "", 1, + dataTypeOptions, tDataType[var], "") + +"
    _FillValue\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "For numeric variables, this is the value that is used in the" + + "
    data file to indicate a missing value for this variable.\n" + + "
    For example, -999 ." + + "
    If the _FillValue is NaN, use NaN .\n" + + "
    For String variables, leave this blank.") + " \n" + +"     		\n" + +widgets.textField("FillValue" + var, //note that field name lacks leading '_' + "", 10, 20, tFillValue[var], "") + +"
    units\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "These are the units of this variable. For example, degree_C" + + "
    This is required for numeric variables, but not used for most String variables." + + "
    • For temperature, use degree_C or degree_F ." + + "
    • For counts of things, use count ." + + "
    • For latitude variables, use degrees_north ." + + "
    • For longitude variables, use degrees_east ." + + "
    • For String date/time variables, paste a sample date/time value here." + + "
      We'll convert it to UDUnits." + + "
    • For numeric date/time variables, describe the values as units since basetime," + + "
      for example, days since 2010-01-01" + + "
    • For all other variables, use UDUNITs unit names if you know them;" + + "
      otherwise, use whatever units you already know.") + " \n" + +"     		\n" + +widgets.textField("units" + var, "", 20, 40, tUnits[var], "") + +"
    range\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "For numeric variables, this specifies the typical range of values." + + "
    For example, minimum=32.0 and maximum=37.0 ." + + "
    The range should include about 98% of the values." + + "
    These should be round numbers. This isn't precise.\n" + + "
    If you don't know the typical range of values, leave this blank." + + "
    For String variables, leave this blank.") + " \n" + +"     		minimum =\n" + +widgets.textField("rangeMin" + var, "", 10, 15, tRangeMin[var], "") + +"   maximum =\n" + +widgets.textField("rangeMax" + var, "", 10, 15, tRangeMax[var], "") + +"
    ioos_category\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, ioosCategoryHelp) + " \n" + +" \n" + +widgets.select("ioos_category" + var, "", 1, + ioosCategoryOptions, tIoosCategory[var], "") + +"
    comment\n" + +"  " + EDStatic.htmlTooltipImage(tLoggedInAs, + "Optional: This is miscellaneous information about this variable, not captured" + + "
    elsewhere. For example," + + "
    This is the difference between today's SST and the previous day's SST.") + " \n" + +"     		\n" + +widgets.textField("comment" + var, "", 80, 160, tComment[var], "") + +"
     \n" + +"
    \n" + //end of variable's table +"\n"); + +//Submit +writer.write( +"

    Finished with part 3?

    \n" + +"Click\n" + +widgets.button("submit", "Submit", "", "Submit", "") + +"to send this information to the ERDDAP administrator and move on to part 4 (of 4).\n" + +"\n"); + +//the end of the overall table that constrains the width of the text +writer.write( +"\n" + +"\n"); + +//end form +writer.write(widgets.endForm()); + + } catch (Throwable t) { + EDStatic.rethrowClientAbortException(t); //first thing in catch{} + if (out == null || writer == null) + throw t; + else writer.write(EDStatic.htmlForException(t)); + } + endHtmlWriter(out, writer, tErddapUrl, false); + return; + } + + /** + * This is Data Provider Form 4 + * + */ + public void doDataProviderForm4(HttpServletRequest request, HttpServletResponse response, + String tLoggedInAs, String ipAddress) throws Throwable { + + String tErddapUrl = EDStatic.erddapUrl(tLoggedInAs); + OutputStream out = null; + Writer writer = null; + + try { + + //get parameters + String + tYourName = request.getParameter("yourName"), + tEmailAddress = request.getParameter("emailAddress"), + tTimestamp = request.getParameter("timestamp"), + tOtherComments = request.getParameter("otherComments"); + + //validate (same order as form) + StringBuilder errorMsgSB = new StringBuilder(); + tYourName = HtmlWidgets.validateIsSomethingNotTooLong( + "Your Name", "?", tYourName, 50, null); + tEmailAddress = HtmlWidgets.validateIsSomethingNotTooLong( + "Email Address", "?", tEmailAddress, 50, null); + tTimestamp = HtmlWidgets.validateIsSomethingNotTooLong( + "Timestamp", Calendar2.getCurrentISODateTimeStringLocal() + "?", + tTimestamp, 20, null); + tOtherComments = HtmlWidgets.validateNotNullNotTooLong( + "Other Comments", "", tOtherComments, 500, errorMsgSB); + if (errorMsgSB.length() > 0) + errorMsgSB.insert(0, + "
    Please fix these problems, then 'Submit' this part of the form again.\n"); + + String fromInfo = tYourName + " <" + tEmailAddress + "> at " + tTimestamp; + + //if this is a submission, + boolean isSubmission = "Submit".equals(request.getParameter("Submit")); + if (isSubmission && errorMsgSB.length() == 0) { + //convert the info into psuedo datasets.xml + String content = + "Data Provider Form - Part 4\n" + //important! Bob's erd.data gmail filter looks for this + " from " + fromInfo + "\n" + + " ipAddress=" + ipAddress + "\n" + + "\n" + + "Other comments:\n" + + tOtherComments + "\n\n"; + + //log the content to /logs/dataProviderForm.log + String error = String2.appendFile( + EDStatic.fullLogsDirectory + "dataProviderForm.log", + "*** " + content); + if (error.length() > 0) + String2.log(String2.ERROR + + " while writing to logs/dataProviderForm.log:\n" + + error); + //email the content to the admin + EDStatic.email(EDStatic.adminEmail, + "Data Provider Form - Part 4, from " + fromInfo, + content); + + //redirect to Done + sendRedirect(response, tErddapUrl + "/dataProviderFormDone.html?" + + "yourName=" + SSR.minimalPercentEncode(tYourName) + + "&emailAddress=" + SSR.minimalPercentEncode(tEmailAddress) + + "×tamp=" + SSR.minimalPercentEncode(tTimestamp)); + + return; + } + + //write the HTML + out = getHtmlOutputStream(request, response); + writer = getHtmlWriter(tLoggedInAs, "Data Provider Form - Part 4", out); + writer.write(EDStatic.youAreHere(tLoggedInAs, "Data Provider Form - Part 4")); + + //begin form + String formName = "f1"; + HtmlWidgets widgets = new HtmlWidgets("", false, //style, false=not htmlTooltips + EDStatic.imageDirUrl(tLoggedInAs)); + widgets.enterTextSubmitsForm = false; + writer.write(widgets.beginForm(formName, + //this could be POST to deal with lots of text. + //but better to change tomcat settings (above) and keep ease of use + "GET", tErddapUrl + "/dataProviderForm4.html", "") + "\n"); + + //hidden fields + writer.write( + widgets.hidden("yourName", XML.encodeAsHTML(tYourName)) + + widgets.hidden("emailAddress", XML.encodeAsHTML(tEmailAddress)) + + widgets.hidden("timestamp", XML.encodeAsHTML(tTimestamp)) + + "\n"); + +//begin text +//the overall table that constrains the width of the text +writer.write( +"\n" + +"\n" + +"\n" + +"
    \n" + +"This is part 4 (of 4) of the Data Provider Form\n" + +"
    from " + XML.encodeAsHTML(fromInfo) + ".\n" + +"
    Need help? Send an email to the administrator of this ERDDAP (" + + XML.encodeAsHTML(SSR.getSafeEmailAddress(EDStatic.adminEmail)) + ").\n" + +"
     \n" + +"\n"); + +//error message? +if (isSubmission && errorMsgSB.length() > 0) +writer.write("" + errorMsgSB.toString() + " " + + "
     \n"); + +//other comments +writer.write( +"

    Other Comments

    \n" + +"Optional: If there are other things you think the ERDDAP administrator\n" + +"should know about this dataset, please add them here.\n" + +"This won't go in the dataset's metadata or be made public. (<500 characters)\n" + +"
    \n" + +"
     \n" + +"\n"); + +//Submit +writer.write( +"

    Finished with part 4?

    \n" + +"Click\n" + +widgets.button("submit", "Submit", "", "Submit", "") + +"to send this information to the ERDDAP administrator and move on to the \"You're done!\" page.\n" + +"\n"); + +//the end of the overall table that constrains the width of the text +writer.write( +"
    \n"); + +//end form +writer.write(widgets.endForm()); + + } catch (Throwable t) { + EDStatic.rethrowClientAbortException(t); //first thing in catch{} + if (out == null || writer == null) + throw t; + else writer.write(EDStatic.htmlForException(t)); + } + endHtmlWriter(out, writer, tErddapUrl, false); + return; + } + + /** + * This is Data Provider Form Done + */ + public void doDataProviderFormDone(HttpServletRequest request, HttpServletResponse response, + String tLoggedInAs) throws Throwable { + + String tErddapUrl = EDStatic.erddapUrl(tLoggedInAs); + OutputStream out = null; + Writer writer = null; + + try { + + //global attributes (and contact info) + String + tYourName = request.getParameter("yourName"), + tEmailAddress = request.getParameter("emailAddress"), + tTimestamp = request.getParameter("timestamp"); + + tYourName = HtmlWidgets.validateIsSomethingNotTooLong( + "Your Name", "?", tYourName, 50, null); + tEmailAddress = HtmlWidgets.validateIsSomethingNotTooLong( + "Email Address", "?", tEmailAddress, 50, null); + tTimestamp = HtmlWidgets.validateIsSomethingNotTooLong( + "Timestamp", Calendar2.getCurrentISODateTimeStringLocal() + "?", + tTimestamp, 20, null); + + //write the HTML + out = getHtmlOutputStream(request, response); + writer = getHtmlWriter(tLoggedInAs, "Data Provider Form - Done", out); + writer.write(EDStatic.youAreHere(tLoggedInAs, "Data Provider Form - Done")); + +//begin text +//the overall table that constrains the width of the text +writer.write( +"\n" + +"\n" + +"\n" + +"
    \n" + +"

    You're done! Congratulations! Thank you!

    \n" + +"The ERDDAP administrator will email you soon to figure out the best way transfer\n" + +" the data and to work out other details.\n" + +" This dataset submission's timestamp is " + tTimestamp + ".\n" + +"

    You can submit another dataset\n" + +"or go back to the ERDDAP home page.\n"); + +//the end of the overall table that constrains the width of the text +writer.write( +"

    \n"); + + } catch (Throwable t) { + EDStatic.rethrowClientAbortException(t); //first thing in catch{} + if (out == null || writer == null) + throw t; + else writer.write(EDStatic.htmlForException(t)); + } + endHtmlWriter(out, writer, tErddapUrl, false); + return; + } + + + + + /** * This responds to a request for status.html. * @@ -1720,7 +3255,7 @@ public void doRestHtml(HttpServletRequest request, HttpServletResponse response, "
  • Subsequent rows have the information you requested.\n" + "\n" + "

    The content in these plain file types is also slightly different from the .html\n" + - "response -- it is intentionally bare-boned, so that it is easier for a computer\n" + + "response — it is intentionally bare-boned, so that it is easier for a computer\n" + "program to work with.\n" + "\n" + "

    A Consistent Data Structure for the Responses\n" + @@ -1807,7 +3342,7 @@ public void doRestHtml(HttpServletRequest request, HttpServletResponse response, " the value in the query.)\n" + "
     \n" + "

  • To get the list of categoryAttributes\n" + - " (e.g., institution, long_name, standard_name), use\n" + + " (for example, institution, long_name, standard_name), use\n" + "
    " + plainLinkExamples(tErddapUrl, "/categorize/index", EDStatic.encodedDefaultPIppQuery) + "
     \n" + @@ -1850,7 +3385,7 @@ public void doRestHtml(HttpServletRequest request, HttpServletResponse response, "
  • Griddap and tabledap have many web services that you can use.\n" + " \n"); if (EDStatic.sosActive || EDStatic.wcsActive || EDStatic.wmsActive) { @@ -2314,8 +3849,7 @@ else if (protocol.equals("tabledap")) endOfRequestUrl = endOfRequestUrl.substring(0, slashPoNP); //currently no nextPath options - if (verbose) String2.log(EDStatic.resourceNotFound + " no nextPath options"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "no nextPath options"); return; } //String2.log(">>nextPath=" + nextPath + " endOfRequestUrl=" + endOfRequestUrl); @@ -2539,7 +4073,7 @@ else if (protocol.equals("tabledap")) } /** - * Process a /files/ request for EDDTableFromFileNames files. + * Process a /files/ request for an accessibleViaFiles dataset. * * @param loggedInAs the name of the logged in user (or null if not logged in) * @param datasetIDStartsAt is the position right after the / at the end of the protocol @@ -2630,8 +4164,7 @@ public void doFiles(HttpServletRequest request, HttpServletResponse response, //beware malformed nextPath, e.g., internal /../ if (File2.addSlash("/" + nextPath + "/").indexOf("/../") >= 0) { - String2.log("MALICIOUS ERROR?! /../ in nextPath=" + nextPath); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "/../ not allowed!"); return; } } else { @@ -2670,7 +4203,7 @@ public void doFiles(HttpServletRequest request, HttpServletResponse response, if (fileDir.length() == 0) { if (verbose) String2.log(EDStatic.resourceNotFound + " accessibleViaFilesDir=\"\""); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "this dataset is not accessible via /files/"); return; } if (!edd.isAccessibleTo(roles)) { //listPrivateDatasets doesn't apply @@ -2681,7 +4214,7 @@ public void doFiles(HttpServletRequest request, HttpServletResponse response, //if nextPath has a subdir, ensure dataset is recursive if (nextPath.indexOf('/') >= 0 && !recursive) { if (verbose) String2.log(EDStatic.resourceNotFound + " subdirectory doesn't exist"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "the subdirectory doesn't exist"); return; } @@ -2697,48 +4230,86 @@ public void doFiles(HttpServletRequest request, HttpServletResponse response, nextPath = nextPath.substring(0, nextPath.length() - justExtension.length()); //request will be handled below - //if (EDStatic.filesActive) ... } else { justExtension = ""; } - //if localFullName is a file, return it - //String2.log(">>localFullName=" + localFullName + "\nfullRequestUrl=" + fullRequestUrl); - if (nextPath.length() > 0 && - !localFullName.endsWith("/") && File2.isFile(localFullName)) { - String localDir = File2.getDirectory(localFullName); - String webDir = File2.getDirectory(fullRequestUrl); - String nameAndExt = File2.getNameAndExtension(localFullName); - if (nameAndExt.matches(fileRegex)) { - String ext = File2.getExtension(nameAndExt); + //get the accessibleViaFilesFileTable + //Formatted like + //FileVisitorDNLS.oneStep(tDirectoriesToo=false, last_mod is LongArray, + //and size is LongArray of epochMillis) + //with valid files (or null if unavailable or any trouble). + //This is a copy of any internal data, so contents can be modified. + Table fileTable = edd.accessibleViaFilesFileTable(); + if (fileTable == null) { + sendResourceNotFoundError(request, response, + "file info for this dataset is currently unavailable"); + return; + } + int fileTableRow; + int fileTableNRows = fileTable.nRows(); + StringArray dirSA = (StringArray)fileTable.getColumn(0); + StringArray nameSA = (StringArray)fileTable.getColumn(1); + + String localDir = File2.getDirectory(localFullName); //where I access source + String webDir = File2.getDirectory(fullRequestUrl); //what user as apparent location + String nameAndExt = File2.getNameAndExtension(localFullName); + String ext = File2.getExtension(nameAndExt); + + //is it a file in the fileTable? + if (nameAndExt.length() > 0) { + fileTableRow = 0; + while (fileTableRow < fileTableNRows && + (!localDir.equals( dirSA.get(fileTableRow)) || + !nameAndExt.equals(nameSA.get(fileTableRow)))) + fileTableRow++; + if (fileTableRow < fileTableNRows) { OutputStreamSource outSource = new OutputStreamFromHttpResponse( request, response, File2.getNameNoExtension(nameAndExt), ext, ext); doTransfer(request, response, localDir, webDir, nameAndExt, outSource.outputStream("", File2.length(localFullName))); - } else { - if (verbose) String2.log(EDStatic.resourceNotFound + " doesn't match regex"); - sendResourceNotFoundError(request, response, ""); - } - //tally - EDStatic.tally.add("files download DatasetID (since startup)", id); - EDStatic.tally.add("files download DatasetID (since last daily report)", id); - return; + //tally + EDStatic.tally.add("files download DatasetID (since startup)", id); + EDStatic.tally.add("files download DatasetID (since last daily report)", id); + return; + } } - - //is it a directory? - if (!File2.isDirectory(localFullName)) { - if (verbose) String2.log(EDStatic.resourceNotFound + " !isDirectory"); - sendResourceNotFoundError(request, response, ""); - return; + + //reduce fileTable to just files in that dir + boolean addedSlash = false; + if (nameAndExt.length() > 0) { + //perhaps user forgot trailing slash + localDir += nameAndExt + "/"; + webDir += nameAndExt + "/"; + nameAndExt = ""; + addedSlash = true; + } + int localDirLength = localDir.length(); + HashSet subdirHash = new HashSet(); //catch all subdirectories + BitSet keep = new BitSet(fileTableNRows); //all false + for (int row = 0; row < fileTableNRows; row++) { + String tDir = dirSA.get(row); + if (tDir.startsWith(localDir)) { + if (tDir.length() == localDirLength) { + keep.set(row); + } else { + //add next-level directory name + subdirHash.add(tDir.substring(localDirLength, tDir.indexOf('/', localDirLength))); + } + } } - if (!fullRequestUrl.endsWith("/")) { //required for table.directoryListing below + fileTable.removeColumn(0); //directory + fileTable.justKeep(keep); + fileTableNRows = fileTable.nRows(); + //yes, there's a match + if (addedSlash) { sendRedirect(response, fullRequestUrl + "/"); return; } //handle justExtension request e.g., datasetID/.csv[?constraintExpression] - if (justExtension.length() > 0) { + /*if (justExtension.length() > 0) { //tally it //make a EDDTableFromAccessibleViaFiles @@ -2757,37 +4328,16 @@ public void doFiles(HttpServletRequest request, HttpServletResponse response, return; - } + }*/ - //get list of dirs and files in that dir - Table table = FileVisitorDNLS.oneStep(localFullName, fileRegex, - false, true); //not recursive, dirsToo - Test.ensureEqual(table.getColumnNamesCSVString(), - "directory,name,lastModified,size", - "Unexpected columnNames"); - //extract the subDirs from the table - StringArray subDirs = new StringArray(); - int nRows = table.nRows(); - BitSet keep = new BitSet(); //all false - keep.set(0, nRows); //all true - StringArray dirPA = (StringArray)table.getColumn(0); - StringArray namePA = (StringArray)table.getColumn(1); - for (int row = 0; row < nRows; row++) { - if (namePA.get(row).length() == 0) { - keep.clear(row); - String td = dirPA.get(row); - subDirs.add(File2.getNameAndExtension(td.substring(0, td.length() - 1))); - } - } - table.removeColumn(0); //directory - table.justKeep(keep); - nRows = table.nRows(); + //show directory index //make columns: "Name" (String), "Last modified" (long), - // "Size" (long), and "Description" (String) - table.setColumnName(0, "Name"); - table.setColumnName(1, "Last modified"); - table.setColumnName(2, "Size"); - table.addColumn("Description", new StringArray(nRows, true)); + // "Size" (long), and "Description" (String) + StringArray subDirs = new StringArray((String[])(subdirHash.toArray(new String[0]))); + fileTable.setColumnName(0, "Name"); + fileTable.setColumnName(1, "Last modified"); + fileTable.setColumnName(2, "Size"); + fileTable.addColumn("Description", new StringArray(fileTableNRows, true)); OutputStream out = getHtmlOutputStream(request, response); Writer writer = getHtmlWriter(loggedInAs, "files/" + id + "/" + nextPath, out); writer.write( @@ -2813,7 +4363,7 @@ public void doFiles(HttpServletRequest request, HttpServletResponse response, edd.writeHtmlDatasetInfo(loggedInAs, writer, true, true, false, true, "", ""); writer.flush(); writer.write( - table.directoryListing( + fileTable.directoryListing( fullRequestUrl, userDapQuery, //may have sort instructions EDStatic.imageDirUrl(loggedInAs) + "fileIcons/", true, subDirs, null)); //addParentDir @@ -2822,6 +4372,7 @@ public void doFiles(HttpServletRequest request, HttpServletResponse response, //tally EDStatic.tally.add("files browse DatasetID (since startup)", id); EDStatic.tally.add("files browse DatasetID (since last daily report)", id); + } @@ -3197,8 +4748,7 @@ Spec questions? Ask Jeff DLb (author of WMS spec!): Jeff.deLaBeaujardiere@noaa.g //ensure it is a SOS server request if (!part1.equals(EDDTable.sosServer) && urlEndParts.length == 2) { - if (verbose) String2.log(EDStatic.resourceNotFound + " not SOS request"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "not a SOS request"); return; } @@ -3550,8 +5100,7 @@ public void doWcs(HttpServletRequest request, HttpServletResponse response, //ensure it is a SOS server request if (!part1.equals(EDDGrid.wcsServer) && urlEndParts.length == 2) { - if (verbose) String2.log(EDStatic.resourceNotFound + " not wcs request"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "not a WCS request"); return; } @@ -3856,14 +5405,12 @@ public void doWms(HttpServletRequest request, HttpServletResponse response, } //error - if (verbose) String2.log(EDStatic.resourceNotFound + " unmatched wms request"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "unmatched WMS request"); return; } //error - if (verbose) String2.log(EDStatic.resourceNotFound + " unmatched wms request #2"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "unmatched WMS request (2)"); } /** @@ -5535,7 +7082,7 @@ public void doWmsOpenLayers(HttpServletRequest request, HttpServletResponse resp EDDGrid eddGrid = gridDatasetHashMap.get(tDatasetID); if (eddGrid == null) { sendResourceNotFoundError(request, response, - "Currently, datasetID=" + tDatasetID + " isn't available."); + "datasetID=" + tDatasetID + " is currently unavailable."); return; } if (!eddGrid.isAccessibleTo(EDStatic.getRoles(loggedInAs))) { //listPrivateDatasets doesn't apply @@ -6420,8 +7967,7 @@ public void doGeoServicesRest(HttpServletRequest request, HttpServletResponse re String tDatasetID = urlParts[2]; EDDGrid tEddGrid = gridDatasetHashMap.get(tDatasetID); if (tEddGrid == null) { - if (verbose) String2.log(EDStatic.resourceNotFound + " eddGrid=null"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "no such dataset"); return; } if (!tEddGrid.isAccessibleTo(EDStatic.getRoles(loggedInAs))) { //authorization (very important) @@ -6511,8 +8057,7 @@ public void doGeoServicesRest(HttpServletRequest request, HttpServletResponse re int tDvi = String2.indexOf(tEddGrid.dataVariableDestinationNames(), tDestName); if (tDvi < 0 || !tDataVariables[tDvi].hasColorBarMinMax()) { //must have colorBarMin/Max - if (verbose) String2.log(EDStatic.resourceNotFound + " no colorBarMin/Max"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "This variable doesn't have predefined colorBarMin/Max."); return; } EDV tEdv = tDataVariables[tDvi]; @@ -6520,7 +8065,7 @@ public void doGeoServicesRest(HttpServletRequest request, HttpServletResponse re //just "/rest/services/[tDatasetID]/[tDestName]" if (nUrlParts == 4) { if (verbose) String2.log(EDStatic.resourceNotFound + " nParts=" + nUrlParts + " !=4"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "nQueryParts!=4"); return; } @@ -6530,7 +8075,7 @@ public void doGeoServicesRest(HttpServletRequest request, HttpServletResponse re //ensure urlParts[4]=ImageServer if (!urlParts[4].equals("ImageServer")) { if (verbose) String2.log(EDStatic.resourceNotFound + " ImageServer expected"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "ImageServer expected"); return; } @@ -7069,14 +8614,14 @@ public void doGeoServicesRest(HttpServletRequest request, HttpServletResponse re } else { if (verbose) String2.log(EDStatic.resourceNotFound + " !isFile " + actualDir + tFileName); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "file doesn't exist"); } return; } else { if (verbose) String2.log(EDStatic.resourceNotFound + " nParts=" + nUrlParts + " !=7"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "incorrect nParts"); return; } //end of /exportImage[/fileName] @@ -7127,9 +8672,7 @@ public static void doTransfer(HttpServletRequest request, HttpServletResponse re requestUrl.substring(datasetIDStartsAt); if (!File2.isFile(dir + fileNameAndExt)) { - if (verbose) String2.log(EDStatic.resourceNotFound + - " !isFile " + dir +fileNameAndExt); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "file doesn't exist"); return; } String ext = File2.getExtension(fileNameAndExt); @@ -7157,9 +8700,9 @@ public static void doTransfer(HttpServletRequest request, HttpServletResponse re gc.add(Calendar2.DATE, nDays); String expires = Calendar2.formatAsRFC822GMT(gc); if (reallyVerbose) String2.log(" setting expires=" + expires + " header"); - response.setHeader("Cache-Control", "PUBLIC, max-age=" + + response.setHeader("Cache-Control", "PUBLIC, max-age=" + (nDays * Calendar2.SECONDS_PER_DAY) + ", must-revalidate"); - response.setHeader("Expires", expires); + response.setHeader("Expires", expires); } doTransfer(request, response, dir, protocol + "/", fileNameAndExt, @@ -7183,10 +8726,11 @@ public static void doTransfer(HttpServletRequest request, HttpServletResponse re //To deal with problems in multithreaded apps //(when deleting and renaming files, for an instant no file with that name exists), int maxAttempts = 3; + String localFullName = localDir + fileNameAndExt; for (int attempt = 1; attempt <= maxAttempts; attempt++) { - if (File2.isFile(localDir + fileNameAndExt)) { - //ok, copy it - File2.copy(localDir + fileNameAndExt, outputStream); + + //ok, copy it + if (SSR.copy(localFullName, outputStream)) { //handles file or URL source outputStream.close(); return; } @@ -7805,8 +9349,7 @@ public void doSearch(HttpServletRequest request, HttpServletResponse response, } //else handle just below here } else { //usually unsupported fileType - if (verbose) String2.log(EDStatic.resourceNotFound + " unsupported endOfRequestUrl"); - sendResourceNotFoundError(request, response, ""); + sendResourceNotFoundError(request, response, "unsupported endOfRequestUrl"); return; } @@ -12943,12 +14486,29 @@ public static OutputStream getHtmlOutputStream(HttpServletRequest request, HttpS * @throws Throwable if trouble */ Writer getHtmlWriter(String loggedInAs, String addToTitle, OutputStream out) throws Throwable { + return getHtmlWriter(loggedInAs, addToTitle, "", out); + } + + /** + * Get a writer for an html file and write up to and including the startHtmlBody + * + * @param loggedInAs the name of the logged in user (or null if not logged in) + * @param addToTitle a string, not yet XML encoded + * @param addToHead additional info for the <head> + * @param out + * @return writer + * @throws Throwable if trouble + */ + Writer getHtmlWriter(String loggedInAs, String addToTitle, String addToHead, + OutputStream out) throws Throwable { Writer writer = new OutputStreamWriter(out, "UTF-8"); //write the information for this protocol (dataset list table and instructions) String tErddapUrl = EDStatic.erddapUrl(loggedInAs); writer.write(EDStatic.startHeadHtml(tErddapUrl, addToTitle)); + if (String2.isSomething(addToHead)) + writer.write("\n" + addToHead); writer.write("\n\n"); writer.write(EDStatic.startBodyHtml(loggedInAs)); writer.write("\n"); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java b/WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java index 548e56edc..f8371ba1b 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java @@ -162,7 +162,8 @@ public String doIt(String args[], boolean loop) throws Throwable { "EDDTableFromOBIS", "EDDTableFromSOS", "EDDTableFromThreddsFiles", - "EDDTableFromWFSFiles"}; + "EDDTableFromWFSFiles", + "EDDsFromFiles"}; StringBuilder sb = new StringBuilder(); int net = eddTypes.length; int net2 = Math2.hiDiv(net, 2); @@ -502,6 +503,11 @@ public String doIt(String args[], boolean loop) throws Throwable { String2.parseInt(s3, EDD.DEFAULT_RELOAD_EVERY_N_MINUTES), s4, s5, s6, s7, null)); + } else if (eddType.equals("EDDsFromFiles")) { + s1 = get(args, 1, s1, "Starting directory"); + String2.log("working..."); + printToBoth(EDD.generateDatasetsXmlFromFiles(s1)); + } else { String2.log("ERROR: eddType=" + eddType + " is not an option."); } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java index f2cc0599a..ead0b2021 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java @@ -30,24 +30,29 @@ import gov.noaa.pfel.coastwatch.sgt.GSHHS; import gov.noaa.pfel.coastwatch.sgt.SgtGraph; import gov.noaa.pfel.coastwatch.sgt.SgtMap; +import gov.noaa.pfel.coastwatch.util.FileVisitorDNLS; +import gov.noaa.pfel.coastwatch.util.FileVisitorSubdir; import gov.noaa.pfel.coastwatch.util.SimpleXMLReader; import gov.noaa.pfel.coastwatch.util.SSR; +import gov.noaa.pfel.coastwatch.util.Tally; import gov.noaa.pfel.erddap.Erddap; import gov.noaa.pfel.erddap.util.*; import gov.noaa.pfel.erddap.variable.*; import java.io.BufferedOutputStream; import java.io.ByteArrayInputStream; +import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.OutputStream; import java.io.OutputStreamWriter; import java.io.StringWriter; import java.io.Writer; +import java.nio.file.Path; import java.text.MessageFormat; import java.util.ArrayList; import java.util.Arrays; -//import java.util.BitSet; +import java.util.BitSet; import java.util.Collections; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.locks.ReentrantLock; @@ -69,6 +74,8 @@ import org.json.JSONString; import org.json.JSONTokener; +import ucar.nc2.NetcdfFile; + /** This class represents an ERDDAP Dataset (EDD) -- a gridded or tabular dataset @@ -1376,6 +1383,17 @@ public boolean accessibleViaFilesRecursive() { return accessibleViaFilesRecursive; } + /** + * This returns a fileTable (formatted like + * FileVisitorDNLS.oneStep(tDirectoriesToo=false, last_mod is LongArray, + * and size is LongArray of epochMillis) + * with valid files (or null if unavailable or any trouble). + * This is a copy of any internal data, so client can modify the contents. + */ + public Table accessibleViaFilesFileTable() { + return null; + } + /** * This indicates why the dataset isn't accessible via the FGDC service * (or "" if it is). @@ -3773,10 +3791,17 @@ public static Attributes makeReadyToUseAddGlobalAttributesForDatasetsXml(Attribu String tPublicSourceUrl = convertToPublicSourceUrl(tLocalSourceUrl); if (tPublicSourceUrl == null) tPublicSourceUrl = ""; - boolean sourceUrlIsThreddsCatalogXml = + boolean sourceUrlIsHyraxFile = + tPublicSourceUrl.startsWith("http") && + tPublicSourceUrl.indexOf("/opendap/") > 0 && + (tPublicSourceUrl.endsWith("/") || tPublicSourceUrl.endsWith("/contents.html")); + boolean sourceUrlIsHyraxCatalog = tPublicSourceUrl.startsWith("http") && - tPublicSourceUrl.indexOf("/thredds/catalog/") > 0 && - tPublicSourceUrl.endsWith(".xml"); + tPublicSourceUrl.indexOf("/opendap/") > 0 && + !sourceUrlIsHyraxFile; + boolean sourceUrlIsThreddsCatalog = + tPublicSourceUrl.startsWith("http") && + tPublicSourceUrl.indexOf("/thredds/catalog/") > 0; String sourceUrlAsTitle = String2.replaceAll( //"extension" may be part of name with internal periods, @@ -3816,12 +3841,15 @@ public static Attributes makeReadyToUseAddGlobalAttributesForDatasetsXml(Attribu //fgdc_metadata_url is fgdc metadata, so not so useful as infoUrl HashSet toRemove = new HashSet(Arrays.asList( //Enter them lowercase here. The search for them is case-insensitive. - "cols", "columns", + "cols", "columns", "cwhdf_version", "data_bins", "data_center", "data_maximum", "data_minimum", "easternmost_longitude", - "end_day", "end_millisec", "end_orbit", "end_time", "end_year", + "end_day", "end_millisec", "end_time", "end_year", + "end_orbit", "endorbitnumber", "end_orbit_number", + "et_affine", "first_index", "format", //e.g., hdf5 "fgdc_metadata_url", "fgdc:metadata_url", + "gctp_datum", "gctp_parm", "gctp_sys", "gctp_zone", "gds_version_id", "georange", "granulepointer", "ice_fraction", "inputpointer", "intercept", @@ -3847,9 +3875,10 @@ public static Attributes makeReadyToUseAddGlobalAttributesForDatasetsXml(Attribu "num_l3_columns", "num_l3_rows", "observation_date", "operationmode", "orbitparameterspointer", "orbit", - "parameter", "percent_rev_data_usage", + "parameter", "pass_date", "percent_rev_data_usage", "period", "period_end_day", "period_end_year", "period_start_day", "period_start_year", + "polygon_latitude", "polygon_longitude", "qagranulepointer", "qapercentmissingdata", "qapercentoutofboundsdata", "range_beginning_date", "rangebeginningdate", "range_beginning_time", "rangebeginningtime", @@ -3863,11 +3892,11 @@ public static Attributes makeReadyToUseAddGlobalAttributesForDatasetsXml(Attribu "spatial_completeness_definition", "spatial_completeness_ratio", "start_date", "start_day", "start_millisec", - "start_orbit", "startorbitnumber", + "start_orbit", "startorbitnumber", "start_orbit_number", "start_time", "start_year", "station_latitude", "station_longitude", "stop_date", "stop_time", - "stop_orbit", "stoporbitnumber", + "stop_orbit", "stoporbitnumber", "stop_orbit_number", "suggested_image_scaling_applied", "suggested_image_scaling_maximum", "suggested_image_scaling_minimum", @@ -4158,8 +4187,12 @@ public static Attributes makeReadyToUseAddGlobalAttributesForDatasetsXml(Attribu if (tPublicSourceUrl.startsWith("http://nomads.ncep.noaa.gov") && tPublicSourceUrl.indexOf("/rtofs/") > 0) { value = "http://polar.ncep.noaa.gov/global/"; - } else if (sourceUrlIsThreddsCatalogXml) { //thredds catalog.xml -> .html - value = tPublicSourceUrl.substring(0, tPublicSourceUrl.length() - 4) + ".html"; + } else if (sourceUrlIsHyraxFile) { + value = File2.getDirectory(tPublicSourceUrl) + ".html"; + } else if (sourceUrlIsHyraxCatalog) { + value = File2.getDirectory(tPublicSourceUrl) + "contents.html"; + } else if (sourceUrlIsThreddsCatalog) { + value = File2.getDirectory(tPublicSourceUrl) + "catalog.html"; } else { value = tPublicSourceUrl + ((tPublicSourceUrl.indexOf("/thredds/") > 0 || //most DAP servers, add .html: THREDDS @@ -4636,7 +4669,7 @@ public static Attributes makeReadyToUseAddGlobalAttributesForDatasetsXml(Attribu if (!isSomething(value)) value = sourceAtts.getString(name); if (!isSomething(value) || //this isn't very sophisticated: - //ensure it is the new ACDD-1.3 style "CF Standard Name Table v27" + //ensure it is the new ACDD-1.3 style "CF Standard Name Table v29" !value.matches("CF Standard Name Table v[0-9]{2,}")) addAtts.add(name, FileNameUtility.getStandardNameVocabulary()); @@ -4644,8 +4677,9 @@ public static Attributes makeReadyToUseAddGlobalAttributesForDatasetsXml(Attribu if (!isSomething(tSummary) || tSummary.length() < 30) { value = isSomething(tInstitution)? tInstitution + " data " : "Data "; - if (sourceUrlIsThreddsCatalogXml) //thredds catalog.xml -> .html - value += "from " + tPublicSourceUrl.substring(0, tPublicSourceUrl.length() - 4) + ".html"; + if (sourceUrlIsHyraxFile || sourceUrlIsHyraxCatalog || + sourceUrlIsThreddsCatalog) + value += "from " + infoUrl; else if (tPublicSourceUrl.startsWith("http")) value += "from " + tPublicSourceUrl + ".das ."; else value += "from a local source."; @@ -4722,13 +4756,11 @@ else if (tPublicSourceUrl.startsWith("http")) } if (debugMode) String2.log(">> 3 title=" + value); - //thredds catalog.xml? use last two directory names - if (!isSomething(value) && sourceUrlIsThreddsCatalogXml) { - value = tPublicSourceUrl; + //hyrax or thredds catalog? use last two directory names + if (!isSomething(value) && + (sourceUrlIsHyraxCatalog || sourceUrlIsThreddsCatalog)) { + value = File2.getDirectory(tPublicSourceUrl); int po = value.lastIndexOf("/"); - if (po > 0) - value = value.substring(0, po); - po = value.lastIndexOf("/"); if (po > 0) po = value.substring(0, po).lastIndexOf("/"); value = po > 0 && po < value.length() - 1? value.substring(po + 1) : @@ -6002,8 +6034,9 @@ public static Attributes makeReadyToUseAddVariableAttributesForDatasetsXml( String tUnits = oUnits; if (tUnits.length() == 0) { //rtofs grads - String from[] = {"degc", "psu", "m/s", "m"}; - String to[] = {"degree_C", "1e-3", "m s-1", "m"}; //PSU -> 1e-3 in CF std names 25 + //sea_water_practical_salinity units = "1" in CF std names 27; I'm sticking with PSU. + String from[] = {"degc", "psu", "m/s", "m", "Presumed Salinity Units"}; + String to[] = {"degree_C", "PSU", "m s-1", "m", "PSU"}; for (int i = 0; i < from.length; i++) { if (oLongName.endsWith(" (" + from[i] + ")")) { //e.g. " (degc)" tUnits = to[i]; @@ -6345,7 +6378,25 @@ else if (tStandardName.equals("wave_period")) testUnits.equals("k"); //udunits and ucum boolean hasTemperatureUnits = isDegreesC || isDegreesF || isDegreesK; - if (!isSomething(tStandardName)) { + if (isSomething(tStandardName)) { + //deal with the mess that is salinity + if (tStandardName.equals("sea_water_salinity") || + tStandardName.equals("sea_surface_salinity")) { + //g/g and kg/kg are very rare + if ("|g/g|kg/kg|g kg-1|g/kg|".indexOf("|" + tUnits + "|") >= 0) { + tStandardName = "sea_water_absolute_salinity"; //canonical is g/kg + } else { + tStandardName = "sea_water_practical_salinity"; + //Possibly changing units is very aggressive. I know. + //1 is CF canonical, but datasets have 1e-3, 1, psu, ... + //It is better to be aggressive and defy CF than have misleading/ + // bizarre units based on previous versions of CF standard names. + if (tUnitsLC.indexOf("pss") < 0) + tUnits = "PSU"; + } + } + } else { + //does the lcSourceName or ttLongName equal a cfName? //special cases String tsn = String2.replaceAll(isSomething(lcSourceName)? lcSourceName : "\r", " ", "_"); //\r won't match anything @@ -6406,6 +6457,7 @@ else if ( lcu.indexOf("precision") >= 0 || lcu.indexOf("error") >= 0 || //"interpolation error fields" lcu.indexOf("number") >= 0 || //"number of observations" + lcu.indexOf("|nobs|") >= 0 || //number of observations lcu.indexOf("radius|influence|grid|points") >= 0 || lcu.indexOf("standard|deviation") >= 0 || lcu.indexOf("standard|error") >= 0) {} @@ -6577,7 +6629,14 @@ else if ((lc.indexOf("salinity") >= 0 || //no generic salt_flux } } else { - tStandardName = "sea_water_salinity"; + if ("|g kg-1|g/kg|".indexOf("|" + tUnits + "|") >= 0) { + tStandardName = "sea_water_absolute_salinity"; + } else { + tStandardName = "sea_water_practical_salinity"; + if (tUnitsLC.indexOf("pss") < 0) + tUnits = "PSU"; //1 is CF canonical, but datasets have 1e-3, 1, psu, ... + //better to defy CF than have misleading bizarre units. + } }} else if (((lc.indexOf("water") >= 0 && lc.indexOf("temp") >= 0) || @@ -6629,6 +6688,13 @@ else if (lc.indexOf("wind") < 0 && lc.indexOf("water|y") >= 0)) tStandardName = "northward_sea_water_velocity"; else if ((lc.indexOf("surface") >= 0 && lc.indexOf("roughness") >= 0 && isMeters)) tStandardName = "surface_roughness_length"; + + else if (lcSourceName.equals("par")) + tStandardName = "downwelling_photosynthetic_photon_radiance_in_sea_water"; + + else if (lcSourceName.equals("ph")) + tStandardName = "sea_water_ph_reported_on_total_scale"; + else if (((lc.indexOf("rel") >= 0 && lc.indexOf("hum") >= 0) || lc.indexOf("humidity") >= 0 || @@ -6993,6 +7059,12 @@ else if (tStandardName.equals("volume_fraction_of_oxygen_in_sea_water") ) {tMin = 0; tMax = 10;} else if (tStandardName.indexOf("oxygen_in_sea_water") >= 0) {tMin = 0; tMax = 500;} else if (tStandardName.indexOf("water_flux_into_ocean") >= 0) {tMin = 0; tMax = 1e-4;} + else if (tStandardName.equals("downwelling_photosynthetic_photon_radiance_in_sea_water") || + lcSourceName.equals("par")) { + if (tUnitsLC.equals("volt") || tUnitsLC.equals("volts")) {tMin = 0; tMax = 3;} + else /* microEinsteins m^-2 s-1 */ {tMin = 0; tMax = 70;}} + else if (tStandardName.equals("sea_water_ph_reported_on_total_scale") || + lcSourceName.equals("ph")) {tMin = 7; tMax = 9;} else if (tStandardName.indexOf("precipitation") >= 0 || tStandardName.indexOf("snowfall") >= 0 || tStandardName.indexOf("rainfall") >= 0 || @@ -7060,11 +7132,19 @@ else if (tStandardName.equals("sea_water_electrical_conductivity" else if (tStandardName.equals("sea_water_pressure" )) {tMin = 0; tMax = 5000;} else if (tStandardName.equals("sea_surface_salinity" ) || tStandardName.equals("sea_water_salinity" ) || + tStandardName.equals("sea_water_absolute_salinity") || + tStandardName.equals("sea_water_cox_salinity") || + tStandardName.equals("sea_water_knudsen_salinity") || tStandardName.equals("sea_water_practical_salinity") || + tStandardName.equals("sea_water_preformed_salinity") || + tStandardName.equals("sea_water_reference_salinity") || + tStandardName.equals("sea_water_salinity") || //lc.indexOf( "salinity") >= 0 || //!but river/bay salinity close to 0 - tUnitsLC.equals("psu" ) || - tUnitsLC.equals("pss" )) { - if (tUnitsLC.equals("kg/kg") || tUnitsLC.equals("g/g")) { + tUnitsLC.equals("psu") || + tUnitsLC.equals("pss78") || tUnitsLC.equals("ipss78") || + tUnitsLC.equals("pss-78") || tUnitsLC.equals("ipss-78") || + tUnitsLC.equals("pss") || tUnitsLC.equals("ipss")) { + if (tUnitsLC.equals("kg/kg") || tUnitsLC.equals("g/g")) { //rare tMin = 0.032; tMax = 0.037; } else { tMin = 32; tMax = 37; @@ -7916,10 +7996,12 @@ public static String suggestConventions(String con) { *
    This seeks to be short, descriptive, and unique (so 2 datasets don't have same datasetID). * * @param tPublicSourceUrl a real URL (starts with "http", e.g., http://oceanwatch.pfeg.noaa.gov/...), - * a fileDirectory (with trailing '/') or directory+fileName (may be a fileNameRegex), - * or a fake fileDirectory (not ideal). - *
    If an OPeNDAP url, it is without the .das, .dds, or .html extension. - *
    If a fileDirectory, the two rightmost directories are important. + * a fileDirectory (with trailing '/') or directory+fileName (may be a fileNameRegex), + * or a fake fileDirectory (not ideal). + *
    If an OPeNDAP url, it is without the .das, .dds, or .html extension. + *
    If a fileDirectory, the two rightmost directories are important. + *
    If you want to add additional information (e.g. dimension names or "EDDTableFromFileNames"), + * add it add the end of the url. * @return a suggested datasetID, e.g., noaa_pfeg######### */ public static String suggestDatasetID(String tPublicSourceUrl) { @@ -7928,12 +8010,20 @@ public static String suggestDatasetID(String tPublicSourceUrl) { //But some datasetIDs would be very long and info is already in sourceUrl in original form. //extract from tPublicSourceUrl - String dir = tPublicSourceUrl.indexOf('/' ) >= 0 || - tPublicSourceUrl.indexOf('\\') >= 0? - File2.getDirectory(tPublicSourceUrl) : - tPublicSourceUrl; - String dsi = String2.modifyToBeFileNameSafe( - String2.toSVString(suggestInstitutionParts(dir), "_", true)); + //is it an Amazon AWS S3 URL? + String dsi = String2.getAwsS3BucketName(tPublicSourceUrl); + if (dsi == null) { + //regular url + String dir = tPublicSourceUrl.indexOf('/' ) >= 0 || + tPublicSourceUrl.indexOf('\\') >= 0? + File2.getDirectory(tPublicSourceUrl) : + tPublicSourceUrl; + dsi = String2.toSVString(suggestInstitutionParts(dir), "_", true); + } else { + //AWS S3 url + dsi = "s3" + dsi + "_"; + } + dsi = String2.modifyToBeFileNameSafe(dsi); dsi = String2.replaceAll(dsi, '-', '_'); dsi = String2.replaceAll(dsi, '.', '_'); return dsi + String2.md5Hex12(tPublicSourceUrl); @@ -8949,6 +9039,278 @@ public Object[] readPngInfo(String loggedInAs, String userDapQuery, String fileT } } + /** + * This walks through the start directory and subdirectories and tries + * to generateDatasetsXml for groups of data files that it finds. + *
    This assumes that when a dataset is found, the dataset includes all + * subdirectories. + *
    If dataset is found, sibling directories will be treated as separate datasets + * (e.g., dir for 1990's, dir for 2000's, dir for 2010's will be separate datasets). + * But they should be easy to combine by hand. + *
    This will only catch one type of file in a directory (e.g., + * a dir with sst files and chl files will just catch one of those). + * + * @return a suggested chunk of xml for all datasets it can find for use in datasets.xml + * @throws Throwable if trouble, e.g., startDir not found or no valid datasets were made. + * If no trouble, then a valid dataset.xml chunk has been returned. + */ + public static String generateDatasetsXmlFromFiles(String startDir) throws Exception { + String2.log("> EDD.generateDatasetsXmlFromFiles(" + startDir + ")"); + StringBuilder resultsSB = new StringBuilder(); + long time = System.currentTimeMillis(); + + //get list of subdirs + //because of the way it recurses, the order is already fine for my use here: + // every parent directory is listed before all of its child directories. + StringArray paths = FileVisitorSubdir.oneStep(startDir); + int nDirs = paths.size(); + //String2.pressEnterToContinue(String2.toNewlineString(paths.toArray())); + + StringArray dirs = new StringArray(nDirs, false); + for (int i = 0; i < nDirs; i++) { + String path = paths.get(i); + if (File.separatorChar == '\\') + path = String2.replaceAll(path, '\\', '/'); + path = File2.addSlash(path); + dirs.add(path); + } + StringArray dirInfo = new StringArray(nDirs, true); + Table dirTable = new Table(); + dirTable.addColumn("dir", dirs); + dirTable.addColumn("dirInfo", dirInfo); + BitSet dirDone = new BitSet(nDirs); //all false + String2.log("> nDirs=" + nDirs + " elapsedTime=" + + (System.currentTimeMillis() - time)); + + //go through dirs, from high level to low level, looking for datafiles/datasets + int nCreated = 0; + int nGridNc = 0; + int nTableNcCF = 0; + int nTableNc = 0; + int nTableAscii = 0; + int nTableFileNames = 0; + String skipThisDir = "> Skip this directory: "; + String success = "> Success: "; + String indent = " "; + for (int diri = 0; diri < nDirs; diri++) { + String tDir = dirs.get(diri); + String2.log("> dir#" + diri + " of " + nDirs + "=" + tDir); + if (dirDone.get(diri)) { + dirInfo.set(diri, indent + "see parent dataset"); + String2.log("> Skip this directory: already covered by a dataset in a parent dir."); + continue; + } + + Table fileTable = FileVisitorDNLS.oneStep(tDir, ".*", + false, false); //tRecursive, tDirectoriesToo + StringArray names = (StringArray)fileTable.getColumn(FileVisitorDNLS.NAME); + StringArray exts = new StringArray(); + int nFiles = names.size(); + if (nFiles == 0) { + dirDone.set(diri); + String msg = "nFiles=0"; + dirInfo.set(diri, indent + msg); + String2.log(skipThisDir + msg); + continue; + } + + //tally the file's extensions + Tally tally = new Tally(); + for (int filei = 0; filei < nFiles; filei++) { + String tName = names.get(filei); + String ext = File2.getExtension(tName); //may be "" + exts.add(ext); + if (ext.equals(".md5") || + tName.toLowerCase().startsWith("readme")) { //readme or read_me + //don't tally .md5, readme, or others? + } else { + tally.add("ext", ext); + } + } + fileTable.addColumn(0, "ext", exts); + + //get the most common file extension + ArrayList tallyArrayList = tally.getSortedNamesAndCounts("ext"); + if (tallyArrayList == null) + return ""; + StringArray tallyExts = (StringArray)tallyArrayList.get(0); + IntArray tallyCounts = (IntArray)tallyArrayList.get(1); + if (tallyCounts.size() == 0) { + dirDone.set(diri); + String msg = "0 of " + nFiles + " have interesting extensions"; + dirInfo.set(diri, indent + msg); + String2.log(skipThisDir + msg); + continue; + } + String topExt = tallyExts.get(0); + int topCount = tallyCounts.get(0); + int sampleRow = exts.indexOf(topExt); + String sampleName = names.get(sampleRow); + String2.log("> topExt=" + topExt + " topCount=" + topCount + " sample=" + sampleName); + String topOfAre = topCount + " of " + nFiles + " files are " + topExt + ": "; + + if (topCount < 4) { + //I'm looking for collections of data files. + //Don't be distracted by e.g., one .txt file. + dirDone.set(diri); + String msg = topOfAre + "That's less than 4."; + dirInfo.set(diri, indent + msg); + String2.log(skipThisDir + msg); + continue; + } + + //try to make datasets.xml for files in this dir (and subdirs) + int tReloadEveryNMinutes = 1440; +//If updateNMillis works, then 1440 is good. If not, then 180? + + //table in .ncCF file + if (topExt.equals(".nc") || topExt.equals(".cdf")) { + String featureType = null; + try { + //does it have featureType metadata? + NetcdfFile ncFile = NcHelper.openFile(tDir + sampleName); + Attributes gAtts = new Attributes(); + NcHelper.getGlobalAttributes(ncFile, gAtts); + featureType = gAtts.getString("featureType"); + ncFile.close(); + if (featureType == null) + throw new RuntimeException("No featureType, so it isn't an .ncCF file."); + + //try to interpret as a .ncCF file + String xmlChunk = EDDTableFromNcCFFiles.generateDatasetsXml( + tDir, ".*\\" + topExt, + tDir + sampleName, tReloadEveryNMinutes, + "", "", "", "", //extract + "", "", "", "", "", null); //other info + resultsSB.append(xmlChunk); //recursive=true + for (int diri2 = diri; diri2 < nDirs; diri2++) + if (dirs.get(diri2).startsWith(tDir)) + dirDone.set(diri2); + String msg = topOfAre + "EDDTableFromNcCFFiles/" + featureType; + dirInfo.set(diri, indent + msg); + String2.log(success + msg); + nTableNcCF++; + nCreated++; + continue; + } catch (Throwable t) { + String2.log("> Attempt with EDDTableFromNcCFFiles (" + + featureType + ") failed:\n" + + MustBe.throwableToString(t)); + } + } + + //grid via netcdf-java + if (topExt.equals(".nc") || topExt.equals(".cdf") || + topExt.equals(".hdf") || + topExt.equals(".grb") || topExt.equals(".grb2") || + topExt.equals(".bufr") || + topExt.equals("")) { //.hdf are sometimes unidentified + try { + String xmlChunk = EDDGridFromNcFiles.generateDatasetsXml( + tDir, ".*\\" + topExt, + tDir + sampleName, + tReloadEveryNMinutes, null); //externalAddGlobalAttributes + resultsSB.append(xmlChunk); //recursive=true + for (int diri2 = diri; diri2 < nDirs; diri2++) + if (dirs.get(diri2).startsWith(tDir)) + dirDone.set(diri2); + String msg = topOfAre + "EDDGridFromNcFiles"; + dirInfo.set(diri, indent + msg); + String2.log(success + msg); + nGridNc++; + nCreated++; + continue; + } catch (Throwable t) { + String2.log("> Attempt with EDDGridFromNcFiles failed:\n" + + MustBe.throwableToString(t)); + } + } + + //table in .nc file + if (topExt.equals(".nc") || topExt.equals(".cdf")) { + try { + String xmlChunk = EDDTableFromNcFiles.generateDatasetsXml( + tDir, ".*\\" + topExt, + tDir + sampleName, "", tReloadEveryNMinutes, + "", "", "", "", //extract + "", "", "", "", "", "", null); //other info + resultsSB.append(xmlChunk); //recursive=true + for (int diri2 = diri; diri2 < nDirs; diri2++) + if (dirs.get(diri2).startsWith(tDir)) + dirDone.set(diri2); + String msg = topOfAre + "EDDTableFromNcFiles"; + dirInfo.set(diri, indent + msg); + String2.log(success + msg); + nTableNc++; + nCreated++; + continue; + } catch (Throwable t) { + String2.log("> Attempt with EDDTableFromNcFiles failed:\n" + + MustBe.throwableToString(t)); + } + } + + //ascii table + if (topExt.equals(".csv") || topExt.equals(".tsv") || + topExt.equals(".txt")) { + try { + String xmlChunk = EDDTableFromAsciiFiles.generateDatasetsXml( + tDir, ".*\\" + topExt, + tDir + sampleName, + "", 1, 2, //charset, columnNamesRow, firstDataRow, + tReloadEveryNMinutes, + "", "", "", "", //extract + "", "", "", "", "", "", null); //other info + resultsSB.append(xmlChunk); //recursive=true + for (int diri2 = diri; diri2 < nDirs; diri2++) + if (dirs.get(diri2).startsWith(tDir)) + dirDone.set(diri2); + String msg = topOfAre + "EDDTableFromAsciiFiles"; + dirInfo.set(diri, indent + msg); + String2.log(success + msg); + nTableAscii++; + nCreated++; + continue; + } catch (Throwable t) { + String2.log("> Attempt with EDDTableFromAscii failed:\n" + + MustBe.throwableToString(t)); + } + } + + //all fail? Use EDDTableFromFileNames and serve all files (not just topExt) + try { + String xmlChunk = EDDTableFromFileNames.generateDatasetsXml( + tDir, ".*", true, //recursive + tReloadEveryNMinutes, + "", "", "", "", null); //other info + resultsSB.append(xmlChunk); //recursive=true + for (int diri2 = diri; diri2 < nDirs; diri2++) + if (dirs.get(diri2).startsWith(tDir)) + dirDone.set(diri2); + String msg = topOfAre + "EDDTableFromFileNames"; + dirInfo.set(diri, indent + msg); + String2.log(success + msg); + nTableFileNames++; + nCreated++; + continue; + } catch (Throwable t) { + String2.log("> Attempt with EDDTableFromFileNames failed! Give up on this dir.\n" + + MustBe.throwableToString(t)); + } + } + + String2.log("\nDirectory Tree:\n"); + String2.log(dirTable.dataToCSVString()); + String2.log("\n> *** EDD.generateDatasetsXmlFromFiles finished successfully. time=" + + Calendar2.elapsedTimeString(System.currentTimeMillis() - time) + "\n" + + "> nDirs=" + nDirs + " nDatasetsCreated=" + nCreated + "\n" + + "> (nGridNc=" + nGridNc + " nTablencCF=" + nTableNcCF + + " nTableNc=" + nTableNc + " nTableAscii=" + nTableAscii + + " nTableFileNames=" + nTableFileNames + ")\n"); + if (nCreated == 0) + throw new RuntimeException("No datasets.xml chunks where successfully constructed."); + return resultsSB.toString(); + } /** * This calls testDasDds(tDatasetID, true). diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java index 58740de79..3c5887d39 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java @@ -108,6 +108,15 @@ public abstract class EDDGrid extends EDD { */ protected EDVGridAxis axisVariables[]; + /** + * This is used to test equality of axis values. + * 0=no testing (not recommended). + * >18 does exact test. default=20. + * 1-18 tests that many digets for doubles and hidiv(n,2) for floats. + */ + public final static int DEFAULT_MATCH_AXIS_N_DIGITS = 20; + + /** These are needed for EDD-required methods of the same name. */ public final static String[] dataFileTypeNames = { // ".asc", ".csv", ".csvp", ".csv0", ".das", ".dds", ".dods", @@ -1853,13 +1862,9 @@ public abstract PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tC * This makes a sibling dataset, based on the new sourceUrl. * * @param tLocalSourceUrl - * @param ensureAxisValuesAreEqual If Integer.MAX_VALUE, no axis sourceValue tests are performed. + * @param firstAxisToMatch * If 0, this tests if sourceValues for axis-variable #0+ are same. * If 1, this tests if sourceValues for axis-variable #1+ are same. - * (This is useful if the, for example, lat and lon values vary slightly and you - * are willing to accept the initial values as the correct values.) - * Actually, the tests are always done but this determines whether - * the error is just logged or whether it throws an exception. * @param shareInfo if true, this ensures that the sibling's * axis and data variables are basically the same as this datasets, * and then makes the new dataset point to the this instance's data structures @@ -1867,8 +1872,8 @@ public abstract PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tC * @return EDDGrid * @throws Throwable if trouble */ - public abstract EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable; + public abstract EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable; /** * This tests if the axisVariables and dataVariables of the other dataset are similar @@ -1876,29 +1881,26 @@ public abstract EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreE * same missing values). * * @param other - * @param ensureAxisValuesAreEqual If Integer.MAX_VALUE, no axis sourceValue - * tests are performed. + * @param firstAxisToMatch * If 0, this tests if sourceValues for axis-variable #0+ are same. * If 1, this tests if sourceValues for axis-variable #1+ are same. - * (This is useful if the, for example, lat and lon values vary slightly and you - * are willing to accept the initial values as the correct values.) - * Actually, the tests are always done but this determines whether - * the error is just logged or whether it throws an exception. * @param strict if !strict, this is less strict * @return "" if similar (same axis and data var names, * same units, same sourceDataType, same missing values) * or a message if not (including if other is null). */ - public String similar(EDDGrid other, int ensureAxisValuesAreEqual, boolean strict) { + public String similar(EDDGrid other, int firstAxisToMatch, + int matchAxisNDigits, boolean strict) { try { if (other == null) return "EDDGrid.similar: There is no 'other' dataset. (Perhaps ERDDAP just restarted.)"; - if (reallyVerbose) String2.log("EDDGrid.similar ensureAxisValuesAreEqual=" + ensureAxisValuesAreEqual); + if (reallyVerbose) String2.log("EDDGrid.similar firstAxisToMatch=" + firstAxisToMatch); String results = super.similar(other); if (results.length() > 0) return results; - return similarAxisVariables(other, ensureAxisValuesAreEqual, strict); + return similarAxisVariables(other, firstAxisToMatch, + matchAxisNDigits, strict); } catch (Throwable t) { return MustBe.throwableToShortString(t); } @@ -2005,23 +2007,18 @@ public String changed(EDD old) { * same missing values). * * @param other - * @param ensureAxisValuesAreEqual If Integer.MAX_VALUE, no axis sourceValue - * tests are performed. + * @param firstAxisToMatch * If 0, this tests if sourceValues for axis-variable #0+ are same. * If 1, this tests if sourceValues for axis-variable #1+ are same. - * (This is useful if the, for example, lat and lon values vary slightly and you - * are willing to accept the initial values as the correct values.) - * Actually, the tests are always done but this determines whether - * the error is just logged or whether it throws an exception. * @param strict if !strict, this is less strict (including allowing different * sourceDataTypes and destinationDataTypes) * @return "" if similar (same axis and data var names, * same units, same sourceDataType, same missing values) or a message if not. */ - public String similarAxisVariables(EDDGrid other, int ensureAxisValuesAreEqual, - boolean strict) { - if (reallyVerbose) String2.log("EDDGrid.similarAxisVariables ensureAxisValuesAreEqual=" + - ensureAxisValuesAreEqual); + public String similarAxisVariables(EDDGrid other, int firstAxisToMatch, + int matchAxisNDigits, boolean strict) { + if (reallyVerbose) String2.log("EDDGrid.similarAxisVariables firstAxisToMatch=" + + firstAxisToMatch); String msg = "EDDGrid.similar: The other dataset has a different "; int nAv = axisVariables.length; if (nAv != other.axisVariables.length) @@ -2080,12 +2077,11 @@ public String similarAxisVariables(EDDGrid other, int ensureAxisValuesAreEqual, } //test sourceValues - String results = av1.sourceValues().almostEqual(av2.sourceValues()); - if (results.length() > 0) { - results = msg + "sourceValue" + msg2 + results + ")"; - if (av >= ensureAxisValuesAreEqual) - return results; - else String2.log("NOTE: " + results); + if (av >= firstAxisToMatch) { + String results = av1.sourceValues().almostEqual(av2.sourceValues(), + matchAxisNDigits); + if (results.length() > 0) + return msg + "sourceValue" + msg2 + results + ")"; } } //they are similar @@ -8054,6 +8050,40 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, "
    and won't ever have a file extension (unlike, for example, .nc for the\n" + "
    sample dataset in the Pydap instructions).\n" + "\n" + + //Python + "

    Python" + + EDStatic.externalLinkHtml(tErddapUrl) + "\n" + + " is a widely-used computer language that is very popular among scientists.\n" + + "
    In addition to the Pydap Client, you can use Python to download various files from ERDDAP\n" + + "
    as you would download other files from the web:\n" + + "

    import urllib\n" +
+            "urllib.urlretrieve(\"http://baseurl/erddap/griddap/datasetID.fileType?query\", \"outputFileName\")
    \n" + + " Or download the content to an object instead of a file:\n" + + "
    import urllib2\n" +
+            "response = urllib2.open(\"http://baseurl/erddap/griddap/datasetID.fileType?query\")\n" +
+            "theContent = response.read()
    \n" + + " There are other ways to do this in Python. Search the web for more information.\n"+ + "
     \n" + + "
    To access a password-protected, private ERDDAP dataset with Python via https, use this\n" + + "
    two-step process after you install the\n" + + "requests library" + + EDStatic.externalLinkHtml(tErddapUrl) + " (\"HTTP for Humans\"):\n" + + "
      \n" + + "
    1. Log in (authenticate) and store the certificate in a 'session' object:\n" + + "
      import requests\n" +
+            "session = requests.session()\n" +
+            "credentials_dct = {'user': 'myUserName', 'password': 'myPassword'}\n" +
+            "p = session.post(\"https://baseurl:8443/erddap/login.html\", credentials_dct, verify=True)
      \n" + + "
    2. Repeatedly make data requests using the session: \n" + + "
      theContent = session.get('https://baseurl:8443/erddap/griddap/datasetID.fileType?query', verify=True)
      \n" + + "
    \n" + + " * Some ERDDAP installations won't need the port number (:8443) in the URL.\n" + + "
    * If the server uses a self-signed certificate and you are okay with that, use verify=False\n" + + "
      to tell Python not to check the server's certificate.\n" + + "
    * That works in Python v2.7. You might need to make slight modifications for other versions.\n" + + "
    * (Thanks to Emilio Mayorga of NANOOS and Paul Janecek of Spyglass Technologies for\n" + + "
      figuring this out.)\n" + + "
     \n" + //R "

    R Statistical Package" + EDStatic.externalLinkHtml(tErddapUrl) + " -\n" + @@ -8197,8 +8227,19 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, "
    to be put into the output fileName.\n" + "
    For example, \n" + "

    curl \"http://coastwatch.pfeg.noaa.gov/erddap/griddap/erdBAssta5day.png?sst%5B%282010-09-[01-05]T12:00:00Z%29%5D%5B%5D%5B%5D%5B%5D&.draw=surface&.vars=longitude|latitude|sst&.colorBar=|||||\" -o BAssta5day201009#1.png
    \n" + - "\n" + - "
     \n"); + "
  • To access a password-protected, private ERDDAP dataset with curl via https, use this\n" + + "
    two-step process:\n" + + "
      \n" + + "
    1. Log in (authenticate) and save the certificate cookie in a cookie-jar file:\n" + + "
      curl -v --data 'user=myUserName&password=myPassword' -c cookies.txt -b cookies.txt -k https://baseurl:8443/erddap/login.html
      \n" + + "
    2. Repeatedly make data requests using the saved cookie: \n" + + "
      curl -v -c cookies.txt -b cookies.txt -k https://baseurl:8443/erddap/griddap/datasetID.fileType?query -o outputFileName
      \n" + + "
    \n" + + " Some ERDDAP installations won't need the port number (:8443) in the URL.\n" + + "
    (Thanks to Liquid Robotics for the starting point and Emilio Mayorga of NANOOS and\n" + + "
    Paul Janecek of Spyglass Technologies for testing.)\n" + + "
     \n" + + "\n"); //query writer.write( diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java index eddd5f8f5..70b50b368 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java @@ -17,6 +17,7 @@ import com.cohort.util.Test; import gov.noaa.pfel.coastwatch.griddata.NcHelper; +import gov.noaa.pfel.coastwatch.util.FileVisitorDNLS; import gov.noaa.pfel.coastwatch.util.SimpleXMLReader; import gov.noaa.pfel.coastwatch.util.SSR; import gov.noaa.pfel.erddap.util.EDStatic; @@ -36,7 +37,14 @@ public class EDDGridAggregateExistingDimension extends EDDGrid { protected EDDGrid childDatasets[]; protected int childStopsAt[]; //the last valid axisVar0 index for each childDataset - protected boolean ensureAxisValuesAreEqual; + + /** + * This is used to test equality of axis values. + * 0=no testing (not recommended). + * >18 does exact test. Default=20. + * 1-18 tests that many digits for doubles and hidiv(n,2) for floats. + */ + protected int matchAxisNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; /** * This constructs an EDDGridAggregateExistingDimension based on the information in an .xml file. @@ -61,7 +69,7 @@ public static EDDGridAggregateExistingDimension fromXml(SimpleXMLReader xmlReade StringArray tOnChange = new StringArray(); String tFgdcFile = null; String tIso19115File = null; - boolean tEnsureAxisValuesAreEqual = true; + int tMatchAxisNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; String tDefaultDataQuery = null; String tDefaultGraphQuery = null; @@ -79,7 +87,6 @@ public static EDDGridAggregateExistingDimension fromXml(SimpleXMLReader xmlReade xmlReader.nextTag(); String tags = xmlReader.allTags(); String content = xmlReader.content(); - //if (reallyVerbose) String2.log("eavae=" + tEnsureAxisValuesAreEqual + " tags=" + tags + content); if (xmlReader.stackSize() == startOfTagsN) break; //the tag String localTags = tags.substring(startOfTagsLength); @@ -115,9 +122,12 @@ else if (localTags.equals( "")) { tSURecursive = tr == null? true : String2.parseBoolean(tr); } else if (localTags.equals("")) tSU = content; - else if (localTags.equals( "")) {} + else if (localTags.equals( "")) {} + else if (localTags.equals("")) + tMatchAxisNDigits = String2.parseInt(content, DEFAULT_MATCH_AXIS_N_DIGITS); + else if (localTags.equals( "")) {} //deprecated else if (localTags.equals("")) - tEnsureAxisValuesAreEqual = String2.parseBoolean(content); + tMatchAxisNDigits = String2.parseBoolean(content)? 20 : 0; else if (localTags.equals( "")) {} else if (localTags.equals("")) tAccessibleTo = content; else if (localTags.equals( "")) {} @@ -140,7 +150,7 @@ else if (localTags.equals( "")) {} tDefaultDataQuery, tDefaultGraphQuery, firstChild, tLocalSourceUrls.toArray(), tSUServerType, tSURegex, tSURecursive, tSU, - tEnsureAxisValuesAreEqual); + tMatchAxisNDigits); } @@ -163,13 +173,9 @@ else if (localTags.equals( "")) {} * @param tIso19115 This is like tFgdcFile, but for the ISO 19119-2/19139 metadata. * @param firstChild * @param tLocalSourceUrls the sourceUrls for the other siblings - * @param tEnsureAxisValuesAreEqual if true (recommended), this ensures - * that the axis sourceValues for axisVar 1+ are almostEqual. - * (Setting this to false is not recommended, but is useful if the, - * for example, lat and lon values vary slightly and you - * are willing to accept the initial values as the correct values.) - * Actually, the tests are always done but if this is false and the test fails, - * the error is just logged and doesn't throw an exception. + * @param tMatchAxisNDigits 0=no checking, 1-18 checks n digits, + * >18 does exact checking. Default is 20. + * This ensures that the axis sourceValues for axisVar 1+ are almostEqual. * @throws Throwable if trouble */ public EDDGridAggregateExistingDimension(String tDatasetID, @@ -177,11 +183,10 @@ public EDDGridAggregateExistingDimension(String tDatasetID, String tDefaultDataQuery, String tDefaultGraphQuery, EDDGrid firstChild, String tLocalSourceUrls[], String tSUServerType, String tSURegex, boolean tSURecursive, String tSU, - boolean tEnsureAxisValuesAreEqual) throws Throwable { + int tMatchAxisNDigits) throws Throwable { if (verbose) String2.log( - "\n*** constructing EDDGridAggregateExistingDimension " + tDatasetID + - " ensureEqual=" + tEnsureAxisValuesAreEqual); + "\n*** constructing EDDGridAggregateExistingDimension " + tDatasetID); long constructionStartMillis = System.currentTimeMillis(); String errorInMethod = "Error in EDDGridGridAggregateExistingDimension(" + tDatasetID + ") constructor:\n"; @@ -195,7 +200,7 @@ public EDDGridAggregateExistingDimension(String tDatasetID, iso19115File = tIso19115File; defaultDataQuery = tDefaultDataQuery; defaultGraphQuery = tDefaultGraphQuery; - ensureAxisValuesAreEqual = tEnsureAxisValuesAreEqual; + matchAxisNDigits = tMatchAxisNDigits; //if no tLocalSourceURLs, generate from hyrax, thredds, or dodsindex catalog? if (tLocalSourceUrls.length == 0 && tSU != null && tSU.length() > 0) { @@ -206,7 +211,7 @@ public EDDGridAggregateExistingDimension(String tDatasetID, if (tSUServerType == null) throw new RuntimeException(errorInMethod + " serverType is null."); else if (tSUServerType.toLowerCase().equals("hyrax")) - tsa.add(EDDGridFromDap.getUrlsFromHyraxCatalog(tSU, tSURegex, tSURecursive)); + tsa.add(FileVisitorDNLS.getUrlsFromHyraxCatalog(tSU, tSURegex, tSURecursive)); else if (tSUServerType.toLowerCase().equals("thredds")) tsa.add(EDDGridFromDap.getUrlsFromThreddsCatalog(tSU, tSURegex, tSURecursive)); else if (tSUServerType.toLowerCase().equals("dodsindex")) @@ -237,7 +242,7 @@ else throw new RuntimeException(errorInMethod + for (int sib = 0; sib < nChildren - 1; sib++) { if (reallyVerbose) String2.log("\n+++ Creating childDatasets[" + (sib+1) + "]\n"); EDDGrid sibling = firstChild.sibling(tLocalSourceUrls[sib], - ensureAxisValuesAreEqual? 1 : Integer.MAX_VALUE, true); + 1, matchAxisNDigits, true); childDatasets[sib + 1] = sibling; PrimitiveArray sourceValues0 = sibling.axisVariables()[0].sourceValues(); cumSV.append(sourceValues0); @@ -297,8 +302,8 @@ else throw new RuntimeException(errorInMethod + * * @throws Throwable always (since this class doesn't support sibling()) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { throw new SimpleException( "Error: EDDGridAggregateExistingDimension doesn't support method=\"sibling\"."); } @@ -460,7 +465,7 @@ public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraint tDataVariables, childConstraints); //childDataset has already checked that axis values are as *it* expects if (cumResults == null) { - if (!ensureAxisValuesAreEqual) { + if (matchAxisNDigits <= 0) { //make axis values exactly as expected by aggregate dataset for (int av = 1; av < nAv; av++) tResults[av] = axisVariables[av].sourceValues().subset( @@ -488,7 +493,8 @@ public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraint /** - * This generates datasets.xml for an aggregated dataset from a Hyrax catalog. + * This generates datasets.xml for an aggregated dataset from a Hyrax or + * THREDDS catalog. * * @param serverType Currently, only "hyrax" and "thredds" are supported (case insensitive) * @param startUrl the url of the current web page (with a hyrax catalog) e.g., @@ -510,11 +516,19 @@ public static String generateDatasetsXml(String serverType, String startUrl, StringBuilder sb = new StringBuilder(); String sa[]; serverType = serverType.toLowerCase(); - if ("hyrax".equals(serverType)) - sa = EDDGridFromDap.getUrlsFromHyraxCatalog(startUrl, fileNameRegex, recursive); - else if ("thredds".equals(serverType)) - sa = EDDGridFromDap.getUrlsFromThreddsCatalog(startUrl, fileNameRegex, recursive); - else throw new RuntimeException("ERROR: serverType must be \"hyrax\" or \"thredds\"."); + if ("hyrax".equals(serverType)) { + Test.ensureTrue(startUrl.endsWith("/contents.html"), + "startUrl must end with '/contents.html'."); + sa = FileVisitorDNLS.getUrlsFromHyraxCatalog( + File2.getDirectory(startUrl), fileNameRegex, recursive); + } else if ("thredds".equals(serverType)) { + Test.ensureTrue(startUrl.endsWith("/catalog.xml"), + "startUrl must end with '/catalog.xml'."); + sa = EDDGridFromDap.getUrlsFromThreddsCatalog( + startUrl, fileNameRegex, recursive); + } else { + throw new RuntimeException("ERROR: serverType must be \"hyrax\" or \"thredds\"."); + } if (sa.length == 0) throw new RuntimeException("ERROR: No matching URLs were found."); @@ -669,13 +683,22 @@ public static void testGenerateDatasetsXml() throws Throwable { " \n" + " null\n" + " COARDS, CF-1.6, ACDD-1.3\n" + +" null\n" + +" null\n" + +" null\n" + +" null\n" + +" null\n" + +" null\n" + " http://thredds1.pfeg.noaa.gov/thredds/dodsC/satellite/MH/chla/1day.html\n" + " NOAA CoastWatch WCN\n" + " 1day, altitude, aqua, chemistry, chla, chlorophyll, chlorophyll-a, coast, coastwatch, color, concentration, concentration_of_chlorophyll_in_sea_water, daily, data, day, degrees, global, imaging, MHchla, moderate, modis, national, noaa, node, npp, ocean, ocean color, oceans,\n" + "Oceans > Ocean Chemistry > Chlorophyll,\n" + "orbiting, partnership, polar, polar-orbiting, quality, resolution, science, science quality, sea, seawater, spectroradiometer, time, water, wcn, west\n" + +" null\n" + +" null\n" + +" null\n" + " null\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " null\n" + " Chlorophyll-a, Aqua MODIS, NPP, 0.05 degrees, Global, Science Quality (1day)\n" + " \n" + @@ -840,7 +863,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " podaac@podaac.jpl.nasa.gov\n" + " NASA GSFC MEaSUREs, NOAA\n" + " http://podaac.jpl.nasa.gov/dataset/CCMP_MEASURES_ATLAS_L4_OW_L3_0_WIND_VECTORS_FLK\n" + -" http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880101_v11l35flk.nc.gz.html\n" + +" http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/contents.html\n" + " NASA GSFC, NOAA\n" + " atlas, atmosphere,\n" + "Atmosphere > Atmospheric Winds > Surface Winds,\n" + @@ -848,7 +871,7 @@ public static void testGenerateDatasetsXml() throws Throwable { "atmospheric, center, component, data, derived, downward, eastward, eastward_wind, flight, flk, goddard, gsfc, level, meters, month, nasa, noaa, nobs, northward, northward_wind, number, observations, oceanography, physical, physical oceanography, pseudostress, space, speed, statistics, stress, surface, surface_downward_eastward_stress, surface_downward_northward_stress, time, u-component, u-wind, upstr, uwnd, v-component, v-wind, v1.1, v11l35flk, vpstr, vwnd, wind, wind_speed, winds, wspd\n" + " GCMD Science Keywords\n" + " [standard]\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " Time average of level3.0 products for the period: 1988-01-01 to 1988-01-31\n" + " Atlas FLK v1.1 derived surface winds (level 3.5) (month 19880101 v11l35flk)\n" + " \n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridCopy.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridCopy.java index 432e141bf..bd30f36c4 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridCopy.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridCopy.java @@ -18,6 +18,7 @@ import com.cohort.util.Test; import gov.noaa.pfel.coastwatch.griddata.NcHelper; +import gov.noaa.pfel.coastwatch.pointdata.Table; import gov.noaa.pfel.coastwatch.util.RegexFilenameFilter; import gov.noaa.pfel.coastwatch.util.SimpleXMLReader; @@ -53,6 +54,14 @@ public class EDDGridCopy extends EDDGrid { protected EDDGrid sourceEdd; protected EDDGridFromNcFiles localEdd; + /** + * This is used to test equality of axis values. + * 0=no testing (not recommended). + * >18 does exact test. default=20. + * 1-18 tests that many digets for doubles and hidiv(n,2) for floats. + */ + protected int matchAxisNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; + /** * Some tests set EDDGridCopy.defaultCheckSourceData = false; * Don't set it here. @@ -78,6 +87,7 @@ public static EDDGridCopy fromXml(SimpleXMLReader xmlReader) throws Throwable { EDDGrid tSourceEdd = null; int tReloadEveryNMinutes = Integer.MAX_VALUE; String tAccessibleTo = null; + int tMatchAxisNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; StringArray tOnChange = new StringArray(); String tFgdcFile = null; String tIso19115File = null; @@ -103,6 +113,12 @@ public static EDDGridCopy fromXml(SimpleXMLReader xmlReader) throws Throwable { //try to make the tag names as consistent, descriptive and readable as possible if (localTags.equals( "")) {} else if (localTags.equals("")) tAccessibleTo = content; + else if (localTags.equals( "")) {} + else if (localTags.equals("")) + tMatchAxisNDigits = String2.parseInt(content, DEFAULT_MATCH_AXIS_N_DIGITS); + else if (localTags.equals( "")) {} //deprecated + else if (localTags.equals("")) + tMatchAxisNDigits = String2.parseBoolean(content)? 20 : 0; else if (localTags.equals( "")) {} else if (localTags.equals("")) tOnChange.add(content); else if (localTags.equals( "")) {} @@ -150,7 +166,8 @@ else if (localTags.equals("")) { } return new EDDGridCopy(tDatasetID, - tAccessibleTo, tOnChange, tFgdcFile, tIso19115File, + tAccessibleTo, tMatchAxisNDigits, + tOnChange, tFgdcFile, tIso19115File, tDefaultDataQuery, tDefaultGraphQuery, tReloadEveryNMinutes, tSourceEdd, tFileTableInMemory, tAccessibleViaFiles); @@ -181,7 +198,7 @@ else if (localTags.equals("")) { * @throws Throwable if trouble */ public EDDGridCopy(String tDatasetID, - String tAccessibleTo, + String tAccessibleTo, int tMatchAxisNDigits, StringArray tOnChange, String tFgdcFile, String tIso19115File, String tDefaultDataQuery, String tDefaultGraphQuery, int tReloadEveryNMinutes, EDDGrid tSourceEdd, @@ -204,6 +221,7 @@ public EDDGridCopy(String tDatasetID, defaultDataQuery = tDefaultDataQuery; defaultGraphQuery = tDefaultGraphQuery; setReloadEveryNMinutes(tReloadEveryNMinutes); + matchAxisNDigits = tMatchAxisNDigits; //ensure copyDatasetDir exists String copyDatasetDir = EDStatic.fullCopyDirectory + datasetID + "/"; @@ -386,7 +404,7 @@ public EDDGridCopy(String tDatasetID, tReloadEveryNMinutes, 0, //updateEveryNMillis copyDatasetDir, recursive, fileNameRegex, EDDGridFromFiles.MF_LAST, - true, //tEnsureAxisValuesAreExactlyEqual, sourceEdd should have made them consistent + matchAxisNDigits, //sourceEdd should have made them consistent tFileTableInMemory, false); //accessibleViaFiles false here //copy things from localEdd @@ -460,12 +478,24 @@ public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraint * * @throws Throwable always (since this class doesn't support sibling()) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { throw new SimpleException("Error: " + "EDDGridCopy doesn't support method=\"sibling\"."); } + /** + * This returns a fileTable (formatted like + * FileVisitorDNLS.oneStep(tDirectoriesToo=false, last_mod is LongArray, + * and size is LongArray of epochMillis) + * with valid files (or null if unavailable or any trouble). + * This is a copy of any internal data, so client can modify the contents. + */ + public Table accessibleViaFilesFileTable() { + return localEdd.accessibleViaFilesFileTable(); + } + + /** * The basic tests of this class (erdGlobecBottle). * diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromBinaryFile.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromBinaryFile.java index d131be221..9acf0ee01 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromBinaryFile.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromBinaryFile.java @@ -32,6 +32,7 @@ /** * THIS IS NOT FINISHED AND NOT ACTIVE. + * IF THIS IS REVIVED, MAKE IT A SUBCLASS OF EDDGridFromFiles. * This class represents a grid dataset from a binary file (e.g., the Etopo1g bathymetry data). * Ideally, this would be quite general. Currently, it is pretty tied to the Etopo1g * data. @@ -253,8 +254,8 @@ else if (EDVTime.hasTimeUnits(tSourceAttributes, tAddAttributes)) * * @throws Throwable always (since this class doesn't support sibling()) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { throw new SimpleException( "Error: EDDGridBinaryFile doesn't support method=\"sibling\"."); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java index 318060799..bee407d48 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java @@ -9,6 +9,7 @@ import com.cohort.array.DoubleArray; import com.cohort.array.FloatArray; import com.cohort.array.IntArray; +import com.cohort.array.LongArray; import com.cohort.array.PrimitiveArray; import com.cohort.array.ShortArray; import com.cohort.array.StringArray; @@ -30,6 +31,7 @@ import gov.noaa.pfel.coastwatch.pointdata.Table; import gov.noaa.pfel.coastwatch.sgt.SgtGraph; import gov.noaa.pfel.coastwatch.sgt.SgtMap; +import gov.noaa.pfel.coastwatch.util.FileVisitorDNLS; import gov.noaa.pfel.coastwatch.util.SimpleXMLReader; import gov.noaa.pfel.coastwatch.util.SSR; @@ -674,13 +676,9 @@ public boolean lowUpdate(String msg, long startUpdateMillis) throws Throwable { * This makes a sibling dataset, based on the new sourceUrl. * * @param tLocalSourceUrl - * @param ensureAxisValuesAreEqual If Integer.MAX_VALUE, no axis sourceValue tests are performed. + * @param firstAxisToMatch * If 0, this tests if sourceValues for axis-variable #0+ are same. * If 1, this tests if sourceValues for axis-variable #1+ are same. - * (This is useful if the, for example, lat and lon values vary slightly and you - * are willing to accept the initial values as the correct values.) - * Actually, the tests are always done but this determines whether - * the error is just logged or whether it throws an exception. * @param shareInfo if true, this ensures that the sibling's * axis and data variables are basically the same as this datasets, * and then makes the new dataset point to the this instance's data structures @@ -689,7 +687,8 @@ public boolean lowUpdate(String msg, long startUpdateMillis) throws Throwable { * @return EDDGrid * @throws Throwable if trouble (e.g., try to shareInfo, but datasets not similar) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { if (verbose) String2.log("EDDGridFromDap.sibling " + tLocalSourceUrl); int nAv = axisVariables.length; @@ -736,7 +735,8 @@ public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, boo //ensure similar boolean testAV0 = false; - String results = similar(newEDDGrid, ensureAxisValuesAreEqual, testAV0); + String results = similar(newEDDGrid, firstAxisToMatch, + matchAxisNDigits, testAV0); if (results.length() > 0) throw new SimpleException("Error in EDDGrid.sibling: " + results); @@ -1351,11 +1351,11 @@ public static void safelyGenerateDatasetsXml(String tLocalSourceUrl, *
    This calls itself recursively, adding into to fileNameInfo as it is found. *
    If there is trouble (e.g., an exception), this catches it and returns. *
    http://www.unidata.ucar.edu/projects/THREDDS/tech/catalog/InvCatalogSpec.html - + * *

    Unsolved problem: this does nothing for detecting groups of files/URLs * that should be aggregated. * - * @param tLocalSourceUrl the tLocalSourceUrl of the current Thredds xml catalog (which usually includes /catalog/), e.g., + * @param tLocalSourceUrl the tLocalSourceUrl of the current Thredds catalog.xml (which usually includes /catalog/), e.g., * http://thredds1.pfeg.noaa.gov/thredds/catalog/catalog.xml * http://thredds1.pfeg.noaa.gov/thredds/catalog/Satellite/aggregsatMH/chla/catalog.xml * (note that comparable .html is at @@ -2147,11 +2147,14 @@ public static void runGenerateDatasetsXmlFromThreddsCatalog( " altitude, aqua, chemistry, chla, chlorophyll, chlorophyll-a, coast, coastwatch, color, concentration, concentration_of_chlorophyll_in_sea_water, data, degrees, global, imaging, MHchla, moderate, modis, national, noaa, node, npp, ocean, ocean color, oceans,\n" + "Oceans > Ocean Chemistry > Chlorophyll,\n" + "orbiting, partnership, polar, polar-orbiting, quality, resolution, science, science quality, sea, seawater, spectroradiometer, time, water, wcn, west\n" + +" null\n" + +" null\n" + +" null\n" + " dave.foley@noaa.gov\n" + " NOAA CoastWatch, West Coast Node\n" + " http://coastwatch.pfel.noaa.gov\n" + " null\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " null\n" + " NOAA CoastWatch distributes chlorophyll-a concentration data from NASA's Aqua Spacecraft. Measurements are gathered by the Moderate Resolution Imaging Spectroradiometer \\(MODIS\\) carried aboard the spacecraft. This is Science Quality data. \\(1-day\\)\n" + " \n" + @@ -2309,211 +2312,6 @@ public static void testUAFSubThreddsCatalog(int which) throws Throwable { UAFSubThreddsCatalogs[which], ".*", -1); //-1 uses suggestReloadEveryNMinutes } - /** - * This gets the file names from Hyrax catalog directory URL. - * - * @param startUrl the url of the current web page (with a hyrax catalog) e.g., - "http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/contents.html" - * @param fileNameRegex e.g., - "pentad.*flk\\.nc\\.gz" - * @param recursive - * @returns a String[] with a list of full URLs of the children (may be new String[0]) - * @throws Throwable if trouble - */ - public static String[] getUrlsFromHyraxCatalog(String startUrl, String fileNameRegex, - boolean recursive) throws Throwable { - if (verbose) String2.log("getUrlsFromHyraxCatalog regex=" + fileNameRegex); - - //call the recursive method - StringArray childUrls = new StringArray(); - DoubleArray lastModified = new DoubleArray(); - addToHyraxUrlList(startUrl, fileNameRegex, recursive, childUrls, lastModified); - - return childUrls.toArray(); - } - - /** - * This does the work for getHyraxUrls. - * This calls itself recursively, adding into to fileNameInfo as it is found. - * - * @param url the url of the current web page (with a hyrax catalog) e.g., - "http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/contents.html" - * @param fileNameRegex e.g., - "pentad.*flk\\.nc\\.gz" - * @param childUrls new children will be added to this - * @param lastModified the lastModified time (secondsSinceEpoch, NaN if not available) - * @return true if completely successful (no access errors, all URLs found) - * @throws Throwable if trouble, e.g., if url doesn't respond - */ - public static boolean addToHyraxUrlList(String url, String fileNameRegex, boolean recursive, - StringArray childUrls, DoubleArray lastModified) throws Throwable { - - if (reallyVerbose) String2.log("\ngetHyraxUrlInfo childUrls.size=" + childUrls.size() + - "\n url=" + url); - boolean completelySuccessful = true; //but any child can set it to false - String response; - try { - response = SSR.getUrlResponseString(url); - } catch (Throwable t) { - String2.log(MustBe.throwableToString(t)); - return false; - } - String responseLC = response.toLowerCase(); - String urlDir = File2.getDirectory(url); - - //skip header line and parent directory - int po = responseLC.indexOf("parent directory"); //Lower Case - if (po < 0 ) { - if (reallyVerbose) String2.log("ERROR: \"parent directory\" not found in Hyrax response."); - return false; - } - po += 18; - - //endPre - int endPre = responseLC.indexOf("", po); //Lower Case - if (endPre < 0) - endPre = response.length(); - - //go through file,dir listings - boolean diagnosticMode = false; - while (true) { - - //EXAMPLE http://data.nodc.noaa.gov/opendap/wod/monthly/ No longer available - - //EXAMPLE http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/M07 - //(reformatted: look for tags, not formatting - /* - month_19870701_v11l35flk.nc.gz - 2007-04-04T07:00:00 - 4807310 - - - - - -
    ddx dds 
    //will exist if exists - - - //may or may not exist - //may or may not exist - //the next row... -
    viewers
    - */ - - //find beginRow and nextRow - int beginRow = responseLC.indexOf(" endPre) - return completelySuccessful; - int endRow = responseLC.indexOf(" endPre) - endRow = endPre; - - //if in the middle, skip table - int tablePo = responseLC.indexOf(" 0 && tablePo < endRow) { - int endTablePo = responseLC.indexOf(" endPre) - endTablePo = endPre; - - //find - endRow = responseLC.indexOf(" endPre) - endRow = endPre; - } - String thisRow = response.substring(beginRow, endRow); - String thisRowLC = responseLC.substring(beginRow, endRow); - if (diagnosticMode) - String2.log("<<>>"); - - //look for .das href="wod_013459339O.nc.das">das< - int dasPo = thisRowLC.indexOf(".das\">das<"); - if (diagnosticMode) - String2.log(" .das " + (dasPo < 0? "not " : "") + "found"); - if (dasPo > 0) { - int quotePo = thisRow.lastIndexOf('"', dasPo); - if (quotePo < 0) { - String2.log("ERROR: invalid .das reference:\n " + thisRow); - po = endRow; - continue; - } - String fileName = thisRow.substring(quotePo + 1, dasPo); - if (diagnosticMode) - String2.log(" filename=" + fileName + - (fileName.matches(fileNameRegex)? " does" : " doesn't") + - " match " + fileNameRegex); - if (fileName.matches(fileNameRegex)) { - - //get lastModified time >2011-06-30T04:43:09< - String stime = String2.extractRegex(thisRow, - ">\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}<", 0); - double dtime = Calendar2.safeIsoStringToEpochSeconds( - stime == null? "" : stime.substring(1, stime.length() - 1)); - - childUrls.add(urlDir + fileName); - lastModified.add(dtime); - //String2.log(" file=" + fileName + " " + stime); - po = endRow; - continue; - } - } - - if (recursive) { - //look for href="199703-199705/contents.html" - int conPo = thisRowLC.indexOf("/contents.html\""); - if (conPo > 0) { - int quotePo = thisRow.lastIndexOf('"', conPo); - if (quotePo < 0) { - String2.log("ERROR: invalid contents.html reference:\n " + thisRow); - po = endRow; - continue; - } - boolean tSuccessful = addToHyraxUrlList( - urlDir + thisRow.substring(quotePo + 1, conPo + 14), - fileNameRegex, recursive, childUrls, lastModified); - if (!tSuccessful) - completelySuccessful = false; - po = endRow; - continue; - } - } - po = endRow; - } - } - - /** - */ - public static void testGetUrlsFromHyraxCatalog() throws Throwable { - String2.log("\n*** EDDGridAggregateExistingDimension.testGetUrlsFromHyraxCatalog()\n"); - - try { - - - String results[] = getUrlsFromHyraxCatalog( - //before 2011-05-18, was - //"http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/flk/1988/contents.html", - "http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/contents.html", - "month.*flk\\.nc\\.gz", true); - String expected[] = new String[]{ -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880101_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880201_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880301_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880401_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880501_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880601_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880701_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880801_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19880901_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881001_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881101_v11l35flk.nc.gz", -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1988/month_19881201_v11l35flk.nc.gz"}; - Test.ensureEqual(results, expected, "results=\n" + results); - } catch (Throwable t) { - String2.pressEnterToContinue(MustBe.throwableToString(t)); - } - - - } /** * This is for use by Bob at ERD -- others don't need it. @@ -3433,7 +3231,7 @@ public static void testBasic2() throws Throwable { " :sensor = \"MODIS\";\n" + " :source = \"satellite observation: Aqua, MODIS\";\n" + " :sourceUrl = \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/MH/chla/8day\";\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"NOAA CoastWatch distributes chlorophyll-a concentration data from NASA's Aqua Spacecraft. Measurements are gathered by the Moderate Resolution Imaging Spectroradiometer (MODIS) carried aboard the spacecraft. This is Science Quality data.\";\n" + " :time_coverage_end = \"2006-12-15T00:00:00Z\";\n" + " :time_coverage_start = \"2002-07-08T00:00:00Z\";\n" + @@ -4127,7 +3925,7 @@ public static void testBasic3() throws Throwable { " :source = \"satellite observation: Aqua, MODIS\";\n" + " :sourceUrl = \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/MH/chla/8day\";\n" + " :Southernmost_Northing = 28.985876360268577; // double\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"NOAA CoastWatch distributes chlorophyll-a concentration data from NASA's Aqua Spacecraft. Measurements are gathered by the Moderate Resolution Imaging Spectroradiometer (MODIS) carried aboard the spacecraft. This is Science Quality data.\";\n" + " :time_coverage_end = \"2007-02-06T00:00:00Z\";\n" + " :time_coverage_start = \"2007-02-06T00:00:00Z\";\n" + @@ -4693,7 +4491,7 @@ public static void testOpendap() throws Throwable { " :source = \"satellite observation: Aqua, MODIS\";\n" + " :sourceUrl = \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/MH/chla/8day\";\n" + " :Southernmost_Northing = -90.0; // double\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"NOAA CoastWatch distributes chlorophyll-a concentration data from NASA's Aqua Spacecraft. Measurements are gathered by the Moderate Resolution Imaging Spectroradiometer \\(MODIS\\) carried aboard the spacecraft. This is Science Quality data.\";\n" + " :time_coverage_end = \"20.{8}T00:00:00Z\";\n" + //changes " :time_coverage_start = \"2002-07-08T00:00:00Z\";\n" + @@ -4870,7 +4668,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " NOAA GFDL\n" + " 18610101-20001231, 20c3m, 20c3m-0, 20th, ar4, assessment, atmosphere, ccsp, century, change, climate, cm2.0, coefficient, coordinate, data, diagnosis, dynamics, experiment, fluid, fourth, geophysical, gfdl, hybrid, intercomparison, intergovernmental, ipcc, laboratory, layer, lev, model, month, monthly, noaa, output, panel, pcmdi, program, report, run, science, sigma, US\n" + " [standard]\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " Geophysical Fluid Dynamics Laboratory (GFDL) CM2.0, 20C3M (run 1) climate of the 20th Century experiment (20C3M) output for Intergovernmental Panel on Climate Change (IPCC) Fourth Assessment Report (AR4) and US Climate Change Science Program (CCSP). GFDL experiment name = CM2Q-d2_1861-2000-AllForc_h1. Program for Climate Model Diagnosis and Intercomparison (PCMDI) experiment name = 20C3M (run1). Initial conditions for this experiment were taken from 1 January of year 1 of the 1860 control model experiment named CM2Q_Control-1860_d2. Several forcing agents varied during the 140 year duration of the CM2Q-d2_1861-2000-AllForc_h1 experiment in a manner based upon observations and reconstructions for the late 19th and 20th centuries. The time varying forcing agents were atmospheric CO2, CH4, N2O, halons, tropospheric and stratospheric O3, anthropogenic tropospheric sulfates, black and organic carbon, volcanic aerosols, solar irradiance, and the distribution of land cover types. The direct effect of tropospheric aerosols is calculated by the model, but not the indirect effects.\n" + " IPCC AR4 CM2.0 R1 20C3M-0 monthly atmos 18610101-20001231 [lev]\n" + " \n" + @@ -5024,7 +4822,7 @@ public static void testGenerateDatasetsXml2() throws Throwable { " satellite observation: GOES, Imager\n" + " http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/GA/ssta/hday\n" + " -44.975\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " NOAA CoastWatch provides SST data from the NOAA Geostationary Operational Environmental Satellites (GOES). Measurements are gathered by the GOES Imager, a multi-channel radiometer carried aboard the satellite. SST is available for hourly Imager measurements, or in composite images of various durations.\n" + " 2013-01-30T15:00:00Z\n" + " 2008-06-02T00:00:00Z\n" + @@ -5576,7 +5374,7 @@ public static void testPmelOscar(boolean doGraphicsTests) throws Throwable { " String source \"Gary Lagerloef, ESR (lager@esr.org) and Fabrice Bonjean (bonjean@esr.org)\";\n" + " String sourceUrl \"http://dapper.pmel.noaa.gov/dapper/oscar/world-unfilter.nc\";\n" + " Float64 Southernmost_Northing -69.5;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"This project is developing a processing system and data center to provide operational ocean surface velocity fields from satellite altimeter and vector wind data. The method to derive surface currents with satellite altimeter and scatterometer data is the outcome of several years NASA sponsored research. The project will transition that capability to operational oceanographic applications. The end product is velocity maps updated daily, with a goal for eventual 2-day maximum delay from time of satellite measurement. Grid resolution is 100 km for the basin scale, and finer resolution in the vicinity of the Pacific Islands.\";\n" + " String time_coverage_end \"" + lastTimeString + "\";\n" + " String time_coverage_start \"1992-10-21T00:00:00Z\";\n" + @@ -5710,7 +5508,7 @@ public static void testPmelOscar(boolean doGraphicsTests) throws Throwable { " :source = \"Gary Lagerloef, ESR (lager@esr.org) and Fabrice Bonjean (bonjean@esr.org)\";\n" + " :sourceUrl = \"http://dapper.pmel.noaa.gov/dapper/oscar/world-unfilter.nc\";\n" + " :Southernmost_Northing = -60.5f; // float\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"This project is developing a processing system and data center to provide operational ocean surface velocity fields from satellite altimeter and vector wind data. The method to derive surface currents with satellite altimeter and scatterometer data is the outcome of several years NASA sponsored research. The project will transition that capability to operational oceanographic applications. The end product is velocity maps updated daily, with a goal for eventual 2-day maximum delay from time of satellite measurement. Grid resolution is 100 km for the basin scale, and finer resolution in the vicinity of the Pacific Islands.\";\n" + " :title = \"OSCAR - Ocean Surface Current Analyses, Real-Time\";\n" + " :VARIABLE = \"Ocean Surface Currents\";\n" + @@ -5956,7 +5754,7 @@ public static void testDescendingLat(boolean doGraphicsTests) throws Throwable { " String reference \"Divins, D.L., and D. Metzger, NGDC Coastal Relief Model, http://www.ngdc.noaa.gov/mgg/coastal/coastal.html\";\n" + " String sourceUrl \"http://geoport.whoi.edu/thredds/dodsC/bathy/crm_vol10.nc\";\n" + " Float64 Southernmost_Northing 18.0;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"This Coastal Relief Gridded database provides the first comprehensive view of the US Coastal Zone; one that extends from the coastal state boundaries to as far offshore as the NOS hydrographic data will support a continuous view of the seafloor. In many cases, this seaward limit reaches out to, and in places even beyond the continental slope. The gridded database contains data for the entire coastal zone of the conterminous US, including Hawaii and Puerto Rico.\";\n" + " String title \"Topography, NOAA Coastal Relief Model, 3 arc second, Vol. 10 (Hawaii)\";\n" + " Float64 Westernmost_Easting -161.0;\n" + @@ -6038,7 +5836,7 @@ public static void testDescendingLat(boolean doGraphicsTests) throws Throwable { " :reference = \"Divins, D.L., and D. Metzger, NGDC Coastal Relief Model, http://www.ngdc.noaa.gov/mgg/coastal/coastal.html\";\n" + " :sourceUrl = \"http://geoport.whoi.edu/thredds/dodsC/bathy/crm_vol10.nc\";\n" + " :Southernmost_Northing = 21.0; // double\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"This Coastal Relief Gridded database provides the first comprehensive view of the US Coastal Zone; one that extends from the coastal state boundaries to as far offshore as the NOS hydrographic data will support a continuous view of the seafloor. In many cases, this seaward limit reaches out to, and in places even beyond the continental slope. The gridded database contains data for the entire coastal zone of the conterminous US, including Hawaii and Puerto Rico.\";\n" + " :title = \"Topography, NOAA Coastal Relief Model, 3 arc second, Vol. 10 (Hawaii)\";\n" + " data:\n" + @@ -8020,41 +7818,6 @@ public static void testDescendingAxisGeotif() throws Throwable { } - /** - * This tests addToHyraxUrlList. - */ - public static void testAddToHyraxUrlList() throws Throwable { - String2.log("\n*** testAddToHyraxUrlList"); - - try{ - StringArray childUrls = new StringArray(); - DoubleArray lastModified = new DoubleArray(); - addToHyraxUrlList( - "http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/", //startUrl, - "month_[0-9]{8}_v11l35flk\\.nc\\.gz", //fileNameRegex, - true, //recursive, - childUrls, lastModified); - - String results = childUrls.toNewlineString(); - String expected = -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19870701_v11l35flk.nc.gz\n" + -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19870801_v11l35flk.nc.gz\n" + -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19870901_v11l35flk.nc.gz\n" + -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19871001_v11l35flk.nc.gz\n" + -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19871101_v11l35flk.nc.gz\n" + -"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/monthly/flk/1987/month_19871201_v11l35flk.nc.gz\n"; - Test.ensureEqual(results, expected, "results=\n" + results); - - results = lastModified.toString(); - expected = "1.336609915E9, 1.336785444E9, 1.336673639E9, 1.336196561E9, 1.336881763E9, 1.336705731E9"; - Test.ensureEqual(results, expected, "results=\n" + results); - } catch (Throwable t) { - String2.pressEnterToContinue(MustBe.throwableToString(t) + - "\nUnexpected error."); - } - - } - /** This tests saveAsNcml. */ public static void testNcml() throws Throwable { testVerboseOn(); @@ -8141,7 +7904,7 @@ public static void testNcml() throws Throwable { " \n" + " \n" + " \n" + -" \n" + +" \n" + " Schroeder, Isaac D., Bryan A. Black, William J. Sydeman, Steven J. Bograd, Elliott L. Hazen, Jarrod A. Santora, and Brian K. Wells. "The North Pacific High and wintertime pre-conditioning of California current productivity", Geophys. Res. Letters, VOL. 40, 541-546, doi:10.1002/grl.50100, 2013\n" + " (local files)\n" + " 23.3\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " time, year, month\n" + " Variations in large-scale atmospheric forcing influence upwelling dynamics and ecosystem productivity in the California Current System (CCS). In this paper, we characterize interannual variability of the North Pacific High over 40 years and investigate how variation in its amplitude and position affect upwelling and biology. We develop a winter upwelling "pre-conditioning" index and demonstrate its utility to understanding biological processes. Variation in the winter NPH can be well described by its areal extent and maximum pressure, which in turn is predictive of winter upwelling. Our winter pre-conditioning index explained 64% of the variation in biological responses (fish and seabirds). Understanding characteristics of the NPH in winter is therefore critical to predicting biological responses in the CCS.\n" + " 2014-01-16\n" + @@ -1311,7 +1311,7 @@ public static void testBasic() throws Throwable { " Float64 Northernmost_Northing 34.04017;\n" + " String sourceUrl \"http://data.nodc.noaa.gov/thredds/catalog/nmsp/wcos/catalog.xml\";\n" + " Float64 Southernmost_Northing 34.04017;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"longitude, latitude\";\n" + " String summary \"The West Coast Observing System (WCOS) project provides access to temperature and currents data collected at four of the five National Marine Sanctuary sites, including Olympic Coast, Gulf of the Farallones, Monterey Bay, and Channel Islands. A semi-automated end-to-end data management system transports and transforms the data from source to archive, making the data acessible for discovery, access and analysis from multiple Internet points of entry.\n" + "\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java index d39896940..50ad09d72 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java @@ -641,13 +641,9 @@ public String getPublicSourceErddapUrl() { * This makes a sibling dataset, based on the new sourceUrl. * * @param tLocalSourceUrl - * @param ensureAxisValuesAreEqual If Integer.MAX_VALUE, no axis sourceValue tests are performed. + * @param firstAxisToMatch * If 0, this tests if sourceValues for axis-variable #0+ are same. * If 1, this tests if sourceValues for axis-variable #1+ are same. - * (This is useful if the, for example, lat and lon values vary slightly and you - * are willing to accept the initial values as the correct values.) - * Actually, the tests are always done but this determines whether - * the error is just logged or whether it throws an exception. * @param shareInfo if true, this ensures that the sibling's * axis and data variables are basically the same as this datasets, * and then makes the new dataset point to the this instance's data structures @@ -656,7 +652,8 @@ public String getPublicSourceErddapUrl() { * @return EDDGrid * @throws Throwable if trouble (e.g., try to shareInfo, but datasets not similar) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { if (verbose) String2.log("EDDGridFromErddap.sibling " + tLocalSourceUrl); int nAv = axisVariables.length; @@ -684,7 +681,8 @@ public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, boo //ensure similar boolean testAV0 = false; - String results = similar(newEDDGrid, ensureAxisValuesAreEqual, false); + String results = similar(newEDDGrid, firstAxisToMatch, + matchAxisNDigits, false); if (results.length() > 0) throw new RuntimeException("Error in EDDGrid.sibling: " + results); @@ -1097,7 +1095,7 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { " String source \"satellite observation: Aqua, MODIS\";\n" + " String sourceUrl \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/MH/chla/8day\";\n" + " Float64 Southernmost_Northing -90.0;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"NOAA CoastWatch distributes chlorophyll-a concentration data from NASA's Aqua Spacecraft. Measurements are gathered by the Moderate Resolution Imaging Spectroradiometer \\(MODIS\\) carried aboard the spacecraft. This is Science Quality data.\";\n" + " String time_coverage_end \"20.{8}T00:00:00Z\";\n" + //changes " String time_coverage_start \"2002-07-08T00:00:00Z\";\n" + @@ -1591,7 +1589,7 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { " :source = \"satellite observation: Aqua, MODIS\";\n" + " :sourceUrl = \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/MH/chla/8day\";\n" + " :Southernmost_Northing = 28.985876360268577; // double\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"NOAA CoastWatch distributes chlorophyll-a concentration data from NASA's Aqua Spacecraft. Measurements are gathered by the Moderate Resolution Imaging Spectroradiometer \\(MODIS\\) carried aboard the spacecraft. This is Science Quality data.\";\n" + " :time_coverage_end = \"2007-02-06T00:00:00Z\";\n" + " :time_coverage_start = \"2007-02-06T00:00:00Z\";\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromEtopo.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromEtopo.java index f90463474..d7e954514 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromEtopo.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromEtopo.java @@ -199,8 +199,8 @@ public EDDGridFromEtopo(String tDatasetID) throws Throwable { * * @throws Throwable always (since this class doesn't support sibling()) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { throw new SimpleException("Error: " + "EDDGridFromEtopo doesn't support method=\"sibling\"."); @@ -529,7 +529,7 @@ public static void test(boolean doGraphicsTests) throws Throwable { " :references = \"Amante, C. and B. W. Eakins, ETOPO1 1 Arc-Minute Global Relief Model: Procedures, Data Sources and Analysis. NOAA Technical Memorandum NESDIS NGDC-24, 19 pp, March 2009.\";\n" + " :sourceUrl = \"(local file)\";\n" + " :Southernmost_Northing = -90.0; // double\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"ETOPO1 is a 1 arc-minute global relief model of Earth's surface that integrates land topography and ocean bathymetry. It was built from numerous global and regional data sets. This is the 'Ice Surface' version, with the top of the Antarctic and Greenland ice sheets. The horizontal datum is WGS-84, the vertical datum is Mean Sea Level. Keywords: Bathymetry, Digital Elevation. This is the grid/node-registered version: the dataset's latitude and longitude values mark the centers of the cells.\";\n" + " :title = \"Topography, ETOPO1, 0.0166667 degrees, Global (longitude -180 to 180), (Ice Sheet Surface)\";\n" + " :Westernmost_Easting = -180.0; // double\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java index 4ee046f1a..d22ed6436 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java @@ -53,7 +53,7 @@ * can be gathered (for each file) * and cached (facilitating handling constraints in data requests). *
    And file data can be cached and reused because each file has a lastModified - * time which can be used to detect if file is unchanged. + * time and size which can be used to detect if file is unchanged. * * @author Bob Simons (bob.simons@noaa.gov) 2008-11-26 */ @@ -61,21 +61,24 @@ public abstract class EDDGridFromFiles extends EDDGrid{ public final static String MF_FIRST = "first", MF_LAST = "last"; public static int suggestedUpdateEveryNMillis = 10000; + public static int suggestUpdateEveryNMillis(String tFileDir) { + return File2.isRemote(tFileDir)? 0 : suggestedUpdateEveryNMillis; + } /** Columns in the File Table */ protected final static int FT_DIR_INDEX_COL=0, //useful that it is #0 (tFileTable uses same positions) FT_FILE_LIST_COL=1, //useful that it is #1 FT_LAST_MOD_COL=2, - FT_N_VALUES_COL=3, FT_MIN_COL=4, FT_MAX_COL=5, FT_CSV_VALUES_COL=6, - FT_START_INDEX_COL = 7; + FT_SIZE_COL=3, + FT_N_VALUES_COL=4, FT_MIN_COL=5, FT_MAX_COL=6, FT_CSV_VALUES_COL=7, + FT_START_INDEX_COL = 8; //set by constructor protected String fileDir; protected boolean recursive; protected String fileNameRegex; protected String metadataFrom; - protected boolean ensureAxisValuesAreExactlyEqual; protected StringArray sourceDataNames; protected StringArray sourceAxisNames; protected String sourceDataTypes[]; @@ -85,11 +88,19 @@ public abstract class EDDGridFromFiles extends EDDGrid{ protected PrimitiveArray sourceAxisValues[]; protected Attributes sourceDataAttributes[]; + /** + * This is used to test equality of axis values. + * 0=no testing (not recommended). + * >18 does exact test. default=20. + * 1-18 tests that many digets for doubles and hidiv(n,2) for floats. + */ + protected int matchAxisNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; + protected WatchDirectory watchDirectory; //dirTable and fileTable inMemory (default=false) protected boolean fileTableInMemory = false; - protected Table dirTable; + protected Table dirTable; //one column with dir names protected Table fileTable; @@ -123,7 +134,7 @@ public static EDDGridFromFiles fromXml(SimpleXMLReader xmlReader) throws Throwab String tFileNameRegex = ".*"; boolean tAccessibleViaFiles = false; String tMetadataFrom = MF_LAST; - boolean tEnsureAxisValuesAreExactlyEqual = true; + int tMatchAxisNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; String tDefaultDataQuery = null; String tDefaultGraphQuery = null; @@ -165,11 +176,12 @@ else if (localTags.equals( "")) {} else if (localTags.equals("")) tMetadataFrom = content; else if (localTags.equals( "")) {} else if (localTags.equals("")) tFileTableInMemory = String2.parseBoolean(content); - //ensureAxisValuesAreExactlyEqual is currently not allowed; - //if false, it is hard to know which are desired values (same as metadataFrom?) - //else if (localTags.equals( "")) {} - //else if (localTags.equals("")) - // tEnsureAxisValuesAreExactlyEqual = String2.parseBoolean(content); + else if (localTags.equals( "")) {} + else if (localTags.equals("")) + tMatchAxisNDigits = String2.parseInt(content, DEFAULT_MATCH_AXIS_N_DIGITS); + else if (localTags.equals( "")) {} //deprecated + else if (localTags.equals("")) + tMatchAxisNDigits = String2.parseBoolean(content)? 20 : 0; else if (localTags.equals( "")) {} else if (localTags.equals("")) tOnChange.add(content); else if (localTags.equals( "")) {} @@ -202,7 +214,7 @@ else if (localTags.equals( "")) {} ttDataVariables, tReloadEveryNMinutes, tUpdateEveryNMillis, tFileDir, tRecursive, tFileNameRegex, tMetadataFrom, - tEnsureAxisValuesAreExactlyEqual, tFileTableInMemory, + tMatchAxisNDigits, tFileTableInMemory, tAccessibleViaFiles); else if (tType.equals("EDDGridFromMergeIRFiles")) return new EDDGridFromMergeIRFiles(tDatasetID, tAccessibleTo, @@ -212,7 +224,7 @@ else if (tType.equals("EDDGridFromMergeIRFiles")) ttDataVariables, tReloadEveryNMinutes, tUpdateEveryNMillis, tFileDir, tRecursive, tFileNameRegex, tMetadataFrom, - tEnsureAxisValuesAreExactlyEqual, tFileTableInMemory, + tMatchAxisNDigits, tFileTableInMemory, tAccessibleViaFiles); else throw new Exception("type=\"" + tType + "\" needs to be added to EDDGridFromFiles.fromXml at end."); @@ -302,9 +314,9 @@ else throw new Exception("type=\"" + tType + * to extract source metadata (first/last based on file list sorted by minimum axis #0 value). * Valid values are "first", "penultimate", "last". * If invalid, "last" is used. - * @param tEnsureAxisValuesAreExactlyEqual if true (default, currently required), - * a file's axis values must exactly equal the others or the file is rejected; - * if false, almostEqual is used. + * @param tMatchAxisNDigits 0=no test, + * 1-18 tests 1-18 digits for doubles and hiDiv(n,2) for floats, + * >18 does an exact test. Default is 20. * @throws Throwable if trouble */ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessibleTo, @@ -315,7 +327,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible Object[][] tDataVariables, int tReloadEveryNMinutes, int tUpdateEveryNMillis, String tFileDir, boolean tRecursive, String tFileNameRegex, - String tMetadataFrom, boolean tEnsureAxisValuesAreExactlyEqual, + String tMetadataFrom, int tMatchAxisNDigits, boolean tFileTableInMemory, boolean tAccessibleViaFiles) throws Throwable { @@ -351,7 +363,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible recursive = tRecursive; fileNameRegex = tFileNameRegex; metadataFrom = tMetadataFrom; - ensureAxisValuesAreExactlyEqual = true; //tEnsureAxisValuesAreExactlyEqual; + matchAxisNDigits = tMatchAxisNDigits; int nav = tAxisVariables.length; int ndv = tDataVariables.length; @@ -415,6 +427,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible if (fileTable.findColumnNumber("dirIndex") != FT_DIR_INDEX_COL) ok = false; else if (fileTable.findColumnNumber("fileList") != FT_FILE_LIST_COL) ok = false; else if (fileTable.findColumnNumber("lastMod") != FT_LAST_MOD_COL) ok = false; + else if (fileTable.findColumnNumber("size") != FT_SIZE_COL) ok = false; else if (fileTable.findColumnNumber("nValues") != FT_N_VALUES_COL) ok = false; else if (fileTable.findColumnNumber("min") != FT_MIN_COL) ok = false; else if (fileTable.findColumnNumber("max") != FT_MAX_COL) ok = false; @@ -423,6 +436,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible else if (!(fileTable.getColumn(FT_DIR_INDEX_COL) instanceof ShortArray)) ok = false; else if (!(fileTable.getColumn(FT_FILE_LIST_COL) instanceof StringArray)) ok = false; else if (!(fileTable.getColumn(FT_LAST_MOD_COL) instanceof DoubleArray)) ok = false; + else if (!(fileTable.getColumn(FT_SIZE_COL) instanceof DoubleArray)) ok = false; else if (!(fileTable.getColumn(FT_N_VALUES_COL) instanceof IntArray)) ok = false; else if (!(fileTable.getColumn(FT_MIN_COL) instanceof DoubleArray)) ok = false; else if (!(fileTable.getColumn(FT_MAX_COL) instanceof DoubleArray)) ok = false; @@ -451,6 +465,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible Test.ensureEqual(fileTable.addColumn(FT_DIR_INDEX_COL, "dirIndex", new ShortArray()), FT_DIR_INDEX_COL, "FT_DIR_INDEX_COL is wrong."); Test.ensureEqual(fileTable.addColumn(FT_FILE_LIST_COL, "fileList", new StringArray()), FT_FILE_LIST_COL, "FT_FILE_LIST_COL is wrong."); Test.ensureEqual(fileTable.addColumn(FT_LAST_MOD_COL, "lastMod", new DoubleArray()), FT_LAST_MOD_COL, "FT_LAST_MOD_COL is wrong."); + Test.ensureEqual(fileTable.addColumn(FT_SIZE_COL, "size", new DoubleArray()), FT_SIZE_COL, "FT_SIZE is wrong."); Test.ensureEqual(fileTable.addColumn(FT_N_VALUES_COL, "nValues", new IntArray()), FT_N_VALUES_COL, "FT_N_VALUES_COL is wrong."); Test.ensureEqual(fileTable.addColumn(FT_MIN_COL, "min", new DoubleArray()), FT_MIN_COL, "FT_MIN_COL is wrong."); Test.ensureEqual(fileTable.addColumn(FT_MAX_COL, "max", new DoubleArray()), FT_MAX_COL, "FT_MAX_COL is wrong."); @@ -465,6 +480,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible ShortArray ftDirIndex = (ShortArray) fileTable.getColumn(FT_DIR_INDEX_COL); StringArray ftFileList = (StringArray)fileTable.getColumn(FT_FILE_LIST_COL); DoubleArray ftLastMod = (DoubleArray)fileTable.getColumn(FT_LAST_MOD_COL); + DoubleArray ftSize = (DoubleArray)fileTable.getColumn(FT_SIZE_COL); IntArray ftNValues = (IntArray) fileTable.getColumn(FT_N_VALUES_COL); DoubleArray ftMin = (DoubleArray)fileTable.getColumn(FT_MIN_COL); DoubleArray ftMax = (DoubleArray)fileTable.getColumn(FT_MAX_COL); @@ -499,7 +515,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible } catch (Throwable t) { String reason = MustBe.throwableToShortString(t); addBadFile(badFileMap, ftDirIndex.get(i), tName, ftLastMod.get(i), reason); - String2.log("Error getting metadata for " + tDir + tName + "\n" + reason); + String2.log(String2.ERROR + " in " + datasetID + " constructor while getting metadata for " + tDir + tName + "\n" + reason); } } //initially there are no files, so haveValidSourceInfo will still be false @@ -509,11 +525,26 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible long elapsedTime = System.currentTimeMillis(); //was tAvailableFiles with dir+name Table tFileTable = getFileInfo(fileDir, fileNameRegex, recursive); - if (updateEveryNMillis > 0) - watchDirectory = WatchDirectory.watchDirectoryAll(fileDir, recursive); + if (updateEveryNMillis > 0) { + try { + watchDirectory = WatchDirectory.watchDirectoryAll(fileDir, recursive); + } catch (Throwable t) { + String subject = String2.ERROR + " in " + datasetID + " constructor (inotify)"; + String msg = MustBe.throwableToString(t); + if (msg.indexOf("inotify instances") >= 0) + msg += + "This may be the problem that is solvable by calling (as root):\n" + + " echo 20000 > /proc/sys/fs/inotify/max_user_watches\n" + + " echo 500 > /proc/sys/fs/inotify/max_user_instances\n" + + "Or, use higher numbers if the problem persists.\n" + + "The default for watches is 8192. The default for instances is 128."; + EDStatic.email(EDStatic.adminEmail, subject, msg); + } + } StringArray tFileDirPA = (StringArray)(tFileTable.getColumn(FileVisitorDNLS.DIRECTORY)); StringArray tFileNamePA = (StringArray)(tFileTable.getColumn(FileVisitorDNLS.NAME)); LongArray tFileLastModPA = (LongArray) (tFileTable.getColumn(FileVisitorDNLS.LASTMODIFIED)); + LongArray tFileSizePA = (LongArray) (tFileTable.getColumn(FileVisitorDNLS.SIZE)); tFileTable.removeColumn(FileVisitorDNLS.SIZE); int ntft = tFileNamePA.size(); String msg = ntft + " files found in " + fileDir + @@ -622,8 +653,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible //update fileTable by processing tFileTable int fileListPo = 0; //next one to look at int tFileListPo = 0; //next one to look at - long lastModCumTime = 0; - int nReadFile = 0, nNoLastMod = 0; + int nReadFile = 0, nNoLastMod = 0, nNoSize = 0; long readFileCumTime = 0; long removeCumTime = 0; int nUnchanged = 0, nRemoved = 0, nDifferentModTime = 0, nNew = 0; @@ -634,13 +664,12 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible int dirI = fileListPo < ftFileList.size()? ftDirIndex.get(fileListPo) : Integer.MAX_VALUE; String fileS = fileListPo < ftFileList.size()? ftFileList.get(fileListPo) : "\uFFFF"; double lastMod = fileListPo < ftFileList.size()? ftLastMod.get(fileListPo) : Double.MAX_VALUE; + double size = fileListPo < ftFileList.size()? ftSize.get(fileListPo) : Double.MAX_VALUE; if (reallyVerbose) String2.log("#" + tFileListPo + " file=" + dirList.get(tDirI) + tFileS); //is tLastMod available for tFile? - long lmcTime = System.currentTimeMillis(); long tLastMod = tFileLastModPA.get(tFileListPo); - lastModCumTime += System.currentTimeMillis() - lmcTime; if (tLastMod == 0) { //0=trouble nNoLastMod++; String2.log("#" + tFileListPo + " reject because unable to get lastMod time: " + @@ -650,6 +679,17 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible continue; } + //is tSize available for tFile? + long tSize = tFileSizePA.get(tFileListPo); + if (tSize < 0) { //-1=trouble + nNoSize++; + String2.log("#" + tFileListPo + " reject because unable to get size: " + + dirList.get(tDirI) + tFileS); + tFileListPo++; + addBadFile(badFileMap, tDirI, tFileS, tLastMod, "Unable to get size."); + continue; + } + //is tFile in badFileMap? Object bfi = badFileMap.get(tDirI + "/" + tFileS); if (bfi != null) { @@ -676,7 +716,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible } //is tFile already in cache? - if (tDirI == dirI && tFileS.equals(fileS) && tLastMod == lastMod) { + if (tDirI == dirI && tFileS.equals(fileS) && tLastMod == lastMod && tSize == size) { if (reallyVerbose) String2.log("#" + tFileListPo + " already in cache"); nUnchanged++; tFileListPo++; @@ -714,6 +754,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible ftDirIndex.setInt(fileListPo, tDirI); ftFileList.set(fileListPo, tFileS); ftLastMod.set(fileListPo, tLastMod); + ftSize.set(fileListPo, tSize); //read axis values nReadFile++; @@ -782,14 +823,12 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible saveDirTableFileTableBadFiles(dirTable, fileTable, badFileMap); //throws Throwable msg = "\n tFileNamePA.size=" + tFileNamePA.size() + - " lastModCumTime=" + Calendar2.elapsedTimeString(lastModCumTime) + - " avg=" + (lastModCumTime / Math.max(1, tFileNamePA.size())) + "ms" + "\n dirTable.nRows=" + dirTable.nRows() + "\n fileTable.nRows=" + fileTable.nRows() + "\n fileTableInMemory=" + fileTableInMemory + "\n nUnchanged=" + nUnchanged + "\n nRemoved=" + nRemoved + " (nNoLastMod=" + nNoLastMod + - ") removedCumTime=" + Calendar2.elapsedTimeString(lastModCumTime) + + ", nNoSize=" + nNoSize + ")" + "\n nReadFile=" + nReadFile + " (nDifferentModTime=" + nDifferentModTime + " nNew=" + nNew + ")" + " readFileCumTime=" + Calendar2.elapsedTimeString(readFileCumTime) + @@ -830,7 +869,7 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible if (combinedGlobalAttributes.getString("cdm_data_type") == null) combinedGlobalAttributes.add("cdm_data_type", "Grid"); if (combinedGlobalAttributes.get("sourceUrl") == null) { - localSourceUrl = "(local files)"; //keep location private + localSourceUrl = "(" + (File2.isRemote(localSourceUrl)? "remote" : "local") + " files)"; //keep location private addGlobalAttributes.set( "sourceUrl", localSourceUrl); combinedGlobalAttributes.set("sourceUrl", localSourceUrl); } @@ -1005,18 +1044,11 @@ protected void validateCompareSet(String dirName, String fileName, //be less strict? PrimitiveArray exp = sourceAxisValues[av]; PrimitiveArray obs = tSourceAxisValues[av]; - boolean equal; - if (ensureAxisValuesAreExactlyEqual) { - String eqError = exp.testEquals(obs); - if (eqError.length() > 0) - throw new RuntimeException("axis=" + av + " values are not exactly equal.\n" + - eqError); - } else { - String eqError = exp.almostEqual(obs); - if (eqError.length() > 0) - throw new RuntimeException("axis=" + av + - " values are not even approximately equal.\n" + eqError); - } + String eqError = exp.almostEqual(obs, matchAxisNDigits); + if (eqError.length() > 0) + throw new RuntimeException( + "Test of matchAxisNDigits=" + matchAxisNDigits + + " failed for axis[" + av + "]=" + sourceAxisNames.get(av) + " failed:\n" + eqError); } //compare axisAttributes @@ -1184,6 +1216,7 @@ public boolean lowUpdate(String msg, long startUpdateMillis) throws Throwable { ShortArray ftDirIndex = (ShortArray) tFileTable.getColumn(FT_DIR_INDEX_COL); StringArray ftFileList = (StringArray)tFileTable.getColumn(FT_FILE_LIST_COL); DoubleArray ftLastMod = (DoubleArray)tFileTable.getColumn(FT_LAST_MOD_COL); + DoubleArray ftSize = (DoubleArray)tFileTable.getColumn(FT_SIZE_COL); IntArray ftNValues = (IntArray) tFileTable.getColumn(FT_N_VALUES_COL); DoubleArray ftMin = (DoubleArray)tFileTable.getColumn(FT_MIN_COL); //sorted by DoubleArray ftMax = (DoubleArray)tFileTable.getColumn(FT_MAX_COL); @@ -1277,6 +1310,7 @@ public boolean lowUpdate(String msg, long startUpdateMillis) throws Throwable { ftDirIndex.setInt(fileListPo, dirIndex); ftFileList.set(fileListPo, fileName); ftLastMod.set(fileListPo, File2.getLastModified(fullName)); + ftSize.set(fileListPo, File2.length(fullName)); ftNValues.set(fileListPo, tnValues); ftMin.set(fileListPo, tSourceAxisValues[0].getNiceDouble(0)); ftMax.set(fileListPo, tSourceAxisValues[0].getNiceDouble(tnValues - 1)); @@ -1431,6 +1465,48 @@ protected Table tryToLoadDirFileTable(String fileName) { } + /** + * This returns a fileTable (formatted like + * FileVisitorDNLS.oneStep(tDirectoriesToo=false, last_mod is LongArray, + * and size is LongArray of epochMillis) + * with valid files (or null if unavailable or any trouble). + * This is a copy of any internal data, so client can modify the contents. + */ + public Table accessibleViaFilesFileTable() { + try { + //get a copy of the source file information + Table tDirTable; + Table tFileTable; + if (fileTableInMemory) { + tDirTable = (Table)dirTable.clone(); + tFileTable = (Table)fileTable.clone(); + } else { + tDirTable = tryToLoadDirFileTable(datasetDir() + DIR_TABLE_FILENAME); //shouldn't be null + tFileTable = tryToLoadDirFileTable(datasetDir() + FILE_TABLE_FILENAME); //shouldn't be null + Test.ensureNotNull(tDirTable, "dirTable"); + Test.ensureNotNull(tFileTable, "fileTable"); + } + + //make the results Table + Table dnlsTable = FileVisitorDNLS.makeEmptyTable(); + dnlsTable.setColumn(0, tFileTable.getColumn(FT_DIR_INDEX_COL)); + dnlsTable.setColumn(1, tFileTable.getColumn(FT_FILE_LIST_COL)); + dnlsTable.setColumn(2, new LongArray(tFileTable.getColumn(FT_LAST_MOD_COL))); //double -> long + dnlsTable.setColumn(3, new LongArray(tFileTable.getColumn(FT_SIZE_COL))); //double -> long + //convert dir Index to dir names + tDirTable.addColumn(0, "dirIndex", new IntArray(0, tDirTable.nRows() - 1)); + dnlsTable.join(1, 0, "", tDirTable); + dnlsTable.removeColumn(0); + dnlsTable.setColumnName(0, FileVisitorDNLS.DIRECTORY); + + return dnlsTable; + } catch (Exception e) { + String2.log(MustBe.throwableToString(e)); + return null; + } + } + + /** * This gets sourceGlobalAttributes and sourceDataAttributes from the specified * source file (or does nothing if that isn't possible). @@ -1526,6 +1602,7 @@ public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraint ShortArray ftDirIndex = (ShortArray) tFileTable.getColumn(FT_DIR_INDEX_COL); StringArray ftFileList = (StringArray)tFileTable.getColumn(FT_FILE_LIST_COL); DoubleArray ftLastMod = (DoubleArray)tFileTable.getColumn(FT_LAST_MOD_COL); + DoubleArray ftSize = (DoubleArray)tFileTable.getColumn(FT_SIZE_COL); IntArray ftNValues = (IntArray) tFileTable.getColumn(FT_N_VALUES_COL); DoubleArray ftMin = (DoubleArray)tFileTable.getColumn(FT_MIN_COL); DoubleArray ftMax = (DoubleArray)tFileTable.getColumn(FT_MAX_COL); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromMatFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromMatFiles.java index 2230a8f9c..48b9bd5a7 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromMatFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromMatFiles.java @@ -262,8 +262,8 @@ public PrimitiveArray[] getSourceDataFromFile(String fileDir, String fileName, * * @throws Throwable always (since this class doesn't support sibling()) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { throw new SimpleException("Error: " + "EDDGridFromMatFiles doesn't support method=\"sibling\"."); @@ -402,7 +402,8 @@ public static String generateDatasetsXml( "\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " " + suggestUpdateEveryNMillis(tFileDir) + + "\n" + " " + tFileDir + "\n" + " true\n" + " " + XML.encodeAsXML(tFileNameRegex) + "\n" + @@ -839,7 +840,7 @@ public static void testMat(boolean deleteCachedDatasetInfo) throws Throwable { " String source \"satellite observation: QuikSCAT, SeaWinds\";\n" + " String sourceUrl \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/QS/ux10/1day\";\n" + " Float64 Southernmost_Northing -89.875;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"Remote Sensing Inc. distributes science quality wind velocity data from the SeaWinds instrument onboard NASA's QuikSCAT satellite. SeaWinds is a microwave scatterometer designed to measure surface winds over the global ocean. Wind velocity fields are provided in zonal, meriodonal, and modulus sets. The reference height for all wind velocities is 10 meters.\";\n" + " String time_coverage_end \"2008-01-10T12:00:00Z\";\n" + " String time_coverage_start \"2008-01-01T12:00:00Z\";\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromMergeIRFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromMergeIRFiles.java index 49c43ac74..85fec21e9 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromMergeIRFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromMergeIRFiles.java @@ -68,7 +68,7 @@ public EDDGridFromMergeIRFiles(String tDatasetID, String tAccessibleTo, Object[][] tDataVariables, int tReloadEveryNMinutes, int tUpdateEveryNMillis, String tFileDir, boolean tRecursive, String tFileNameRegex, - String tMetadataFrom, boolean tEnsureAxisValuesAreExactlyEqual, + String tMetadataFrom, int tMatchAxisNDigits, boolean tFileTableInMemory, boolean tAccessibleViaFiles) throws Throwable { @@ -80,7 +80,7 @@ public EDDGridFromMergeIRFiles(String tDatasetID, String tAccessibleTo, tDataVariables, tReloadEveryNMinutes, tUpdateEveryNMillis, tFileDir, tRecursive, tFileNameRegex, - tMetadataFrom, tEnsureAxisValuesAreExactlyEqual, + tMetadataFrom, tMatchAxisNDigits, tFileTableInMemory, tAccessibleViaFiles); if (verbose) String2.log("\n*** constructing EDDGridFromMergeIRFiles(xmlReader)..."); @@ -452,8 +452,8 @@ private static double T2F(double pT) { * * @throws Throwable always (since this class doesn't support sibling()) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { throw new SimpleException( "Error: EDDGridFromMergeIRFiles doesn't support method=\"sibling\"."); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFiles.java index 8f0bb776d..c5ee57c28 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFiles.java @@ -32,6 +32,8 @@ import java.text.MessageFormat; import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** @@ -58,6 +60,9 @@ */ public class EDDGridFromNcFiles extends EDDGridFromFiles { + /** Used by Bob only. Don't set this to true here -- do it in the calling code. */ + public static boolean generateDatasetsXmlCoastwatchErdMode = false; + /** The constructor just calls the super constructor. */ public EDDGridFromNcFiles(String tDatasetID, String tAccessibleTo, @@ -68,7 +73,7 @@ public EDDGridFromNcFiles(String tDatasetID, String tAccessibleTo, Object[][] tDataVariables, int tReloadEveryNMinutes, int tUpdateEveryNMillis, String tFileDir, boolean tRecursive, String tFileNameRegex, String tMetadataFrom, - boolean tEnsureAxisValuesAreExactlyEqual, boolean tFileTableInMemory, + int tMatchAxisNDigits, boolean tFileTableInMemory, boolean tAccessibleViaFiles) throws Throwable { @@ -80,7 +85,7 @@ public EDDGridFromNcFiles(String tDatasetID, String tAccessibleTo, tDataVariables, tReloadEveryNMinutes, tUpdateEveryNMillis, tFileDir, tRecursive, tFileNameRegex, tMetadataFrom, - tEnsureAxisValuesAreExactlyEqual, tFileTableInMemory, + tMatchAxisNDigits, tFileTableInMemory, tAccessibleViaFiles); } @@ -279,8 +284,8 @@ public PrimitiveArray[] getSourceDataFromFile(String fileDir, String fileName, * * @throws Throwable always (since this class doesn't support sibling()) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { throw new SimpleException("Error: " + "EDDGridFromNcFiles doesn't support method=\"sibling\"."); @@ -309,7 +314,14 @@ public static String generateDatasetsXml( String2.log("EDDGridFromNcFiles.generateDatasetsXml" + "\n sampleFileName=" + sampleFileName); - tFileDir = File2.addSlash(tFileDir); //ensure it has trailing slash + if (tFileDir.endsWith("/catalog.html")) //thredds catalog + tFileDir = tFileDir.substring(0, tFileDir.length() - 12); + else if (tFileDir.endsWith("/catalog.xml")) //thredds catalog + tFileDir = tFileDir.substring(0, tFileDir.length() - 11); + else if (tFileDir.endsWith("/contents.html")) //hyrax catalog + tFileDir = tFileDir.substring(0, tFileDir.length() - 11); + else tFileDir = File2.addSlash(tFileDir); //otherwise, assume tFileDir is missing final slash + if (tReloadEveryNMinutes <= 0 || tReloadEveryNMinutes == Integer.MAX_VALUE) tReloadEveryNMinutes = 1440; //1440 works well with suggestedUpdateEveryNMillis @@ -326,7 +338,8 @@ public static String generateDatasetsXml( StringBuilder sb = new StringBuilder(); //get source global Attributes - NcHelper.getGlobalAttributes(ncFile, axisSourceTable.globalAttributes()); + Attributes globalSourceAtts = axisSourceTable.globalAttributes(); + NcHelper.getGlobalAttributes(ncFile, globalSourceAtts); try { //look at all variables with dimensions, find ones which share same max nDim @@ -376,7 +389,7 @@ public static String generateDatasetsXml( String destName = String2.modifyToBeVariableNameSafe(axisName); axisAddTable.addColumn( avi, destName, new DoubleArray(), //type doesn't matter makeReadyToUseAddVariableAttributesForDatasetsXml( - axisSourceTable.globalAttributes(), + globalSourceAtts, sourceAtts, destName, false, true)); //addColorBarMinMax, tryToFindLLAT } @@ -408,14 +421,18 @@ public static String generateDatasetsXml( String destName = String2.modifyToBeVariableNameSafe(varName); dataAddTable.addColumn( dataAddTable.nColumns(), destName, pa, makeReadyToUseAddVariableAttributesForDatasetsXml( - axisSourceTable.globalAttributes(), + globalSourceAtts, sourceAtts, destName, true, false)); //addColorBarMinMax, tryToFindLLAT } + if (dataAddTable.nColumns() == 0) + throw new RuntimeException("No dataVariables found."); + //after dataVariables known, add global attributes in the axisAddTable - axisAddTable.globalAttributes().set( + Attributes globalAddAtts = axisAddTable.globalAttributes(); + globalAddAtts.set( makeReadyToUseAddGlobalAttributesForDatasetsXml( - axisSourceTable.globalAttributes(), + globalSourceAtts, "Grid", //another cdm type could be better; this is ok tFileDir, externalAddGlobalAttributes, EDD.chopUpCsvAndAdd(axisAddTable.getColumnNamesCSVString(), @@ -423,7 +440,63 @@ public static String generateDatasetsXml( //gather the results String tDatasetID = suggestDatasetID(tFileDir + tFileNameRegex); - sb.append(directionsForGenerateDatasetsXml()); + boolean accViaFiles = false; + int tMatchNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; + + if (generateDatasetsXmlCoastwatchErdMode) { + accViaFiles = true; + tMatchNDigits = 15; + // /u00/satellite/AT/ssta/1day/ + Pattern pattern = Pattern.compile("/u00/satellite/([^/]+)/([^/]+)/([^/]+)day/"); + Matcher matcher = pattern.matcher(tFileDir); + String m12, m1_2; //ATssta AT_ssta + String cl; //composite length + if (matcher.matches()) { + m12 = matcher.group(1) + matcher.group(2); + m1_2 = matcher.group(1) + "_" + matcher.group(2); + cl = matcher.group(3); + } else { + // /u00/satellite/MPIC/1day/ + pattern = Pattern.compile("/u00/satellite/([^/]+)/([^/]+)day/"); + matcher = pattern.matcher(tFileDir); + if (matcher.matches()) { + m12 = matcher.group(1); + m1_2 = m12; + cl = matcher.group(2); + } else { + throw new RuntimeException(tFileDir + " doesn't match the pattern!"); + } + } + + tDatasetID = "erd" + m12 + cl + "day"; + globalAddAtts.set("creator_name", "NOAA NMFS SWFSC ERD"); + globalAddAtts.set("creator_email", "erd.data@noaa.gov"); + globalAddAtts.set("creator_url", "http://www.pfeg.noaa.gov"); + globalAddAtts.set("publisher_name", "NOAA NMFS SWFSC ERD"); + globalAddAtts.set("publisher_email", "erd.data@noaa.gov"); + globalAddAtts.set("publisher_url", "http://www.pfeg.noaa.gov"); + globalAddAtts.set("id", "null"); + globalAddAtts.set("infoUrl", "http://coastwatch.pfeg.noaa.gov/infog/" + + m1_2 + "_las.html"); + globalAddAtts.set("institution", "NOAA NMFS SWFSC ERD"); + globalAddAtts.set("license", "[standard]"); + globalAddAtts.remove("summary"); + globalAddAtts.set("title", + globalSourceAtts.getString("title") + " (" + + (cl.equals("h")? "Single Scan" : + cl.equals("m")? "Monthly Composite" : + cl + " Day Composite") + + ")"); + + for (int dv = 0; dv < dataSourceTable.nColumns(); dv++) { + dataAddTable.columnAttributes(dv).set("long_name", "!!! FIX THIS !!!"); + if (dataSourceTable.columnAttributes(dv).get("actual_range") != null) + dataAddTable.columnAttributes(dv).set("actual_range", "null"); + } + + } else { + sb.append(directionsForGenerateDatasetsXml()); + } if (nGridsAtSource > dataAddTable.nColumns()) sb.append( @@ -431,20 +504,23 @@ public static String generateDatasetsXml( "but this dataset will only serve " + dataAddTable.nColumns() + " because the others use different dimensions.\n"); sb.append( - "-->\n\n" + + (generateDatasetsXmlCoastwatchErdMode? "": "-->\n") + + "\n" + "\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " " + suggestUpdateEveryNMillis(tFileDir) + + "\n" + " " + tFileDir + "\n" + " true\n" + " " + XML.encodeAsXML(tFileNameRegex) + "\n" + " last\n" + + " " + tMatchNDigits + "\n" + " false\n" + - " false\n"); + " " + accViaFiles + "\n"); - sb.append(writeAttsForDatasetsXml(false, axisSourceTable.globalAttributes(), " ")); - sb.append(writeAttsForDatasetsXml(true, axisAddTable.globalAttributes(), " ")); + sb.append(writeAttsForDatasetsXml(false, globalSourceAtts, " ")); + sb.append(writeAttsForDatasetsXml(true, globalAddAtts, " ")); //last 3 params: includeDataType, tryToFindLLAT, questionDestinationName sb.append(writeVariablesForDatasetsXml(axisSourceTable, axisAddTable, "axisVariable", false, true, false)); @@ -507,6 +583,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " true\n" + " .*_03\\.nc\n" + " last\n" + +" 20\n" + " false\n" + " false\n" + " \n" + +"\n" + +"\n" + +" 1000000\n" + +" 0\n" + +" http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/\n" + +" true\n" + +" .*_CESM1-CAM5_.*\\.nc\n" + +" last\n" + +" 20\n" + +" false\n" + +" false\n" + +" \n" + +" \n" + +" Grid\n" + +" CF-1.6, COARDS, ACDD-1.3\n" + +" rama.nemani@nasa.gov\n" + +" Rama Nemani\n" + +" http://www.nasa.gov/\n" + +" http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/\n" + +" 800m, air, air_temperature, ames, atmosphere,\n" + +"Atmosphere > Atmospheric Temperature > Air Temperature,\n" + +"Atmosphere > Atmospheric Temperature > Surface Air Temperature,\n" + +"atmospheric, center, climate, cmip5, continental, daily, data, day, downscaled, earth, exchange, field, intercomparison, minimum, model, moffett, nasa, near, near-surface, nex, project, projections, research, surface, tasmin, temperature, time, US\n" + +" GCMD Science Keywords\n" + +" [standard]\n" + +" CF Standard Name Table v29\n" + +" 800m Downscaled NEX Climate Model Intercomparison Project 5 (CMIP5) Climate Projections for the Continental US\n" + +" \n" + +" \n" + +" time\n" + +" time\n" + +" \n" + +" \n" + +" null\n" + +" null\n" + +" Time\n" + +" \n" + +" \n" + +" \n" + +" lat\n" + +" latitude\n" + +" \n" + +" \n" + +" null\n" + +" null\n" + +" Location\n" + +" Latitude\n" + +" \n" + +" \n" + +" \n" + +" lon\n" + +" longitude\n" + +" \n" + +" \n" + +" null\n" + +" null\n" + +" Location\n" + +" Longitude\n" + +" \n" + +" \n" + +" \n" + +" tasmin\n" + +" tasmin\n" + +" float\n" + +" \n" + +" \n" + +" null\n" + +" 313.0\n" + +" 263.0\n" + +" null\n" + +" Temperature\n" + +" \n" + +" \n" + +"\n" + +"\n\n"; + Test.ensureEqual(results, expected, results.length() + " " + expected.length() + + "\nresults=\n" + results); + + //ensure it is ready-to-use by making a dataset from it + EDD edd = oneFromXmlFragment(results); + Test.ensureEqual(edd.datasetID(), suggDatasetID, ""); + Test.ensureEqual(edd.title(), "800m Downscaled NEX CMIP5 Climate Projections for the Continental US", ""); + Test.ensureEqual(String2.toCSSVString(edd.dataVariableDestinationNames()), + "tasmin", ""); + + String2.log("\nEDDGridFromNcFiles.testGenerateDatasetsXmlAwsS3 passed the test."); + } + + /** + * This tests reading NetCDF .nc files with this class. + * + * @throws Throwable if trouble + */ + public static void testAwsS3(boolean deleteCachedDatasetInfo) throws Throwable { + String2.log("\n****************** EDDGridFromNcFiles.testNc() *****************\n"); + testVerboseOn(); + String name, tName, results, tResults, expected, userDapQuery, tQuery; + String error = ""; + EDVGridAxis edvga; + String id = "testAwsS3"; + if (deleteCachedDatasetInfo) + deleteCachedDatasetInfo(id); + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml(id); + String today = Calendar2.getCurrentISODateTimeStringLocal().substring(0, 10); + String tDir = EDStatic.fullTestCacheDirectory; + + //*** test getting das for entire dataset + String2.log("\n*** .nc test das dds for entire dataset\n"); + tName = eddGrid.makeNewFileForDapQuery(null, null, "", EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_Entire", ".das"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"Attributes {\n" + +" time {\n" + +" String _CoordinateAxisType \"Time\";\n" + +" Float64 actual_range 1.1374128e+9, 4.1011056e+9;\n" + +" String axis \"T\";\n" + +" String calendar \"standard\";\n" + +" String ioos_category \"Time\";\n" + +" String long_name \"Time\";\n" + +" String standard_name \"time\";\n" + +" String time_origin \"01-JAN-1970 00:00:00\";\n" + +" String units \"seconds since 1970-01-01T00:00:00Z\";\n" + +" }\n" + +" latitude {\n" + +" String _CoordinateAxisType \"Lat\";\n" + +" Float64 actual_range 24.0625, 49.92916665632;\n" + +" String axis \"Y\";\n" + +" String ioos_category \"Location\";\n" + +" String long_name \"Latitude\";\n" + +" String standard_name \"latitude\";\n" + +" String units \"degrees_north\";\n" + +" }\n" + +" longitude {\n" + +" String _CoordinateAxisType \"Lon\";\n" + +" Float64 actual_range 234.97916666666998, 293.51249997659;\n" + +" String axis \"X\";\n" + +" String ioos_category \"Location\";\n" + +" String long_name \"Longitude\";\n" + +" String standard_name \"longitude\";\n" + +" String units \"degrees_east\";\n" + +" Float64 valid_range 0.0, 360.0;\n" + +" }\n" + +" tasmin {\n" + +" Float32 _FillValue 1.0E20;\n" + +" String associated_files \"baseURL: http://cmip-pcmdi.llnl.gov/CMIP5/dataLocation gridspecFile: gridspec_atmos_fx_CESM1-CAM5_rcp26_r0i0p0.nc areacella: areacella_fx_CESM1-CAM5_rcp26_r0i0p0.nc\";\n" + +" String cell_measures \"area: areacella\";\n" + +" String cell_methods \"time: minimum (interval: 30 days) within days time: mean over days\";\n" + +" Float64 colorBarMaximum 313.0;\n" + +" Float64 colorBarMinimum 263.0;\n" + +" String comment \"TREFMNAV no change, CMIP5_table_comment: monthly mean of the daily-minimum near-surface air temperature.\";\n" + +" String history \"2012-06-09T00:36:32Z altered by CMOR: Treated scalar dimension: 'height'. 2012-06-09T00:36:32Z altered by CMOR: Reordered dimensions, original order: lat lon time. 2012-06-09T00:36:32Z altered by CMOR: replaced missing value flag (-1e+32) with standard missing value (1e+20).\";\n" + +" String ioos_category \"Temperature\";\n" + +" String long_name \"Daily Minimum Near-Surface Air Temperature\";\n" + +" Float32 missing_value 1.0E20;\n" + +" String original_name \"TREFMNAV\";\n" + +" String standard_name \"air_temperature\";\n" + +" String units \"K\";\n" + +" }\n" + +" NC_GLOBAL {\n" + +" String cdm_data_type \"Grid\";\n" + +" String CMIPtable \"Amon\";\n" + +" String contact \"Dr. Rama Nemani: rama.nemani@nasa.gov, Dr. Bridget Thrasher: bridget@climateanalyticsgroup.org, and Dr. Mark Snyder: mark@climateanalyticsgroup.org\";\n" + +" String Conventions \"CF-1.6, COARDS, ACDD-1.3\";\n" + +" String creation_date \"Wed Sep 12 14:44:43 PDT 2012\";\n" + +" String creator_email \"rama.nemani@nasa.gov\";\n" + +" String creator_name \"Rama Nemani\";\n" + +" String creator_url \"http://www.nasa.gov/\";\n" + +" String DOI \"http://dx.doi.org/10.7292/W0WD3XH4\";\n" + +" String downscalingModel \"BCSD\";\n" + +" String driving_data_tracking_ids \"N/A\";\n" + +" String driving_experiment \"historical\";\n" + +" String driving_experiment_name \"historical\";\n" + +" String driving_model_ensemble_member \"r3i1p1\";\n" + +" String driving_model_id \"CESM1-CAM5\";\n" + +" Float64 Easternmost_Easting 293.51249997659;\n" + +" String experiment \"RCP2.6\";\n" + +" String experiment_id \"rcp26\";\n" + +" String frequency \"mon\";\n" + +" Float64 geospatial_lat_max 49.92916665632;\n" + +" Float64 geospatial_lat_min 24.0625;\n" + +" Float64 geospatial_lat_resolution 0.00833333333;\n" + +" String geospatial_lat_units \"degrees_north\";\n" + +" Float64 geospatial_lon_max 293.51249997659;\n" + +" Float64 geospatial_lon_min 234.97916666666998;\n" + +" Float64 geospatial_lon_resolution 0.008333333330000001;\n" + +" String geospatial_lon_units \"degrees_east\";\n" + +" String history \"" + today; +//2015-06-24T17:36:33Z (local files) + tResults = results.substring(0, Math.min(results.length(), expected.length())); + Test.ensureEqual(tResults, expected, "results=\n" + results); + +// + " http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/QS/ux10/1day\n" + +//today + + +expected = +//"2015-06-24T17:36:33Z http://127.0.0.1:8080/cwexperimental/griddap/testAwsS3.das\";\n" + + "String infoUrl \"http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/\";\n" + +" String initialization_method \"1\";\n" + +" String institute_id \"NASA-Ames\";\n" + +" String institution \"NASA Earth Exchange, NASA Ames Research Center, Moffett Field, CA 94035\";\n" + +" String keywords \"800m, air, air_temperature, ames, atmosphere,\n" + +"Atmosphere > Atmospheric Temperature > Air Temperature,\n" + +"Atmosphere > Atmospheric Temperature > Surface Air Temperature,\n" + +"atmospheric, center, climate, cmip5, continental, daily, data, day, downscaled, earth, exchange, field, intercomparison, minimum, model, moffett, nasa, near, near-surface, nex, project, projections, research, surface, tasmin, temperature, time, US\";\n" + +" String keywords_vocabulary \"GCMD Science Keywords\";\n" + +" String license \"The data may be used and redistributed for free but is not intended\n" + +"for legal use, since it may contain inaccuracies. Neither the data\n" + +"Contributor, ERD, NOAA, nor the United States Government, nor any\n" + +"of their employees or contractors, makes any warranty, express or\n" + +"implied, including warranties of merchantability and fitness for a\n" + +"particular purpose, or assumes any legal liability for the accuracy,\n" + +"completeness, or usefulness, of this information.\";\n" + +" String model_id \"BCSD\";\n" + +" String modeling_realm \"atmos\";\n" + +" Float64 Northernmost_Northing 49.92916665632;\n" + +" String parent_experiment \"historical\";\n" + +" String parent_experiment_id \"historical\";\n" + +" String parent_experiment_rip \"r1i1p1\";\n" + +" String physics_version \"1\";\n" + +" String product \"downscaled\";\n" + +" String project_id \"NEX\";\n" + +" String realization \"1\";\n" + +" String realm \"atmos\";\n" + +" String references \"BCSD method: Wood AW, Maurer EP, Kumar A, Lettenmaier DP, 2002, J Geophys Res 107(D20):4429 & \n" + +" Wood AW, Leung LR, Sridhar V, Lettenmaier DP, 2004, Clim Change 62:189-216\n" + +" Reference period obs: PRISM (http://www.prism.oregonstate.edu/)\";\n" + +" String region \"CONUS\";\n" + +" String region_id \"CONUS\";\n" + +" String region_lexicon \"http://en.wikipedia.org/wiki/Contiguous_United_States\";\n" + +" String resolution_id \"800m\";\n" + +" String sourceUrl \"(local files)\";\n" + +" Float64 Southernmost_Northing 24.0625;\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + +" String summary \"800m Downscaled NEX Climate Model Intercomparison Project 5 (CMIP5) Climate Projections for the Continental US\";\n" + +" String table_id \"Table Amon\";\n" + +" String time_coverage_end \"2099-12-16T12:00:00Z\";\n" + +" String time_coverage_start \"2006-01-16T12:00:00Z\";\n" + +" String title \"800m Downscaled NEX CMIP5 Climate Projections for the Continental US\";\n" + +" String tracking_id \"d62220e2-af11-11e2-bdc9-e41f13ef5cee\";\n" + +" String variableName \"tasmin\";\n" + +" String version \"1.0\";\n" + +" Float64 Westernmost_Easting 234.97916666666998;\n" + +" }\n" + +"}\n"; + int tPo = results.indexOf(expected.substring(0, 25)); + Test.ensureTrue(tPo >= 0, "tPo=-1 results=\n" + results); + Test.ensureEqual( + results.substring(tPo, Math.min(results.length(), tPo + expected.length())), + expected, "results=\n" + results); + + //*** test getting dds for entire dataset + tName = eddGrid.makeNewFileForDapQuery(null, null, "", EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_Entire", ".dds"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"Dataset {\n" + +" Float64 time[time = 1128];\n" + +" Float64 latitude[latitude = 3105];\n" + +" Float64 longitude[longitude = 7025];\n" + +" GRID {\n" + +" ARRAY:\n" + +" Float32 tasmin[time = 1128][latitude = 3105][longitude = 7025];\n" + +" MAPS:\n" + +" Float64 time[time = 1128];\n" + +" Float64 latitude[latitude = 3105];\n" + +" Float64 longitude[longitude = 7025];\n" + +" } tasmin;\n" + +"} testAwsS3;\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.csv with data from one file + String2.log("\n*** .nc test read from one file\n"); + userDapQuery = "tasmin[1127][(40):100:(43)][(260):100:(263)]"; + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_Data1", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time,latitude,longitude,tasmin\n" + +"UTC,degrees_north,degrees_east,K\n" + +"2099-12-16T12:00:00Z,40.00416666029,260.00416665666,263.7364\n" + +"2099-12-16T12:00:00Z,40.00416666029,260.83749998966,263.4652\n" + +"2099-12-16T12:00:00Z,40.00416666029,261.67083332266,263.61017\n" + +"2099-12-16T12:00:00Z,40.00416666029,262.50416665566,264.5588\n" + +"2099-12-16T12:00:00Z,40.837499993289995,260.00416665666,263.3948\n" + +"2099-12-16T12:00:00Z,40.837499993289995,260.83749998966,263.02103\n" + +"2099-12-16T12:00:00Z,40.837499993289995,261.67083332266,263.31967\n" + +"2099-12-16T12:00:00Z,40.837499993289995,262.50416665566,263.83475\n" + +"2099-12-16T12:00:00Z,41.670833326289994,260.00416665666,262.9155\n" + +"2099-12-16T12:00:00Z,41.670833326289994,260.83749998966,261.65094\n" + +"2099-12-16T12:00:00Z,41.670833326289994,261.67083332266,261.9907\n" + +"2099-12-16T12:00:00Z,41.670833326289994,262.50416665566,262.13275\n" + +"2099-12-16T12:00:00Z,42.50416665929,260.00416665666,263.9838\n" + +"2099-12-16T12:00:00Z,42.50416665929,260.83749998966,262.93536\n" + +"2099-12-16T12:00:00Z,42.50416665929,261.67083332266,262.64273\n" + +"2099-12-16T12:00:00Z,42.50416665929,262.50416665566,261.5762\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //img + tName = eddGrid.makeNewFileForDapQuery(null, null, + "tasmin[(2099-12-16T12:00:00Z)][(24.0625):(49.92916665632)][(234.97916666666998):" + + "(293.51249997659)]&.draw=surface&.vars=longitude|latitude|tasmin" + + "&.colorBar=|||||&.land=under", + tDir, "testAwsS3", ".png"); + SSR.displayInBrowser("file://" + tDir + tName); + + } /** * This tests reading NetCDF .nc files with this class. @@ -1074,7 +1566,7 @@ public static void testNc(boolean deleteCachedDatasetInfo) throws Throwable { //numeric IP because these are files captured long ago " String sourceUrl \"http://192.168.31.18/thredds/dodsC/satellite/QS/ux10/1day\";\n" + " Float64 Southernmost_Northing -89.875;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"Remote Sensing Inc. distributes science quality wind velocity data from the SeaWinds instrument onboard NASA's QuikSCAT satellite. SeaWinds is a microwave scatterometer designed to measure surface winds over the global ocean. Wind velocity fields are provided in zonal, meriodonal, and modulus sets. The reference height for all wind velocities is 10 meters.\";\n" + " String time_coverage_end \"2008-01-10T12:00:00Z\";\n" + " String time_coverage_start \"2008-01-01T12:00:00Z\";\n" + @@ -1401,7 +1893,7 @@ public static void testGrib_42(boolean deleteCachedDatasetInfo) throws Throwable " String source \"Initialized analysis product\";\n" + " String sourceUrl \"(local files)\";\n" + " Float64 Southernmost_Northing -88.75;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"This is a test of EDDGridFromNcFiles with GRIB files.\";\n" + " String time_coverage_end \"1990-12-01T12:00:00Z\";\n" + " String time_coverage_start \"1981-01-01T12:00:00Z\";\n" + @@ -1823,7 +2315,7 @@ public static void testGrib2_42(boolean deleteCachedDatasetInfo) throws Throwabl " String source \"Type: Forecast products Status: Operational products\";\n" + " String sourceUrl \"(local files)\";\n" + " Float64 Southernmost_Northing -77.5;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"???\";\n" + " String time_coverage_end \"2009-06-08T18:00:00Z\";\n" + " String time_coverage_start \"2009-06-01T06:00:00Z\";\n" + @@ -2116,7 +2608,7 @@ public static void testGrib_43(boolean deleteCachedDatasetInfo) throws Throwable " String Product_Type \"Initialized analysis product\";\n" + " String sourceUrl \"(local files)\";\n" + " Float64 Southernmost_Northing -88.74999;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"This is a test of EDDGridFromNcFiles with GRIB files.\";\n" + " String time_coverage_end \"1990-12-01T12:00:00Z\";\n" + " String time_coverage_start \"1981-01-01T12:00:00Z\";\n" + @@ -2467,7 +2959,7 @@ public static void testGrib2_43(boolean deleteCachedDatasetInfo) throws Throwabl " String Originating_or_generating_Subcenter \"0\";\n" + " String sourceUrl \"(local files)\";\n" + " Float64 Southernmost_Northing -77.5;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"???\";\n" + " String time_coverage_end \"2009-06-08T18:00:00Z\";\n" + " String time_coverage_start \"2009-06-01T06:00:00Z\";\n" + @@ -2753,7 +3245,7 @@ public static void testCwHdf(boolean deleteCachedDatasetInfo) throws Throwable { " String satellite \"noaa-18\";\n" + " String sensor \"avhrr\";\n" + " String sourceUrl \"(local files)\";\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"???\";\n" + " String title \"Test of CoastWatch HDF files\";\n" + " }\n" + @@ -3613,7 +4105,7 @@ public static void testSimpleTestNc() throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " :sourceUrl = \"(local files)\";\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"My summary.\";\n" + " :title = \"My Title\";\n" + " data:\n" + @@ -3917,7 +4409,7 @@ public static void testSimpleTestNc2() throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " :sourceUrl = \"(local files)\";\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :summary = \"My summary.\";\n" + " :title = \"My Title\";\n" + " data:\n" + @@ -4249,34 +4741,737 @@ public static void testUpdate() throws Throwable { "\nNot an error, just FYI."); } + /** + * !!! NOT FINISHED/TESTABLE: this test server doesn't support byte ranges!!! + * + * This tests if netcdf-java can work with remote Hyrax files + * and thus if a dataset can be made from remote Hyrax files. + * + * @throws Throwable if trouble + */ + public static void testGenerateDatasetsXmlWithRemoteHyraxFiles() throws Throwable { + String2.log("\n****** EDDGridFromNcFiles.testGenerateDatasetsXmlWithRemoteHyraxFiles() *****************\n"); + testVerboseOn(); + String results, expected; + String today = Calendar2.getCurrentISODateTimeStringZulu().substring(0, 14); //14 is enough to check hour. Hard to check min:sec. + String dir = "http://podaac-opendap.jpl.nasa.gov/opendap/hyrax/allData/avhrr/L4/reynolds_er/v3b/monthly/netcdf/2014/"; + + results = generateDatasetsXml( //dir is a HYRAX catalog.html URL! + dir + "contents.html", + ".*\\.nc", + dir + "ersst.201401.nc", + -1, null); + //String2.log(results); + expected = "zztop"; + Test.ensureEqual(results, expected, "results=\n" + results); + String2.log("\n*** EDDGridFromNcFiles.testGenerateDatasetsXmlWithRemoteHyraxFiles() finished successfully\n"); + + } /** - * This tests this class. + * !!! NOT FINISHED/TESTABLE: this test server doesn't support byte ranges!!! + * + * This tests if netcdf-java can work with remote files + * and thus if a dataset can be made from remote files. * * @throws Throwable if trouble */ - public static void test(boolean deleteCachedDatasetInfo) throws Throwable { - /* */ - testNc(deleteCachedDatasetInfo); - testCwHdf(deleteCachedDatasetInfo); - testHdf(); - testNcml(); - testGrib_43(deleteCachedDatasetInfo); //42 or 43 for netcdfAll 4.2- or 4.3+ - testGrib2_43(deleteCachedDatasetInfo); //42 or 43 for netcdfAll 4.2- or 4.3+ - testGenerateDatasetsXml(); - testGenerateDatasetsXml2(); - testSpeed(-1); //-1 = all - testAVDVSameSource(); - test2DVSameSource(); - testAVDVSameDestination(); - test2DVSameDestination(); - testTimePrecisionMillis(); - testSimpleTestNc(); - testSimpleTestNc2(); -//finish this testRTechHdf(); - testUpdate(); - /* */ + public static void testRemoteHyraxFiles(boolean deleteCachedInfo) throws Throwable { + String2.log("\n****** EDDGridFromNcFiles.testRemoteHyraxFiles(" + deleteCachedInfo + + ") *****************\n"); + testVerboseOn(); + String name, tName, results, tResults, expected, userDapQuery, tQuery; + String error = ""; + int po; + EDV edv; + String today = Calendar2.getCurrentISODateTimeStringZulu().substring(0, 14); //14 is enough to check hour. Hard to check min:sec. + String id = "testRemoteHyraxFiles"; + +/* + //*** test getting das for entire dataset + String2.log("\n*** testCwHdf test das dds for entire dataset\n"); + tName = eddGrid.makeNewFileForDapQuery(null, null, "", EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_CwHdfEntire", ".das"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"Attributes {\n" + +" rows {\n" + +" Int32 actual_range 0, 1247;\n" + +" String ioos_category \"Location\";\n" + +" String long_name \"Rows\";\n" + +" String units \"count\";\n" + +" }\n" + +" cols {\n" + +" Int16 actual_range 0, 1139;\n" + +" String ioos_category \"Location\";\n" + +" String long_name \"Cols\";\n" + +" String units \"count\";\n" + +" }\n" + +" avhrr_ch1 {\n" + +" Float64 _FillValue -327.68;\n" + +" Float64 add_offset_err 0.0;\n" + +" Int32 calibrated_nt 0;\n" + +" String coordsys \"Mercator\";\n" + +" Int32 fraction_digits 2;\n" + +" String ioos_category \"Other\";\n" + +" String long_name \"AVHRR Channel 1\";\n" + +" Float64 missing_value -327.68;\n" + +" Float64 scale_factor_err 0.0;\n" + +" String standard_name \"isotropic_spectral_radiance_in_air\";\n" + +" String units \"percent\";\n" + +" }\n" + +" sst {\n" + +" Float64 _FillValue -327.68;\n" + +" Float64 add_offset_err 0.0;\n" + +" Int32 calibrated_nt 0;\n" + +" Float64 colorBarMaximum 32.0;\n" + +" Float64 colorBarMinimum 0.0;\n" + +" String coordsys \"Mercator\";\n" + +" Int32 fraction_digits 2;\n" + +" String ioos_category \"Temperature\";\n" + +" String long_name \"Sea Surface Temperature\";\n" + +" Float64 missing_value -327.68;\n" + +" Float64 scale_factor_err 0.0;\n" + +" String sst_equation_day \"nonlinear split window\";\n" + +" String sst_equation_night \"linear triple window modified\";\n" + +" String standard_name \"sea_surface_temperature\";\n" + +" String units \"celsius\";\n" + +" }\n" + +" NC_GLOBAL {\n" + +" String _History \"Direct read of HDF4 file through CDM library\";\n" + +" String autonav_performed \"true\";\n" + +" Int32 autonav_quality 2;\n" + +" String cdm_data_type \"Grid\";\n" + +" String Conventions \"COARDS, CF-1.6, ACDD-1.3\";\n" + +" String history \"Direct read of HDF4 file through CDM library\n" + +today; + tResults = results.substring(0, Math.min(results.length(), expected.length())); + Test.ensureEqual(tResults, expected, "results=\n" + results); + +//+ " (local files)\n" + +//today + + +expected = +" http://127.0.0.1:8080/cwexperimental/griddap/testCwHdf.das\";\n" + +" String infoUrl \"???\";\n" + +" String institution \"NOAA CoastWatch\";\n" + +" String keywords \"Oceans > Ocean Temperature > Sea Surface Temperature\";\n" + +" String keywords_vocabulary \"GCMD Science Keywords\";\n" + +" String license \"The data may be used and redistributed for free but is not intended\n" + +"for legal use, since it may contain inaccuracies. Neither the data\n" + +"Contributor, ERD, NOAA, nor the United States Government, nor any\n" + +"of their employees or contractors, makes any warranty, express or\n" + +"implied, including warranties of merchantability and fitness for a\n" + +"particular purpose, or assumes any legal liability for the accuracy,\n" + +"completeness, or usefulness, of this information.\";\n" + +" String origin \"USDOC/NOAA/NESDIS CoastWatch\";\n" + +" String pass_type \"day\";\n" + +" String projection \"Mercator\";\n" + +" String projection_type \"mapped\";\n" + +" String satellite \"noaa-18\";\n" + +" String sensor \"avhrr\";\n" + +" String sourceUrl \"(local files)\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + +" String summary \"???\";\n" + +" String title \"Test of CoastWatch HDF files\";\n" + +" }\n" + +"}\n"; + int tPo = results.indexOf(expected.substring(0, 17)); + Test.ensureTrue(tPo >= 0, "tPo=-1 results=\n" + results); + Test.ensureEqual( + results.substring(tPo, Math.min(results.length(), tPo + expected.length())), + expected, "results=\n" + results); + + //*** test getting dds for entire dataset + tName = eddGrid.makeNewFileForDapQuery(null, null, "", EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_CwHdfEntire", ".dds"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"Dataset {\n" + +" Int32 rows[rows = 1248];\n" + +" Int16 cols[cols = 1140];\n" + +" GRID {\n" + +" ARRAY:\n" + +" Float64 avhrr_ch1[rows = 1248][cols = 1140];\n" + +" MAPS:\n" + +" Int32 rows[rows = 1248];\n" + +" Int16 cols[cols = 1140];\n" + +" } avhrr_ch1;\n" + +" GRID {\n" + +" ARRAY:\n" + +" Float64 sst[rows = 1248][cols = 1140];\n" + +" MAPS:\n" + +" Int32 rows[rows = 1248];\n" + +" Int16 cols[cols = 1140];\n" + +" } sst;\n" + +"} testCwHdf;\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + + //.csv with data from one file + String2.log("\n*** testCwHdf test read from one file\n"); + userDapQuery = "sst[600:2:606][500:503]"; + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_CwHdfData1", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"rows,cols,sst\n" + +"count,count,celsius\n" + +"600,500,21.07\n" + +"600,501,20.96\n" + +"600,502,21.080000000000002\n" + +"600,503,20.93\n" + +"602,500,21.16\n" + +"602,501,21.150000000000002\n" + +"602,502,21.2\n" + +"602,503,20.95\n" + +"604,500,21.34\n" + +"604,501,21.13\n" + +"604,502,21.13\n" + +"604,503,21.25\n" + +"606,500,21.37\n" + +"606,501,21.11\n" + +"606,502,21.0\n" + +"606,503,21.02\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + // */ + } + + /** + * This tests if netcdf-java can work with remote Thredds files + * and thus if a dataset can be made from remote Thredds files. + * + * @throws Throwable if trouble + */ + public static void testGenerateDatasetsXmlWithRemoteThreddsFiles() throws Throwable { + String2.log("\n****** EDDGridFromNcFiles.testGenerateDatasetsXmlWithRemoteThreddsFiles() *****************\n"); + testVerboseOn(); + String results, expected; + String today = Calendar2.getCurrentISODateTimeStringZulu().substring(0, 14); //14 is enough to check hour. Hard to check min:sec. + String dir = "http://data.nodc.noaa.gov/thredds/catalog/aquarius/nodc_binned_V3.0/"; //catalog.html + + //dir is a /thredds/catalog/.../ [implied catalog.html] URL! + //file is a /thredds/fileServer/... not compressed data file. + results = generateDatasetsXml( + dir, + "sss_binned_L3_MON_SCI_V3.0_\\d{4}\\.nc", + //sample file is a thredds/fileServer/.../...nc URL! + "http://data.nodc.noaa.gov/thredds/fileServer/aquarius/nodc_binned_V3.0/monthly/sss_binned_L3_MON_SCI_V3.0_2011.nc", + -1, null); + //String2.log(results); +String2.setClipboardString(results); + + expected = +"\n" + +"\n" + +"\n" + +" 1440\n" + +" 0\n" + +" http://data.nodc.noaa.gov/thredds/catalog/aquarius/nodc_binned_V3.0/\n" + +" true\n" + +" sss_binned_L3_MON_SCI_V3.0_\\d{4}\\.nc\n" + +" last\n" + +" 20\n" + +" false\n" + +" false\n" + +" \n" + +" \n" + +" Grid\n" + +" CF-1.6, COARDS, ACDD-1.3\n" + +" http://data.nodc.noaa.gov/thredds/catalog/aquarius/nodc_binned_V3.0/catalog.html\n" + +" JPL, California Institute of Technology\n" + +" aquarius, calculate, calculated, california, center, data, earth,\n" + +"Earth Science >Oceans > Surface Salinity,\n" + +"gridded, institute, jet, jpl, laboratory, level, level-2, mean, month, monthly, national, ncei, noaa, nodc, number, observation, observations, ocean, oceanographic, oceans, propulsion, salinity, sci, science, sea, sea_surface_salinity, sea_surface_salinity_number_of_observations, sss, sss_obs, statistics, surface, swath, technology, time, used, v3.0, valid\n" + +" GCMD Science Keywords\n" + +" null\n" + +" CF Standard Name Table v29\n" + +" This dataset is created by National Oceanographic Data Center (NODC) Satellite Oceanography Group from Aquarius level-2 SCI V3.0 data,using 1.0x1.0 (lon/lat) degree box average\n" + +" \n" + +" \n" + +" time\n" + +" time\n" + +" \n" + +" \n" + +" Time\n" + +" \n" + +" \n" + +" \n" + +" lat\n" + +" latitude\n" + +" \n" + +" \n" + +" Location\n" + +" \n" + +" \n" + +" \n" + +" lon\n" + +" longitude\n" + +" \n" + +" \n" + +" Location\n" + +" \n" + +" \n" + +" \n" + +" sss\n" + +" sss\n" + +" float\n" + +" \n" + +" \n" + +" null\n" + +" 37.0\n" + +" 32.0\n" + +" null\n" + +" Salinity\n" + +" null\n" + +" \n" + +" \n" + +" \n" + +" sss_obs\n" + +" sss_obs\n" + +" float\n" + +" \n" + +" \n" + +" null\n" + +" 100.0\n" + +" 0.0\n" + +" null\n" + +" Statistics\n" + +" null\n" + +" \n" + +" \n" + +"\n" + +"\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + String2.log("\n*** EDDGridFromNcFiles.testGenerateDatasetsXmlWithRemoteThreddsFiles() finished successfully\n"); + + } + + /** + * This tests if netcdf-java can work with remote files + * and thus if a dataset can be made from remote files. + * + * @throws Throwable if trouble + */ + public static void testRemoteThreddsFiles(boolean deleteCachedInfo) throws Throwable { + String2.log("\n****** EDDGridFromNcFiles.testRemoteThreddsFiles(" + deleteCachedInfo + + ") *****************\n"); + testVerboseOn(); + String name, tName, results, tResults, expected, userDapQuery, tQuery; + String error = ""; + int po; + String today = Calendar2.getCurrentISODateTimeStringZulu().substring(0, 14); //14 is enough to check hour. Hard to check min:sec. + String id = "testRemoteThreddsFiles"; //from generateDatasetsXml above but different datasetID + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml(id); + + //*** test getting das for entire dataset + String2.log("\n*** test das dds for entire dataset\n"); + tName = eddGrid.makeNewFileForDapQuery(null, null, "", EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_trtf", ".das"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"Attributes {\n" + +" time {\n" + +" String _CoordinateAxisType \"Time\";\n" + +" Float64 actual_range 1.3133664e+9, 1.4263776e+9;\n" + +" String axis \"T\";\n" + +" String calendar \"gregorian\";\n" + +" String ioos_category \"Time\";\n" + +" String long_name \"Time\";\n" + +" String standard_name \"time\";\n" + +" String time_origin \"01-JAN-1970 00:00:00\";\n" + +" String units \"seconds since 1970-01-01T00:00:00Z\";\n" + +" }\n" + +" latitude {\n" + +" String _CoordinateAxisType \"Lat\";\n" + +" Float32 actual_range 89.5, -89.5;\n" + +" String axis \"Y\";\n" + +" String coordinate_defines \"center\";\n" + +" String ioos_category \"Location\";\n" + +" String long_name \"Latitude\";\n" + +" String standard_name \"latitude\";\n" + +" String units \"degrees_north\";\n" + +" }\n" + +" longitude {\n" + +" String _CoordinateAxisType \"Lon\";\n" + +" Float32 actual_range -179.5, 179.5;\n" + +" String axis \"X\";\n" + +" String coordinate_defines \"center\";\n" + +" String ioos_category \"Location\";\n" + +" String long_name \"Longitude\";\n" + +" String standard_name \"longitude\";\n" + +" String units \"degrees_east\";\n" + +" }\n" + +" sss {\n" + +" Float32 _FillValue -9999.0;\n" + +" String ancillary_variables \"sss_obs \";\n" + +" String cell_methods \"time: mean over one month\";\n" + +" Float64 colorBarMaximum 37.0;\n" + +" Float64 colorBarMinimum 32.0;\n" + +" String grid_mapping \"crs\";\n" + +" String ioos_category \"Salinity\";\n" + +" String long_name \" Monthly mean Sea Surface Salinity from level-2 swath data\";\n" + +" String standard_name \"Sea_Surface_Salinity\";\n" + +" String units \"psu\";\n" + +" }\n" + +" sss_obs {\n" + +" Float32 _FillValue -9999.0;\n" + +" String cell_methods \"time: sum over one month\";\n" + +" Float64 colorBarMaximum 100.0;\n" + +" Float64 colorBarMinimum 0.0;\n" + +" String grid_mapping \"crs\";\n" + +" String ioos_category \"Statistics\";\n" + +" String long_name \"Valid observation number used to calculate SSS from level-2 swath data\";\n" + +" String standard_name \"Sea_Surface_Salinity_number_of_observations\";\n" + +" String units \"count\";\n" + +" }\n" + +" NC_GLOBAL {\n" + +" String cdm_data_type \"Grid\";\n" + +" String Conventions \"CF-1.6, COARDS, ACDD-1.3\";\n" + +" String creator_email \"Yongsheng.Zhang@noaa.gov\";\n" + +" String creator_institution \"US National Oceanographic Data Center\";\n" + +" String creator_name \"Yongsheng Zhang, Ph.D.\";\n" + +" String creator_url \"http://www.nodc.noaa.gov/SatelliteData\";\n" + +" String date_created \"20150424T093015Z\";\n" + +" Float64 Easternmost_Easting 179.5;\n" + +" Float64 geospatial_lat_max 89.5;\n" + +" Float64 geospatial_lat_min -89.5;\n" + +" Float64 geospatial_lat_resolution 1.0;\n" + +" String geospatial_lat_units \"degrees_north\";\n" + +" Float64 geospatial_lon_max 179.5;\n" + +" Float64 geospatial_lon_min -179.5;\n" + +" Float64 geospatial_lon_resolution 1.0;\n" + +" String geospatial_lon_units \"degrees_east\";\n" + +" String grid_mapping_name \"latitude_longitude\";\n" + +" String history \"Aquarius Level-2 SCI CAP V3.0\n" + +today; + tResults = results.substring(0, Math.min(results.length(), expected.length())); + Test.ensureEqual(tResults, expected, "results=\n" + results); + +//+ " (local files)\n" + +//today + + +expected = +"Z http://127.0.0.1:8080/cwexperimental/griddap/testRemoteThreddsFiles.das\";\n" + +" String infoUrl \"http://data.nodc.noaa.gov/thredds/catalog/aquarius/nodc_binned_V3.0/catalog.html\";\n" + +" String institution \"JPL, California Institute of Technology\";\n" + +" String keywords \"aquarius, calculate, calculated, california, center, data, earth,\n" + +"Earth Science >Oceans > Surface Salinity,\n" + +"gridded, institute, jet, jpl, laboratory, level, level-2, mean, month, monthly, national, ncei, noaa, nodc, number, observation, observations, ocean, oceanographic, oceans, propulsion, salinity, sci, science, sea, sea_surface_salinity, sea_surface_salinity_number_of_observations, sss, sss_obs, statistics, surface, swath, technology, time, used, v3.0, valid\";\n" + +" String keywords_vocabulary \"GCMD Science Keywords\";\n" + +" String license \"These data are available for use without restriction.\";\n" + +" String mission \"SAC-D Aquarius\";\n" + +" String necdf_version_id \"3.6.3\";\n" + +" String nodc_template_version \"NODC_NetCDF_Grid_Template_v1.0\";\n" + +" Float64 Northernmost_Northing 89.5;\n" + +" String publisher_name \"NOAA/NESDIS/NODC - US National Oceanographic Data Center\";\n" + +" String references \"Aquarius users guide, V6.0, PO.DAAC, JPL/NASA. Jun 2, 2014\";\n" + +" String sensor \"Aquarius\";\n" + +" String source \"Jet Propulsion Laboratory, California Institute of Technology\";\n" + +" String sourceUrl \"(local files)\";\n" + +" Float64 Southernmost_Northing -89.5;\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + +" String summary \"This dataset is created by National Oceanographic Data Center (NODC) Satellite Oceanography Group from Aquarius level-2 SCI V3.0 data,using 1.0x1.0 (lon/lat) degree box average\";\n" + +" String time_coverage_end \"2015-03-15T00:00:00Z\";\n" + +" String time_coverage_start \"2011-08-15T00:00:00Z\";\n" + +" String title \"Gridded monthly mean Sea Surface Salinity calculated from Aquarius level-2 SCI V3.0 data\";\n" + +" Float64 Westernmost_Easting -179.5;\n" + +" }\n" + +"}\n"; + int tPo = results.indexOf(expected.substring(0, 17)); + Test.ensureTrue(tPo >= 0, "tPo=-1 results=\n" + results); + Test.ensureEqual( + results.substring(tPo, Math.min(results.length(), tPo + expected.length())), + expected, "results=\n" + results); + + //*** test getting dds for entire dataset + tName = eddGrid.makeNewFileForDapQuery(null, null, "", EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_trtf", ".dds"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"Dataset {\n" + +" Float64 time[time = 44];\n" + +" Float32 latitude[latitude = 180];\n" + +" Float32 longitude[longitude = 360];\n" + +" GRID {\n" + +" ARRAY:\n" + +" Float32 sss[time = 44][latitude = 180][longitude = 360];\n" + +" MAPS:\n" + +" Float64 time[time = 44];\n" + +" Float32 latitude[latitude = 180];\n" + +" Float32 longitude[longitude = 360];\n" + +" } sss;\n" + +" GRID {\n" + +" ARRAY:\n" + +" Float32 sss_obs[time = 44][latitude = 180][longitude = 360];\n" + +" MAPS:\n" + +" Float64 time[time = 44];\n" + +" Float32 latitude[latitude = 180];\n" + +" Float32 longitude[longitude = 360];\n" + +" } sss_obs;\n" + +"} testRemoteThreddsFiles;\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + + //.csv with data from one file + String2.log("\n*** test read from one file\n"); + userDapQuery = "sss[43][100:2:106][50:53]"; + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, EDStatic.fullTestCacheDirectory, + eddGrid.className() + "_trtf", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time,latitude,longitude,sss\n" + +"UTC,degrees_north,degrees_east,psu\n" + +"2015-03-15T00:00:00Z,-10.5,-129.5,35.764122\n" + +"2015-03-15T00:00:00Z,-10.5,-128.5,35.747765\n" + +"2015-03-15T00:00:00Z,-10.5,-127.5,35.75875\n" + +"2015-03-15T00:00:00Z,-10.5,-126.5,35.791206\n" + +"2015-03-15T00:00:00Z,-12.5,-129.5,35.893097\n" + +"2015-03-15T00:00:00Z,-12.5,-128.5,35.968\n" + +"2015-03-15T00:00:00Z,-12.5,-127.5,35.920654\n" + +"2015-03-15T00:00:00Z,-12.5,-126.5,35.961067\n" + +"2015-03-15T00:00:00Z,-14.5,-129.5,36.069397\n" + +"2015-03-15T00:00:00Z,-14.5,-128.5,36.19855\n" + +"2015-03-15T00:00:00Z,-14.5,-127.5,36.24694\n" + +"2015-03-15T00:00:00Z,-14.5,-126.5,36.085762\n" + +"2015-03-15T00:00:00Z,-16.5,-129.5,36.118313\n" + +"2015-03-15T00:00:00Z,-16.5,-128.5,36.377663\n" + +"2015-03-15T00:00:00Z,-16.5,-127.5,36.298218\n" + +"2015-03-15T00:00:00Z,-16.5,-126.5,36.25797\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + // */ + } + + /** This tests matchAxisNDigits */ + public static void testMatchAxisNDigits() throws Throwable { + + String2.log("\n *** EDDGridFromNcFiles.testMatchAxisNDigits() ***"); + + //force reload files + File2.delete("/u00/cwatch/erddap2/dataset/ay/erdATssta3day/fileTable.nc"); + + //load dataset + testVerboseOn(); + String name, tName, results, tResults, expected, userDapQuery, tQuery; + String cDir = EDStatic.fullTestCacheDirectory; + String error = ""; + int po; + String id = "erdATssta3day"; + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml(id); + Table table; + PrimitiveArray pa1, pa2; + + //there should be two time values + String2.log("\n*** test all time values\n"); + tName = eddGrid.makeNewFileForDapQuery(null, null, "time", + cDir, eddGrid.className() + "_tmnd0", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time\n" + +"UTC\n" + +"2004-02-18T12:00:00Z\n" + +"2004-03-20T12:00:00Z\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + +//a further test by hand: +//change erdATssta3day's to 20 +// and above test will fail because n time points will be 1 +// (because axes don't match: because lat [641](30.0125 != 30.012500000000003) + + //*** even though the 2 files have slightly different axis values + // test that lats returned from requests are always the same + String2.log("\n*** test get one lon, all lats\n"); + tName = eddGrid.makeNewFileForDapQuery(null, null, "sst[0][0][][0]", + cDir, eddGrid.className() + "_tmnd1", ".csv"); + table = new Table(); + table.readASCII(cDir + tName, 0, 2); + pa1 = table.getColumn("latitude"); + + tName = eddGrid.makeNewFileForDapQuery(null, null, "sst[1][0][][0]", + cDir, eddGrid.className() + "_tmnd2", ".csv"); + table = new Table(); + table.readASCII(cDir + tName, 0, 2); + pa2 = table.getColumn("latitude"); + + String2.log("pa1[0:10]=" + pa1.subset(0, 1, 10).toString()); + Test.ensureEqual(pa1.testEquals(pa2), "", ""); + + //*** even though the 2 files have slightly different axis values + // test that lons returned from requests are always the same + String2.log("\n*** test get one lat, all lons\n"); + tName = eddGrid.makeNewFileForDapQuery(null, null, "sst[0][0][0][]", + cDir, eddGrid.className() + "_tmnd3", ".csv"); + table = new Table(); + table.readASCII(cDir + tName, 0, 2); + pa1 = table.getColumn("longitude"); + + tName = eddGrid.makeNewFileForDapQuery(null, null, "sst[1][0][0][]", + cDir, eddGrid.className() + "_tmnd4", ".csv"); + table = new Table(); + table.readASCII(cDir + tName, 0, 2); + pa2 = table.getColumn("longitude"); + + String2.log("pa1[0:10]=" + pa1.subset(0, 1, 10).toString()); + Test.ensureEqual(pa1.testEquals(pa2), "", ""); + + + } + + + /** + * This tests this class. + * + * @throws Throwable if trouble + */ + public static void test(boolean deleteCachedDatasetInfo) throws Throwable { +/* */ + testNc(deleteCachedDatasetInfo); + testCwHdf(deleteCachedDatasetInfo); + testHdf(); + testNcml(); + testGrib_43(deleteCachedDatasetInfo); //42 or 43 for netcdfAll 4.2- or 4.3+ + testGrib2_43(deleteCachedDatasetInfo); //42 or 43 for netcdfAll 4.2- or 4.3+ + testGenerateDatasetsXml(); + testGenerateDatasetsXml2(); + testSpeed(-1); //-1 = all + testAVDVSameSource(); + test2DVSameSource(); + testAVDVSameDestination(); + test2DVSameDestination(); + testTimePrecisionMillis(); + testSimpleTestNc(); + testSimpleTestNc2(); +//finish this testRTechHdf(); + testUpdate(); + + //tests of remote sources on-the-fly + testGenerateDatasetsXmlAwsS3(); + testAwsS3(false); //deleteCachedInfo + testGenerateDatasetsXmlWithRemoteThreddsFiles(); + testRemoteThreddsFiles(false); //deleteCachedInfo + testMatchAxisNDigits(); + + /* */ + + //NOT FINISHED + //none + + //Remote Hyrax + // These were tests of reading remote data files on Hyrax without downloading, + // but test Hyrax server doesn't support Byte Ranges. + //testGenerateDatasetsXmlWithRemoteHyraxFiles(); //Test server doesn't support Byte Ranges. + //testRemoteHyraxFiles(false); //deleteCachedInfo //Test server doesn't support Byte Ranges. //one time tests //String fiName = "/erddapTestBig/geosgrib/multi_1.glo_30m.all.grb2"; diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridLonPM180.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridLonPM180.java new file mode 100644 index 000000000..67d51aae4 --- /dev/null +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridLonPM180.java @@ -0,0 +1,629 @@ +/* + * EDDGridLonPM180 Copyright 2015, NOAA. + * See the LICENSE.txt file in this file's directory. + */ +package gov.noaa.pfel.erddap.dataset; + +import com.cohort.array.Attributes; +//import com.cohort.array.ByteArray; +//import com.cohort.array.IntArray; +import com.cohort.array.PrimitiveArray; +//import com.cohort.array.StringArray; +import com.cohort.util.Calendar2; +import com.cohort.util.File2; +import com.cohort.util.MustBe; +import com.cohort.util.SimpleException; +import com.cohort.util.String2; +import com.cohort.util.Test; + +//import gov.noaa.pfel.coastwatch.griddata.NcHelper; +//import gov.noaa.pfel.coastwatch.util.FileVisitorDNLS; +import gov.noaa.pfel.coastwatch.util.SimpleXMLReader; +//import gov.noaa.pfel.coastwatch.util.SSR; +import gov.noaa.pfel.erddap.util.EDStatic; +import gov.noaa.pfel.erddap.variable.*; + +//import java.util.ArrayList; + + +/** + * THIS IS NOT FINISHED. + * This class creates an EDDGrid with longitude values in the range -180 to 180 + * from the enclosed dataset with longitude values in the range 0 to 360. + * + * @author Bob Simons (bob.simons@noaa.gov) 2015-08-05 + */ +public class EDDGridLonPM180 extends EDDGrid { + + protected EDDGrid childDataset; + + /** + * This constructs an EDDGridLonPM180 based on the information in an .xml file. + * + * @param xmlReader with the <erddapDatasets><dataset type="EDDGridLonPM180"> + * having just been read. + * @return an EDDGridLonPM180. + * When this returns, xmlReader will have just read <erddapDatasets></dataset> . + * @throws Throwable if trouble + */ + public static EDDGridLonPM180 fromXml(SimpleXMLReader xmlReader) throws Throwable { + + if (verbose) String2.log("\n*** constructing EDDGridAggregateExistingDimension(xmlReader)..."); + String tDatasetID = xmlReader.attributeValue("datasetID"); + + //data to be obtained while reading xml + EDDGrid tChildDataset = null; + String tAccessibleTo = null; + StringArray tOnChange = new StringArray(); + String tFgdcFile = null; + String tIso19115File = null; + + //process the tags + String startOfTags = xmlReader.allTags(); + int startOfTagsN = xmlReader.stackSize(); + int startOfTagsLength = startOfTags.length(); + + while (true) { + xmlReader.nextTag(); + String tags = xmlReader.allTags(); + String content = xmlReader.content(); + if (xmlReader.stackSize() == startOfTagsN) + break; //the tag + String localTags = tags.substring(startOfTagsLength); + + //try to make the tag names as consistent, descriptive and readable as possible + if (localTags.equals("")) { + if (tChildDataset == null) { + EDD edd = EDD.fromXml(xmlReader.attributeValue("type"), xmlReader); + if (edd instanceof EDDGrid) { + tChildDataset = (EDDGrid)edd; + } else { + throw new RuntimeException("Datasets.xml error: " + + "The dataset defined in an " + + "EDDGridLonPM180 must be a subclass of EDDGrid."); + } + } else { + throw new RuntimeException("Datasets.xml error: " + + "There can be only one defined within an " + + "EDDGridLonPM180 ."); + } + + } + else if (localTags.equals( "")) {} + else if (localTags.equals("")) + tLocalSourceUrls.add(content); + + //http://ourocean.jpl.nasa.gov:8080/thredds/dodsC/g1sst/catalog.xml + else if (localTags.equals( "")) { + tSUServerType = xmlReader.attributeValue("serverType"); + tSURegex = xmlReader.attributeValue("regex"); + String tr = xmlReader.attributeValue("recursive"); + tSURecursive = tr == null? true : String2.parseBoolean(tr); + } + else if (localTags.equals("")) tSU = content; + else if (localTags.equals( "")) {} + else if (localTags.equals("")) tAccessibleTo = content; + else if (localTags.equals( "")) {} + else if (localTags.equals("")) tOnChange.add(content); + else if (localTags.equals( "")) {} + else if (localTags.equals("")) tFgdcFile = content; + else if (localTags.equals( "")) {} + else if (localTags.equals("")) tIso19115File = content; + else if (localTags.equals( "")) {} + else if (localTags.equals("")) tDefaultDataQuery = content; + else if (localTags.equals( "")) {} + else if (localTags.equals("")) tDefaultGraphQuery = content; + + else xmlReader.unexpectedTagException(); + } + + //make the main dataset based on the information gathered + return new EDDGridLonPM180(tDatasetID, tAccessibleTo, + tOnChange, tFgdcFile, tIso19115File, + tChildDataset); + + } + + /** + * The constructor. + * The axisVariables must be the same and in the same + * order for each dataVariable. + * + * @param tDatasetID + * @param tAccessibleTo is a space separated list of 0 or more + * roles which will have access to this dataset. + *
    If null, everyone will have access to this dataset (even if not logged in). + *
    If "", no one will have access to this dataset. + * @param tOnChange 0 or more actions (starting with "http://" or "mailto:") + * to be done whenever the dataset changes significantly + * @param tFgdcFile This should be the fullname of a file with the FGDC + * that should be used for this dataset, or "" (to cause ERDDAP not + * to try to generate FGDC metadata for this dataset), or null (to allow + * ERDDAP to try to generate FGDC metadata for this dataset). + * @param tIso19115 This is like tFgdcFile, but for the ISO 19119-2/19139 metadata. + * @param tChildDataset + * @throws Throwable if trouble + */ + public EDDGridLonPM180(String tDatasetID, + String tAccessibleTo, StringArray tOnChange, String tFgdcFile, String tIso19115File, + EDDGrid tChildDataset) throws Throwable { + + if (verbose) String2.log( + "\n*** constructing EDDGridLonPM180 " + tDatasetID); + long constructionStartMillis = System.currentTimeMillis(); + String errorInMethod = "Error in EDDGridGridAggregateExistingDimension(" + + tDatasetID + ") constructor:\n"; + + //save some of the parameters + className = "EDDGridLonPM180"; + datasetID = tDatasetID; + setAccessibleTo(tAccessibleTo); + onChange = tOnChange; + fgdcFile = tFgdcFile; + iso19115File = tIso19115File; + childDataset = tChildDataset; + + if (childDataset == null) + throw new RuntimeException(String2.ERROR + + ": The child dataset doesn't have a longitude axis variable."); + MustBe.ensureFileNameSafe(datasetID, "datasetID"); + if (datasetID.equals(childDataset.datasetID())) + throw new RuntimeException(String2.ERROR + + ": This dataset's datasetID must not be the same as the child's datasetID."); + + localSourceUrl = childDataset.localSourceUrl(); + addGlobalAttributes = childDataset.addGlobalAttributes(); + sourceGlobalAttributes = childDataset.sourceGlobalAttributes(); + combinedGlobalAttributes = childDataset.combinedGlobalAttributes(); + //and clear searchString of children? + + //make the local axisVariables + int nAv = childDataset.axisVariables.length; + axisVariables = new EDVGridAxis[nAv]; + System.arraycopy(childDataset.axisVariables, 0, axisVariables, 0, nAv); + lonIndex = childDataset.lonIndex; + latIndex = childDataset.latIndex; + altIndex = childDataset.altIndex; + depthIndex = childDataset.depthIndex; + timeIndex = childDataset.timeIndex; + if (lonIndex < 0) + throw new RuntimeException(String2.ERROR + + ": The child dataset doesn't have a longitude axis variable."); + + //make the new lon axisVariable from child's destinationValues + EDVLonGridAxis childEDVLon = axisVariables[lonIndex]; + PrimitiveArray childEDVLonDestValues = childEDVLon.destinationValues(); +... + EDVLonGridAxis newEDVLon = new EDVLonGridAxis(EDV.LON_NAME, + new Attributes(sourceEDVLon.combinedAttributes()), new Attributes(), + PrimitiveArray tSourceValues) throws Throwable { + axisVariables[lonIndex] = newEDVLon; + + //ensure the setup is valid + ensureValid(); + + //finally + if (verbose) String2.log( + (reallyVerbose? "\n" + toString() : "") + + "\n*** EDDGridLonPM180 " + datasetID + " constructor finished. TIME=" + + (System.currentTimeMillis() - constructionStartMillis) + "\n"); + } + + /** + * This makes a sibling dataset, based on the new sourceUrl. + * + * @throws Throwable always (since this class doesn't support sibling()) + */ + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { + throw new SimpleException( + "Error: EDDGridLonPM180 doesn't support method=\"sibling\"."); + } + + /** + * This gets data (not yet standardized) from the data + * source for this EDDGrid. + * Because this is called by GridDataAccessor, the request won't be the + * full user's request, but will be a partial request (for less than + * EDStatic.partialRequestMaxBytes). + * + * @param tDataVariables EDV[] with just the requested data variables + * @param tConstraints int[nAxisVariables*3] + * where av*3+0=startIndex, av*3+1=stride, av*3+2=stopIndex. + * AxisVariables are counted left to right, e.g., sst[0=time][1=lat][2=lon]. + * @return a PrimitiveArray[] where the first axisVariables.length elements + * are the axisValues and the next tDataVariables.length elements + * are the dataValues. + * Both the axisValues and dataValues are straight from the source, + * not modified. + * @throws Throwable if trouble (notably, WaitThenTryAgainException) + */ + public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraints) + throws Throwable { + + //parcel first dimension's constraints to appropriate datasets + int nChildren = childStopsAt.length; + int nAv = axisVariables.length; + int nDv = tDataVariables.length; + PrimitiveArray[] cumResults = null; + int index = tConstraints.get(0); + int stride = tConstraints.get(1); + int stop = tConstraints.get(2); + //find currentDataset (associated with index) + int currentStart = index; + int currentDataset = 0; + while (index > childStopsAt[currentDataset]) + currentDataset++; + IntArray childConstraints = (IntArray)tConstraints.clone(); + + //walk through the requested index values + while (index <= stop) { + //find nextDataset (associated with next iteration's index) + int nextDataset = currentDataset; + while (nextDataset < nChildren && index + stride > childStopsAt[nextDataset]) + nextDataset++; //ok if >= nDatasets + + //get a chunk of data related to current chunk of indexes? + if (nextDataset != currentDataset || //next iteration will be a different dataset + index + stride > stop) { //this is last iteration + //get currentStart:stride:index + int currentDatasetStartsAt = currentDataset == 0? 0 : childStopsAt[currentDataset - 1] + 1; + childConstraints.set(0, currentStart - currentDatasetStartsAt); + childConstraints.set(2, index - currentDatasetStartsAt); + if (reallyVerbose) String2.log(" currentDataset=" + currentDataset + + " datasetStartsAt=" + currentDatasetStartsAt + + " localStart=" + childConstraints.get(0) + + " localStop=" + childConstraints.get(2)); + PrimitiveArray[] tResults = childDatasets[currentDataset].getSourceData( + tDataVariables, childConstraints); + //childDataset has already checked that axis values are as *it* expects + if (cumResults == null) { + if (!ensureAxisValuesAreEqual ..switch to matchAxisNDigits) { + //make axis values exactly as expected by aggregate dataset + for (int av = 1; av < nAv; av++) + tResults[av] = axisVariables[av].sourceValues().subset( + childConstraints.get(av * 3 + 0), + childConstraints.get(av * 3 + 1), + childConstraints.get(av * 3 + 2)); + } + cumResults = tResults; + } else { + cumResults[0].append(tResults[0]); + for (int dv = 0; dv < nDv; dv++) + cumResults[nAv + dv].append(tResults[nAv + dv]); + } + + currentDataset = nextDataset; + currentStart = index + stride; + } + + //increment index + index += stride; + } + + return cumResults; + } + + + /** + * This tests the basic methods in this class. + * + * @throws Throwable if trouble + */ + public static void testBasic() throws Throwable { + + String2.log("\n****************** EDDGridLonPM180.testBasic() *****************\n"); + testVerboseOn(); + String name, tName, userDapQuery, results, expected, error; + + //one time + + //*** NDBC is also IMPORTANT UNIQUE TEST of >1 variable in a file + EDDGrid gridDataset = (EDDGrid)oneFromDatasetXml("ndbcCWind41002"); + + //min max + EDV edv = gridDataset.findAxisVariableByDestinationName("longitude"); + Test.ensureEqual(edv.destinationMin(), -75.42, ""); + Test.ensureEqual(edv.destinationMax(), -75.42, ""); + + String ndbcDapQuery = "wind_speed[1:5][0][0]"; + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, + EDStatic.fullTestCacheDirectory, gridDataset.className(), ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time, latitude, longitude, wind_speed\n" + +"UTC, degrees_north, degrees_east, m s-1\n" + +"1989-06-13T16:10:00Z, 32.27, -75.42, 15.7\n" + +"1989-06-13T16:20:00Z, 32.27, -75.42, 15.0\n" + +"1989-06-13T16:30:00Z, 32.27, -75.42, 14.4\n" + +"1989-06-13T16:40:00Z, 32.27, -75.42, 14.0\n" + +"1989-06-13T16:50:00Z, 32.27, -75.42, 13.6\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + ndbcDapQuery = "wind_speed[1:5][0][0]"; + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, EDStatic.fullTestCacheDirectory, + gridDataset.className(), ".nc"); + results = NcHelper.dumpString(EDStatic.fullTestCacheDirectory + tName, true); + int dataPo = results.indexOf("data:"); + String today = Calendar2.getCurrentISODateTimeStringLocal().substring(0, 10); + expected = +"netcdf EDDGridLonPM180.nc {\n" + +" dimensions:\n" + +" time = 5;\n" + // (has coord.var)\n" + //changed when switched to netcdf-java 4.0, 2009-02-23 +" latitude = 1;\n" + // (has coord.var)\n" + +" longitude = 1;\n" + // (has coord.var)\n" + +" variables:\n" + +" double time(time=5);\n" + +" :_CoordinateAxisType = \"Time\";\n"; + Test.ensureEqual(results.substring(0, expected.length()), expected, "results=\n" + results); + +//BUG???!!! +//" :actual_range = 6.137568E8, 7.258458E8; // double\n" + //this changes depending on how many files aggregated + +expected = +":axis = \"T\";\n" + +" :ioos_category = \"Time\";\n" + +" :long_name = \"Epoch Time\";\n" + +" :short_name = \"time\";\n" + +" :standard_name = \"time\";\n" + +" :time_origin = \"01-JAN-1970 00:00:00\";\n" + +" :units = \"seconds since 1970-01-01T00:00:00Z\";\n" + +" float latitude(latitude=1);\n" + +" :_CoordinateAxisType = \"Lat\";\n" + +" :actual_range = 32.27f, 32.27f; // float\n" + +" :axis = \"Y\";\n" + +" :ioos_category = \"Location\";\n" + +" :long_name = \"Latitude\";\n" + +" :short_name = \"latitude\";\n" + +" :standard_name = \"latitude\";\n" + +" :units = \"degrees_north\";\n" + +" float longitude(longitude=1);\n" + +" :_CoordinateAxisType = \"Lon\";\n" + +" :actual_range = -75.42f, -75.42f; // float\n" + +" :axis = \"X\";\n" + +" :ioos_category = \"Location\";\n" + +" :long_name = \"Longitude\";\n" + +" :short_name = \"longitude\";\n" + +" :standard_name = \"longitude\";\n" + +" :units = \"degrees_east\";\n" + +" float wind_speed(time=5, latitude=1, longitude=1);\n" + +" :_FillValue = 99.0f; // float\n" + +" :ioos_category = \"Wind\";\n" + +" :long_name = \"Wind Speed\";\n" + +" :missing_value = 99.0f; // float\n" + +" :short_name = \"wspd\";\n" + +" :standard_name = \"wind_speed\";\n" + +" :units = \"m s-1\";\n" + +"\n" + +" :cdm_data_type = \"Grid\";\n" + +" :comment = \"S HATTERAS - 250 NM East of Charleston, SC\";\n" + +" :contributor_name = \"NOAA NDBC\";\n" + +" :contributor_role = \"Source of data.\";\n" + +" :Conventions = \"COARDS, CF-1.6, ACDD-1.3\";\n" + +" :Easternmost_Easting = -75.42f; // float\n" + +" :geospatial_lat_max = 32.27f; // float\n" + +" :geospatial_lat_min = 32.27f; // float\n" + +" :geospatial_lon_max = -75.42f; // float\n" + +" :geospatial_lon_min = -75.42f; // float\n" + +" :history = \"" + today + " http://dods.ndbc.noaa.gov/thredds/dodsC/data/cwind/41002/41002c1989.nc\n" + +today + " " + EDStatic.erddapUrl + //in tests, always non-https url +"/griddap/ndbcCWind41002.nc?wind_speed[1:5][0][0]\";\n" + +" :infoUrl = \"http://www.ndbc.noaa.gov/cwind.shtml\";\n" + +" :institution = \"NOAA NDBC\";\n" + +" :license = \"The data may be used and redistributed for free but is not intended\n" + +"for legal use, since it may contain inaccuracies. Neither the data\n" + +"Contributor, ERD, NOAA, nor the United States Government, nor any\n" + +"of their employees or contractors, makes any warranty, express or\n" + +"implied, including warranties of merchantability and fitness for a\n" + +"particular purpose, or assumes any legal liability for the accuracy,\n" + +"completeness, or usefulness, of this information.\";\n" + +" :location = \"32.27 N 75.42 W \";\n" + +" :Northernmost_Northing = 32.27f; // float\n" + +" :quality = \"Automated QC checks with manual editing and comprehensive monthly QC\";\n" + +" :sourceUrl = \"http://dods.ndbc.noaa.gov/thredds/dodsC/data/cwind/41002/41002c1989.nc\";\n" + +" :Southernmost_Northing = 32.27f; // float\n" + +" :station = \"41002\";\n" + +" :summary = \"These continuous wind measurements from the NOAA National Data Buoy Center (NDBC) stations are 10-minute average values of wind speed (in m/s) and direction (in degrees clockwise from North).\";\n" + +" :time_coverage_end = \"1989-06-13T16:50:00Z\";\n" + +" :time_coverage_start = \"1989-06-13T16:10:00Z\";\n" + +" :title = \"Wind Data from NDBC 41002\";\n" + +" :url = \"http://dods.ndbc.noaa.gov\";\n" + +" :Westernmost_Easting = -75.42f; // float\n" + +" data:\n" + +"time =\n" + +" {6.137574E8, 6.13758E8, 6.137586E8, 6.137592E8, 6.137598E8}\n" + +"latitude =\n" + +" {32.27}\n" + +"longitude =\n" + +" {-75.42}\n" + +"wind_speed =\n" + +" {\n" + +" {\n" + +" {15.7}\n" + +" },\n" + +" {\n" + +" {15.0}\n" + +" },\n" + +" {\n" + +" {14.4}\n" + +" },\n" + +" {\n" + +" {14.0}\n" + +" },\n" + +" {\n" + +" {13.6}\n" + +" }\n" + +" }\n" + +"}\n"; + int po = results.indexOf(":axis ="); + Test.ensureEqual(results.substring(po), expected, "results=\n" + results); + + ndbcDapQuery = "wind_direction[1:5][0][0]"; + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, EDStatic.fullTestCacheDirectory, + gridDataset.className() + "2", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time, latitude, longitude, wind_direction\n" + +"UTC, degrees_north, degrees_east, degrees_true\n" + +"1989-06-13T16:10:00Z, 32.27, -75.42, 234\n" + +"1989-06-13T16:20:00Z, 32.27, -75.42, 233\n" + +"1989-06-13T16:30:00Z, 32.27, -75.42, 233\n" + +"1989-06-13T16:40:00Z, 32.27, -75.42, 235\n" + +"1989-06-13T16:50:00Z, 32.27, -75.42, 235\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + ndbcDapQuery = "wind_speed[1:5][0][0],wind_direction[1:5][0][0]"; + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, EDStatic.fullTestCacheDirectory, + gridDataset.className() + "3", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time, latitude, longitude, wind_speed, wind_direction\n" + +"UTC, degrees_north, degrees_east, m s-1, degrees_true\n" + +"1989-06-13T16:10:00Z, 32.27, -75.42, 15.7, 234\n" + +"1989-06-13T16:20:00Z, 32.27, -75.42, 15.0, 233\n" + +"1989-06-13T16:30:00Z, 32.27, -75.42, 14.4, 233\n" + +"1989-06-13T16:40:00Z, 32.27, -75.42, 14.0, 235\n" + +"1989-06-13T16:50:00Z, 32.27, -75.42, 13.6, 235\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + ndbcDapQuery = "wind_direction[1:5][0][0],wind_speed[1:5][0][0]"; + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, EDStatic.fullTestCacheDirectory, + gridDataset.className() + "4", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time, latitude, longitude, wind_direction, wind_speed\n" + +"UTC, degrees_north, degrees_east, degrees_true, m s-1\n" + +"1989-06-13T16:10:00Z, 32.27, -75.42, 234, 15.7\n" + +"1989-06-13T16:20:00Z, 32.27, -75.42, 233, 15.0\n" + +"1989-06-13T16:30:00Z, 32.27, -75.42, 233, 14.4\n" + +"1989-06-13T16:40:00Z, 32.27, -75.42, 235, 14.0\n" + +"1989-06-13T16:50:00Z, 32.27, -75.42, 235, 13.6\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, EDStatic.fullTestCacheDirectory, + gridDataset.className() + "4", ".nc"); + results = NcHelper.dumpString(EDStatic.fullTestCacheDirectory + tName, true); + dataPo = results.indexOf("data:"); + results = results.substring(dataPo); + expected = +"data:\n" + +"time =\n" + +" {6.137574E8, 6.13758E8, 6.137586E8, 6.137592E8, 6.137598E8}\n" + +"latitude =\n" + +" {32.27}\n" + +"longitude =\n" + +" {-75.42}\n" + +"wind_direction =\n" + +" {\n" + +" {\n" + +" {234}\n" + +" },\n" + +" {\n" + +" {233}\n" + +" },\n" + +" {\n" + +" {233}\n" + +" },\n" + +" {\n" + +" {235}\n" + +" },\n" + +" {\n" + +" {235}\n" + +" }\n" + +" }\n" + +"wind_speed =\n" + +" {\n" + +" {\n" + +" {15.7}\n" + +" },\n" + +" {\n" + +" {15.0}\n" + +" },\n" + +" {\n" + +" {14.4}\n" + +" },\n" + +" {\n" + +" {14.0}\n" + +" },\n" + +" {\n" + +" {13.6}\n" + +" }\n" + +" }\n" + +"}\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + //test seam of two datasets + ndbcDapQuery = "wind_direction[22232:22239][0][0],wind_speed[22232:22239][0][0]"; + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, EDStatic.fullTestCacheDirectory, + gridDataset.className() + "5", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time, latitude, longitude, wind_direction, wind_speed\n" + +"UTC, degrees_north, degrees_east, degrees_true, m s-1\n" + +"1989-12-05T05:20:00Z, 32.27, -75.42, 232, 23.7\n" + +"1989-12-05T05:30:00Z, 32.27, -75.42, 230, 24.1\n" + +"1989-12-05T05:40:00Z, 32.27, -75.42, 225, 23.5\n" + +"1989-12-05T05:50:00Z, 32.27, -75.42, 233, 23.3\n" + +"1992-04-28T23:00:00Z, 32.27, -75.42, 335, 29.4\n" + +"1992-04-28T23:10:00Z, 32.27, -75.42, 335, 30.5\n" + +"1992-04-28T23:20:00Z, 32.27, -75.42, 330, 32.3\n" + +"1992-04-28T23:30:00Z, 32.27, -75.42, 331, 33.2\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + //test seam of two datasets with stride + ndbcDapQuery = "wind_direction[22232:2:22239][0][0],wind_speed[22232:2:22239][0][0]"; + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, EDStatic.fullTestCacheDirectory, + gridDataset.className() + "6", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time, latitude, longitude, wind_direction, wind_speed\n" + +"UTC, degrees_north, degrees_east, degrees_true, m s-1\n" + +"1989-12-05T05:20:00Z, 32.27, -75.42, 232, 23.7\n" + +"1989-12-05T05:40:00Z, 32.27, -75.42, 225, 23.5\n" + +"1992-04-28T23:00:00Z, 32.27, -75.42, 335, 29.4\n" + +"1992-04-28T23:20:00Z, 32.27, -75.42, 330, 32.3\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + // */ + + //test last data value (it causes a few problems -- including different axis values) + ndbcDapQuery = "wind_direction[(2007-12-31T22:40:00):1:(2007-12-31T22:55:00)][0][0]," + + "wind_speed[(2007-12-31T22:40:00):1:(2007-12-31T22:55:00)][0][0]"; + tName = gridDataset.makeNewFileForDapQuery(null, null, ndbcDapQuery, EDStatic.fullTestCacheDirectory, + gridDataset.className() + "7", ".csv"); + results = new String((new ByteArray(EDStatic.fullTestCacheDirectory + tName)).toArray()); + //String2.log(results); + expected = +"time, latitude, longitude, wind_direction, wind_speed\n" + +"UTC, degrees_north, degrees_east, degrees_true, m s-1\n" + +"2007-12-31T22:40:00Z, 32.27, -75.42, 36, 4.0\n" + +"2007-12-31T22:50:00Z, 32.27, -75.42, 37, 4.3\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + + String2.log("\n*** EDDGridLonPM180.test finished."); + + } + + /** + * This tests the methods in this class. + * + * @throws Throwable if trouble + */ + public static void test() throws Throwable { + + String2.log("\n****************** EDDGridLonPM180.test() *****************\n"); + testBasic(); + + //not usually run + } + + + +} diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridSideBySide.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridSideBySide.java index 3647a7dc8..9c6ce8cdc 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridSideBySide.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridSideBySide.java @@ -50,6 +50,14 @@ public class EDDGridSideBySide extends EDDGrid { protected int childStopsAt[]; //the last valid dataVariables index for each childDataset protected IntArray indexOfAxis0Value[]; //an IntArray for each child; a row for each axis0 value + /** + * This is used to test equality of axis values. + * 0=no testing (not recommended). + * >18 does exact test. default=20. + * 1-18 tests that many digets for doubles and hidiv(n,2) for floats. + */ + protected int matchAxisNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; + /** * This constructs an EDDGridSideBySide based on the information in an .xml file. @@ -71,6 +79,7 @@ public static EDDGridSideBySide fromXml(SimpleXMLReader xmlReader) throws Throwa ArrayList tChildDatasets = new ArrayList(); StringBuilder messages = new StringBuilder(); String tAccessibleTo = null; + int tMatchAxisNDigits = DEFAULT_MATCH_AXIS_N_DIGITS; StringArray tOnChange = new StringArray(); String tFgdcFile = null; String tIso19115File = null; @@ -120,6 +129,12 @@ public static EDDGridSideBySide fromXml(SimpleXMLReader xmlReader) throws Throwa } else if (localTags.equals( "")) {} else if (localTags.equals("")) tOnChange.add(content); + else if (localTags.equals( "")) {} + else if (localTags.equals("")) + tMatchAxisNDigits = String2.parseInt(content, DEFAULT_MATCH_AXIS_N_DIGITS); + else if (localTags.equals( "")) {} //deprecated + else if (localTags.equals("")) + tMatchAxisNDigits = String2.parseBoolean(content)? 20 : 0; else if (localTags.equals( "")) {} else if (localTags.equals("")) tAccessibleTo = content; else if (localTags.equals( "")) {} @@ -145,7 +160,7 @@ else if (localTags.equals( "")) {} //make the main dataset based on the information gathered return new EDDGridSideBySide(tDatasetID, tAccessibleTo, - tOnChange, tFgdcFile, tIso19115File, + tMatchAxisNDigits, tOnChange, tFgdcFile, tIso19115File, tDefaultDataQuery, tDefaultGraphQuery, tcds); } @@ -171,6 +186,7 @@ else if (localTags.equals( "")) {} * @throws Throwable if trouble */ public EDDGridSideBySide(String tDatasetID, String tAccessibleTo, + int tMatchAxisNDigits, StringArray tOnChange, String tFgdcFile, String tIso19115File, String tDefaultDataQuery, String tDefaultGraphQuery, EDDGrid tChildDatasets[]) throws Throwable { @@ -192,13 +208,15 @@ public EDDGridSideBySide(String tDatasetID, String tAccessibleTo, childDatasets = tChildDatasets; int nChildren = tChildDatasets.length; childStopsAt = new int[nChildren]; + matchAxisNDigits = tMatchAxisNDigits; //check the siblings and create childStopsAt EDDGrid firstChild = childDatasets[0]; int nAV = firstChild.axisVariables.length; for (int c = 0; c < nChildren; c++) { if (c > 0) { - String similar = firstChild.similarAxisVariables(childDatasets[c], 1, false); //test axes 1+ + String similar = firstChild.similarAxisVariables(childDatasets[c], 1, //test axes 1+ + matchAxisNDigits, false); if (similar.length() > 0) throw new RuntimeException("Error: Datasets #0 and #" + c + " are not similar: " + similar); @@ -371,8 +389,8 @@ public long creationTimeMillis() { * * @throws Throwable always (since this class doesn't support sibling()) */ - public EDDGrid sibling(String tLocalSourceUrl, int ensureAxisValuesAreEqual, - boolean shareInfo) throws Throwable { + public EDDGrid sibling(String tLocalSourceUrl, int firstAxisToMatch, + int matchAxisNDigits, boolean shareInfo) throws Throwable { throw new SimpleException("Error: " + "EDDGridSideBySide doesn't support method=\"sibling\"."); } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTable.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTable.java index 238981c8e..83a53d9a4 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTable.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTable.java @@ -6385,6 +6385,40 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, "
    Pydap instructions). Also, the name of the sequence in tabledap datasets will always be \"" + SEQUENCE_NAME + "\"\n" + "
    (unlike \"location\" for the sample dataset in the Pydap instructions).\n" + "\n" + + //Python + "

    Python" + + EDStatic.externalLinkHtml(tErddapUrl) + "\n" + + " is a widely-used computer language that is very popular among scientists.\n" + + "
    In addition to the Pydap Client, you can use Python to download various files from ERDDAP\n" + + "
    as you would download other files from the web:\n" + + "

    import urllib\n" +
+            "urllib.urlretrieve(\"http://baseurl/erddap/tabledap/datasetID.fileType?query\", \"outputFileName\")
    \n" + + " Or download the content to an object instead of a file:\n" + + "
    import urllib2\n" +
+            "response = urllib2.open(\"http://baseurl/erddap/tabledap/datasetID.fileType?query\")\n" +
+            "theContent = response.read()
    \n" + + " There are other ways to do this in Python. Search the web for more information.\n"+ + "
     \n" + + "
    To access a password-protected, private ERDDAP dataset with Python via https, use this\n" + + "
    two-step process after you install the\n" + + "requests library" + + EDStatic.externalLinkHtml(tErddapUrl) + " (\"HTTP for Humans\"):\n" + + "
      \n" + + "
    1. Log in (authenticate) and store the certificate in a 'session' object:\n" + + "
      import requests\n" +
+            "session = requests.session()\n" +
+            "credentials_dct = {'user': 'myUserName', 'password': 'myPassword'}\n" +
+            "p = session.post(\"https://baseurl:8443/erddap/login.html\", credentials_dct, verify=True)
      \n" + + "
    2. Repeatedly make data requests using the session: \n" + + "
      theContent = session.get('https://baseurl:8443/erddap/tabledap/datasetID.fileType?query', verify=True)
      \n" + + "
    \n" + + " * Some ERDDAP installations won't need the port number (:8443) in the URL.\n" + + "
    * If the server uses a self-signed certificate and you are okay with that, use verify=False\n" + + "
      to tell Python not to check the server's certificate.\n" + + "
    * That works in Python v2.7. You might need to make slight modifications for other versions.\n" + + "
    * (Thanks to Emilio Mayorga of NANOOS and Paul Janecek of Spyglass Technologies for\n" + + "
      figuring this out.)\n" + + "
     \n" + //R "

    R Statistical Package" + EDStatic.externalLinkHtml(tErddapUrl) + " -\n" + @@ -6510,8 +6544,19 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, "
    to be put into the output fileName.\n" + "
    For example, \n" + "

    curl \"http://coastwatch.pfeg.noaa.gov/erddap/tabledap/cwwcNDBCMet.png?time,atmp&time%3E=2010-09-03T00:00:00Z&time%3C=2010-09-06T00:00:00Z&station=%22{TAML1,41009,46088}%22&.draw=linesAndMarkers&.marker=5|5&.color=0x000000&.colorBar=|||||\" -o NDBCatmp#1.png
    \n" + - "\n" + - "
     \n"); + "
  • To access a password-protected, private ERDDAP dataset with curl via https, use this\n" + + "
    two-step process:\n" + + "
      \n" + + "
    1. Log in (authenticate) and save the certificate cookie in a cookie-jar file:\n" + + "
      curl -v --data 'user=myUserName&password=myPassword' -c cookies.txt -b cookies.txt -k https://baseurl:8443/erddap/login.html
      \n" + + "
    2. Repeatedly make data requests using the saved cookie: \n" + + "
      curl -v -c cookies.txt -b cookies.txt -k https://baseurl:8443/erddap/tabledap/datasetID.fileType?query -o outputFileName
      \n" + + "
    \n" + + " Some ERDDAP installations won't need the port number (:8443) in the URL.\n" + + "
    (Thanks to Liquid Robotics for the starting point and Emilio Mayorga of NANOOS and\n" + + "
    Paul Janecek of Spyglass Technologies for testing.)\n" + + "
     \n" + + "\n"); //query @@ -6670,8 +6715,8 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, "
    display date/time values as \n" + " ISO 8601:2004 \"extended\" date/time strings" + EDStatic.externalLinkHtml(tErddapUrl) + "\n" + - "
    (e.g., 2002-08-03T12:30:00Z, but some variables include milliseconds, e.g.,\n" + - "
    2002-08-03T12:30:00.123Z).\n" + + "
    (e.g., 2002-08-03T12:30:00Z, but some time variables in some datasets include\n" + + "
    milliseconds, e.g., 2002-08-03T12:30:00.123Z).\n" + (EDStatic.convertersActive? "
    ERDDAP has a utility to\n" + " Convert\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableCopy.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableCopy.java index f6c8cb246..d9122e46a 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableCopy.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableCopy.java @@ -529,6 +529,17 @@ public void getDataForDapQuery(String loggedInAs, String requestUrl, String user localEdd.getDataForDapQuery(loggedInAs, requestUrl, userDapQuery, tableWriter); } + /** + * This returns a fileTable (formatted like + * FileVisitorDNLS.oneStep(tDirectoriesToo=false, last_mod is LongArray, + * and size is LongArray of epochMillis) + * with valid files (or null if unavailable or any trouble). + * This is a copy of any internal data, so client can modify the contents. + */ + public Table accessibleViaFilesFileTable() { + return localEdd.accessibleViaFilesFileTable(); + } + /** * The basic tests of this class (erdGlobecBottle). * diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromAsciiFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromAsciiFiles.java index a7723e8c1..aa7ff7f83 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromAsciiFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromAsciiFiles.java @@ -231,7 +231,8 @@ public static String generateDatasetsXml(String tFileDir, String tFileNameRegex, if (tInstitution != null && tInstitution.length() > 0) externalAddGlobalAttributes.add("institution", tInstitution); if (tSummary != null && tSummary.length() > 0) externalAddGlobalAttributes.add("summary", tSummary); if (tTitle != null && tTitle.length() > 0) externalAddGlobalAttributes.add("title", tTitle); - externalAddGlobalAttributes.setIfNotAlreadySet("sourceUrl", "(local files)"); + externalAddGlobalAttributes.setIfNotAlreadySet("sourceUrl", + "(" + (File2.isRemote(tFileDir)? "remote" : "local") + " files)"); //externalAddGlobalAttributes.setIfNotAlreadySet("subsetVariables", "???"); boolean dateTimeAlreadyFound = false; @@ -298,7 +299,8 @@ public static String generateDatasetsXml(String tFileDir, String tFileNameRegex, suggestDatasetID(tFileDir + tFileNameRegex) + "\" active=\"true\">\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " " + suggestUpdateEveryNMillis(tFileDir) + + "\n" + " " + tFileDir + "\n" + " true\n" + " " + XML.encodeAsXML(tFileNameRegex) + "\n" + @@ -407,7 +409,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " GCMD Science Keywords\n" + " [standard]\n" + " (local files)\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " The new summary! NOAA National Data Buoy Center (NDBC) data from a local source.\n" + " The Newer Title!\n" + " \n" + @@ -717,7 +719,7 @@ public static void testBasic(boolean deleteCachedDatasetInfo) throws Throwable { " Float64 Northernmost_Northing 37.75;\n" + " String sourceUrl \"The source URL.\";\n" + " Float64 Southernmost_Northing -27.7;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"station, longitude, latitude, altitude\";\n" + " String summary \"The summary.\";\n" + " String time_coverage_end \"2006-12-31T23:00:00Z\";\n" + @@ -935,7 +937,7 @@ public static void testFixedValue() throws Throwable { " Float64 Northernmost_Northing 30.368;\n" + " String sourceUrl \"(local files)\";\n" + " Float64 Southernmost_Northing 26.6255;\n" + - " String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + + " String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + (test == 0? " String subsetVariables \"ship_call_sign\";\n" : "") + " String summary \"NOAA Ship Pisces Realtime Data updated every hour\";\n" + " String time_coverage_end \"2013-05-23T18:04:00Z\";\n" + @@ -1090,7 +1092,7 @@ public static void testBasic2() throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " String sourceUrl \"(local files)\";\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"five, fileName\";\n" + " String summary \"The new summary!\";\n" + " String title \"The Newer Title!\";\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromAwsXmlFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromAwsXmlFiles.java index 47f8ac37d..a684299c8 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromAwsXmlFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromAwsXmlFiles.java @@ -239,7 +239,8 @@ public static String generateDatasetsXml(String tFileDir, String tFileNameRegex, suggestDatasetID(tFileDir + tFileNameRegex) + "\" active=\"true\">\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " " + suggestUpdateEveryNMillis(tFileDir) + + "\n" + " " + tFileDir + "\n" + " true\n" + " " + XML.encodeAsXML(tFileNameRegex) + "\n" + @@ -348,7 +349,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " GCMD Science Keywords\n" + " [standard]\n" + " (local files)\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " The new summary! exploratorium data from a local source.\n" + " The Newer Title!\n" + " \n" + @@ -1065,7 +1066,7 @@ public static void testBasic(boolean deleteCachedDatasetInfo) throws Throwable { " gust_time {\n" + " Float64 actual_range 1.3519746e+9, 1.3519746e+9;\n" + " String ioos_category \"Time\";\n" + -" String long_name \"Time\";\n" + +" String long_name \"Gust-time\";\n" + " String standard_name \"time\";\n" + " String time_origin \"01-JAN-1970 00:00:00\";\n" + " String units \"seconds since 1970-01-01T00:00:00Z\";\n" + @@ -1241,7 +1242,7 @@ public static void testBasic(boolean deleteCachedDatasetInfo) throws Throwable { " sunrise {\n" + " Float64 actual_range 1.351953497e+9, 1.351953497e+9;\n" + " String ioos_category \"Time\";\n" + -" String long_name \"Time\";\n" + +" String long_name \"Sunrise\";\n" + " String standard_name \"time\";\n" + " String time_origin \"01-JAN-1970 00:00:00\";\n" + " String units \"seconds since 1970-01-01T00:00:00Z\";\n" + @@ -1249,7 +1250,7 @@ public static void testBasic(boolean deleteCachedDatasetInfo) throws Throwable { " sunset {\n" + " Float64 actual_range 1.351991286e+9, 1.351991286e+9;\n" + " String ioos_category \"Time\";\n" + -" String long_name \"Time\";\n" + +" String long_name \"Sunset\";\n" + " String standard_name \"time\";\n" + " String time_origin \"01-JAN-1970 00:00:00\";\n" + " String units \"seconds since 1970-01-01T00:00:00Z\";\n" + @@ -1323,7 +1324,7 @@ public static void testBasic(boolean deleteCachedDatasetInfo) throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " String sourceUrl \"(local files)\";\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"fileName, station_id, station, city_state_zip, city_state, site_url, altitude\";\n" + " String summary \"The new summary!\";\n" + " String time_coverage_end \"2012-11-03T20:30:00Z\";\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromBMDE.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromBMDE.java index 900512d2d..64bd5d122 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromBMDE.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromBMDE.java @@ -661,7 +661,7 @@ public static void testPrbo() throws Throwable { " Float64 Northernmost_Northing 38.5;\n" + " String sourceUrl \"http://digir.prbo.org/digir/DiGIR.php\";\n" + " Float64 Southernmost_Northing 36.5;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"Shorebird observation data from the Southeast Farallon Island from the PRBO Conservation Service.\n" + "\n" + "DiGIR is an engine which takes XML requests for data and returns a data\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromCassandra.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromCassandra.java index f0e136c6d..a833196af 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromCassandra.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromCassandra.java @@ -1675,7 +1675,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " bob, bobtable, canada, cascii, cassandra, cboolean, cbyte, cdecimal, cdouble, cfloat, cint, clong, cmap, cset, cshort, ctext, currents, cvarchar, data, date, depth, deviceid, networks, ocean, sampletime, test, time, title, u, v, velocity, vertical, w\n" + " [standard]\n" + " (local Cassandra)\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " deviceid, date\n" + " The summary for Bob's great Cassandra test data.\n" + " The Title for Bob's Cassandra Test Data (bobTable)\n" + @@ -1977,7 +1977,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " canada, cassandra, data, date, depth, deviceid, latitude, longitude, networks, ocean, sampletime, static, test, time, u, v\n" + " [standard]\n" + " (local Cassandra)\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " deviceid, date, latitude, longitude\n" + " The summary for Bob's great Cassandra test data.\n" + " Cassandra Static Test\n" + @@ -2275,7 +2275,7 @@ public static void testBasic(boolean pauseBetweenTests) throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " String sourceUrl \"(Cassandra)\";\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"deviceid, date\";\n" + " String summary \"The summary for Bob's Cassandra test data.\";\n" + " String title \"Bob's Cassandra Test Data\";\n" + @@ -2917,7 +2917,7 @@ public static void testCass1Device(boolean pauseBetweenTests) throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " String sourceUrl \"(Cassandra)\";\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"deviceid, date\";\n" + " String summary \"The summary for Bob's Cassandra test data.\";\n" + " String title \"Bob's Cassandra Test Data\";\n" + @@ -3276,7 +3276,7 @@ public static void testStatic(boolean pauseBetweenTests) throws Throwable { " Float64 Northernmost_Northing 34.0;\n" + " String sourceUrl \"(Cassandra)\";\n" + " Float64 Southernmost_Northing 33.0;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"deviceid, date, latitude, longitude\";\n" + " String summary \"The summary for Bob's Cassandra test data.\";\n" + " String title \"Cassandra Static Test\";\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromColumnarAsciiFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromColumnarAsciiFiles.java index f72660ced..8702fde8d 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromColumnarAsciiFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromColumnarAsciiFiles.java @@ -271,7 +271,8 @@ public static String generateDatasetsXml(String tFileDir, String tFileNameRegex, if (tInstitution != null && tInstitution.length() > 0) externalAddGlobalAttributes.add("institution", tInstitution); if (tSummary != null && tSummary.length() > 0) externalAddGlobalAttributes.add("summary", tSummary); if (tTitle != null && tTitle.length() > 0) externalAddGlobalAttributes.add("title", tTitle); - externalAddGlobalAttributes.setIfNotAlreadySet("sourceUrl", "(local files)"); + externalAddGlobalAttributes.setIfNotAlreadySet("sourceUrl", + "(" + (File2.isRemote(tFileDir)? "remote" : "local") + " files)"); //externalAddGlobalAttributes.setIfNotAlreadySet("subsetVariables", "???"); boolean dateTimeAlreadyFound = false; @@ -340,7 +341,8 @@ public static String generateDatasetsXml(String tFileDir, String tFileNameRegex, suggestDatasetID(tFileDir + tFileNameRegex) + "\" active=\"true\">\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " " + suggestUpdateEveryNMillis(tFileDir) + + "\n" + " " + tFileDir + "\n" + " true\n" + " " + XML.encodeAsXML(tFileNameRegex) + "\n" + @@ -448,7 +450,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " aBoolean, aByte, aChar, aDouble, aFloat, aLong, anInt, aShort, aString, boolean, buoy, byte, center, char, data, double, float, int, long, national, ndbc, newer, noaa, short, string, title\n" + " [standard]\n" + " (local files)\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " The new summary! NOAA National Data Buoy Center (NDBC) data from a local source.\n" + " The Newer Title!\n" + " \n" + @@ -706,7 +708,7 @@ public static void testBasic() throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " String sourceUrl \"(local files)\";\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"five, fileName\";\n" + " String summary \"The new summary!\";\n" + " String title \"The Newer Title!\";\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromDapSequence.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromDapSequence.java index 9be6b7163..fa528ee4b 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromDapSequence.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromDapSequence.java @@ -885,10 +885,6 @@ varName, new StringArray(), - - - - /** * testGenerateDatasetsXml */ @@ -924,10 +920,10 @@ public static void testGenerateDatasetsXml() throws Throwable { " DYNDNS CIMT\n" + " acceleration, anomaly, average, avg_sound_velocity, center, cimt, cimt.dyndns.org, currents, data, density, depth, dods, drds, dyndns, fluorescence, geopotential, geopotential_anomaly, identifier, integrated, latitude, longitude, marine, ocean, oceans,\n" + "Oceans > Salinity/Density > Salinity,\n" + -"optical, optical properties, properties, salinity, sea, sea_water_salinity, seawater, sigma, sigma_t, sound, station, technology, temperature, time, vctd, vctd.das, velocity, water\n" + +"optical, optical properties, practical, properties, salinity, sea, sea_water_practical_salinity, seawater, sigma, sigma_t, sound, station, technology, temperature, time, vctd, vctd.das, velocity, water\n" + " GCMD Science Keywords\n" + " [standard]\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " time, latitude, longitude, station, depth, temperature, salinity, fluorescence, avg_sound_velocity, sigma_t, acceleration, geopotential_anomaly\n" + " vCTD. DYNDNS Center for Integrated Marine Technology (CIMT) data from http://cimt.dyndns.org:8080/dods/drds/vCTD.das .\n" + " vCTD. DYNDNS CIMT data from http://cimt.dyndns.org:8080/dods/drds/vCTD.das .\n" + @@ -1029,8 +1025,9 @@ public static void testGenerateDatasetsXml() throws Throwable { " 37.0\n" + " 32.0\n" + " Salinity\n" + -" Sea Water Salinity\n" + -" sea_water_salinity\n" + +" Sea Water Practical Salinity\n" + +" sea_water_practical_salinity\n" + +" PSU\n" + " \n" + " \n" + " \n" + @@ -1748,7 +1745,7 @@ public static void testSubsetVariablesRange() throws Throwable { " Float64 Northernmost_Northing 48.969085693359375;[10]\n" + " String sourceUrl \"http://nwioos.coas.oregonstate.edu:8080/dods/drds/Coral%201980-2005\";[10]\n" + " Float64 Southernmost_Northing 32.570838928222656;[10]\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";[10]\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";[10]\n" + " String subsetVariables \"longitude, latitude, depth, time, institution, institution_id, species_code, taxa_scientific, taxonomic_order, order_abbreviation, taxonomic_family, family_abbreviation, taxonomic_genus\";[10]\n" + " String summary \"This data contains the locations of some observations of[10]\n" + "cold-water/deep-sea corals off the west coast of the United States.[10]\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromDatabase.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromDatabase.java index eccb18000..11203bb91 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromDatabase.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromDatabase.java @@ -1417,7 +1417,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " birthdate, category, center, data, erd, first, fisheries, height, height_cm, identifier, local, marine, national, nmfs, noaa, science, service, source, southwest, swfsc, time, weight, weight_kg\n" + " [standard]\n" + " (local database)\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " NOAA National Marine Fisheries Service (NMFS) Southwest Fisheries Science Center (SWFSC) ERD data from a local source.\n" + " NOAA NMFS SWFSC ERD data from a local source.\n" + " \n" + @@ -1606,7 +1606,7 @@ public static void testBasic() throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " String sourceUrl \"\\(source database\\)\";\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"This is Bob's test for reading from a database table.\";\n" + " String title \"mydatabase myschema mytable\";\n" + " \\}\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromEDDGrid.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromEDDGrid.java index 986c95272..f1163fbbb 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromEDDGrid.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromEDDGrid.java @@ -738,7 +738,7 @@ public static void testBasic() throws Throwable { " String source \"satellite observation: Aqua, GOES, POES, AMSR-E, MODIS, Imager, AVHRR\";\n" + " String sourceUrl \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/BA/ssta/5day\";\n" + " Float64 Southernmost_Northing -75.0;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"NOAA OceanWatch provides a blended sea surface temperature " + "\\(SST\\) products derived from both microwave and infrared sensors carried " + "on multiple platforms. The microwave instruments can measure ocean " + @@ -1234,7 +1234,7 @@ public static void testTableFromGriddap() throws Throwable { " String source \"satellite observation: Aqua, GOES, POES, AMSR-E, MODIS, Imager, AVHRR\";\n" + " String sourceUrl \"http://oceanwatch.pfeg.noaa.gov/thredds/dodsC/satellite/BA/ssta/5day\";\n" + " Float64 Southernmost_Northing -75.0;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String summary \"NOAA OceanWatch provides a blended sea surface temperature " + "\\(SST\\) products derived from both microwave and infrared sensors carried " + "on multiple platforms. The microwave instruments can measure ocean " + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromErddap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromErddap.java index 9ea335f43..0e80fee3e 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromErddap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromErddap.java @@ -794,9 +794,9 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { " Float64 colorBarMaximum 37.0;\n" + " Float64 colorBarMinimum 32.0;\n" + " String ioos_category \"Salinity\";\n" + -" String long_name \"Salinity from T0 and C0 Sensors\";\n" + +" String long_name \"Practical Salinity from T0 and C0 Sensors\";\n" + " Float32 missing_value -9999.0;\n" + -" String standard_name \"sea_water_salinity\";\n" + +" String standard_name \"sea_water_practical_salinity\";\n" + " String units \"PSU\";\n" + " }\n" + " sal11 {\n" + @@ -805,9 +805,9 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { " Float64 colorBarMaximum 37.0;\n" + " Float64 colorBarMinimum 32.0;\n" + " String ioos_category \"Salinity\";\n" + -" String long_name \"Salinity from T1 and C1 Sensors\";\n" + +" String long_name \"Practical Salinity from T1 and C1 Sensors\";\n" + " Float32 missing_value -9999.0;\n" + -" String standard_name \"sea_water_salinity\";\n" + +" String standard_name \"sea_water_practical_salinity\";\n" + " String units \"PSU\";\n" + " }\n" + " temperature0 {\n" + @@ -980,7 +980,7 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { "Oceans > Ocean Optics > Attenuation/Transmission,\n" + "Oceans > Ocean Temperature > Water Temperature,\n" + "Oceans > Salinity/Density > Salinity,\n" + -"active, after, ammonia, ammonium, attenuation, biosphere, bottle, cast, chemistry, chlorophyll, chlorophyll-a, color, concentration, concentration_of_chlorophyll_in_sea_water, cruise, data, density, dissolved, dissolved nutrients, dissolved o2, fluorescence, fraction, from, globec, identifier, mass, mole, mole_concentration_of_ammonium_in_sea_water, mole_concentration_of_nitrate_in_sea_water, mole_concentration_of_nitrite_in_sea_water, mole_concentration_of_phosphate_in_sea_water, mole_concentration_of_silicate_in_sea_water, moles, moles_of_nitrate_and_nitrite_per_unit_mass_in_sea_water, n02, nep, nh4, nitrate, nitrite, nitrogen, no3, number, nutrients, o2, ocean, ocean color, oceans, optical, optical properties, optics, oxygen, passing, per, phaeopigments, phosphate, photosynthetically, pigments, plus, po4, properties, radiation, rosette, salinity, screen, sea, sea_water_salinity, sea_water_temperature, seawater, sensor, sensors, ship, silicate, temperature, time, total, transmission, transmissivity, unit, vegetation, voltage, volume, volume_fraction_of_oxygen_in_sea_water, water\";\n" + +"active, after, ammonia, ammonium, attenuation, biosphere, bottle, cast, chemistry, chlorophyll, chlorophyll-a, color, concentration, concentration_of_chlorophyll_in_sea_water, cruise, data, density, dissolved, dissolved nutrients, dissolved o2, fluorescence, fraction, from, globec, identifier, mass, mole, mole_concentration_of_ammonium_in_sea_water, mole_concentration_of_nitrate_in_sea_water, mole_concentration_of_nitrite_in_sea_water, mole_concentration_of_phosphate_in_sea_water, mole_concentration_of_silicate_in_sea_water, moles, moles_of_nitrate_and_nitrite_per_unit_mass_in_sea_water, n02, nep, nh4, nitrate, nitrite, nitrogen, no3, number, nutrients, o2, ocean, ocean color, oceans, optical, optical properties, optics, oxygen, passing, per, phaeopigments, phosphate, photosynthetically, pigments, plus, po4, properties, radiation, rosette, salinity, screen, sea, sea_water_practical_salinity, sea_water_temperature, seawater, sensor, sensors, ship, silicate, temperature, time, total, transmission, transmissivity, unit, vegetation, voltage, volume, volume_fraction_of_oxygen_in_sea_water, water\";\n" + " String keywords_vocabulary \"GCMD Science Keywords\";\n" + " String license \"The data may be used and redistributed for free but is not intended\n" + "for legal use, since it may contain inaccuracies. Neither the data\n" + @@ -992,7 +992,7 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { " Float64 Northernmost_Northing 44.65;\n" + " String sourceUrl \"(local files; contact erd.data@noaa.gov)\";\n" + " Float64 Southernmost_Northing 41.9;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"cruise_id, ship, cast, longitude, latitude, time\";\n" + " String summary \"GLOBEC (GLOBal Ocean ECosystems Dynamics) NEP (Northeast Pacific)\n" + "Rosette Bottle Data from New Horizon Cruise (NH0207: 1-19 August 2002).\n" + @@ -1263,7 +1263,7 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { "Oceans > Ocean Optics > Attenuation/Transmission,\n" + "Oceans > Ocean Temperature > Water Temperature,\n" + "Oceans > Salinity/Density > Salinity,\n" + -"active, after, ammonia, ammonium, attenuation, biosphere, bottle, cast, chemistry, chlorophyll, chlorophyll-a, color, concentration, concentration_of_chlorophyll_in_sea_water, cruise, data, density, dissolved, dissolved nutrients, dissolved o2, fluorescence, fraction, from, globec, identifier, mass, mole, mole_concentration_of_ammonium_in_sea_water, mole_concentration_of_nitrate_in_sea_water, mole_concentration_of_nitrite_in_sea_water, mole_concentration_of_phosphate_in_sea_water, mole_concentration_of_silicate_in_sea_water, moles, moles_of_nitrate_and_nitrite_per_unit_mass_in_sea_water, n02, nep, nh4, nitrate, nitrite, nitrogen, no3, number, nutrients, o2, ocean, ocean color, oceans, optical, optical properties, optics, oxygen, passing, per, phaeopigments, phosphate, photosynthetically, pigments, plus, po4, properties, radiation, rosette, salinity, screen, sea, sea_water_salinity, sea_water_temperature, seawater, sensor, sensors, ship, silicate, temperature, time, total, transmission, transmissivity, unit, vegetation, voltage, volume, volume_fraction_of_oxygen_in_sea_water, water\";\n" + +"active, after, ammonia, ammonium, attenuation, biosphere, bottle, cast, chemistry, chlorophyll, chlorophyll-a, color, concentration, concentration_of_chlorophyll_in_sea_water, cruise, data, density, dissolved, dissolved nutrients, dissolved o2, fluorescence, fraction, from, globec, identifier, mass, mole, mole_concentration_of_ammonium_in_sea_water, mole_concentration_of_nitrate_in_sea_water, mole_concentration_of_nitrite_in_sea_water, mole_concentration_of_phosphate_in_sea_water, mole_concentration_of_silicate_in_sea_water, moles, moles_of_nitrate_and_nitrite_per_unit_mass_in_sea_water, n02, nep, nh4, nitrate, nitrite, nitrogen, no3, number, nutrients, o2, ocean, ocean color, oceans, optical, optical properties, optics, oxygen, passing, per, phaeopigments, phosphate, photosynthetically, pigments, plus, po4, properties, radiation, rosette, salinity, screen, sea, sea_water_practical_salinity, sea_water_temperature, seawater, sensor, sensors, ship, silicate, temperature, time, total, transmission, transmissivity, unit, vegetation, voltage, volume, volume_fraction_of_oxygen_in_sea_water, water\";\n" + " String keywords_vocabulary \"GCMD Science Keywords\";\n" + " String license \"The data may be used and redistributed for free but is not intended\n" + "for legal use, since it may contain inaccuracies. Neither the data\n" + @@ -1275,7 +1275,7 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { " Float64 Northernmost_Northing 44.65;\n" + " String sourceUrl \"(local files; contact erd.data@noaa.gov)\";\n" + " Float64 Southernmost_Northing 41.9;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"cruise_id, ship, cast, longitude, latitude, time\";\n" + " String summary \"GLOBEC (GLOBal Ocean ECosystems Dynamics) NEP (Northeast Pacific)\n" + "Rosette Bottle Data from New Horizon Cruise (NH0207: 1-19 August 2002).\n" + @@ -1461,7 +1461,7 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { "Oceans > Ocean Optics > Attenuation/Transmission,\n" + "Oceans > Ocean Temperature > Water Temperature,\n" + "Oceans > Salinity/Density > Salinity,\n" + -"active, after, ammonia, ammonium, attenuation, biosphere, bottle, cast, chemistry, chlorophyll, chlorophyll-a, color, concentration, concentration_of_chlorophyll_in_sea_water, cruise, data, density, dissolved, dissolved nutrients, dissolved o2, fluorescence, fraction, from, globec, identifier, mass, mole, mole_concentration_of_ammonium_in_sea_water, mole_concentration_of_nitrate_in_sea_water, mole_concentration_of_nitrite_in_sea_water, mole_concentration_of_phosphate_in_sea_water, mole_concentration_of_silicate_in_sea_water, moles, moles_of_nitrate_and_nitrite_per_unit_mass_in_sea_water, n02, nep, nh4, nitrate, nitrite, nitrogen, no3, number, nutrients, o2, ocean, ocean color, oceans, optical, optical properties, optics, oxygen, passing, per, phaeopigments, phosphate, photosynthetically, pigments, plus, po4, properties, radiation, rosette, salinity, screen, sea, sea_water_salinity, sea_water_temperature, seawater, sensor, sensors, ship, silicate, temperature, time, total, transmission, transmissivity, unit, vegetation, voltage, volume, volume_fraction_of_oxygen_in_sea_water, water\";\n" + +"active, after, ammonia, ammonium, attenuation, biosphere, bottle, cast, chemistry, chlorophyll, chlorophyll-a, color, concentration, concentration_of_chlorophyll_in_sea_water, cruise, data, density, dissolved, dissolved nutrients, dissolved o2, fluorescence, fraction, from, globec, identifier, mass, mole, mole_concentration_of_ammonium_in_sea_water, mole_concentration_of_nitrate_in_sea_water, mole_concentration_of_nitrite_in_sea_water, mole_concentration_of_phosphate_in_sea_water, mole_concentration_of_silicate_in_sea_water, moles, moles_of_nitrate_and_nitrite_per_unit_mass_in_sea_water, n02, nep, nh4, nitrate, nitrite, nitrogen, no3, number, nutrients, o2, ocean, ocean color, oceans, optical, optical properties, optics, oxygen, passing, per, phaeopigments, phosphate, photosynthetically, pigments, plus, po4, properties, radiation, rosette, salinity, screen, sea, sea_water_practical_salinity, sea_water_temperature, seawater, sensor, sensors, ship, silicate, temperature, time, total, transmission, transmissivity, unit, vegetation, voltage, volume, volume_fraction_of_oxygen_in_sea_water, water\";\n" + " :keywords_vocabulary = \"GCMD Science Keywords\";\n" + " :license = \"The data may be used and redistributed for free but is not intended\n" + "for legal use, since it may contain inaccuracies. Neither the data\n" + @@ -1471,7 +1471,7 @@ public static void testBasic(boolean testLocalErddapToo) throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " :sourceUrl = \"(local files; contact erd.data@noaa.gov)\";\n" + -" :standard_name_vocabulary = \"CF Standard Name Table v27\";\n" + +" :standard_name_vocabulary = \"CF Standard Name Table v29\";\n" + " :subsetVariables = \"cruise_id, ship, cast, longitude, latitude, time\";\n" + " :summary = \"GLOBEC (GLOBal Ocean ECosystems Dynamics) NEP (Northeast Pacific)\n" + "Rosette Bottle Data from New Horizon Cruise (NH0207: 1-19 August 2002).\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFileNames.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFileNames.java index eacb7fd1c..c898af46e 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFileNames.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFileNames.java @@ -54,8 +54,9 @@ public class EDDTableFromFileNames extends EDDTable{ protected boolean recursive; protected String extractRegex[]; protected byte extractGroup[]; + protected boolean useCachedDNLSInfo = false; //DNLS info with directoriesToo=false - //variable names + //standard variable names public final static String URL = FileVisitorDNLS.URL; //"url"; public final static String DIRECTORY = FileVisitorDNLS.DIRECTORY; //"directory"; public final static String NAME = FileVisitorDNLS.NAME; //"name"; @@ -63,6 +64,7 @@ public class EDDTableFromFileNames extends EDDTable{ public final static String SIZE = FileVisitorDNLS.SIZE; //"size"; + /** * This constructs an EDDTableFromFileNames based on the information in an .xml file. * @@ -271,9 +273,69 @@ public EDDTableFromFileNames(String tDatasetID, String tAccessibleTo, String2.replaceAll(tLicense, "[standard]", EDStatic.standardLicense)); combinedGlobalAttributes.removeValue("null"); + //useCachedInfo? + Table tCachedDNLSTable = null; + useCachedDNLSInfo = File2.isRemote(fileDir); + if (useCachedDNLSInfo) { + String qrName = quickRestartFullFileName(); + + if (EDStatic.quickRestart && + EDStatic.initialLoadDatasets() && + File2.isFile(qrName)) { + + //try to do quickRestart + //set creationTimeMillis to time of previous creation, so next time + //to be reloaded will be same as if ERDDAP hadn't been restarted. + long tCreationTime = File2.getLastModified(qrName); //0 if trouble + if (verbose) + String2.log(" quickRestart " + tDatasetID + " previous=" + + Calendar2.millisToIsoZuluString(tCreationTime) + "Z"); + + //Ensure quickRestart information is recent. + //If too old: abandon construction, delete quickRestart file, flag dataset reloadASAP + ensureQuickRestartInfoIsRecent(tDatasetID, + tReloadEveryNMinutes == Integer.MAX_VALUE? DEFAULT_RELOAD_EVERY_N_MINUTES : + tReloadEveryNMinutes, + tCreationTime, qrName); + + //use cached info + tCachedDNLSTable = getCachedDNLSTable(); + } + + if (tCachedDNLSTable == null) { + + //get the info to be cached + tCachedDNLSTable = FileVisitorDNLS.oneStep(fileDir, fileNameRegex, + recursive, false); //tDirectoriesToo + tCachedDNLSTable.setColumn(2, new DoubleArray(tCachedDNLSTable.getColumn(2))); //long -> double + tCachedDNLSTable.setColumn(3, new DoubleArray(tCachedDNLSTable.getColumn(3))); //long -> double + if (tCachedDNLSTable.nRows() == 0) + throw new SimpleException(MustBe.THERE_IS_NO_DATA + + " (0 matching files)"); + + //store it + File2.makeDirectory(datasetDir()); + tCachedDNLSTable.saveAsFlatNc(qrName, "row"); //throws exceptions + + //prepare for below + tCachedDNLSTable.setColumn(2, new LongArray(tCachedDNLSTable.getColumn(2))); //double -> long + tCachedDNLSTable.setColumn(3, new LongArray(tCachedDNLSTable.getColumn(3))); //double -> long + } + } + //get a source table (this also ensures there are valid files in fileDir) - Table sourceBasicTable = getBasicTable(fileDir, fileNameRegex, recursive, - null, datasetID); //loggedInAs - irrelevant since just getting this for metadata, not data + Table sourceBasicTable; + if (useCachedDNLSInfo) { + //use tCachedDNLSTable since I just got it (above) + //and perhaps time consuming to get it + sourceBasicTable = FileVisitorDNLS.oneStepDoubleWithUrlsNotDirs( + FileVisitorDNLS.oneStepDouble(tCachedDNLSTable), + fileDir, + EDStatic.erddapUrl(null) + "/files/" + datasetID + "/"); //loggedInAs=null doesn't matter here + } else { + sourceBasicTable = getBasicTable(fileDir, fileNameRegex, recursive, + null, datasetID); //loggedInAs - irrelevant since just getting this for metadata, not data + } //create dataVariables[] int ndv = tDataVariables.length; @@ -375,7 +437,7 @@ public EDDTableFromFileNames(String tDatasetID, String tAccessibleTo, edv.setDestinationMinMax(twawm.columnMinValue(dv), twawm.columnMaxValue(dv)); } } - twawm.releaseResources(); + twawm.releaseResources(); //accessibleViaFiles if (EDStatic.filesActive) { @@ -400,6 +462,37 @@ public EDDTableFromFileNames(String tDatasetID, String tAccessibleTo, public String fileNameRegex() {return fileNameRegex;} public boolean recursive() {return recursive;} + /** + * This gets the cached DNLS file table (last_mod and size are longs). + * Only call this if useCachedDNLSInfo==true. + * + * @throws Exception + */ + public Table getCachedDNLSTable() throws Exception { + Table table = new Table(); + table.readFlatNc(quickRestartFullFileName(), null, 0); + table.setColumn(2, new LongArray(table.getColumn(2))); //double -> long + table.setColumn(3, new LongArray(table.getColumn(3))); //double -> long + return table; + } + + /** + * This returns a fileTable (formatted like + * FileVisitorDNLS.oneStep(tDirectoriesToo=false, last_mod is LongArray, + * and size is LongArray of epochMillis) + * with valid files (or null if unavailable or any trouble). + * This is a copy of any internal data, so client can modify the contents. + */ + public Table accessibleViaFilesFileTable() { + try { + return useCachedDNLSInfo? + getCachedDNLSTable() : + FileVisitorDNLS.oneStep(fileDir, fileNameRegex, recursive, false); //dirToo=false + } catch (Exception e) { + String2.log(MustBe.throwableToString(e)); + return null; + } + } /** * Get low level data: URL, NAME, LASTMODIFIED (as double epoch seconds), SIZE (as double) @@ -408,7 +501,7 @@ public EDDTableFromFileNames(String tDatasetID, String tAccessibleTo, */ public static Table getBasicTable(String fileDir, String fileNameRegex, boolean recursive, String loggedInAs, String datasetID) throws Exception { - Table table = FileVisitorDNLS.oneStepAccessibleViaFiles( + Table table = FileVisitorDNLS.oneStepDoubleWithUrlsNotDirs( fileDir, fileNameRegex, recursive, EDStatic.erddapUrl(loggedInAs) + "/files/" + datasetID + "/"); int nRows = table.nRows(); @@ -442,8 +535,16 @@ public void getDataForDapQuery(String loggedInAs, String requestUrl, constraintVariables, constraintOps, constraintValues); //timeStamp constraints other than regex are epochSeconds //get low level data: URL, NAME, LASTMODIFIED (as double epoch seconds), SIZE (as double) - Table table = getBasicTable(fileDir, fileNameRegex, recursive, - loggedInAs, datasetID); + Table table; + if (useCachedDNLSInfo) { + table = FileVisitorDNLS.oneStepDoubleWithUrlsNotDirs( + FileVisitorDNLS.oneStepDouble(getCachedDNLSTable()), + fileDir, + EDStatic.erddapUrl(loggedInAs) + "/files/" + datasetID + "/"); + } else { + table = getBasicTable(fileDir, fileNameRegex, recursive, + loggedInAs, datasetID); //loggedInAs - irrelevant since just getting this for metadata, not data + } int nRows = table.nRows(); //create other results variables as needed @@ -507,14 +608,18 @@ public static String generateDatasetsXml( String2.log("EDDTableFromFileNames.generateDatasetsXml" + "\n tFileDir=" + tFileDir); tFileDir = File2.addSlash(String2.replaceAll(tFileDir, '\\', '/')); - String tDatasetID = suggestDatasetID("EDDTableFromFileNames(" + tFileDir + "," + - tFileNameRegex + ")"); + String tDatasetID = suggestDatasetID( + //if awsS3, important that it start with tFileDir + File2.addSlash(tFileDir) + tFileNameRegex + + //distinguish from e.g., EDDGridFromNcFiles for same files + "(EDDTableFromFileNames)"); + boolean remoteFiles = File2.isRemote(tFileDir); if (tReloadEveryNMinutes < suggestReloadEveryNMinutesMin || - tReloadEveryNMinutes > suggestReloadEveryNMinutesMax) - tReloadEveryNMinutes = DEFAULT_RELOAD_EVERY_N_MINUTES; + tReloadEveryNMinutes > suggestReloadEveryNMinutesMax) + tReloadEveryNMinutes = remoteFiles? 120 : DEFAULT_RELOAD_EVERY_N_MINUTES; //make the sourceTable and addTable - Table sourceTable = FileVisitorDNLS.oneStepAccessibleViaFiles( + Table sourceTable = FileVisitorDNLS.oneStepDoubleWithUrlsNotDirs( tFileDir, tFileNameRegex, tRecursive, EDStatic.erddapUrl(null) + "/files/" + tDatasetID + "/"); @@ -527,7 +632,7 @@ public static String generateDatasetsXml( .add("history", "null") .add("infoUrl", String2.isSomething(tInfoUrl )? tInfoUrl : "???") .add("institution", String2.isSomething(tInstitution)? tInstitution : "???") - .add("sourceUrl", "(local files)") + .add("sourceUrl", "(" + (remoteFiles? "remote" : "local") + " files)") .add("summary", String2.isSomething(tSummary )? tSummary : "???") .add("title", String2.isSomething(tTitle )? tTitle : "???"); int nCols = sourceTable.nColumns(); @@ -602,6 +707,7 @@ public static String generateDatasetsXml( * testGenerateDatasetsXml */ public static void testGenerateDatasetsXml() throws Throwable { + String2.log("\n*** EDDTableFromFileNames.testGenerateDatasetsXml()"); testVerboseOn(); String tDir = EDStatic.unitTestDataDir + "fileNames"; @@ -612,17 +718,16 @@ public static void testGenerateDatasetsXml() throws Throwable { String tSummary = "Images from JPL MUR SST Daily."; String tTitle = "JPL MUR SST Images"; //datasetID changes with different unitTestDataDir - String tDatasetID = suggestDatasetID("EDDTableFromFileNames(" + - tDir + "/," + tRegex + ")"); + String tDatasetID = "fileNames_e21d_ef79_13da"; String expected = directionsForGenerateDatasetsXml() + "-->\n" + "\n" + "\n" + -" " + EDStatic.unitTestDataDir + "fileNames/\n" + +" " + tDir + "/\n" + " .*\\.png\n" + " true\n" + -" 1440\n" + +" 10080\n" + " \n" + " \n" + @@ -631,12 +736,157 @@ public static void testGenerateDatasetsXml() throws Throwable { " null\n" + " null\n" + " null\n" + -" http://mur.jpl.nasa.gov/\n" + -" NASA JPL\n" + +" " + tInfoUrl + "\n" + +" " + tInstitution + "\n" + " data, file, high, identifier, images, jet, jpl, laboratory, lastModified, modified, multi, multi-scale, mur, name, nasa, propulsion, resolution, scale, sea, size, sst, surface, temperature, time, ultra, ultra-high\n" + " (local files)\n" + -" Images from JPL MUR SST Daily.\n" + -" JPL MUR SST Images\n" + +" " + tSummary + "\n" + +" " + tTitle + "\n" + +" \n" + +" \n" + +" url\n" + +" url\n" + +" String\n" + +" \n" + +" \n" + +" \n" + +" \n" + +" \n" + +" name\n" + +" name\n" + +" String\n" + +" \n" + +" \n" + +" \n" + +" \n" + +" \n" + +" lastModified\n" + +" lastModified\n" + +" double\n" + +" \n" + +" \n" + +" \n" + +" \n" + +" \n" + +" size\n" + +" size\n" + +" double\n" + +" \n" + +" \n" + +" \n" + +" \n" + +" \n" + +"\n\n\n"; + String results = generateDatasetsXml(tDir, tRegex, tRecursive, -1, + tInfoUrl, tInstitution, tSummary, tTitle, null) + "\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + //GenerateDatasetsXml + String gdxResults = (new GenerateDatasetsXml()).doIt(new String[]{"-verbose", + "EDDTableFromFileNames", + tDir, tRegex, "" + tRecursive, "-1", + tInfoUrl, tInstitution, tSummary, tTitle}, + false); //doIt loop? + Test.ensureEqual(gdxResults, results, + "Unexpected results from GenerateDatasetsXml.doIt."); + + //ensure it is ready-to-use by making a dataset from it + String2.log("results=\n" + results); + EDD edd = oneFromXmlFragment(results); + Test.ensureEqual(edd.datasetID(), tDatasetID, ""); + Test.ensureEqual(edd.title(), tTitle, ""); + Test.ensureEqual(String2.toCSSVString(edd.dataVariableDestinationNames()), + "url, name, lastModified, size", + ""); + String2.log("\n*** EDDTableFromFileNames.testGenerateDatasetsXml() finished successfully."); + } + + /** + * testGenerateDatasetsXmlAwsS3 + * Your S3 credentials must be in + *
    ~/.aws/credentials on Linux, OS X, or Unix + *
    C:\Users\USERNAME\.aws\credentials on Windows + * See http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html . + */ + public static void testGenerateDatasetsXmlAwsS3() throws Throwable { + String2.log("\n*** EDDTableFromFileNames.testGenerateDatasetsXmlAwsS3()"); + try { + + testVerboseOn(); + + String tDir = "http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS"; + //tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_CESM1-CAM5_200601-201012.nc + String tRegex = ".*_CESM1-CAM5_.*\\.nc"; + boolean tRecursive = true; + String tInfoUrl = "https://nex.nasa.gov/nex/"; + String tInstitution = "NASA Earth Exchange"; + String tSummary = "My great summary"; + String tTitle = "My Great Title"; + String tDatasetID = "s3nasanex_803b_6c09_f004"; +String expected = +directionsForGenerateDatasetsXml() + +"-->\n" + +"\n" + +"\n" + +" " + tDir + "/\n" + +" " + tRegex + "\n" + +" true\n" + +" 120\n" + +" \n" + +" \n" + +" Other\n" + +" null\n" + +" null\n" + +" null\n" + +" null\n" + +" " + tInfoUrl + "\n" + +" " + tInstitution + "\n" + +" data, earth, exchange, file, great, identifier, lastModified, modified, name, nasa, size, time, title\n" + +" (remote files)\n" + +" " + tSummary + "\n" + +" " + tTitle + "\n" + " \n" + " \n" + " url\n" + @@ -713,14 +963,14 @@ public static void testGenerateDatasetsXml() throws Throwable { " \n" + " -->\n" + "\n\n\n"; - String results = generateDatasetsXml(tDir, tRegex, tRecursive, 1440, + String results = generateDatasetsXml(tDir, tRegex, tRecursive, -1, tInfoUrl, tInstitution, tSummary, tTitle, null) + "\n"; Test.ensureEqual(results, expected, "results=\n" + results); //GenerateDatasetsXml String gdxResults = (new GenerateDatasetsXml()).doIt(new String[]{"-verbose", "EDDTableFromFileNames", - tDir, tRegex, "" + tRecursive, "1440", + tDir, tRegex, "" + tRecursive, "-1", tInfoUrl, tInstitution, tSummary, tTitle}, false); //doIt loop? Test.ensureEqual(gdxResults, results, @@ -734,13 +984,20 @@ public static void testGenerateDatasetsXml() throws Throwable { Test.ensureEqual(String2.toCSSVString(edd.dataVariableDestinationNames()), "url, name, lastModified, size", ""); + + String2.log("\n*** EDDTableFromFileNames.testGenerateDatasetsXmlAwsS3() finished successfully."); + + } catch (Throwable t) { + String2.pressEnterToContinue(MustBe.throwableToString(t) + + "\nUnexpected error. (Did you create your AWS S3 credentials file?)"); + } } /** - * Do basic tests of this class. + * Do tests of local file system. */ - public static void basicTest() throws Throwable { - String2.log("\n*** EDDTableFromFileNames\n"); + public static void testLocal() throws Throwable { + String2.log("\n*** EDDTableFromFileNames.testLocal\n"); testVerboseOn(); String dir = EDStatic.fullTestCacheDirectory; String results, expected, query, tName; @@ -862,6 +1119,149 @@ public static void basicTest() throws Throwable { "jplMURSST20150104090000.png,4,46586.0\n"; Test.ensureEqual(results, expected, "results=\n" + results); + String2.log("\n EDDTableFromFileNames.testLocal finished successfully"); + } + + /** + * Do tests of an Amazon AWS S3 file system. + * Your S3 credentials must be in + *
    ~/.aws/credentials on Linux, OS X, or Unix + *
    C:\Users\USERNAME\.aws\credentials on Windows + * See http://docs.aws.amazon.com/AWSSdkDocsJava/latest/DeveloperGuide/java-dg-setup.html . + */ + public static void testAwsS3() throws Throwable { + try { + String2.log("\n*** EDDTableFromFileNames.testAwsS3\n"); + testVerboseOn(); + String dir = EDStatic.fullTestCacheDirectory; + String results, expected, query, tName; + + EDDTable tedd = (EDDTable)oneFromDatasetXml("testFileNamesAwsS3"); + + //.dds + tName = tedd.makeNewFileForDapQuery(null, null, "", dir, + tedd.className() + "_all", ".dds"); + results = new String((new ByteArray(dir + tName)).toArray()); + expected = +"Dataset {\n" + +" Sequence {\n" + +" Float32 five;\n" + +" String url;\n" + +" String name;\n" + +" Float64 startMonth;\n" + +" Float64 endMonth;\n" + +" Float64 lastModified;\n" + +" Float64 size;\n" + +" } s;\n" + +"} s;\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + //.das + tName = tedd.makeNewFileForDapQuery(null, null, "", dir, + tedd.className() + "_all", ".das"); + results = new String((new ByteArray(dir + tName)).toArray()); + expected = +"Attributes \\{\n" + +" s \\{\n" + +" five \\{\n" + +" String ioos_category \"Other\";\n" + +" String long_name \"Five\";\n" + +" String units \"m\";\n" + +" \\}\n" + +" url \\{\n" + +" String ioos_category \"Identifier\";\n" + +" String long_name \"URL\";\n" + +" \\}\n" + +" name \\{\n" + +" String ioos_category \"Identifier\";\n" + +" String long_name \"File Name\";\n" + +" \\}\n" + +" startMonth \\{\n" + +" String ioos_category \"Time\";\n" + +" String long_name \"Start Month\";\n" + +" String time_origin \"01-JAN-1970 00:00:00\";\n" + +" String units \"seconds since 1970-01-01T00:00:00Z\";\n" + +" \\}\n" + +" endMonth \\{\n" + +" String ioos_category \"Time\";\n" + +" String long_name \"End Month\";\n" + +" String time_origin \"01-JAN-1970 00:00:00\";\n" + +" String units \"seconds since 1970-01-01T00:00:00Z\";\n" + +" \\}\n" + +" lastModified \\{\n" + +" String ioos_category \"Time\";\n" + +" String long_name \"Last Modified\";\n" + +" String time_origin \"01-JAN-1970 00:00:00\";\n" + +" String units \"seconds since 1970-01-01T00:00:00Z\";\n" + +" \\}\n" + +" size \\{\n" + +" String ioos_category \"Other\";\n" + +" String long_name \"Size\";\n" + +" String units \"bytes\";\n" + +" \\}\n" + +" \\}\n" + +" NC_GLOBAL \\{\n" + +" String cdm_data_type \"Other\";\n" + +" String creator_name \"NASA Earth Exchange\";\n" + +" String creator_url \"https://nex.nasa.gov/nex/\";\n" + +" String history \".{19}Z \\(remote files\\)\n" + +".{19}Z http://127.0.0.1:8080/cwexperimental/tabledap/testFileNamesAwsS3.das\";\n" + +" String infoUrl \"https://nex.nasa.gov/nex/\";\n" + +" String institution \"NASA Earth Exchange\";\n" + +" String keywords \"data, earth, exchange, file, great, identifier, lastModified, modified, name, nasa, size, time, title\";\n" + +" String sourceUrl \"\\(remote files\\)\";\n" + +" String summary \"File Names from http://nasanex.s3.amazonaws.com/NEX-DCP30/BCSD/rcp26/mon/atmos/tasmin/r1i1p1/v1.0/CONUS/\";\n" + +" String title \"File Names from Amazon AWS S3 NASA NEX tasmin Files\";\n" + +" \\}\n" + +"\\}\n"; + Test.ensureLinesMatch(results, expected, "results=\n" + results); + + //get all as .csv + tName = tedd.makeNewFileForDapQuery(null, null, "", dir, + tedd.className() + "_all", ".csv"); + results = new String((new ByteArray(dir + tName)).toArray()); + expected = +"five,url,name,startMonth,endMonth,lastModified,size\n" + +"m,,,UTC,UTC,UTC,bytes\n" + +"5.0,http://127.0.0.1:8080/cwexperimental/files/testFileNamesAwsS3/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_CESM1-CAM5_200601-201012.nc,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_CESM1-CAM5_200601-201012.nc,2006-01-01T00:00:00Z,2010-12-01T00:00:00Z,2013-10-25T20:46:53Z,1.372730447E9\n" + +"5.0,http://127.0.0.1:8080/cwexperimental/files/testFileNamesAwsS3/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_CESM1-CAM5_201101-201512.nc,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_CESM1-CAM5_201101-201512.nc,2011-01-01T00:00:00Z,2015-12-01T00:00:00Z,2013-10-25T20:47:18Z,1.373728987E9\n" + +"5.0,http://127.0.0.1:8080/cwexperimental/files/testFileNamesAwsS3/tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_CESM1-CAM5_201601-202012.nc,tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_CESM1-CAM5_201601-202012.nc,2016-01-01T00:00:00Z,2020-12-01T00:00:00Z,2013-10-25T20:51:23Z,1.373747344E9\n"; + Test.ensureEqual(results.substring(0, expected.length()), expected, "results=\n" + results); + + //test that min and max are being set by the constructor + EDV edv = tedd.findVariableByDestinationName("startMonth"); + Test.ensureEqual(edv.destinationMinString(), "2006-01-01T00:00:00Z", "min"); + Test.ensureEqual(edv.destinationMaxString(), "2096-01-01T00:00:00Z", "max"); + + edv = tedd.findVariableByDestinationName("endMonth"); + Test.ensureEqual(edv.destinationMinString(), "2010-12-01T00:00:00Z", "min"); + Test.ensureEqual(edv.destinationMaxString(), "2099-12-01T00:00:00Z", "max"); + + edv = tedd.findVariableByDestinationName("lastModified"); + Test.ensureEqual(edv.destinationMinString(), "2013-10-25T20:45:24Z", "min"); + Test.ensureEqual(edv.destinationMaxString(), "2013-10-25T20:54:20Z", "max"); + + edv = tedd.findVariableByDestinationName("size"); + Test.ensureEqual(""+edv.destinationMin(), "1.098815646E9", "min"); //exact test + Test.ensureEqual(""+edv.destinationMax(), "1.373941204E9", "max"); + + //a constraint on an extracted variable, and fewer results variables + tName = tedd.makeNewFileForDapQuery(null, null, "name,startMonth,size&size=1098815646", dir, + tedd.className() + "_subset", ".csv"); + results = new String((new ByteArray(dir + tName)).toArray()); + expected = +"name,startMonth,size\n" + +",UTC,bytes\n" + +"tasmin_amon_BCSD_rcp26_r1i1p1_CONUS_CESM1-CAM5_209601-209912.nc,2096-01-01T00:00:00Z,1.098815646E9\n"; + Test.ensureEqual(results, expected, "results=\n" + results); + + String2.log("\n EDDTableFromFileNames.testAwsS3 finished successfully"); + + } catch (Throwable t) { + String2.pressEnterToContinue(MustBe.throwableToString(t) + + "\nUnexpected error. (Did you create your AWS S3 credentials file?)"); + } + } @@ -876,7 +1276,9 @@ public static void test() throws Throwable { /* */ //always done testGenerateDatasetsXml(); - basicTest(); + testGenerateDatasetsXmlAwsS3(); + testLocal(); + testAwsS3(); } } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java index 659e595d9..b9e85827d 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromFiles.java @@ -50,8 +50,8 @@ * (from the local files, unlike remote data) and all variable's min and max info * can be gathered (for each file) * and cached (facilitating handling constraints in data requests). - *
    And file data can be cached and reused because each file has a lastModified - * time which can be used to detect if file is unchanged. + *
    And file data can be cached and reused because each file has a lastModified + * time and size which can be used to detect if file is unchanged. * * @author Bob Simons (bob.simons@noaa.gov) 2008-04-12 */ @@ -59,6 +59,10 @@ public abstract class EDDTableFromFiles extends EDDTable{ public final static String MF_FIRST = "first", MF_LAST = "last"; public static int suggestedUpdateEveryNMillis = 10000; + public static int suggestUpdateEveryNMillis(String tFileDir) { + return File2.isRemote(tFileDir)? 0 : suggestedUpdateEveryNMillis; + } + //set by constructor protected String fileDir; @@ -97,13 +101,14 @@ public abstract class EDDTableFromFiles extends EDDTable{ protected double addAttMissingValueNEC[]; /** Columns in the File Table */ - protected final static int dv0 = 4; protected final static int FT_DIR_INDEX_COL=0, //useful that it is #0 (tFileTable uses same positions) FT_FILE_LIST_COL=1, //useful that it is #1 FT_LAST_MOD_COL=2, - FT_SORTED_SPACING_COL=3; - //then 3 cols for each dataVariable: sourceName + _min_|_max_|_hasNaN + FT_SIZE_COL=3, + FT_SORTED_SPACING_COL=4; + //then 3 cols for each dataVariable: sourceName + _min_|_max_|_hasNaN starting at dv0 + protected final static int dv0 = 5; int fileTableSortColumns[]; //null if not active boolean fileTableSortAscending[]; //size matches fileTableSortcolumns, all true @@ -123,7 +128,7 @@ public abstract class EDDTableFromFiles extends EDDTable{ //dirTable and fileTable inMemory (default=false) protected boolean fileTableInMemory = false; - protected Table dirTable; + protected Table dirTable; //one column with dir names protected Table fileTable; /** @@ -832,10 +837,12 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, if (fileTable.findColumnNumber("dirIndex") != FT_DIR_INDEX_COL) ok = false; else if (fileTable.findColumnNumber("fileName") != FT_FILE_LIST_COL) ok = false; else if (fileTable.findColumnNumber("lastMod") != FT_LAST_MOD_COL) ok = false; + else if (fileTable.findColumnNumber("size") != FT_SIZE_COL) ok = false; else if (fileTable.findColumnNumber("sortedSpacing") != FT_SORTED_SPACING_COL) ok = false; else if (!(fileTable.getColumn(FT_DIR_INDEX_COL) instanceof ShortArray)) ok = false; else if (!(fileTable.getColumn(FT_FILE_LIST_COL) instanceof StringArray)) ok = false; else if (!(fileTable.getColumn(FT_LAST_MOD_COL) instanceof DoubleArray)) ok = false; + else if (!(fileTable.getColumn(FT_SIZE_COL) instanceof DoubleArray)) ok = false; else if (!(fileTable.getColumn(FT_SORTED_SPACING_COL) instanceof DoubleArray)) ok = false; else for (int dv = 0; dv < ndv; dv++) { String sdt = sourceDataTypes[dv]; @@ -874,7 +881,8 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, fileTable.addColumn("dirIndex", new ShortArray()); //col 0=FT_DIR_INDEX_COL fileTable.addColumn("fileName", new StringArray()); //col 1=FT_FILE_NAME_COL fileTable.addColumn("lastMod", new DoubleArray()); //col 2=FT_LAST_MOD_COL - fileTable.addColumn("sortedSpacing", new DoubleArray()); //col 3=FT_SORTED_SPACING_COL + fileTable.addColumn("size", new DoubleArray()); //col 3=FT_SIZE_COL + fileTable.addColumn("sortedSpacing", new DoubleArray()); //col 4=FT_SORTED_SPACING_COL for (int dv = 0; dv < ndv; dv++) { String sdt = sourceDataTypes[dv]; //booleans handled correctly below fileTable.addColumn(safeSourceDataNames.get(dv) + "_min_", @@ -905,17 +913,33 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, ShortArray ftDirIndex = (ShortArray)fileTable.getColumn( FT_DIR_INDEX_COL); //0 StringArray ftFileList = (StringArray)fileTable.getColumn(FT_FILE_LIST_COL); //1 DoubleArray ftLastMod = (DoubleArray)fileTable.getColumn(FT_LAST_MOD_COL); //2 - DoubleArray ftSortedSpacing = (DoubleArray)fileTable.getColumn(FT_SORTED_SPACING_COL); //3 + DoubleArray ftSize = (DoubleArray)fileTable.getColumn(FT_SIZE_COL); //3 + DoubleArray ftSortedSpacing = (DoubleArray)fileTable.getColumn(FT_SORTED_SPACING_COL); //4 //get tFileList of available data files long elapsedTime = System.currentTimeMillis(); //was tFileNames with dir+name Table tFileTable = getFileInfo(fileDir, fileNameRegex, recursive); - if (updateEveryNMillis > 0) - watchDirectory = WatchDirectory.watchDirectoryAll(fileDir, recursive) ; + if (updateEveryNMillis > 0) { + try { + watchDirectory = WatchDirectory.watchDirectoryAll(fileDir, recursive); + } catch (Throwable t) { + String subject = String2.ERROR + " in " + datasetID + " constructor (inotify)"; + String msg = MustBe.throwableToString(t); + if (msg.indexOf("inotify instances") >= 0) + msg += + "This may be the problem that is solvable by calling (as root):\n" + + " echo 20000 > /proc/sys/fs/inotify/max_user_watches\n" + + " echo 500 > /proc/sys/fs/inotify/max_user_instances\n" + + "Or, use higher numbers if the problem persists.\n" + + "The default for watches is 8192. The default for instances is 128."; + EDStatic.email(EDStatic.adminEmail, subject, msg); + } + } StringArray tFileDirPA = (StringArray)(tFileTable.getColumn(FileVisitorDNLS.DIRECTORY)); StringArray tFileNamePA = (StringArray)(tFileTable.getColumn(FileVisitorDNLS.NAME)); LongArray tFileLastModPA = (LongArray) (tFileTable.getColumn(FileVisitorDNLS.LASTMODIFIED)); + LongArray tFileSizePA = (LongArray) (tFileTable.getColumn(FileVisitorDNLS.SIZE)); tFileTable.removeColumn(FileVisitorDNLS.SIZE); int ntft = tFileNamePA.size(); String msg = ntft + " files found in " + fileDir + @@ -1041,7 +1065,10 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, String dir = dirList.get(ftDirIndex.get(f)); String name = ftFileList.get(f); long lastMod = getLastModified(dir, name); - if (lastMod == 0 || ftLastMod.get(f) != lastMod) //unavailable or changed + if (lastMod == 0 || ftLastMod.get(f) != lastMod) //0=trouble: unavailable or changed + continue; + long size = getSize(dir, name); + if (size < 0 || ftSize.get(f) != size) //-1=touble: unavailable or changed continue; try { @@ -1100,8 +1127,7 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, //update fileTable by processing tFileNamePA int fileListPo = 0; //next one to look at int tFileListPo = 0; //next one to look at - long lastModCumTime = 0; - int nReadFile = 0, nNoLastMod = 0; + int nReadFile = 0, nNoLastMod = 0, nNoSize = 0; long readFileCumTime = 0; long removeCumTime = 0; int nUnchanged = 0, nRemoved = 0, nDifferentModTime = 0, nNew = 0; @@ -1112,6 +1138,7 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, int dirI = fileListPo < ftFileList.size()? ftDirIndex.get(fileListPo) : Integer.MAX_VALUE; String fileS = fileListPo < ftFileList.size()? ftFileList.get(fileListPo) : "\uFFFF"; double lastMod = fileListPo < ftFileList.size()? ftLastMod.get(fileListPo) : Double.MAX_VALUE; + double size = fileListPo < ftFileList.size()? ftSize.get(fileListPo) : Double.MAX_VALUE; boolean logThis = (reallyVerbose && tFileListPo <= 100) || ((reallyVerbose || verbose) && ((tFileListPo <= 1000 && tFileListPo % 100 == 0) || @@ -1120,9 +1147,7 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, String2.log("EDDTableFromFiles file #" + tFileListPo + "=" + dirList.get(tDirI) + tFileS); //is tLastMod available for tFile? - long lmcTime = System.currentTimeMillis(); long tLastMod = tFileLastModPA.get(tFileListPo); - lastModCumTime += System.currentTimeMillis() - lmcTime; if (tLastMod == 0) { //0=trouble nNoLastMod++; String2.log(tFileListPo + " reject because unable to get lastMod time: " + @@ -1132,6 +1157,17 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, continue; } + //is tSize available for tFile? + long tSize = tFileSizePA.get(tFileListPo); + if (tSize < 0) { //-1=trouble + nNoSize++; + String2.log(tFileListPo + " reject because unable to get size: " + + dirList.get(tDirI) + tFileS); + tFileListPo++; + addBadFile(badFileMap, tDirI, tFileS, tLastMod, "Unable to get size."); + continue; + } + //is tFile in badFileMap? Object bfi = badFileMap.get(tDirI + "/" + tFileS); if (bfi != null) { @@ -1160,7 +1196,7 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, } //is tFile already in cache? - if (tDirI == dirI && tFileS.equals(fileS) && tLastMod == lastMod) { + if (tDirI == dirI && tFileS.equals(fileS) && tLastMod == lastMod && tSize == size) { if (logThis) String2.log(tFileListPo + " already in cached fileList"); nUnchanged++; @@ -1209,7 +1245,7 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, readFileCumTime += System.currentTimeMillis() - rfcTime; //set the values on the fileTable row throws throwable - setFileTableRow(fileTable, fileListPo, tDirI, tFileS, tLastMod, + setFileTableRow(fileTable, fileListPo, tDirI, tFileS, tLastMod, tSize, tTable, logThis? tFileListPo : -1); tFileListPo++; fileListPo++; @@ -1276,14 +1312,12 @@ public EDDTableFromFiles(String tClassName, boolean tFilesAreLocal, minMaxTable = tMinMaxTable; msg = "\n tFileNamePA.size()=" + tFileNamePA.size() + - " lastModCumTime=" + Calendar2.elapsedTimeString(lastModCumTime) + - " avg=" + (lastModCumTime / Math.max(1, tFileNamePA.size())) + "ms" + "\n dirTable.nRows()=" + dirTable.nRows() + "\n fileTable.nRows()=" + fileTable.nRows() + "\n fileTableInMemory=" + fileTableInMemory + "\n nUnchanged=" + nUnchanged + "\n nRemoved=" + nRemoved + " (nNoLastMod=" + nNoLastMod + - ") removedCumTime=" + Calendar2.elapsedTimeString(lastModCumTime) + + ", nNoSize=" + nNoSize + ")" + "\n nReadFile=" + nReadFile + " (nDifferentModTime=" + nDifferentModTime + " nNew=" + nNew + ")" + " readFileCumTime=" + Calendar2.elapsedTimeString(readFileCumTime) + @@ -1640,16 +1674,18 @@ protected void testIfNewFileAttsAreCompatible(String dvName, int dvNEC, * @throws throwable if trouble */ protected void setFileTableRow(Table fileTable, int fileListPo, - int tDirI, String tFileS, double tLastMod, Table tTable, int logAsRowNumber) { + int tDirI, String tFileS, double tLastMod, double tSize, Table tTable, int logAsRowNumber) { ShortArray ftDirIndex = (ShortArray)fileTable.getColumn(FT_DIR_INDEX_COL); //0 StringArray ftFileList = (StringArray)fileTable.getColumn(FT_FILE_LIST_COL); //1 DoubleArray ftLastMod = (DoubleArray)fileTable.getColumn(FT_LAST_MOD_COL); //2 - DoubleArray ftSortedSpacing = (DoubleArray)fileTable.getColumn(FT_SORTED_SPACING_COL); //3 + DoubleArray ftSize = (DoubleArray)fileTable.getColumn(FT_SIZE_COL); //3 + DoubleArray ftSortedSpacing = (DoubleArray)fileTable.getColumn(FT_SORTED_SPACING_COL); //4 ftDirIndex.setInt(fileListPo, tDirI); ftFileList.set(fileListPo, tFileS); ftLastMod.set(fileListPo, tLastMod); + ftSize.set(fileListPo, tSize); ftSortedSpacing.set(fileListPo, -1); //default, usually set below //get min,max for dataVariables @@ -1931,7 +1967,8 @@ public boolean lowUpdate(String msg, long startUpdateMillis) throws Throwable { ShortArray ftDirIndex = (ShortArray) tFileTable.getColumn(FT_DIR_INDEX_COL); //0 StringArray ftFileList = (StringArray)tFileTable.getColumn(FT_FILE_LIST_COL); //1 DoubleArray ftLastMod = (DoubleArray)tFileTable.getColumn(FT_LAST_MOD_COL); //2 - DoubleArray ftSortedSpacing = (DoubleArray)tFileTable.getColumn(FT_SORTED_SPACING_COL); //3 + DoubleArray ftSize = (DoubleArray)tFileTable.getColumn(FT_SIZE_COL); //3 + DoubleArray ftSortedSpacing = (DoubleArray)tFileTable.getColumn(FT_SORTED_SPACING_COL); //4 //for each changed file int nChanges = 0; //BadFiles or FileTable @@ -2022,7 +2059,7 @@ public boolean lowUpdate(String msg, long startUpdateMillis) throws Throwable { tFileTable.insertBlankRow(fileListPo); } //else use same row it was on before (can be inappropriate, but will sort below) setFileTableRow(tFileTable, fileListPo, dirIndex, fileName, - File2.getLastModified(fullName), tTable, + File2.getLastModified(fullName), File2.length(fullName), tTable, debugMode? evi : -1); } else { @@ -2156,7 +2193,7 @@ public boolean lowUpdate(String msg, long startUpdateMillis) throws Throwable { /** * Try to load the dirTable or fileTable. - * fileTable PrimitiveArrays: 0=ftDirIndex 1=ftFileList 2=ftLastMod 3=ftSortedSpacing, + * fileTable PrimitiveArrays: 0=ftDirIndex 1=ftFileList 2=ftLastMod 3=ftSize 4=ftSortedSpacing, * then sourceMin, sourceMax, hasNaN columns for each dv. * * @param fileName dirTableFileName or fileTableFileName @@ -2179,6 +2216,46 @@ protected Table tryToLoadDirFileTable(String fileName) { return table; } + /** + * This returns a fileTable (formatted like + * FileVisitorDNLS.oneStep(tDirectoriesToo=false, last_mod is LongArray, + * and size is LongArray of epochMillis) + * with valid files (or null if unavailable or any trouble). + * This is a copy of any internal data, so client can modify the contents. + */ + public Table accessibleViaFilesFileTable() { + try { + //get a copy of the source file information + Table tDirTable; + Table tFileTable; + if (fileTableInMemory) { + tDirTable = (Table)dirTable.clone(); + tFileTable = (Table)fileTable.clone(); + } else { + tDirTable = tryToLoadDirFileTable(datasetDir() + DIR_TABLE_FILENAME); //shouldn't be null + tFileTable = tryToLoadDirFileTable(datasetDir() + FILE_TABLE_FILENAME); //shouldn't be null + Test.ensureNotNull(tDirTable, "dirTable"); + Test.ensureNotNull(tFileTable, "fileTable"); + } + + //make the results Table + Table dnlsTable = FileVisitorDNLS.makeEmptyTable(); + dnlsTable.setColumn(0, tFileTable.getColumn(FT_DIR_INDEX_COL)); + dnlsTable.setColumn(1, tFileTable.getColumn(FT_FILE_LIST_COL)); + dnlsTable.setColumn(2, new LongArray(tFileTable.getColumn(FT_LAST_MOD_COL))); //double -> long + dnlsTable.setColumn(3, new LongArray(tFileTable.getColumn(FT_SIZE_COL))); //double -> long + //convert dir Index to dir names + tDirTable.addColumn(0, "dirIndex", new IntArray(0, tDirTable.nRows() - 1)); + dnlsTable.join(1, 0, "", tDirTable); + dnlsTable.removeColumn(0); + dnlsTable.setColumnName(0, FileVisitorDNLS.DIRECTORY); + + return dnlsTable; + } catch (Exception e) { + String2.log(MustBe.throwableToString(e)); + return null; + } + } /** * This tests if 'old' is different from this in any way. @@ -2216,7 +2293,7 @@ public Table getFileInfo(String fileDir, String fileNameRegex, boolean recursive /** * This is the default implementation of getFileLastModified, which * gets lastModified for files in local directory. - * Some subclasses override this. + * Subclasses can override this. (Currently, none.) * * @return the time (millis since the start of the Unix epoch) * the file was last modified @@ -2226,6 +2303,17 @@ public long getLastModified(String tDir, String tName) { return File2.getLastModified(tDir + tName); } + /** + * This is the default implementation of getSize, which + * gets size for files in local directory. + * Subclasses can override this. (Currently, none.) + * + * @return the size (in bytes) of the file (-1 if trouble) + */ + public long getSize(String tDir, String tName) { + return File2.length(tDir + tName); + } + /** * This is the low level method to get source data from one file. * This is only called by getSourceDataFromFile(); @@ -2478,7 +2566,8 @@ public void getDataForDapQuery(String loggedInAs, String requestUrl, ShortArray ftDirIndex = (ShortArray)tFileTable.getColumn(0); StringArray ftFileList = (StringArray)tFileTable.getColumn(1); DoubleArray ftLastMod = (DoubleArray)tFileTable.getColumn(2); - DoubleArray ftSortedSpacing = (DoubleArray)tFileTable.getColumn(3); + DoubleArray ftSize = (DoubleArray)tFileTable.getColumn(3); + DoubleArray ftSortedSpacing = (DoubleArray)tFileTable.getColumn(4); //no need to further prune constraints. diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromHyraxFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromHyraxFiles.java index 75297ab2d..2a8dcee1a 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromHyraxFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromHyraxFiles.java @@ -26,6 +26,7 @@ import gov.noaa.pfel.coastwatch.griddata.OpendapHelper; import gov.noaa.pfel.coastwatch.pointdata.Table; +import gov.noaa.pfel.coastwatch.util.FileVisitorDNLS; import gov.noaa.pfel.coastwatch.util.RegexFilenameFilter; import gov.noaa.pfel.coastwatch.util.SSR; @@ -155,9 +156,10 @@ public static void makeDownloadFileTasks(String tDatasetID, //gather all sourceFile info StringArray sourceFileName = new StringArray(); DoubleArray sourceFileLastMod = new DoubleArray(); - boolean completelySuccessful = EDDGridFromDap.addToHyraxUrlList( - catalogUrl, fileNameRegex, recursive, - sourceFileName, sourceFileLastMod); + LongArray fSize = new LongArray(); + boolean completelySuccessful = FileVisitorDNLS.addToHyraxUrlList( + catalogUrl, fileNameRegex, recursive, false, //dirsToo + sourceFileName, sourceFileLastMod, fSize); //Rename local files that shouldn't exist? //If completelySuccessful and found some files, @@ -450,7 +452,7 @@ public static String generateDatasetsXml(String tLocalDirUrl, suggestDatasetID(tPublicDirUrl + tFileNameRegex) + "\" active=\"true\">\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " 0\n" + //files are only added by full reload " \n" + " true\n" + " " + XML.encodeAsXML(tFileNameRegex) + "\n" + @@ -500,7 +502,7 @@ public static void testGenerateDatasetsXml() throws Throwable { "\n" + "\n" + " 2880\n" + -" 10000\n" + +" 0\n" + " \n" + " true\n" + " pentad.*\\.nc\\.gz\n" + @@ -539,7 +541,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " GCMD Science Keywords\n" + " [standard]\n" + " http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/pentad/flk/1987/07/\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " Time average of level3.0 products for the period: 1987-07-05 to 1987-07-09\n" + " \n" + " \n" + @@ -801,21 +803,39 @@ public static void testGenerateDatasetsXml2() throws Throwable { * * @throws Throwable if trouble */ - public static void testJpl(boolean deleteCachedInfo) throws Throwable { - String2.log("\n****************** EDDTableFromHyraxFiles.testJpl() *****************\n"); + public static void testJpl(boolean deleteCachedInfoAndOneFile) throws Throwable { + String2.log("\n****** EDDTableFromHyraxFiles.testJpl(deleteCachedInfoAndOneFile=" + + deleteCachedInfoAndOneFile + ")\n"); testVerboseOn(); String name, tName, results, tResults, expected, userDapQuery, tQuery; String error = ""; int po; EDV edv; String today = Calendar2.getCurrentISODateTimeStringZulu().substring(0, 14); //14 is enough to check hour. Hard to check min:sec. + String id = "testEDDTableFromHyraxFiles"; + String deletedFile = "pentad_19870928_v11l35flk.nc.gz"; try { - String id = "testEDDTableFromHyraxFiles"; - if (deleteCachedInfo) + + //delete the last file in the collection + if (deleteCachedInfoAndOneFile) { deleteCachedDatasetInfo(id); + File2.delete(EDStatic.fullCopyDirectory + id + "/" + deletedFile); + } + EDDTable eddTable = (EDDTable)oneFromDatasetXml(id); + if (deleteCachedInfoAndOneFile) { + String2.pressEnterToContinue( + "\n****** BOB! ******\n" + + "This test just deleted a file: " + deletedFile + "\n" + + "The background task to re-download it should have already started.\n" + + "The remote dataset is really slow.\n" + + "Wait for it to finish background tasks.\n\n"); + eddTable = (EDDTable)oneFromDatasetXml(id); //redownload the dataset + } + + //*** test getting das for entire dataset try { String2.log("\n****************** EDDTableFromHyraxFiles das and dds for entire dataset\n"); @@ -929,10 +949,8 @@ public static void testJpl(boolean deleteCachedInfo) throws Throwable { " }\n" + " }\n" + " NC_GLOBAL {\n" + -" Int16 base_date 1987, 9, 28;\n" + " String cdm_data_type \"Point\";\n" + " String Conventions \"COARDS, CF-1.6, ACDD-1.3\";\n" + -" String description \"Time average of level3.0 products for the period: 1987-09-28 to 1987-10-02\";\n" + " Float64 Easternmost_Easting 359.875;\n" + " String featureType \"Point\";\n" + " Float64 geospatial_lat_max 78.375;\n" + @@ -951,7 +969,7 @@ public static void testJpl(boolean deleteCachedInfo) throws Throwable { //today + " http://127.0.0.1:8080/cwexperimental/ expected = "tabledap/testEDDTableFromHyraxFiles.das\";\n" + -" String infoUrl \"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/pentad/flk/1987/M09/.html\";\n" + +" String infoUrl \"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/pentad/flk/1987/09/.html\";\n" + " String institution \"NASA JPL\";\n" + " String keywords \"Atmosphere > Atmospheric Winds > Surface Winds,\n" + "Atmosphere > Atmospheric Winds > Wind Stress,\n" + @@ -965,10 +983,10 @@ public static void testJpl(boolean deleteCachedInfo) throws Throwable { "particular purpose, or assumes any legal liability for the accuracy,\n" + "completeness, or usefulness, of this information.\";\n" + " Float64 Northernmost_Northing 78.375;\n" + -" String sourceUrl \"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/pentad/flk/1987/M09/\";\n" + +" String sourceUrl \"http://podaac-opendap.jpl.nasa.gov/opendap/allData/ccmp/L3.5a/pentad/flk/1987/09/\";\n" + " Float64 Southernmost_Northing -78.375;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + -" String summary \"Time average of level3.0 products for the period: 1987-09-08 to 1987-09-12\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + +" String summary \"Time average of level3.0 products.\";\n" + " String time_coverage_end \"1987-09-28T00:00:00Z\";\n" + " String time_coverage_start \"1987-09-03T00:00:00Z\";\n" + " String title \"Atlas FLK v1.1 derived surface winds (level 3.5)\";\n" + @@ -1089,7 +1107,7 @@ public static void testJpl(boolean deleteCachedInfo) throws Throwable { /* */ } - + /** * This tests the methods in this class. @@ -1101,9 +1119,8 @@ public static void test() throws Throwable { //usually run /* */ testGenerateDatasetsXml(); - testJpl(true); //deleteCachedInfo (the file info, not the data files) + testJpl(true); //deleteCachedInfoAndOneFile testJpl(false); - } } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromMWFS.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromMWFS.java index 07599f2e7..bfc5ebb5a 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromMWFS.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromMWFS.java @@ -586,7 +586,7 @@ Markup Language (GML) Simple Feature Profile to transport in-situ http://www.csc.noaa.gov/DTL/dtl_proj4_gmlsfp_wfs.html NOAA CSC [standard] - CF Standard Name Table v27 + CF Standard Name Table v29 -97.22 -70.43 @@ -739,7 +739,7 @@ Markup Language (GML) Simple Feature Profile to transport in-situ " Float64 Northernmost_Northing 38.48;[10]\n" + " String sourceUrl \"http://csc-s-ial-p.csc.noaa.gov/cgi-bin/microwfs/microWFS.cgi?SERVICENAME=dtlservicesubType=gml/3.1.1/profiles/gmlsf/1.0.0/1\";[10]\n" + " Float64 Southernmost_Northing 24.55;[10]\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";[10]\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";[10]\n" + " String summary \"[Normally, the summary describes the dataset. Here, it describes [10]\n" + "the server.][10]\n" + "The mission of the NOAA CSC Data Transport Laboratory (DTL) is to[10]\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNOS.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNOS.java index 0bb039171..c69203add 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNOS.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNOS.java @@ -1027,7 +1027,7 @@ public static void test(boolean doLongTest) throws Throwable { " Float64 Northernmost_Northing 71.3601;[10]\n" + " String sourceUrl \"http://opendap.co-ops.nos.noaa.gov/axis/services/Wind\";[10]\n" + " Float64 Southernmost_Northing -14.28;[10]\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";[10]\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";[10]\n" + " String summary \"[Normally, the summary describes the dataset. Here, it describes[10]\n" + "the server.][10]\n" + "NOS CO-OPS has developed a web-based application that serves as an[10]\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcCFFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcCFFiles.java index 42bc1ee88..fb926a247 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcCFFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcCFFiles.java @@ -188,7 +188,8 @@ public static String generateDatasetsXml( if (tInstitution != null && tInstitution.length() > 0) externalAddGlobalAttributes.add("institution", tInstitution); if (tSummary != null && tSummary.length() > 0) externalAddGlobalAttributes.add("summary", tSummary); if (tTitle != null && tTitle.length() > 0) externalAddGlobalAttributes.add("title", tTitle); - externalAddGlobalAttributes.setIfNotAlreadySet("sourceUrl", "(local files)"); + externalAddGlobalAttributes.setIfNotAlreadySet("sourceUrl", + "(" + (File2.isRemote(tFileDir)? "remote" : "local") + " files)"); //after dataVariables known, add global attributes in the dataAddTable dataAddTable.globalAttributes().set( makeReadyToUseAddGlobalAttributesForDatasetsXml( @@ -221,7 +222,8 @@ public static String generateDatasetsXml( suggestDatasetID(tFileDir + suggestedRegex) + //dirs can't be made public "\" active=\"true\">\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " " + suggestUpdateEveryNMillis(tFileDir) + + "\n" + " " + tFileDir + "\n" + " true\n" + " " + XML.encodeAsXML(suggestedRegex) + "\n" + @@ -369,7 +371,7 @@ public static void testGenerateDatasetsXml() throws Throwable { "Oceans > Aquatic Sciences > Fisheries,\n" + "order, sciences, scientific, ship, start, station, time, tow, units, value, vertebrates\n" + " null\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " \n" + " \n" + " line_station\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcFiles.java index 63445afed..bae14fd6d 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcFiles.java @@ -311,7 +311,8 @@ public static String generateDatasetsXml( if (tInstitution != null && tInstitution.length() > 0) externalAddGlobalAttributes.add("institution", tInstitution); if (tSummary != null && tSummary.length() > 0) externalAddGlobalAttributes.add("summary", tSummary); if (tTitle != null && tTitle.length() > 0) externalAddGlobalAttributes.add("title", tTitle); - externalAddGlobalAttributes.setIfNotAlreadySet("sourceUrl", "(local files)"); + externalAddGlobalAttributes.setIfNotAlreadySet("sourceUrl", + "(" + (File2.isRemote(tFileDir)? "remote" : "local") + " files)"); //externalAddGlobalAttributes.setIfNotAlreadySet("subsetVariables", "???"); //after dataVariables known, add global attributes in the dataAddTable dataAddTable.globalAttributes().set( @@ -347,7 +348,8 @@ public static String generateDatasetsXml( suggestDatasetID(tFileDir + suggestedRegex) + //dirs can't be made public "\" active=\"true\">\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " " + suggestUpdateEveryNMillis(tFileDir) + + "\n" + " " + tFileDir + "\n" + " true\n" + " " + XML.encodeAsXML(suggestedRegex) + "\n" + @@ -435,8 +437,8 @@ public static void testGenerateDatasetsXml() throws Throwable { " dave.foley@noaa.gov\n" + " NOAA CoastWatch, West Coast Node\n" + " http://coastwatch.pfeg.noaa.gov\n" + -" 2014-06-11Z\n" + //changes -" 2014-06-11Z\n" + //changes +" 2015-07-20Z\n" + //changes +" 2015-07-20Z\n" + //changes " -79.099\n" + " 32.501\n" + " 32.501\n" + @@ -465,7 +467,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " The National Data Buoy Center (NDBC) distributes meteorological data from moored buoys maintained by NDBC and others. Moored buoys are the weather sentinels of the sea. They are deployed in the coastal and offshore waters from the western Atlantic to the Pacific Ocean around Hawaii, and from the Bering Sea to the South Pacific. NDBC's moored buoys measure and transmit barometric pressure; wind direction, speed, and gust; air and sea temperature; and wave energy spectra from which significant wave height, dominant wave period, and average wave period are derived. Even the direction of wave propagation is measured on many moored buoys. \n" + "\n" + //changes 2 places... date is old, but this is what's in the file "This dataset has both historical data (quality controlled, before 2011-05-01T00:00:00) and near real time data (less quality controlled, from 2011-05-01T00:00:00 on).\n" + -" 2014-06-11T14:00:00Z\n" + //changes +" 2015-07-20T15:00:00Z\n" + //changes " P1H\n" + " 1978-06-27T13:00:00Z\n" + " NOAA NDBC Standard Meteorological\n" + @@ -497,7 +499,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " GCMD Science Keywords\n" + " null\n" + " (local files)\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " \n" + " \n" + " stationID\n" + @@ -516,7 +518,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " double\n" + " \n" + " true\n" + " " + XML.encodeAsXML(tFileNameRegex) + "\n" + @@ -761,7 +761,7 @@ public static void testGenerateDatasetsXml() throws Throwable { "\n" + "\n" + " 1440\n" + -" 10000\n" + +" 0\n" + " \n" + " true\n" + " .*MTBD.*\\.nc\n" + @@ -800,7 +800,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " GCMD Science Keywords\n" + " [standard]\n" + " http://data.nodc.noaa.gov/thredds/catalog/nmsp/wcos/WES001/2008/catalog.xml\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " NOAA National Oceanographic Data Center (NODC) data from http://data.nodc.noaa.gov/thredds/catalog/nmsp/wcos/WES001/2008/catalog.html\n" + " NOAA NODC data from http://data.nodc.noaa.gov/thredds/catalo ...\n" + " \n" + @@ -1105,7 +1105,7 @@ public static void testWcosTemp(boolean deleteCachedInfo) throws Throwable { " Float64 Northernmost_Northing 48.325001;\n" + " String sourceUrl \"http://data.nodc.noaa.gov/thredds/catalog/nmsp/wcos/catalog.xml\";\n" + " Float64 Southernmost_Northing 33.89511;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"station, longitude, latitude\";\n" + " String summary \"The West Coast Observing System (WCOS) project provides access to temperature and currents data collected at four of the five National Marine Sanctuary sites, including Olympic Coast, Gulf of the Farallones, Monterey Bay, and Channel Islands. A semi-automated end-to-end data management system transports and transforms the data from source to archive, making the data acessible for discovery, access and analysis from multiple Internet points of entry.\n" + "\n" + @@ -1479,14 +1479,14 @@ public static void testShipWTEP(boolean deleteCachedInfo) throws Throwable { " Float32 height -9999.0;\n" + " String instrument \"unknown\";\n" + " String ioos_category \"Salinity\";\n" + -" String long_name \"Salinity\";\n" + +" String long_name \"Sea Water Practical Salinity\";\n" + " Float32 missing_value -9999.0;\n" + " String observation_type \"calculated\";\n" + " String original_units \"PSU\";\n" + " Int32 qcindex 15;\n" + " Float32 sampling_rate -9999.0;\n" + " Float32 special_value -8888.0;\n" + -" String standard_name \"sea_water_salinity\";\n" + +" String standard_name \"sea_water_practical_salinity\";\n" + " String units \"PSU\";\n" + " \\}\n" + " seaTemperature \\{\n" + @@ -1749,7 +1749,7 @@ public static void testShipWTEP(boolean deleteCachedInfo) throws Throwable { "Atmosphere > Atmospheric Winds > Surface Winds,\n" + "Oceans > Salinity/Density > Conductivity,\n" + "Oceans > Salinity/Density > Salinity,\n" + -"air, air_pressure, air_temperature, atmosphere, atmospheric, calender, conductivity, control, course, data, date, day, density, direction, dyson, earth, electrical, file, flags, from, fsu, ground, heading, history, humidity, information, level, measurements, meteorological, meteorology, oceans, oscar, over, platform, pressure, quality, relative, relative_humidity, salinity, sea, sea_water_electrical_conductivity, sea_water_salinity, seawater, speed, static, surface, temperature, time, vapor, water, wind, wind_from_direction, wind_speed, winds\";\n" + +"air, air_pressure, air_temperature, atmosphere, atmospheric, calender, conductivity, control, course, data, date, day, density, direction, dyson, earth, electrical, file, flags, from, fsu, ground, heading, history, humidity, information, level, measurements, meteorological, meteorology, oceans, oscar, over, platform, pressure, quality, relative, relative_humidity, salinity, sea, sea_water_electrical_conductivity, sea_water_practical_salinity, seawater, speed, static, surface, temperature, time, vapor, water, wind, wind_from_direction, wind_speed, winds\";\n" + " String keywords_vocabulary \"GCMD Science Keywords\";\n" + " String license \"The data may be used and redistributed for free but is not intended\n" + "for legal use, since it may contain inaccuracies. Neither the data\n" + @@ -1763,7 +1763,7 @@ public static void testShipWTEP(boolean deleteCachedInfo) throws Throwable { " String receipt_order \"01\";\n" + " String sourceUrl \"http://coaps.fsu.edu/thredds/catalog/samos/data/research/WTEP/catalog.xml\";\n" + " Float64 Southernmost_Northing -46.45;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"cruise_id, expocode, facility, ID, IMO, platform, platform_version, site\";\n" + " String summary \"NOAA Ship Oscar Dyson Underway Meteorological Data " + "\\(delayed ~10 days for quality control\\) are from the Shipboard " + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromWFSFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromWFSFiles.java index 0d1d97d4b..1479eb200 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromWFSFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromWFSFiles.java @@ -257,7 +257,7 @@ public static String generateDatasetsXml(String tSourceUrl, String tRowElementXP "\n" + " " + tReloadEveryNMinutes + "\n" + - " " + suggestedUpdateEveryNMillis + "\n" + + " 0\n" + //files are only added by full reload //" false\n" + //" " + EDStatic.fullCopyDirectory + tDatasetID + "/\n" + //" .*\\.tsv\n" + @@ -294,7 +294,9 @@ public static String generateDatasetsXml(String tSourceUrl, String tRowElementXP */ public static void testGenerateDatasetsXml() throws Throwable { testVerboseOn(); -developmentMode = true; + boolean oDevelopmentMode = developmentMode; + developmentMode = false; + try { Attributes externalAddAttributes = new Attributes(); externalAddAttributes.add("title", "Old Title!"); @@ -321,8 +323,6 @@ public static void testGenerateDatasetsXml() throws Throwable { false); //doIt loop? Test.ensureEqual(gdxResults, results, "Unexpected results from GenerateDatasetsXml.doIt."); -developmentMode = true; - String expected = directionsForGenerateDatasetsXml() + " * Since the source files don't have any metadata, you must add metadata\n" + @@ -331,7 +331,7 @@ public static void testGenerateDatasetsXml() throws Throwable { "\n" + "\n" + " 10080\n" + -" 10000\n" + +" 0\n" + " last\n" + " false\n" + " false\n" + @@ -352,7 +352,7 @@ public static void testGenerateDatasetsXml() throws Throwable { " [standard]\n" + " /wfs:FeatureCollection/gml:featureMember\n" + " http://kgs.uky.edu/arcgis/services/aasggeothermal/WVBoreholeTemperatures/MapServer/WFSServer?request=GetFeature&service=WFS&typename=aasg:BoreholeTemperature&format="text/xml;%20subType=gml/3.1.1/profiles/gmlsf/1.0.0/0"\n" + -" CF Standard Name Table v27\n" + +" CF Standard Name Table v29\n" + " The summary. Kentucky Geological Survey data from a local source.\n" + " The Title\n" + " \n" + @@ -863,6 +863,13 @@ public static void testGenerateDatasetsXml() throws Throwable { pa1 = new FloatArray(table.getColumn("aasg:BoreholeTemperature/aasg:LongDegree")); pa2 = new FloatArray(table.getColumn("aasg:BoreholeTemperature/aasg:Shape/gml:Point/longitude")); Test.ensureEqual(pa1, pa2, ""); + + } catch (Throwable t) { + String2.pressEnterToContinue(MustBe.throwableToString(t) + + "*** The server disappeared ~July 2015. Find another?"); + } + developmentMode = oDevelopmentMode; + } @@ -876,6 +883,7 @@ public static void testBasic() throws Throwable { String tName, error, results, tResults, expected; int po; String today = Calendar2.getCurrentISODateTimeStringZulu().substring(0, 14); //14 is enough to check hour. Hard to check min:sec. + try { //*** .das tName = tedd.makeNewFileForDapQuery(null, null, "", EDStatic.fullTestCacheDirectory, @@ -1080,7 +1088,7 @@ public static void testBasic() throws Throwable { " String rowElementXPath \"/wfs:FeatureCollection/gml:featureMember\";\n" + " String sourceUrl \"http://kgs.uky.edu/arcgis/services/aasggeothermal/WVBoreholeTemperatures/MapServer/WFSServer?request=GetFeature&service=WFS&typename=aasg:BoreholeTemperature&format=\\\"text/xml;%20subType=gml/3.1.1/profiles/gmlsf/1.0.0/0\\\"\";\n" + " Float64 Southernmost_Northing 37.24673;\n" + -" String standard_name_vocabulary \"CF Standard Name Table v27\";\n" + +" String standard_name_vocabulary \"CF Standard Name Table v29\";\n" + " String subsetVariables \"WellName,APINo,Label,Operator,WellType,Field,County,State,FormationTD,OtherName\";\n" + " String summary \"Borehole temperature measurements in West Virginia\";\n" + " String time_coverage_end \"2012-08-05\";\n" + @@ -1149,6 +1157,10 @@ public static void testBasic() throws Throwable { Test.ensureEqual(results.substring(0, expected.length()), expected, "\nresults=\n" + results); + } catch (Throwable t) { + String2.pressEnterToContinue(MustBe.throwableToString(t) + + "Unexpected error"); + } } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/CfToGcmd.txt b/WEB-INF/classes/gov/noaa/pfel/erddap/util/CfToGcmd.txt index 44162ecaa..c0c7e12e2 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/CfToGcmd.txt +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/CfToGcmd.txt @@ -5715,6 +5715,9 @@ Oceans > Salinity/Density > Potential Density sea_water_potential_temperature Oceans > Ocean Temperature > Potential Temperature +sea_water_practical_salinity +Oceans > Salinity/Density > Salinity + sea_water_pressure Oceans > Ocean Pressure > Water Pressure diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java b/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java index ed67d580c..f67ae89f3 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java @@ -134,9 +134,10 @@ public class EDStatic { *
    1.56 released on 2014-12-16 *
    1.58 released on 2015-02-25 *
    1.60 released on 2015-03-12 - *
    1.62 released on 2015-05-08 + *
    1.62 released on 2015-06-08 + *
    1.64 released on 2015-08-19 */ - public static String erddapVersion = "1.62"; + public static String erddapVersion = "1.64"; /** * This is almost always false. @@ -422,7 +423,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { postShortDescriptionActive, //if true, PostIndexHtml is on home page and /post/index.html redirects there subscriptionSystemActive, convertersActive, slideSorterActive, fgdcActive, iso19115Active, geoServicesRestActive, - filesActive, sosActive, wcsActive, wmsActive, + filesActive, dataProviderFormActive, sosActive, wcsActive, wmsActive, quickRestart, useOriginalSearchEngine, useLuceneSearchEngine, //exactly one will be true variablesMustHaveIoosCategory, @@ -719,6 +720,8 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { errorOdvLLTGrid, errorOdvLLTTable, errorOnWebPage, + errorXWasntSpecified, + errorXWasTooLong, externalLink, externalWebSite, fileHelp_asc, @@ -1489,6 +1492,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { //until geoServicesRest is finished, it is always inactive geoServicesRestActive = false; //setup.getBoolean( "geoServicesRestActive", false); filesActive = setup.getBoolean( "filesActive", true); + dataProviderFormActive = setup.getBoolean( "dataProviderFormActive", true); //until SOS is finished, it is always inactive sosActive = false;// sosActive = setup.getBoolean( "sosActive", false); if (sosActive) { @@ -1903,6 +1907,10 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { errorOdvLLTGrid = messages.getNotNothingString("errorOdvLLTGrid", ""); errorOdvLLTTable = messages.getNotNothingString("errorOdvLLTTable", ""); errorOnWebPage = messages.getNotNothingString("errorOnWebPage", ""); + errorXWasntSpecified = messages.getNotNothingString("errorXWasntSpecified", ""); + HtmlWidgets.errorXWasntSpecified = errorXWasntSpecified; + errorXWasTooLong = messages.getNotNothingString("errorXWasTooLong", ""); + HtmlWidgets.errorXWasTooLong = errorXWasTooLong; externalLink = " " + messages.getNotNothingString("externalLink", ""); externalWebSite = messages.getNotNothingString("externalWebSite", ""); fileHelp_asc = messages.getNotNothingString("fileHelp_asc", ""); @@ -2387,8 +2395,7 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { convertUnitsService = MessageFormat.format(convertUnitsService, erddapUrl) + "\n"; //standardContact is used by legal - String tEmail = String2.replaceAll(adminEmail, "@", " at "); - tEmail = String2.replaceAll(tEmail, ".", " dot "); + String tEmail = SSR.getSafeEmailAddress(adminEmail); filesDocumentation = String2.replaceAll(filesDocumentation, "&adminEmail;", tEmail); standardContact = String2.replaceAll(standardContact, "&adminEmail;", tEmail); legal = String2.replaceAll(legal,"[standardContact]", standardContact + "\n\n"); @@ -2812,9 +2819,10 @@ public static String email(String emailAddresses[], String subject, String conte try { //catch common problem: sending email to one invalid address if (emailAddresses.length == 1 && - !String2.isEmailAddress(emailAddresses[0]) || - emailAddresses[0].startsWith("your.name") || - emailAddresses[0].startsWith("your.email")) { + (!String2.isEmailAddress(emailAddresses[0]) || + emailAddresses[0].startsWith("nobody@") || + emailAddresses[0].startsWith("your.name") || + emailAddresses[0].startsWith("your.email"))) { errors = "Error in EDStatic.email: invalid emailAddresses=" + emailAddressesCSSV; String2.log(errors); } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/FishBase.java b/WEB-INF/classes/gov/noaa/pfel/erddap/util/FishBase.java index bda8cc901..8e45055e3 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/FishBase.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/FishBase.java @@ -98,7 +98,7 @@ public static void convertHtmlToNc(String tableRegex) throws Throwable { "\n" + EDStatic.standardLicense); gatts.set("sourceUrl", "(local files)"); - gatts.set("standard_name_vocabulary", "CF Standard Name Table v27"); + gatts.set("standard_name_vocabulary", "CF Standard Name Table v29"); String startReference = "To give due credit to the original authors, please cite:\n" + "Froese, R. and D. Pauly, Editors. 2004. FishBase 2004 DVD: the " + fName + " table.\n"; diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/HtmlWidgets.java b/WEB-INF/classes/gov/noaa/pfel/erddap/util/HtmlWidgets.java index 7cfdf89dd..2f0b3447a 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/HtmlWidgets.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/HtmlWidgets.java @@ -14,6 +14,7 @@ import java.awt.Color; import java.io.FileWriter; import java.io.Writer; +import java.text.MessageFormat; /** * HtmlWidgets has methods to simplify creation of widgets in @@ -104,6 +105,9 @@ public class HtmlWidgets { public static String twoClickMapDefaultTooltip = "Specify a rectangle by clicking on two diagonal corners. Do it again if needed."; + public static String errorXWasntSpecified = "Error: \"{0}\" wasn''t specified."; + public static String errorXWasTooLong = "\"{0}\" was more than {1} characters long."; + /** * If you want to use Tip for big html tooltips (see below), * include this in the 'body' of a web page (preferably right after the body tag). @@ -770,18 +774,20 @@ private String selectButton(String name, int nRows, int test, * @param fieldLength the size of the field, in mspaces(?), e.g., 10. * If the fieldLength <= 0, no 'size' value is specified in the html. * @param maxLength usually 255 - * @param initialTextValue the initial text value, or "" if none + * @param initialTextValue the initial text value, or null or "" if none * @param other e.g., "onchange=\"pleaseWait();\"" * For textField(), use onchange, not onclick. * @return the HTML code for a textField. */ public String textField(String name, String tooltip, int fieldLength, int maxLength, String initialTextValue, String other) { - + return " submit //was the keypress event's keycode 'Enter'? @@ -1436,6 +1442,107 @@ public static String cleanInput(String text) { continue; */ + /** + * This ensure that the string value isn't null and isn't too long. + * + * @param cumulativeHtmlErrorMsg (may be null) + * @return the corrected value (default (if it was null), previous value.trim(), or + * previous value shortened to maxLength) + */ + public static String validateNotNullNotTooLong(String name, String defaultValue, + String value, int maxLength, StringBuilder cumulativeHtmlErrorMsg) { + + if (value == null) { + if (cumulativeHtmlErrorMsg != null) + cumulativeHtmlErrorMsg.append("
    • " + + MessageFormat.format(errorXWasntSpecified, XML.encodeAsHTML(name)) + + "\n"); + return defaultValue; + } else { + value = value.trim(); + } + if (value.length() > maxLength) { + if (cumulativeHtmlErrorMsg != null) + cumulativeHtmlErrorMsg.append("
    • " + + MessageFormat.format(errorXWasTooLong, XML.encodeAsHTML(name), "" + maxLength) + + "\n"); + return value.substring(0, maxLength); + } + return value; + } + + /** + * This ensure that the string value isSomething and isn't too long. + * + * @param cumulativeHtmlErrorMsg (may be null) + * @return the corrected value (default (if value was null or ""), previous value.trim(), or + * previous value shortened to maxLength) + */ + public static String validateIsSomethingNotTooLong(String name, String defaultValue, + String value, int maxLength, StringBuilder cumulativeHtmlErrorMsg) { + + if (!String2.isSomething(value)) { + if (cumulativeHtmlErrorMsg != null) + cumulativeHtmlErrorMsg.append("
    • " + + MessageFormat.format(errorXWasntSpecified, XML.encodeAsHTML(name)) + + "\n"); + return defaultValue; + } else { + value = value.trim(); + } + if (value.length() > maxLength) { + if (cumulativeHtmlErrorMsg != null) + cumulativeHtmlErrorMsg.append("
    • " + + MessageFormat.format(errorXWasTooLong, XML.encodeAsHTML(name), "" + maxLength) + + "\n"); + return value.substring(0, maxLength); + } + return value; + } + + /** + * This ensure that the string value isn't too long. + * + * @param cumulativeHtmlErrorMsg (may be null) + * @return the corrected value (default (if value was null), previous value.trim(), or + * previous value shortened to maxLength) + */ + public static String validateNotTooLong(String name, String defaultValue, + String value, int maxLength, StringBuilder cumulativeHtmlErrorMsg) { + + if (value == null) + value = defaultValue; + else + value = value.trim(); + if (value.length() > maxLength) { + if (cumulativeHtmlErrorMsg != null) + cumulativeHtmlErrorMsg.append("
    • " + + MessageFormat.format(errorXWasTooLong, XML.encodeAsHTML(name), "" + maxLength) + + "\n"); + return value.substring(0, maxLength); + } + return value; + } + + /** + * This ensure that value is >=0 and <=maxValue. + * + * @param cumulativeHtmlErrorMsg (may be null) + * @return the corrected value (default (if it was null), previous value, or + * previous value shortened to maxLength) + */ + public static int validate0ToMax(String name, int defaultValue, + int value, int max, StringBuilder cumulativeHtmlErrorMsg) { + + if (value < 0 || value > max) { + if (cumulativeHtmlErrorMsg != null) + cumulativeHtmlErrorMsg.append("
    • " + + MessageFormat.format(errorXWasntSpecified, XML.encodeAsHTML(name)) + + "\n"); + return defaultValue; + } + return value; + } /** * This makes a test document and displays it in the browser. @@ -1508,7 +1615,7 @@ public static void test() throws Throwable { widgets.textField("textFieldName2", "", 20, 255, "has big tooltip ", htmlTooltip("Hi bold!\n
    There \\/'\"&¦!")) + "
    \n" + - "\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/OceanicAtmosphericAcronyms.tsv b/WEB-INF/classes/gov/noaa/pfel/erddap/util/OceanicAtmosphericAcronyms.tsv index 148ec4dcb..9cc8dc722 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/OceanicAtmosphericAcronyms.tsv +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/OceanicAtmosphericAcronyms.tsv @@ -109,6 +109,7 @@ CRM Coastal Relief Model CRMN Coral Reef Monitoring Network CRU Climatic Research Unit CRW Coral Reef Watch +CSIRO Commonwealth Scientific and Industrial Research Organisation CTD Conductivity, Temperature, Depth CWCGOM CoastWatch Caribbean and Gulf of Mexico CW CoastWatch @@ -223,6 +224,7 @@ IHB International Hydrographic Bureau IHO International Hydrographic Organisation ILP International Lithosphere Programme IMI Irish Marine Institute +IMOS Integrated Marine Observing System INMARSAT International Maritime Satellite Organisation INODC Indian National Oceanographic Data Center INQUA International Union for Quaternary Research diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/Projects2.java b/WEB-INF/classes/gov/noaa/pfel/erddap/util/Projects2.java index 75e7b0a5b..670efe22c 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/Projects2.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/Projects2.java @@ -327,8 +327,10 @@ public static void copyHyraxFiles(String urlDir, String fileNameRegex, String lo //parse the hyrax catalog StringArray childUrls = new StringArray(); DoubleArray lastModified = new DoubleArray(); - EDDGridFromDap.addToHyraxUrlList(urlDir + "contents.html", fileNameRegex, - true, childUrls, lastModified); + LongArray fSize = new LongArray(); + FileVisitorDNLS.addToHyraxUrlList( + urlDir + "contents.html", fileNameRegex, true, false, //recursive, dirsToo + childUrls, lastModified, fSize); //copy the files int n = childUrls.size(); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/cfStdNames.txt b/WEB-INF/classes/gov/noaa/pfel/erddap/util/cfStdNames.txt index f745e3155..9c6f79490 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/cfStdNames.txt +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/cfStdNames.txt @@ -1454,6 +1454,7 @@ sea_water_neutral_density sea_water_ph_reported_on_total_scale sea_water_potential_density sea_water_potential_temperature +sea_water_practical_salinity sea_water_pressure sea_water_pressure_at_sea_floor sea_water_pressure_at_sea_water_surface diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/messages.xml b/WEB-INF/classes/gov/noaa/pfel/erddap/util/messages.xml index 9117cc430..fbc27ef7f 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/messages.xml +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/messages.xml @@ -1575,6 +1575,9 @@ This constraint variable has a limited number of values. They are listed here. For an ODV .txt file, the dataset MUST include longitude, latitude, and time axes. Data requests for ODV .txt files must include longtitude, latitude, and time data. An error occurred while writing this web page: + +Error: "{0}" wasn''t specified. +Error: "{0}" was more than {1} characters long. (external link) This link to an external web site does not constitute an endorsement. diff --git a/download/changes.html b/download/changes.html index 0885230bd..b7c953961 100644 --- a/download/changes.html +++ b/download/changes.html @@ -42,6 +42,144 @@

    ERDDAP Changes

    • + + + + +
    +

    Changes in ERDDAP version 1.64 (released 2015-08-19)

    +
      +
    • New Features (for users): +
        +
      • There is now guidance for accessing the password-protected private ERDDAP + datasets (https://) via curl and Python. + See the + curl + and + Python + instructions. +
        Thanks to Emilio Mayorga of NANOOS and Paul Janecek of Spyglass Technologies. +
          +
      + +
    • Things ERDDAP Administrators Need to Know and Do: +
        +
      • ERDDAP now requires Java 1.8+. +
        Java 1.7 reached its + end of life (external link) + (no more security updates) in April 2015. + This version of ERDDAP will not work with versions of Java below 1.8. + If you update from Java 1.7x (or earlier), you should also update Tomcat. + See the ERDDAP Set Up Instructions for download links and advice. + +
      • New Data Provider Form. +
        When a data provider comes to you hoping to add some data to your ERDDAP, + it can be difficult and time consuming to collect all of the metadata + needed to add the dataset into ERDDAP. Many data sources + (for example, .csv files, Excel files, databases) have no internal metadata. + So ERDDAP has a new Data Provider Form which gathers metadata from the data + provider and gives the data provider some other guidance, + including extensive guidance for Data In Databases. + The information submitted is converted into the datasets.xml format + and then emailed to the ERDDAP administrator (you) and written (appended) + to bigParentDirectory/logs/dataProviderForm.log . + Thus, the form semi-automates the process of getting a dataset into ERDDAP, + but the ERDDAP administrator still has to complete the datasets.xml chunk + and deal with getting the data file(s) from the provider or connecting to the database. + For more information, see the +Data Provider Form description. + +
      • New <matchAxisNDigits> +
        can be used by EDDGridFromFiles (and thus fromNcFiles and fromMergeIRFiles), + EDDGridAggregateExistingDimension, EDDGridCopy, and EDDGridSideBySide datasets + to specify how precisely equal the axis values in different files must be + (how many digits): 0=no checking (don't use this!), 1-18 for increasing precision, + or 20 (the default) for exact equality. + For n=1-18, ERDDAP ensures that the first n digits of double values + (or (n+1) div 2 for float values) are equal. +
        <matchAxisNDigits> replaces <ensureAxisValuesAreEqual>, which is now deprecated. + A value of 'true' will be converted to matchAxisNDigits=20. + A value of 'false' (don't do this!) will be converted to matchAxisNDigits=0. + +
      • EDDGridFromFiles and EDDTableFromFiles will load very slowly the first time + you use this version of ERDDAP. +
        ERDDAP now stores the internal file information a little differently, + so the internal file table for each of these datasets has to be rebuilt. + So don't worry. Nothing is wrong. It's a one time thing. + +
      • Remote Source Files +
        EDDGridFromNcFiles, EDDTableFromNcFiles, EDDTableFromNcCFFiles now + allow the files to be remote files in a directory accessible by + http:// (and probably https:// and ftp://, but they are untested) + if the remote server supports + Range Requests (external link) + in the request header. THREDDS and Amazon S3 support Range Requests, + Hyrax does not. + This system allows you to access data in remote files without downloading the files + (which is helpful if the remote files are too voluminous), + but access to these files will be far slower than access to local files + or even to a remote OPeNDAP source. + +
        This includes "files" in an Amazon S3 bucket since they are accessible via http://. + If the S3 object names are like file names (with internal /'s like a Linux + directory tree), ERDDAP can also make the files accessible via ERDDAP's "files" system. + For this to work, your S3 credentials must be in + ~/.aws/credentials (on Linux, OS X, or Unix), or C:\Users\USERNAME\.aws\credentials (on Windows) + on the server with ERDDAP. See the + Amazon SDK documentation (external link). + +
      • GenerateDatasetsXml has a new, unusual option: EDDsFromFiles. +
        This will go through a file system (even a remote system like an Amazon S3 + if the objects have file-like names) and create the datasets.xml chunks for + a series of datasets. + Your mileage may vary. + This works well if the files are organized so that all the data files in a given directory + (and its subdirectories) are suitable for one dataset (e.g., all SST 1-day composites). + Otherwise (e.g., if a diretory contains some SST files and some Chlorophyll-a files), + this works poorly but may still be useful. + +
      • Programmers: new /lib .jar files. +
        If you compile ERDDAP, please note the new .jar files in the + classpath -cp parameter listed in the ERDDAP + Programmer's Guide. + +
      • sea_water_practical_salinity +
        If you use the CF standard name sea_water_salinity for any variable, + I encourage you to switch to sea_water_practical_salinity which + is available in version 29 of the CF Standard Name Table + (and some previous versions -- I didn't know that). + This name indicates that this is indeed a Practical Salinity value using + Practical Salinity Units (PSU), as opposed to an older g/kg value. + The canonical units are different, but still incredibly unhelpful: + 1 (presumably implying PSU/PSS-78), as opposed to 1e-3 + (presumably implying g/kg) for sea_water_salinity. + [Hey, Unidata and CF: We identify values that use other scales, + for example Fahrenheit or Celsius, + via a units string that is the name of the scale or some variation. + Why can't we identify salinity units via their scale, e.g., PSS-78? + I know: PSS-78 values are "unitless", but there is an implied scale, + isn't there? If I invent a new practical salinity scale where the values + are 0.875 times the PSS-78 values, should the canonical units still be "1"? + How could a user tell them apart? + Units of 1e-3 and 1 are neither descriptive nor helpful to users + who are trying to figure out what the numbers indicate.] + +
      +
    + +
    diff --git a/download/grids.html b/download/grids.html index 1c55b710e..8ec9fb348 100644 --- a/download/grids.html +++ b/download/grids.html @@ -48,16 +48,33 @@

    ERDDAP: Cloud Computing

    ERDDAP -is a web application and a web service that aggregates scientific data from diverse local and -remote sources and offers a simple, consistent way to download subsets of the data in common file +is a web application and a web service that aggregates scientific data from +diverse local and +remote sources and offers a simple, consistent way to download subsets of the +data in common file formats and make graphs and maps. +This web page discusses issues related to heavy ERDDAP usage loads +and explores possibilities for dealing with extremely heavy loads +via grids, clusters, federations, and cloud computing. +

    DISCLAIMER - +The contents of this web page are my (Bob Simons) personal opinions and +do not necessarily reflect any position of the +Government or the National Oceanic and Atmospheric Administration. +The calculations are simplistic, but I think the conclusions are correct. +Did I use faulty logic or make a mistake in my calculations? +If so, the fault is mine alone. +Please send an email with the correction to bob dot simons at noaa dot gov. +The original version was written in June 2009. There have been no significant +changes. This was last updated 2015-06-25.

    Heavy Loads / Constraints

    With heavy use, a standalone ERDDAP will be constrained (from most to least likely) by:
      -
    1. A remote data source's bandwidth - Unless a remote data source has a very high bandwidth +
    2. A remote data source's bandwidth - + Even with an efficient connection (e.g., via OPeNDAP), + unless a remote data source has a very high bandwidth Internet connection, ERDDAP's responses will be constrained by how fast ERDDAP can get data from the data source. A solution is to copy the dataset onto ERDDAP's hard drive, perhaps with @@ -70,7 +87,8 @@

      Heavy Loads / Constraints

      is to get a faster Internet connection.
    3. Memory - If there are many simultaneous requests, ERDDAP can run out of memory and temporarily refuse new requests. - (ERDDAP has a couple of mechanisms to avoid this and to minimize the consequences if it does + (ERDDAP has a couple of mechanisms to avoid this and to minimize the + consequences if it does happen.) So the more memory in the server the better. On a 32-bit server, 4+ GB is really good, 2 GB is okay, less is not recommended. @@ -79,10 +97,18 @@

      Heavy Loads / Constraints

      See the -Xmx and -Xms settings for ERDDAP/Tomcat. -
    4. Hard drive bandwidth - Accessing data stored on the server's hard drive is vastly faster than - accessing remote data. Even so, if the ERDDAP server has a very high bandwidth connection, - it is possible that accessing data on the hard drive will be a bottleneck. A partial solution - is to use fast (e.g., 10,000 RPM) hard drives. + An ERDDAP getting heavy usage on a computer with a 64-bit server + with 8GB of memory and -Xmx set to 4000M is rarely, if ever, constrained by memory. +
    5. Hard drive bandwidth - Accessing data stored on the server's hard drive + is vastly faster than + accessing remote data. Even so, if the ERDDAP server has a very high + bandwidth Internet connection, + it is possible that accessing data on the hard drive will be a bottleneck. + A partial solution + is to use faster (e.g., 10,000 RPM) magnetic hard drives + or SSD drives (if it makes + sense cost-wise). Another solution is to store different datasets + on different drives, so that the cumulative hard drive bandwidth is much higher.
    6. Too many files in a cache directory - ERDDAP caches all images, but only caches the data for certain types of data requests. It is possible for the cache directory for a @@ -90,9 +116,11 @@

      Heavy Loads / Constraints

      if a file is in the cache (really!). <cacheMinutes> in setup.xml lets you set how - long a file can be in the cache before it is deleted. Setting a smaller number would - minimize this problem. -
    7. CPU - Making graphs is the only thing that takes significant CPU time (roughly 1 second per graph). + long a file can be in the cache before it is deleted. Setting a smaller + number would minimize this problem. +
    8. CPU - Making graphs (including maps) + is the only thing that takes significant CPU time + (roughly 0.2 - 1 second per graph). So if there were many simultaneous unique requests for graphs (WMS clients!), there could be a CPU limitation. On a multi-core server, it would take a lot of requests before this became a problem. @@ -103,22 +131,24 @@

      Heavy Loads / Constraints

      Grids, Clusters, and Federations

      Under very heavy use, a single standalone ERDDAP will run into one or more of the constraints listed - above and even the suggested solutions will be insufficient. For such situations, ERDDAP has + above and even the suggested solutions will be insufficient. For such situations, + ERDDAP has features that make it easy to construct scalable grids (also called clusters or federations) of ERDDAPs which allow the system to handle very heavy use (e.g., for a large data center). -

      We're using +

      I'm using grid (external link) - as a general term to indicate a type of computer cluster where all of the + as a general term to indicate a type of computer cluster where all of the parts may or may not be physically located in one facility and may or may not be centrally administered. An advantage of co-located, centrally owned and administered grids (clusters) is that they benefit from economies of scale (especially the human workload) and simplify making the parts of the system work well together. An advantage of non-co-located grids, - non-centrally owned and administered (federations) is that they distribute the human work load - and the cost, and may provide some additional fault tolerance. The solution we propose below - works well for all grid topographies. + non-centrally owned and administered (federations) + is that they distribute the human work load + and the cost, and may provide some additional fault tolerance. + The solution I propose below works well for all grid topographies.

      The basic idea of designing a scalable system is to identify the potential bottlenecks and then design the system so that parts of the system can be replicated as needed to @@ -137,16 +167,20 @@

      Grids, Clusters, and Federations

      The goals of this design are:

        -
      • To make a scalable architecture (one that is easily extensible by replicating any part that - becomes over-burdened). To make an efficient system that maximizes the availability and +
      • To make a scalable architecture + (one that is easily extensible by replicating any part that + becomes over-burdened). To make an efficient system that maximizes the + availability and throughput of the data given the available computing resources. (Cost is almost always an issue.) -
      • To balance the capabilities of the parts of the system so that one part of the system won't - overwhelm another part. +
      • To balance the capabilities of the parts of the system so that one part + of the system won't overwhelm another part.
      • To make a simple architecture so that the system is easy to set up and administer.
      • To make an architecture that works well with all grid topographies. -
      • To make a system that fails gracefully and in a limited way if any part becomes over-burdened. - (The time required to copy a large datasets will always limit the system's ability to deal +
      • To make a system that fails gracefully + and in a limited way if any part becomes over-burdened. + (The time required to copy a large datasets will always limit + the system's ability to deal with sudden increases in the demand for a specific dataset.)
      • (If possible) To make an architecture that isn't tied any specific cloud computing service @@ -156,9 +190,12 @@

        Grids, Clusters, and Federations

        Our recommendations are:
        grid/cluster diagram

          -
        • Basically, we suggest setting up a Composite ERDDAP (D in the diagram), which is a - regular ERDDAP except that it just serves data from other ERDDAPs. The grid's architecture - is designed to shift as much work as possible (CPU usage, memory usage, bandwidth usage) +
        • Basically, I suggest setting up a Composite ERDDAP + (D in the diagram), which is a + regular ERDDAP except that it just serves data from other ERDDAPs. + The grid's architecture + is designed to shift as much work as possible + (CPU usage, memory usage, bandwidth usage) from the Composite ERDDAP to the other ERDDAPs.
        • ERDDAP has two special dataset types, EDDGridFromErddap @@ -166,8 +203,9 @@

          Grids, Clusters, and Federations

          EDDTableFromErddap, which refer to
          datasets on other ERDDAPs. -
        • When the composite ERDDAP receives a request for data or images from these datasets, the - composite ERDDAP redirectsWhen the composite ERDDAP receives a request for data or images from + these datasets, the composite ERDDAP + redirects (external link) the data request to the other ERDDAP server. The result is: @@ -175,70 +213,87 @@

          Grids, Clusters, and Federations

        • This is very efficient (CPU, memory, and bandwidth), because otherwise
          1. The composite ERDDAP has to send the data request to the other ERDDAP. -
          2. The other ERDDAP has to get the data, reformat it, and transmit the data to the composite - ERDDAP. -
          3. The composite ERDDAP has to receive the data (using extra bandwidth), reformat it - (using extra CPU time and memory), and transmit the data to the user (using extra bandwidth). +
          4. The other ERDDAP has to get the data, reformat it, + and transmit the data to the composite ERDDAP. +
          5. The composite ERDDAP has to receive the data (using extra bandwidth), + reformat it (using extra CPU time and memory), + and transmit the data to the user (using extra bandwidth).
          - By redirecting the data request and allowing the other ERDDAP to send the response directly - to the user, the composite ERDDAP spends essentially no CPU time, memory, or bandwidth - on data requests. -
        • The redirect is transparent to the user regardless of the client software (a browser or - any other software or command line tool). + By redirecting the data request and allowing the other ERDDAP to send the + response directly + to the user, the composite ERDDAP spends essentially no CPU time, memory, + or bandwidth on data requests. +
        • The redirect is transparent to the user regardless of the client software + (a browser or any other software or command line tool).

      The parts of the grid are: -

      A) For every ERDDAP data source that has a high-bandwidth server, - use EDDGridFromErddap or +

      A) For every ERDDAP data source that + has a high-bandwidth server, use EDDGridFromErddap or EDDTableFromERDDAP to serve the data in the Composite ERDDAP. -

      B) For every ERDDAP-able data source (a data source from which ERDDAP - can read data) that has a high-bandwidth server, set up another ERDDAP in the grid which +

      B) For every ERDDAP-able data source + (a data source from which ERDDAP + can read data) that has a high-bandwidth server, set up another ERDDAP in + the grid which is responsible for serving the data from this data source.

        -
      • If several such ERDDAPs aren't getting many requests for data, you can consolidate them into - one ERDDAP. -
      • If the ERDDAP dedicated to getting data from one remote source is getting too many requests, - there is a temptation to add additional ERDDAPs to access the remote data source. In special - cases this may make sense, but it is more likely that this will overwhelm the remote data - source (which is self-defeating) and also prevent other users from accessing the remote data - source (which isn't nice). In such a case, consider setting up another ERDDAP to serve that - one dataset and copy the dataset on that ERDDAP's hard drive (see C) using +
      • If several such ERDDAPs aren't getting many requests for data, you can + consolidate them into one ERDDAP. +
      • If the ERDDAP dedicated to getting data from one remote source is + getting too many requests, + there is a temptation to add additional ERDDAPs to access the remote + data source. In special cases this may make sense, + but it is more likely that this will overwhelm the remote data + source (which is self-defeating) and also prevent other users + from accessing the remote data source (which isn't nice). + In such a case, consider setting up another ERDDAP to serve that + one dataset and copy the dataset on that ERDDAP's hard drive (see C), + perhaps with EDDGridCopy and/or EDDTableCopy.
      -

      C) For every ERDDAP-able data source that has a low-bandwidth server - (or is a slow service for - other reasons), consider setting up another ERDDAP and storing a copy of the dataset - on that ERDDAP's hard drives using +

      C) For every ERDDAP-able data source + that has a low-bandwidth server + (or is a slow service for other reasons), + consider setting up another ERDDAP and storing a copy of the dataset + on that ERDDAP's hard drives, perhaps with EDDGridCopy and/or EDDTableCopy. If several such ERDDAPs aren't getting many requests for data, you can consolidate them into one ERDDAP. -

      D) The composite ERDDAP is a regular ERDDAP except that it just serves data from other ERDDAPs. +

      D) The composite ERDDAP is a regular + ERDDAP except that it just serves data from other ERDDAPs.

        -
      • Because the composite ERDDAP has information in memory about all of the datasets, it can +
      • Because the composite ERDDAP has information in memory about all of the + datasets, it can quickly respond to requests for lists of datasets (full text searches, category searches, the list of all datasets), and requests for an individual dataset's Data Access Form, Make A Graph form, or WMS info page. These are all small, dynamically generated, HTML pages based on information which is held in memory. So the responses are very fast. -
      • Because requests for actual data are quickly redirected to the other ERDDAPs, the composite - ERDDAP can quickly respond to requests for actual data without using any CPU time, memory, - or bandwidth. -
      • By shifting as much work as possible (CPU, memory, bandwidth) from the Composite ERDDAP to - the other ERDDAPs, the composite ERDDAP can appear to serve data from all of the datasets - and yet still keep up with very large numbers of data requests from a large number of users. -
      • Preliminary tests indicate that the composite ERDDAP can respond to any request in ~1ms of - CPU time, or 1000 requests/second. So an 8 core processor should be able to respond to - about 8000 requests/second. Although it is possible to envision bursts of higher activity - which would cause slowdowns, that is a lot of throughput. It is likely that data center +
      • Because requests for actual data are quickly redirected to the other ERDDAPs, + the composite + ERDDAP can quickly respond to requests for actual data without using any CPU time, memory, or bandwidth. +
      • By shifting as much work as possible (CPU, memory, bandwidth) + from the Composite ERDDAP to + the other ERDDAPs, the composite ERDDAP can appear to serve data + from all of the datasets + and yet still keep up with very large numbers of data requests + from a large number of users. +
      • Preliminary tests indicate that the composite ERDDAP can respond to + most requests in ~1ms of + CPU time, or 1000 requests/second. So an 8 core processor should be able + to respond to about 8000 requests/second. + Although it is possible to envision bursts of higher activity + which would cause slowdowns, that is a lot of throughput. + It is likely that data center bandwidth will be the bottleneck long before the composite ERDDAP becomes the bottleneck.
      • In very, very extreme cases, you may want to set up more than one composite ERDDAP. It is likely that other parts of the system (notably, the data center's bandwidth) @@ -257,11 +312,12 @@

        Grids, Clusters, and Federations

        title="This link to an external web site does not constitute an endorsement."/>.]
      -

      Datasets In Very High Demand - In the really unusual case that one of the +

      Datasets In Very High Demand - + In the really unusual case that one of the A, B, or C ERDDAPs can't keep up with the requests because of bandwidth or hard drive limitations, - it makes sense to copy the data (again) on to another server+hardDrive+ERDDAP, using - + it makes sense to copy the data (again) on to another server+hardDrive+ERDDAP, + perhaps with EDDGridCopy and/or EDDTableCopy. @@ -269,13 +325,14 @@

      Grids, Clusters, and Federations

      copied dataset appear seamlessly as one dataset in the composite ERDDAP, this is difficult because the two datasets will be in slightly different states at different times (notably, after the original gets new data, but before the copied dataset gets its copy). - Therefore, we recommend that the datasets be given slightly different titles (e.g., + Therefore, I recommend that the datasets be given slightly different titles (e.g., "... (copy #1)" and "... (copy #2)", or perhaps "(mirror #n)" or "(server #n)") and - appear as separate datasets in the composite ERDDAP. Users are used to seeing lists of + appear as separate datasets in the composite ERDDAP. + Users are used to seeing lists of mirror sites (external link) - at popular file download sites, so this shouldn't surprise or disappoint them. + at popular file download sites, so this shouldn't surprise or disappoint them. Because of bandwidth limitations at a given site, it may make sense to have the mirror located at another site. If the mirror copy is at a different data center, accessed just by that data center's composite ERDDAP, the different titles (e.g., "mirror #1) aren't @@ -283,16 +340,16 @@

      Grids, Clusters, and Federations

      RAIDs vs. Regular Hard Drives - If a large dataset or a group of datasets are not heavily used, - it may make sense to store the data on a RAID since it offers fault tolerance and since - you don't need the processing power or bandwidth of another server. But if a dataset is - heavily used, it may make more sense to copy the data on another server + ERDDAP + hard - drive (similar to - what Google does (external link)) - rather than to use one server and a RAID to store - multiple datasets since you get to use both server+hardDrive+ERDDAPs in the grid until - one of them fails. + it may make sense to store the data on a RAID since it offers fault tolerance and since + you don't need the processing power or bandwidth of another server. But if a dataset is + heavily used, it may make more sense to copy the data on another server + ERDDAP + hard + drive (similar to + what Google does (external link)) + rather than to use one server and a RAID to store + multiple datasets since you get to use both server+hardDrive+ERDDAPs in the grid until + one of them fails.

      Failures - What happens if...

        @@ -313,19 +370,23 @@

        Grids, Clusters, and Federations

        src="../images/external.png" align="bottom" alt=" (external link)" title="This link to an external web site does not constitute an endorsement."/>, the system is effectively down until you set up - a replacement. That's not good. So if this is a concern, set up multiple composite ERDDAPs - or a hot or warm failover. Then the effect is minimal. Failures of the composite ERDDAP + a replacement. That's not good. So if this is a concern, + set up multiple composite ERDDAPs + or a hot or warm failover. Then the effect is minimal. + Failures of the composite ERDDAP should be rare because it has minimal hard drive activity.
      -

      Simple, Scalable - This system is easy to set up and administer, and easily extensible when - any part of it becomes over-burdened. The only real limitation for a given data center - is the data center's bandwidth. +

      Simple, Scalable - This system is easy to set up and administer, + and easily extensible when + any part of it becomes over-burdened. The only real limitation for a given data center + is the data center's bandwidth.

      Bandwidth - Note the approximate bandwidth of commonly used components of the system: - - + + + @@ -338,35 +399,36 @@

      Grids, Clusters, and Federations

      And one Gigabit Ethernet LAN (0.1GB/s) can probably saturate an OC-12 Internet connection (0.06GB/s). And at least one source lists OC-12 lines costing about $100,000 per month. - (Yes, these calculations are based on pushing the system to its limits, which is not good - because it leads to very sluggish responses. + (Yes, these calculations are based on pushing the system to its limits, + which is not good because it leads to very sluggish responses. But these calculations are useful for planning and for balancing parts of the system.) - Clearly, a suitably fast Internet connection for your data center is by far the most - expensive part of the system. - You can easily and relatively cheaply build a grid with a dozen servers running a dozen ERDDAPs - which is capable of pumping out lots of data quickly, but a suitably fast Internet - connection will be very, very expensive. - (Fun fact: ERD, which hosts several data servers including an ERDDAP, currently has just - a fractional OC-3, ~0.001 GB/s. So, ERD is bandwidth limited.) + Clearly, a suitably fast Internet connection for your data center is + by far the most expensive part of the system. + You can easily and relatively cheaply build a grid with a dozen servers + running a dozen ERDDAPs + which is capable of pumping out lots of data quickly, + but a suitably fast Internet connection will be very, very expensive. The partial solutions are:
      • Encourage clients to request subsets of the data if that is all that is needed. - If the client only needs data for a small region or at a lower resolution, that is what - they should request. Subsetting is a central focus of the protocols ERDDAP supports for + If the client only needs data for a small region or at a lower resolution, + that is what they should request. + Subsetting is a central focus of the protocols ERDDAP supports for requesting data.
      • Encourage transmitting compressed data. ERDDAP compresses - a data transmission if it + a data transmission if it finds "accept-encoding" in the HTTP GET request header. All web browsers use - "accept-encoding" and automatically decompress the response. Other clients (e.g., - computer programs) have to use it explicitly. -
      • Colocate your servers at an ISP or other site that offers relatively less expensive - bandwidth costs. -
      • Disperse the servers with the ERDDAPs to different institutions so that the costs are - dispersed. You can then link your composite ERDDAP to their ERDDAPs. + "accept-encoding" and automatically decompress the response. Other clients + (e.g., computer programs) have to use it explicitly. +
      • Colocate your servers at an ISP or other site that offers relatively + less expensive bandwidth costs. +
      • Disperse the servers with the ERDDAPs to different institutions so that + the costs are dispersed. + You can then link your composite ERDDAP to their ERDDAPs.
      Note that Cloud Computing and web hosting services - offer all the Internet bandwidth + offer all the Internet bandwidth you need, but don't solve the price problem.

      For general information on designing scalable, high capacity, fault-tolerant systems, @@ -375,7 +437,8 @@

      Grids, Clusters, and Federations

      src="../images/external.png" align="bottom" alt=" (external link)" title="This link to an external web site does not constitute an endorsement."/>. -

      [Yes, these are simplistic calculations, but I think the conclusions are correct. +

      [These are my opinions. +Yes, the calculations are simplistic, but I think the conclusions are correct. Did I use faulty logic or make a mistake in my calculations? If so, the fault is mine alone. Please send an email with the correction to bob dot simons at noaa dot gov.]
        @@ -383,79 +446,125 @@

      Grids, Clusters, and Federations

      Cloud Computing

      -Several companies are starting to offer cloud computing services +Several companies offer cloud computing services (e.g., Amazon Web Services (external link) and - Google App EngineGoogle Cloud Platform (external link)). Web hosting companies (external link) - have offered a range of roughly similar - services since the mid-1990's. Since the ERDDAP grid just consists of ERDDAPs and + have offered simpler services since the mid-1990's, + but the "cloud" services have greatly expanded the flexibility + of the systems and the range of services offered. + Since the ERDDAP grid just consists of ERDDAPs and since ERDDAPs are Java web applications that can run in Tomcat (the most common application server) or other application servers, it should be relatively easy to - set up an ERDDAP grid on a cloud service or web hosting site. The advantages of these - services are: + set up an ERDDAP grid on a cloud service or web hosting site. + The advantages of these services are:
        -
      • They offer access to very high bandwidth Internet connections. This alone may justify - using these services. -
      • They only charge for the services you use. For example, you get access to a very high +
      • They offer access to very high bandwidth Internet connections. + This alone may justify using these services. +
      • They only charge for the services you use. + For example, you get access to a very high bandwidth Internet connection, but you only pay for actual data transferred. That lets you build a system that rarely gets overwhelmed (even at peak demand), - without have to pay for capacity that is rarely used. -
      • They are easily extensible. -
      • They free you from many of the administrative duties of running the servers and networks. + without having to pay for capacity that is rarely used. +
      • They are easily extensible. You can change server types or add + as many servers or as much storage as you want, in less than a minute. + This alone may justify using these services. +
      • They free you from many of the administrative duties of running the + servers and networks. + This alone may justify using these services.
      The disadvantages of these services are:
        -
      • They charge for their services, sometimes a lot (in absolute terms; not that it isn't a - good value). The prices listed here are for - Amazon EC2They charge for their services, sometimes a lot + (in absolute terms; not that it isn't a good value). + The prices listed here are for + Amazon EC2 (external link). - These prices (as of March 2009) will come down, but there will also probably be more - data and more requests for data in the future. + These prices (as of June 2015) will come down. +
        In the past, prices were higher, + but data files and the number of requests were smaller. +
        In the future, prices will be lower, + but data files and the number of requests will be larger. +
        So the details change, but the situation stays relatively constant. +
        And it isn't that the service is over-priced, + it is that we are using and buying a lot of the service.
          -
        • Data Transfer - $0.10/GB for data transfers into the system and $0.10/GB for data - transfers out of the system. - Storing 1 TB in the cloud will cost $100 for the initial data transfer fees alone. - But the ongoing cost for data transfer is the bigger problem. - One SATA hard drive (0.3GB/s) on one server with one ERDDAP can probably +
        • Data Transfer - Data transfers into the system are now free (Yea!). +
          Data transfers out of the system are $0.09/GB. +
          One SATA hard drive (0.3GB/s) on one server with one ERDDAP can probably saturate a Gigabit Ethernet LAN (0.1GB/s). - And one Gigabit Ethernet LAN (0.1GB/s) can probably saturate an OC-12 Internet +
          One Gigabit Ethernet LAN (0.1GB/s) can probably saturate an OC-12 Internet connection (0.06GB/s). - If one OC-12 connection can transmit ~150,000 GB/month, the Data Transfer costs - will be about 150,000 GB @ $0.10/GB = $15,000/month. - Clearly, if you have a dozen hard-working ERDDAP's on a cloud service, your - monthly Data Transfer fees could be substantial (up to $180,000/month). - (Again, it isn't that the service is over-priced, it is that we are buying a lot of - the service.) -
        • Data storage - Amazon charges $150/month per TB. (Compare that to buying a drive - outright for ~$150/TB). So if you need to store lots of data in the cloud, it might - it might be fairly expensive (e.g., 20TB would cost $3000/month). But this is a - minor issue compared to bandwidth/data transfer costs. +
          If one OC-12 connection can transmit ~150,000 GB/month, the Data Transfer costs + could be as much as 150,000 GB @ $0.09/GB = $13,500/month, + which is a significant cost. + Clearly, if you have a dozen hard-working ERDDAP's on a cloud service, your + monthly Data Transfer fees could be substantial (up to $162,000/month). + (Again, it isn't that the service is over-priced, + it is that we are using and buying a lot of the service.) +
        • Data storage - Amazon charges $50/month per TB. + (Compare that to buying a 4TB enterprise drive outright for ~$50/TB, + although the RAID to put it in and administrative costs add to the total cost.) + So if you need to store lots of data in the cloud, + it might be fairly expensive (e.g., 100TB would cost $5000/month). + But unless you have a really large amount of data, + this is a smaller issue than the bandwidth/data transfer costs. + (Again, it isn't that the service is over-priced, + it is that we are using and buying a lot of the service.) +
           
      • The subsetting problem: - The only way to efficiently distribute data which is stored in the cloud, is to have - the program which is distributing the data (ERDDAP) running on a server which has the - data stored on a local hard drive (or a SAN). Many data requests (notably gridded data - requests where the stride value is > 1) can't be done efficiently if the program - has to requests big chunks of a dataset from a non-local data storage system and then - extract a subset. + The only way to efficiently distribute data from data files + is to have the program which is distributing the data (e.g., ERDDAP) running on + a server which has the data stored on a local hard drive + (or similarly fast access to a SAN or local RAID). + Local file systems allow ERDDAP (and underlying libraries, such as netcdf-java) + to request specific byte ranges from the files and get responses very quickly. + Many types of data requests from ERDDAP to the file + (notably gridded data requests where the stride value + is > 1) can't be done efficiently if the program + has to request the entire file or big chunks of a file + from a non-local (hence slower) data storage system and then extract a subset. + If the cloud setup doesn't give ERDDAP fast access to byte ranges of the files + (as with local files), + ERDDAP's access to the data will be a severe bottleneck + and negate other benefits of using a cloud service. + +

        One possible solution is that ERDDAP and the underlying libraries + could be rewritten to use a + MapReduce (external link) + approach to the problem, + i.e., make multiple, simultaneous requests to the file system + and then merge the responses. + Unfortunately, that would be a big project for multiple groups. + It competes with many other projects for the limited development + resources available for ERDDAP, netcdf-java, and other libraries. + And it seems like a lot of effort given that the common solution + (getting local access to the files) already exists. +

      -

      Thanks - Many thanks to Matthew Arrott and his group for their work on putting ERDDAP in +

      Thanks - + Many thanks to Matthew Arrott and his group for their work on putting ERDDAP in the cloud and the resulting discussions. -

      -

      Contact

      -Yes, these are simplistic calculations, but I think the conclusions are correct. +

      Contact

      +The contents of this web page are my (Bob Simons) personal opinions and +do not necessarily reflect any position of the +Government or the National Oceanic and Atmospheric Administration. +The calculations are simplistic, but I think the conclusions are correct. Did I use faulty logic or make a mistake in my calculations? If so, the fault is mine alone. Please send an email with the correction to bob dot simons at noaa dot gov. diff --git a/download/setup.html b/download/setup.html index 60a13bb26..b4adc015c 100644 --- a/download/setup.html +++ b/download/setup.html @@ -82,8 +82,10 @@

      Why use ERDDAP to distribute your data?

      ERDDAP has been installed by over 50 organizations worldwide,

      including:
        -
      • IPSC JRC EC (Maritime Affairs Unit, Institute for the Protection and Security of the Citizen, -Joint Research Centre of the European Commission) +
      • CSIRO and IMOS (Australia's Commonwealth Scientific and Industrial Research Organisation and the Integrated Marine Observing System) + +
      • IPSC JRC EC (Maritime Affairs Unit, Institute for the Protection and Security of the Citizen, +Joint Research Centre of the European Commission)
      • IRD (Institut de Recherche pour le Développement, France)
        CNRS (Centre National de la Recherche Scientifique, France) @@ -94,40 +96,41 @@

        ERDDAP has been installed by over 50 organizatio
        IPSL (Institut Pierre Simon Laplace des sciences de l'environnement, Paris, France)
        LMI ECLAIRS (Laboratoire Mixte International «Etude du Climat en Afrique de l’Ouest et de ses Interactions avec l’Environnement Régional, et appui aux services climatiques») -
      • The Marine Institute (Ireland) +
      • The Marine Institute (Ireland)
      • Marine Instruments S.A. (Spain) -
      • NOAA CW CGOM (CoastWatch Caribbean/Gulf of Mexico Node) +
      • NOAA CW CGOM (CoastWatch Caribbean/Gulf of Mexico Node) -
      • NOAA CW WC (CoastWatch West Coast Node) which is co-located with and works with -
        NOAA ERD (Environmental Research Division of SWFSC of NMFS) +
      • NOAA CW WC (CoastWatch West Coast Node) which is co-located with and works with -
      • NOAA IOOS CeNCOOS (Central and Northern California Ocean Observing System) +
        NOAA ERD (Environmental Research Division of SWFSC of NMFS) + +
      • NOAA IOOS CeNCOOS (Central and Northern California Ocean Observing System) -
      • NOAA IOOS NERACOOS (Northeastern Regional Association of Coastal and Ocean Observing Systems) +
      • NOAA IOOS NERACOOS (Northeastern Regional Association of Coastal and Ocean Observing Systems) -
      • NOAA IOOS NGDAC (National Glider Data Assembly Center) +
      • NOAA IOOS NGDAC (National Glider Data Assembly Center) -
      • NOAA IOOS PacIOOS (Pacific Islands Ocean Observing System) at the University of Hawaii (UH) +
      • NOAA IOOS PacIOOS (Pacific Islands Ocean Observing System) at the University of Hawaii (UH) -
      • NOAA IOOS SECOORA (Southeast Coastal Ocean Observing Regional Association) +
      • NOAA IOOS SECOORA (Southeast Coastal Ocean Observing Regional Association) -
      • NOAA NCEI (National Centers for Environmental Information) / NCDDC +
      • NOAA NCEI (National Centers for Environmental Information) / NCDDC
      • NOAA NGDC STP (National Geophysical Data Center, Solar - Terrestrial Physics) -
      • NOAA OSMC (Observing System Monitoring Center) +
      • NOAA OSMC (Observing System Monitoring Center) -
      • NOAA UAF (Unified Access Framework) +
      • NOAA UAF (Unified Access Framework) -
      • ONC (Ocean Networks Canada) +
      • ONC (Ocean Networks Canada) -
      • R.Tech Engineering, France +
      • R.Tech Engineering, France
      • Stanford University, Hopkins Marine Station -
      • UC Davis BML (University of California at Davis, Bodega Marine Laboratory) +
      • UC Davis BML (University of California at Davis, Bodega Marine Laboratory)
      • USGS Coastal and Marine Geology Program @@ -151,7 +154,10 @@

        Is the installation procedure hard? Can I do it?

        How To Do the Initial Setup of ERDDAP on Your Server

        ERDDAP can run on any server that supports Java and Tomcat (and perhaps other application servers). -ERDDAP has been tested on Linux, Mac, and Windows computers. +ERDDAP has been tested on Linux, Mac, and Windows computers. If you are installing +ERDDAP on an Amazon Web Services EC2 instance, see this +Amazon Web Services Overview +first.

        Running ERDDAP on Windows may have problems: notably, ERDDAP may be unable to delete and/or rename files quickly. This is probably due to antivirus software (e.g., from MacAfee @@ -166,8 +172,12 @@

        How To Do the Initial Setup of ERDDAP on Your Ser Java (external link) - (JRE or JDK) version 1.7 or higher installed (1.8 is STRONGLY RECOMMENDED, - because Java 1.7 will reach its end-of-life in April 2015). + (JRE or JDK) version 1.8 or higher installed + (Java 1.7 is no longer supported because it reached its + end of life (external link) + and so is no longer supported by Oracle). For security reasons, it is almost always best to use the latest version of Java. ERDDAP works with 32 bit or 64 bit Java. 64 bit is preferred for 64-bit operating systems. On Linux, we recommend that you download and install Java even if your computer came with Java @@ -176,14 +186,6 @@

        How To Do the Initial Setup of ERDDAP on Your Ser instructions (external link). - -

        This version of ERDDAP will not work with Java 1.6 (AKA 6). - Java 1.6 is past its official end-of-life and so is no longer supported by Oracle. - As Oracle's Java SE 6 Downloads web page says: "WARNING: These older versions - of the JRE and JDK are provided to help developers debug issues in older systems. - They are not updated with the latest security patches and are not recommended for - use in production." -
         

      • Set up TomcatHow To Do the Initial Setup of ERDDAP on Your Ser (read/write privileges for the apache-tomcat directory tree and <bigParentDirectory> and read-only privileges for directories with data that ERDDAP needs access to). - Set up that user account now. - Then change ownership of the apache-tomcat directory tree to the tomcat user. - From the parent of the apache-tomcat directory, type +
          +
        • You can create the tomcat user account (which has no password) by using the command +
          sudo useradd tomcat -s /bin/bash -p '*' +
        • You can switch to working as user tomcat by using the command +
          sudo su - tomcat +
          (It will ask you for the superuser password for permission to do this.) +
        • You can stop working as user tomcat by using the command +
          exit +
        • You should change ownership of the apache-tomcat directory tree to the tomcat user. + From the parent of the apache-tomcat directory, type
          chown -R tomcat apache-tomcat-8.0.9
          chgrp -R tomcat apache-tomcat-8.0.9
          (but substitute the actual name of your tomcat directory) right after unpacking Tomcat. @@ -242,6 +251,7 @@

          How To Do the Initial Setup of ERDDAP on Your Ser Later, run the startup.sh and shutdown.sh scripts as user "tomcat" so that Tomcat has permission to write to its log files.
            +

      • Set Tomcat's Environmental Variables @@ -251,7 +261,7 @@

        How To Do the Initial Setup of ERDDAP on Your Ser This file will be used by tomcat/bin/startup.sh and shutdown.sh (or \'s and .bat in Windows). The file should contain: -
        export JAVA_HOME=/usr/local/jdk1.8.0_20/jre +
        export JAVA_HOME=/usr/local/jdk1.8.0_51/jre
        export JAVA_OPTS='-server -Djava.awt.headless=true -Xmx1500M -Xms1500M'
        export TOMCAT_HOME=/usr/local/apache-tomcat-8.0.9
        export CATALINA_HOME=/usr/local/apache-tomcat-8.0.9         @@ -360,9 +370,9 @@

        How To Do the Initial Setup of ERDDAP on Your Ser Please check back here tomorrow. Sorry for the inconvenience, but the new version is worth the wait. -->
        On Linux, Mac, and Windows, download - + erddapContent.zip - (version 1.62, size=25,035 bytes, MD5=4B17A93CE52087C53AF6AEA905391418, dated 2015-06-08) + (version 1.64, size=24,996 bytes, MD5=8E375D13D388E0D290110E196117E3DC, dated 2015-08-19) and unzip it into tomcat, creating tomcat/content/erddap . @@ -372,20 +382,12 @@

        How To Do the Initial Setup of ERDDAP on Your Ser in ~tomcat/conf/tomcat8.conf so ERDDAP can find the directory.

        [Some previous versions are also available: -
        1.46 - (size=22,268 bytes, MD5=FE827B9C411ECAC535F8C65392A5CDFD, dated 2013-07-09) -
        1.50 - (size=22,132 bytes, MD5=25697F19B1AE7C595EFE0E3F25DFFDE2, dated 2014-09-06) -
        1.52 - (size=22,132 bytes, MD5=9CFB41D3FA7F6A267B600D629FABBC17, dated 2014-10-03) -
        1.54 - (size=22,132 bytes, MD5=38F003E2E270DA8F01446CB74ABE41E0, dated 2014-10-24) -
        1.56 - (size=24,076 bytes, MD5=93E1272069DDF6C858C48F7EDB0CC690, dated 2014-12-17)
        1.58 (size=25,000 bytes, MD5=620A54B6A52D7C95C0D1635A9FF39DC6, dated 2015-02-25)
        1.60 (size=25,035 bytes, MD5=10332FC959B72F17E6A00AD432A38578, dated 2015-03-12) +
        1.62 + (size=25,035 bytes, MD5=4B17A93CE52087C53AF6AEA905391418, dated 2015-06-08) ]

        Then, @@ -424,11 +426,11 @@

        How To Do the Initial Setup of ERDDAP on Your Ser Please check back here tomorrow. Sorry for the inconvenience, but the new version is worth the wait. -->
        On Linux, Mac, and Windows, download - erddap.war - (version 1.62, size=491,038,285 bytes, MD5=1FDF3F349EC7D87A4A88AD14A7B5F41A, dated 2015-06-09) + (version 1.64, size=511,898,199 bytes, MD5=0E11741BF06EE5388C022F3632EE1915, dated 2015-08-19) into tomcat/webapps . The .war file is big because it contains high resolution coastline, boundary, @@ -437,26 +439,14 @@

        How To Do the Initial Setup of ERDDAP on Your Ser

        [Some previous versions are also available. If you download one of these, rename it erddap.war.
        1.46 - (size=482,115,063 bytes, MD5=26957D4A6866F5DBAF3238E7A39BD0FE, dated 2013-07-09) -
        1.50 - (size=486,076,835 bytes, MD5=917B108FDF089B6426027168A2DC2B8A, dated 2014-09-06) -
        1.52 - (size=486,085,688 bytes, MD5=4CDAF259D9E51792D75C6CE66570BF2C, dated 2014-10-03) -
        1.54 - (size=486,119,509 bytes, MD5=8118C3C4059C341DFC2488B50DAFCB31, dated 2014-10-24) -
        1.56 - (size=490,780,244 bytes, MD5=B35EB5F1774ECB6F098583E757EF3DBA, dated 2014-12-16) -
        1.58 (size=490,902,857 bytes, MD5=B4D1BBDA2FADD88FDCA29461C2578D3D, dated 2015-02-25)
        1.60 (size=490,919,033 bytes, MD5=99319916DA726030C25E0045A65B0971, dated 2015-03-12) +
        1.62 + (size=491,038,285 bytes, MD5=1FDF3F349EC7D87A4A88AD14A7B5F41A, dated 2015-06-09) ]

      • Use ProxyPass so users don't have to put the port number, e.g., :8080, in the URL. @@ -549,7 +539,7 @@

        How To Do an Update of an Existing ERDDAP on Your Serve
      • Download erddap.war - (version 1.62, size=491,038,285 bytes, MD5=1FDF3F349EC7D87A4A88AD14A7B5F41A, dated 2015-06-09) + (version 1.64, size=511,898,199 bytes, MD5=0E11741BF06EE5388C022F3632EE1915, dated 2015-08-19) into a temporary directory.
          @@ -1701,6 +1691,7 @@

        Things You Don't Need To Know

      +
    9. Heavy Loads / Constraints - With heavy use, a standalone ERDDAP may be constrained by various problems. For more information, see the @@ -1719,20 +1710,110 @@

      Things You Don't Need To Know


       
    10. Cloud Computing - - Several companies are starting to offer cloud computing services + Several companies are starting to offer + cloud computing services (external link) (e.g., Amazon Web Services (external link)). Web hosting companies (external link) - have offered a range of roughly similar services since the mid-1990's. - You can use these services to set up a grid/cluster of ERDDAPs to handle very heavy use. + have offered simpler services since the mid-1990's, but the "cloud" services + have greatly expanded the flexibility of the systems and the range of + services offered. + You can use these services to set up a single ERDDAP + or a grid/cluster of ERDDAPs to handle very heavy use. For more information, see cloud computing with ERDDAP.
        +
    11. Amazon Web Services (AWS) EC2 Installation Overview - + Amazon Web Services (AWS) (external link) + is a + cloud computing service (external link) + that offers a wide range of computer infrastructure that you can rent by the hour. + You can install ERDDAP on an + Elastic Compute Cloud (EC2) (external link) + instance (their name for a computer that you can rent by the hour). + AWS has an excellent + AWS User Guide (external link) + and you can use Google to find answers to specific questions you might have. + Brace yourself -- + it is a fair amount of work to get started. But once you get one server up and running, + you can easily rent as many additional resources (servers, databases, SSD-space, etc.) + as you need, at a reasonable price. + [This isn't a recommendation or endorsement of Amazon Web Services. + There are other cloud providers.] + +

      An overview of things you need to do to get ERDDAP running on AWS is: +

        +
      • In general, you will do all the things described in the + AWS User Guide (external link). +
      • Set up an AWS account. +
      • Set up an AWS user within that account with administrator privileges. + Log in as this user to do all the following steps. +
      • Elastic Block Storage (EBS) is the AWS equivalent of a hard drive attached to + your server. + Some EBS space will be allocated when you first create an EC2 instance. + It is persistent storage -- the information isn't lost when you stop + your EC2 instance. And if you change instance types, your EBS space + automatically gets attached to the new instance. +
      • Create an Elastic IP address so that your EC2 instance has a stable, + public URL (as opposed to just a private URL that changes every time you + restart your instance). +
      • Create and start up an EC2 instance (computer). + There are a wide range of + instance types (external link), + each at a different price. + An m4.large or m4.xlarge instance is powerful and is probably suitable + for most uses, but choose whatever meets your needs. + You will probably want to use Amazon's Linux as the operating system. +
      • If your desktop/laptop computer is a Windows computer, you can use + PuTTY (external link), + a free SSH client for Windows, to get access to your EC2 instance's command line. + Or, you may have some other SSH program that you prefer. +
      • When you log into your EC2 instance, you will be logged in as the + administrative user with the user name "ec2-user". + ec2-user has sudo privileges. + So, when you need to do something as the root user, use: sudo someCommand +
      • If your desktop/laptop computer is a Windows computer, you can use + FileZilla (external link), + a free SFTP program, to transfer files + to/from your EC2 instance. Or, you may have some other SFTP program that you prefer. +
      • Install Apache (external link) + on your EC2 instance. +
      • Follow the regular ERDDAP installation instructions. +
          +
      +
    12. WaitThenTryAgain Exception - In unusual circumstances, a user may get an error message like @@ -1920,16 +2001,16 @@

      Programmer's Guide

      (No, we don't use Eclipse, Ant, Maven, or ...; nor do we offer ERDDAP-related support for them. If we required you to use Ant and you preferred Maven (or vice versa), you wouldn't be very happy about it.)
    13. We use a batch file which deletes all of the .class files in the source tree. -
    14. We currently use javac 1.7.0_67 to compile gov.noaa.pfel.coastwatch.TestAll +
    15. We currently use javac 1.8.0_51 to compile gov.noaa.pfel.coastwatch.TestAll (it has links to a few classes that wouldn't be compiled otherwise) - and java 1.7.0_67 and 1.8.0_20 to run the tests. + and java 1.8.0_51 to run the tests. For security reasons, it is almost always best to use the latest versions of Java and Tomcat.
      • When we run javac or java, the current directory is tomcat/webapps/erddap/WEB-INF .
      • Our javac and java classpath (including some unnecessary items) is currently
        -./classes;../../../lib/servlet-api.jar;lib/axis.jar;lib/cassandra-driver-core.jar;lib/netty-all.jar;lib/guava.jar;lib/metrics-core.jar;lib/lz4.jar;lib\snappy-java.jar;lib/commons-compress.jar;lib/commons-discovery.jar;lib/itext-1.3.1.jar;lib/jaxrpc.jar;lib/joid.jar;lib/lucene-core.jar;lib/mail.jar;lib/netcdfAll-latest.jar;lib/postgresql.jdbc.jar;lib/saaj.jar;lib/tsik.jar;lib/wsdl4j.jar +./classes;../../../lib/servlet-api.jar;lib/axis.jar;lib/cassandra-driver-core.jar;lib/netty-all.jar;lib/guava.jar;lib/metrics-core.jar;lib/lz4.jar;lib/snappy-java.jar;lib/commons-compress.jar;lib/commons-discovery.jar;lib/itext-1.3.1.jar;lib/jaxrpc.jar;lib/joid.jar;lib/lucene-core.jar;lib/mail.jar;lib/netcdfAll-latest.jar;lib/postgresql.jdbc.jar;lib/saaj.jar;lib/tsik.jar;lib/wsdl4j.jar;lib/aws-java-sdk.jar;lib/commons-codec.jar;lib/commons-logging.jar;lib/fluent-hc.jar;lib/httpclient.jar;lib/httpclient-cache.jar;lib/httpcore.jar;lib/httpmime.jar;lib/jna.jar;lib/jna-platform.jar;lib/jackson-annotations.jar;lib/jackson-core.jar;lib/jackson-databind.jar
      • So your javac command line will be something like
        javac -cp (classpath above) classes/gov/noaa/pfel/coastwatch/TestAll.java @@ -1943,7 +2024,7 @@

        Programmer's Guide

        (There are some unfinished/unused classes.)
    16. We use some 3rd party source code instead of .jar files (notably for SSHTools and Dods) - and have modified them slightly to avoid problems compiling with Java 1.7. + and have modified them slightly to avoid problems compiling with Java 1.8. We have often made other slight modifications (notably to Dods) for other reasons.
    17. Most classes have test methods. We run lots of tests via TestAll. Unfortunately, many of the tests are specific to our set up. (Sorry.) diff --git a/download/setupDatasetsXml.html b/download/setupDatasetsXml.html index 0cdb80d0e..c421a0088 100644 --- a/download/setupDatasetsXml.html +++ b/download/setupDatasetsXml.html @@ -56,6 +56,7 @@

      Table of Contents

    18. Introduction (Please read all of this.)
      • Some Assembly Required +
      • Data Provider Form
      • Tools
      • The basic structure of the datasets.xml file
          @@ -153,8 +154,8 @@

        Table of Contents

        Introduction

        -

        Some Assembly Required - -Setting up a dataset in ERDDAP isn't just a matter of pointing to the dataset's +

        Some Assembly Required +
        Setting up a dataset in ERDDAP isn't just a matter of pointing to the dataset's directory or URL. You have to write a chunk of XML for datasets.xml which describes the dataset.

        • For gridded datasets, in order to make the dataset conform to ERDDAP's data structure for gridded data, @@ -189,8 +190,75 @@

          Introduction

          dataset in datasets.xml. And if you get stuck, please send an email with the details to bob dot simons at noaa dot gov. -

          Tools - - There are two command line programs which are tools to help you create the XML +

          Data Provider Form +
          When a data provider comes to you hoping to add some data to your ERDDAP, + it can be difficult and time consuming to collect all of the metadata + (information about the dataset) needed to add the dataset into ERDDAP. + Many data sources (for example, .csv files, Excel files, databases) + have no internal metadata. + So ERDDAP has a Data Provider Form which gathers metadata + from the data provider and gives the data provider + some other guidance, including extensive guidance for + Data In Databases. + The information submitted is converted into the datasets.xml format and then + emailed to the ERDDAP administrator (you) and written (appended) to + bigParentDirectory/logs/dataProviderForm.log . + Thus, the form semi-automates the process of getting a dataset into ERDDAP, + but the ERDDAP administrator still has to complete the datasets.xml chunk + and deal with getting the data file(s) from the provider or connecting to the database. + +

          The submission of actual data files from external sources is a huge security risk, + so ERDDAP does not deal with that. You have to figure out a solution that + works for you and the data provider, for example, email (for small files), + pull from the cloud (for example, DropBox or Google Drive), + an sftp site (with passwords), or sneakerNet (a USB thumb drive or external hard drive). + You should probably only accept files from people you know. + You will need to scan the files for viruses and take other security precautions. + +

          There isn't a link in ERDDAP to the Data Provider Form + (for example, on the ERDDAP home page). + Instead, when someone tells you they want to have their data served by your ERDDAP, + you can send them an email saying something like: +

          Yes, we can get your data into ERDDAP. To get started, please fill out the form at
+http://yourUrl/erddap/dataProviderForm.html 
+After you finish, I'll contact you to work out the final details.
+
          + If you just want to look at the form (without filling it out), + you can see the form on ERD's ERDDAP: + Introduction, + Part 1, + Part 2, + Part 3, and + Part 4. + These links on the ERD ERDDAP send information to me, not you, so don't submit + information with them unless you actually want to add data to the ERD ERDDAP. + +

          If you want to remove the Data Provider Form from your ERDDAP, put +
          <dataProviderFormActive>false</dataProviderFormActive> +
          in your setup.xml file. + +

          The impetus for this was NOAA's 2014 Public Access to Research Results (PARR) directive (external link), + which requires that all NOAA environmental data funded through taxpayer dollars + be made available via a data service (not just files) within 12 months of creation. + So there is increased interest in using ERDDAP to make datasets available via a service ASAP. + We needed a more efficient way to deal with a large number of data providers. + +

          Feedback/Suggestions? + This form is new, so please email bob dot simons at noaa dot gov + if you have any feedback or suggestions for improving this. + +

          Tools +
          There are two command line programs which are tools to help you create the XML for each dataset that you want your ERDDAP to serve. Once you have set up ERDDAP and run it (at least one time), you can find and use these programs in the tomcat/webapps/erddap/WEB-INF directory. @@ -304,7 +372,7 @@

          Introduction

          EDDGridFromThreddsCatalog - In general, the options in GenerateDatasetsXml generate a datasets.xml chunk for one dataset from one specific data source. An exception to this is the - EDDGridFromThreddsCatalog option. + EDDType=EDDGridFromThreddsCatalog option. It generates all of the datasets.xml chunks needed for all of the EDDGridFromDap datasets that it can find by crawling recursively through a THREDDS (sub) catalog. There are many forms of THREDDS catalog URLs. @@ -326,9 +394,58 @@

          Introduction

        • If you have problems that you can't solve, send an email to Bob with as much information as possible. +
        + +

        EDDsFromFiles - + In general, the options in GenerateDatasetsXml generate a datasets.xml chunk + for one dataset from one specific data source. Another exception to this is the + EDDType=EDDsFromFiles option. Given a start directory, + this traverses the directory and all subdirectories and tries + to create a dataset for each group of data files that it finds. +

          +
        • This assumes that when a dataset is found, the dataset includes all + subdirectories. +
        • If dataset is found, similar sibling directories will be treated + as separate datasets + (for example, directories for the 1990's, the 2000's, the 2010's, + will generate separate datasets). + They should be easy to combine by hand -- just change the + first dataset's <fileDir> to the parent directory and delete all the + subsequent sibling datasets. +
        • This will only try to generate a chunk of datasets.xml for the most + common type of file extension in a directory (not counting .md5, which is ignored). + So, given a directory with 10 .nc files and 5 .txt files, + a dataset will be generated for the .nc files only. +
        • This assumes that all files in a directory with the same extension + belong in the same dataset. If a directory has some .nc files with SST data + and some .nc files with chlorophyll data, just one sample .nc + file will be read (SST? chlorophyll?) and just one dataset + will be created for that type of file. That dataset will probably fail + to load because of complications from trying to load two types + of files into the same dataset. +
        • If there are fewer than 4 files with the most common extension in a directory, + this assumes that they aren't data files and just skips the directory. +
        • If there are 4 or more files in a directory, + but this can't successfully generate a chunk of datasets.xml for the files + (for example, an unsupported file type), this will generate an + EDDTableFromFileNames + dataset for the files. +
        • At the end of the diagnostics that this writes to the log file, just before + the datasets.xml chunks, this will print a table with a summary of information + gathered by traversing all the subdirectories. The table will + list every subdirectory and indicate the most common type of file extension, + the total number of files, and which type of dataset was create for + these files (if any). If you are faced with a complex, deeply nested + file structure, consider running GenerateDatasetsXml with EDDType=EDDsFromFiles + just to generate this information, +
        • This option may not do a great job of guessing the best EDDType for a given + group of data files, but it is a quick, easy, and generally good first + step in generating the datasets.xml for a file system with lots of + subdirectories, each with data files from different datasets.
           
        +
      • DasDds is a command line program that you can use after you have created a first attempt at the XML for a new dataset in datasets.xml. With DasDds, you can repeatedly test and refine the XML. @@ -479,7 +596,7 @@

        Notes

      • Use the destinationName "altitude" to identify the data's distance above sea level (positive values). Optionally, you may use "altitude" for distances below sea level if the values are negative - below the sea (or if you use, e.g., + below the sea (or if you use, for example,
        <att name="scale_factor" type="int">-1</att> @@ -772,8 +889,9 @@

        Notes

      • EDDGridSideBySide aggregates two or more EDDGrid datasets side by side.
      • EDDGridAggregateExistingDimension - aggregates two or more EDDGrid datasets based - on different values of the first dimension. + aggregates two or more EDDGrid datasets, each of which has + a different range of values for the first dimension, + but identical values for the other dimensions.
      • EDDGridCopy makes a local copy of any EDDGrid's data and serves data from the local copy.
          @@ -983,7 +1101,7 @@

        Detailed Descriptions of Dataset Types

        <att name="axisValues" type="doubleList">2, 2.5, 3, 3.5, 4</att> -
        Note the use of a list data type. Also, the type of list (e.g., double), +
        Note the use of a list data type. Also, the type of list (for example, double), MUST match the dataType of the variable in the EDDTable and EDDGridFromEDDTable datasets.
      • axisValuesStartStrideStop - lets you specify a sequence of @@ -991,7 +1109,7 @@

        Detailed Descriptions of Dataset Types

        axisValues example above:
        <att name="axisValuesStartStrideStop" type="doubleList">2, 0.5, 4</att> -
        Again, note the use of a list data type. Also, the type of list (e.g., double), +
        Again, note the use of a list data type. Also, the type of list (for example, double), MUST match the dataType of the variable in the EDDTable and EDDGridFromEDDTable datasets.
          @@ -1033,7 +1151,7 @@

        Detailed Descriptions of Dataset Types

        not matching exactly (external link). - (e.g., 0.2 vs 0.199999999999996). + (for example, 0.2 vs 0.199999999999996). To (try to) deal with this, EDDGridFromTable lets you specify a precision attribute for any of the axisVariables, which specifies the total number of decimal digits which must be identical. @@ -1325,7 +1443,8 @@

        Detailed Descriptions of Dataset Types

        Currently, no other file types are supported. -But it is usually relatively easy to support other file types. Contact us if you have requests. +But it is usually relatively easy to add support for other file types. +Contact us if you have a request. Or, if your data is in an old file format that you would like to move away from, we recommend converting the files to be NetCDF .nc files. NetCDF is a widely supported format, allows fast random access to the data, and is already supported by ERDDAP. @@ -1334,19 +1453,21 @@

        Detailed Descriptions of Dataset Types

      • Aggregation - This class aggregates data from local files. The resulting dataset appears as if all of the file's data had been combined. - The local files all MUST have the same dataVariables (as defined in the dataset's datasets.xml). - All of the dataVariables MUST use the same axisVariables/dimensions (as defined in the dataset's - datasets.xml). - The files will be aggregated based on the first (left-most) dimension, sorted in ascending order. - Each file MAY have data for one or more values of the first dimension, but there can't be any - overlap between files. - If a file has more than one value for the first dimension, the values MUST be sorted in ascending - order, with no ties. + The local files all MUST have the same dataVariables + (as defined in the dataset's datasets.xml). + All of the dataVariables MUST use the same axisVariables/dimensions + (as defined in the dataset's datasets.xml). + The files will be aggregated based on the first (left-most) dimension, + sorted in ascending order. + Each file MAY have data for one or more values of the first dimension, + but there can't be any overlap between files. + If a file has more than one value for the first dimension, + the values MUST be sorted in ascending order, with no ties. All files MUST have exactly the same values for all of the other dimensions. - All files MUST have exactly the same units metadata for all axisVariables - and dataVariables. - For example, the dimensions might be [time][altitude][latitude][longitude], and the files might - have the data for one time (or more) value(s) per file. + All files MUST have exactly the same units + metadata for all axisVariables and dataVariables. + For example, the dimensions might be [time][altitude][latitude][longitude], + and the files might have the data for one time (or more) value(s) per file. The big advantages of aggregation are:
        • The size of the aggregated data set can be much larger than a single file can be conveniently (~2GB).
        • For near-real-time data, it is easy to add a new file with the latest chunk of data. @@ -1373,8 +1494,8 @@

          Detailed Descriptions of Dataset Types

          Directories - The files MAY be in one directory, or in a directory and its subdirectories (recursively). - If there are a large number of files (for example, >1,000), the operating system (and thus - EDDGridFromFiles) will operate much more efficiently if you store the files + If there are a large number of files (for example, >1,000), the operating system + (and thus EDDGridFromFiles) will operate much more efficiently if you store the files in a series of subdirectories (one per year, or one per month for datasets with very frequent files), so that there are never a huge number of files in a given directory. @@ -1392,11 +1513,13 @@

          Detailed Descriptions of Dataset Types

          <fileTableInMemory>true</fileTableInMemory>
          to tell ERDDAP to keep a copy of the file information tables in memory. -
        • The copy of the file information tables on disk is also useful when ERDDAP is shut down and restarted: +
        • The copy of the file information tables on disk is also useful + when ERDDAP is shut down and restarted: it saves EDDGridFromFiles from having to re-read all of the data files.
        • When a dataset is reloaded, ERDDAP only needs to read the data in new files and files that have changed. -
        • If a file has a different structure from the other files (for example, different data type +
        • If a file has a different structure from the other files + (for example, different data type for one of the variables, different value for the "units" attribute), ERDDAP adds the file to the list of "bad" files. Information about the problem with the file @@ -1418,41 +1541,47 @@

          Detailed Descriptions of Dataset Types

          Handling Requests - When a client's request for data is processed, EDDGridFromFiles can quickly look in the table with the valid file information to see which files have the requested data. -
        • Updating the Cached File Information - Whenever the dataset is reloaded, the cached file - information is updated. +
        • Updating the Cached File Information - Whenever the dataset is reloaded, + the cached file information is updated. - When the dataset is reloaded, ERDDAP compares the currently available files to the cached file - information tables. + When the dataset is reloaded, ERDDAP compares the currently available files + to the cached file information tables. New files are read and added to the valid files table. Files that no longer exist are dropped from the valid files table. Files where the file timestamp has changed are read and their information is updated. The new tables replace the old tables in memory and on disk. -
        • Bad Files - The table of bad files and the reasons the files were declared bad (corrupted file, +
        • Bad Files - The table of bad files and the reasons the files were + declared bad (corrupted file, missing variables, etc.) is emailed to the emailEverythingTo email address (probably you) - every time the dataset is reloaded. You should replace or repair these files as soon as possible. -
        • FTP Trouble/Advice - If you FTP new data files to the ERDDAP server while ERDDAP is running, + every time the dataset is reloaded. You should replace or repair these files + as soon as possible. +
        • FTP Trouble/Advice - If you FTP new data files to the ERDDAP server + while ERDDAP is running, there is the chance that ERDDAP will be reloading the dataset during the FTP process. It happens more often than you might think! - If it happens, the file will appear to be valid (it has a valid name), but the file isn't yet valid. - If ERDDAP tries to read data from that invalid file, the resulting error will cause the file - to be added to the table of invalid files. + If it happens, the file will appear to be valid (it has a valid name), + but the file isn't yet valid. + If ERDDAP tries to read data from that invalid file, the resulting error will + cause the file to be added to the table of invalid files. This is not good. To avoid this problem, use a temporary file name when FTP'ing the file, for example, ABC2005.nc_TEMP . Then, the fileNameRegex test (see below) will indicate that this is not a relevant file. After the FTP process is complete, rename the file to the correct name. The renaming process will cause the file to become relevant in an instant. -
        • The skeleton XML for all EDDGridFromFiles subclasses is: +
        • The skeleton XML + for all EDDGridFromFiles subclasses is: 
           <dataset type="EDDGridFrom...Files" datasetID="..." active="..." >
   <accessibleTo>...</accessibleTo> <!-- 0 or 1 -->
@@ -1519,7 +1648,7 @@ 
          Detailed Descriptions of Dataset Types
          If you have compressed and uncompressed versions of the same data files
   in the same directory, please make sure the <fileNameRegex> for your dataset 
   matches the file names that you want it to match and doesn't match file names that
@@ -1528,7 +1657,7 @@ 
          Detailed Descriptions of Dataset Types
          Compressed source data files must have a file extension, but ERDDAP determines 
   the type of compression by inspecting the contents of the file, not by looking at 
-  the file's file extension (e.g., ".Z"). The supported compression types include
+  the file's file extension (for example, ".Z"). The supported compression types include
   "gz", "bzip2", "xz", "lzma", "snappy-raw", "snappy-framed", "pack200", and "z".
   When ERDDAP reads compressed files, it decompresses on-the-fly, without writing
   to a temporary file.
@@ -1581,23 +1710,30 @@ 
          Detailed Descriptions of Dataset Types
          The parent dataset and all of the child datasets MUST have different datasetIDs.
   If any names in a family are exactly the same, the dataset will fail to load
   (with the error message that the values of the aggregated axis are not in sorted order).
-
        • All children MUST have the same source values for axisVariables[1+] (for example, latitude, longitude). 
-
        • The children may have different source values for axisVariables[0] (for example, time), but they are
-  usually largely the same.
-
        • The parent dataset will appear to have all of the axisVariables[0] source values from all of the children.
-
        • For example, this lets you combine a source dataset with a vector's u-component and another source
-  dataset with a vector's v-component, so the combined data can be served.
+
        • All children MUST have the same source values for axisVariables[1+] 
+  (for example, latitude, longitude). 
+  The precision of the testing is determined by the matchAxisNDigits.
+
        • The children may have different source values for axisVariables[0]
+  (for example, time), but they are usually largely the same.
+
        • The parent dataset will appear to have all of the axisVariables[0] source
+  values from all of the children.
+
        • For example, this lets you combine a source dataset with a vector's 
+  u-component and another source dataset with a vector's v-component, 
+  so the combined data can be served.
 
        • Children created by this method are held privately.
-  They are not separately accessible datasets (for example, by client data requests or by 
+  They are not separately accessible datasets (for example, by client data
+  requests or by 
     flag files).
-
        • The global metadata and settings for the parent comes from the global metadata and settings for
-  the first child.
-
        • If there is an exception while creating the first child, the parent will not be created.
+
        • The global metadata and settings for the parent comes from the global
+  metadata and settings for the first child.
+
        • If there is an exception while creating the first child, 
+  the parent will not be created.
 
        • If there is an exception while creating other children, this sends an email to
   emailEverythingTo (as specified in 
   setup.xml) 
   and continues with the other children.
-
        • The skeleton XML for an EDDGridSideBySide dataset is:
+
        • The skeleton XML for an 
+  EDDGridSideBySide dataset is:
 
           <dataset type="EDDGridSideBySide" datasetID="..." active="..." >
   <accessibleTo>...</accessibleTo> <!-- 0 or 1 -->
@@ -1613,13 +1749,16 @@ 
          Detailed Descriptions of Dataset Types
          
 
 
          EDDGridAggregateExistingDimension 
-aggregates two or more EDDGrid datasets based on different
-  values of the first dimension.
+aggregates two or more EDDGrid datasets each of which has 
+a different range of values for the first dimension, 
+but identical values for the other dimensions.
 
            
-
          • For example, one child dataset might have 366 values (for 2004) for the time dimension and another
-  child might have 365 values (for 2005) for the time dimension.
-
          • All the values for all of the other dimensions (for example, latitude, longitude) MUST be identical
-  for all of the children.
+
          • For example, one child dataset might have 366 values (for 2004) for the time
+dimension and another child might have 365 values (for 2005) for the time dimension.
+
          • All the values for all of the other dimensions (for example, latitude, longitude)
+  MUST be identical for all of the children.
+  The precision of the test is determined by 
+  matchAxisNDigits.  
 
          • Sorted Dimension Values -
    The values for each dimension MUST be in sorted order (ascending or descending). 
    The values can be irregularly spaced. There can be no ties.
@@ -1646,24 +1785,18 @@ 
            Detailed Descriptions of Dataset Types
            The aggregate dataset gets its metadata from the first child.
-
          • ensureAxisValuesAreEqual - This tag is OPTIONAL. 
-  If true (the default), the non-first-axis values MUST be exactly equal in all children. 
-  If false, some minor variation is allowed(for example, it would allow 0.1 in one child and
-  0.1000000002 in another).
-  Only use false if you need to and if you are certain that the variation that is present 
-  is acceptable to you.
 
          • The GenerateDatasetsXml program can make a rough draft of the 
     datasets.xml for an
   EDDGridAggregateExistingDimension based on a set of files served by a Hyrax or THREDDS server.
   For example, use this input for the program (the "/1988" in the URL makes the example run faster):
   
              EDDType? EDDGridAggregateExistingDimension
   
              Server type (hyrax, thredds, or dodsindex)? hyrax
-  
              Parent URL (e.g., for hyrax, ending in "contents.html"; 
+  
              Parent URL (for example, for hyrax, ending in "contents.html"; 
   
                for thredds, ending in "catalog.xml")
   
              ? http://dods.jpl.nasa.gov/opendap/ocean_wind/ccmp/L3.5a/data/
   
                flk/1988/contents.html
-  
              File name regex (e.g., ".*\.nc")? month.*flk\.nc\.gz
-  
              ReloadEveryNMinutes (e.g., 10080)? 10080            
+  
              File name regex (for example, ".*\.nc")? month.*flk\.nc\.gz
+  
              ReloadEveryNMinutes (for example, 10080)? 10080
   
            You can use the resulting <sourceUrl> tags or 
   delete them and uncomment the <sourceUrl> tag 
   (so that new files are noticed each time
@@ -1710,8 +1843,7 @@ 
            Detailed Descriptions of Dataset Types
            <accessibleTo>...</accessibleTo> <!-- 0 or 1 -->
   <defaultDataQuery>...</defaultDataQuery> <!-- 0 or 1 -->
   <defaultGraphQuery>...</defaultGraphQuery> <!-- 0 or 1 -->
@@ -1724,75 +1856,85 @@ 
            Detailed Descriptions of Dataset Types
            
 
 
-
            EDDGridCopy makes and maintains a local copy of another EDDGrid's data and
-serves data from the local copy.
+
            EDDGridCopy makes and maintains a local copy 
+of another EDDGrid's data and serves data from the local copy.
 
              
 
            • EDDGridCopy (and for tabular data, EDDTableCopy)
     is a very easy to use and a very effective
-  solution to some of the biggest problems with serving data from remote data sources:
+  
              solution to some of the biggest problems with serving data from a remote data source:
   
                
   
              • Accessing data from a remote data source can be slow.
     
                  
-    
                • They may be slow because they are inherently slow (for example, an inefficient type of server), 
-    
                • because they are overwhelmed by too many requests,
+    
                • It may be slow because it is inherently slow (for example, an
+      inefficient type of server), 
+    
                • because it is overwhelmed by too many requests,
     
                • or because your server or the remote server is bandwidth limited.
     
                
   
              • The remote dataset is sometimes unavailable (again, for a variety of reasons).
-  
              • Relying on one source for the data doesn't scale well (for example, when many users and many ERDDAPs
-    utilize it).
+  
              • Relying on one source for the data doesn't scale well (for example, 
+    when many users and many ERDDAPs utilize it).
     
                 
   
              
 
-
            • How It Works - EDDGridCopy solves these problems by automatically making and maintaining 
-  a local copy of the data and serving data from the local copy.
+
            • How It Works - EDDGridCopy solves these problems by automatically making and
+  maintaining a local copy of the data and serving data from the local copy.
   ERDDAP can serve data from the local copy very, very quickly.
   And making a local copy relieves the burden on the remote server.
-  And the local copy is a backup of the original, which is useful in case something happens to the
-  original.
+  And the local copy is a backup of the original, which is useful in case
+  something happens to the original.
 
-  
              There is nothing new about making a local copy of a dataset. What is new here is that this class
+  
              There is nothing new about making a local copy of a dataset. 
+  What is new here is that this class
   makes it *easy* to create and *maintain* a local copy of data from a *variety* of types 
   of remote data sources and *add metadata* while copying the data.
 
-
            • Chunks of Data - EDDGridCopy makes the local copy of the data by requesting chunks of data from the
-  remote <dataset> .
+
            • Chunks of Data - EDDGridCopy makes the local copy of the data by requesting 
+  chunks of data from the remote <dataset> .
   There will be a chunk for each value of the leftmost axis variable.
-  Note that EDDGridCopy doesn't rely on the remote dataset's index numbers for the axis -- those
-  may change.
+  Note that EDDGridCopy doesn't rely on the remote dataset's index numbers 
+  for the axis -- those may change.
 
-  
              WARNING: If the size of a chunk of data is so big that it causes problems (> 1GB?), EDDGridCopy
-  can't be used. (Sorry, we hope to have a solution for this problem in the future.)
+  
              WARNING: If the size of a chunk of data is so big (> 2GB) that it causes 
+  problems, EDDGridCopy can't be used. 
+  (Sorry, we hope to have a solution for this problem in the future.)
 
-
            • Local Files - Each chunk of data is stored in a separate netCDF file in a subdirectory of
+
            • Local Files - Each chunk of data is stored in a separate netCDF file in a 
+  subdirectory of
   bigParentDirectory/copy/datasetID/ (as specified in 
     setup.xml).
   File names created from axis values are modified to make them file-name-safe 
   (for example, hyphens are replaced by "x2D") -- this doesn't affect the actual data.
   
               
 
-
            • New Data - Each time EDDGridCopy is reloaded, it checks the remote <dataset> to see what
-  chunks are available.
-  If the file for a chunk of data doesn't already exist, a request to get the chunk is added to a queue.
-  ERDDAP's taskThread processes all the queued requests for chunks of data, 
-    one-by-one.
+
            • New Data - Each time EDDGridCopy is reloaded, 
+  it checks the remote <dataset> to see what chunks are available.
+  If the file for a chunk of data doesn't already exist, a request to get the 
+  chunk is added to a queue.
+  ERDDAP's taskThread processes all the queued 
+  requests for chunks of data, one-by-one.
   You can see statistics for the taskThread's activity on the
      Status Page and in the 
      Daily Report.
-  (Yes, ERDDAP could assign multiple tasks to this process, but that would use up lots of the
-  remote data source's bandwidth, memory, and CPU time, and lots of the local ERDDAP's bandwidth,
+  (Yes, ERDDAP could assign multiple tasks to this process, but that would use 
+  up lots of the remote data source's bandwidth, memory, and CPU time,
+  and lots of the local ERDDAP's bandwidth,
   memory, and CPU time, neither of which is a good idea.)
 
-  
              NOTE: The very first time an EDDGridCopy is loaded, (if all goes well) lots of requests for chunks
-  of data will be added to the taskThread's queue, but no local data files will have been created.
+  
              NOTE: The very first time an EDDGridCopy is loaded, (if all goes well) 
+  lots of requests for chunks
+  of data will be added to the taskThread's queue, but no local data files will
+  have been created.
   So the constructor will fail but taskThread will continue to work and create local files.
   If all goes well, the taskThread will make some local data files and the next attempt to 
-  reload the dataset (in ~15 minutes) will succeed, but initially with a very limited amount of data.
+  reload the dataset (in ~15 minutes) will succeed, but initially with a very 
+  limited amount of data.
 
   
              WARNING: If the remote dataset is large and/or the remote server is slow
-  (that's the problem, isn't it?!), it will take a long time to make a complete local copy.
+  (that's the problem, isn't it?!), it will take a long time to make a complete
+  local copy.
   In some cases, the time needed will be unacceptable.
-  For example, transmitting 1 TB of data over a T1 line (0.15 GB/s) takes at least 60 days,
-  under optimal conditions.
+  For example, transmitting 1 TB of data over a T1 line (0.15 GB/s) takes at 
+  least 60 days, under optimal conditions.
   Plus, it uses lots of bandwidth, memory, and CPU time on the remote and local computers.
   The solution is to mail a hard drive to the administrator of the remote data set so that
   s/he can make a copy of the dataset and mail the hard drive back to you.
@@ -1804,8 +1946,9 @@ 
              Detailed Descriptions of Dataset Types
              WARNING: If a given value for the leftmost axis variable disappears from the remote dataset,
-  EDDGridCopy does NOT delete the local copied file. If you want to, you can delete it yourself.
+  
              WARNING: If a given value for the leftmost axis variable disappears from 
+  the remote dataset, EDDGridCopy does NOT delete the local copied file. 
+  If you want to, you can delete it yourself.
 
 
            • Recommended use -
   
                
@@ -1818,7 +1961,8 @@ 
                Detailed Descriptions of Dataset Types
                Copy the <accessibleTo>, <reloadEveryNMinutes> and 
       <onChange> from the
       remote EDDGrid's XML to the EDDGridCopy's XML.
-      (Their values for EDDGridCopy matter; their values for the inner dataset become irrelevant.)
+      (Their values for EDDGridCopy matter; their values for the inner dataset 
+      become irrelevant.)
     
            
   
          • ERDDAP will make and maintain a local copy of the data.
     
             
@@ -1832,11 +1976,19 @@ 
            Detailed Descriptions of Dataset Types
             
 
-
          • Change Metadata - If you need to change any addAttributes or change the order of the variables
-  associated with the source dataset:
+
          • All axis values must be equal.
+  
            For each of the axes except the leftmost, all of the values must be equal for all children.
+  The precision of the test is determined by matchAxisNValues.
+
+
          • Settings, Metadata, Variables - EDDGridCopy uses settings, metadata,
+  and variables from the enclosed source dataset. 
+
+
          • Change Metadata - If you need to change any addAttributes or change the 
+  order of the variables associated with the source dataset:
   
              
   
            1. Change the addAttributes for the source dataset in datasets.xml, as needed. 
   
            2. Delete one of the copied files.
@@ -2059,7 +2211,7 @@ 
              Detailed Descriptions of Dataset Types
               
   
              Not so simple - Unfortunately, CQL has many restrictions on which columns can be queried
-    with which types of constraints, e.g., partition key columns can be constrained with = and IN,
+    with which types of constraints, for example, partition key columns can be constrained with = and IN,
     so ERDDAP sends some constraints to Cassandra and applies all constraints 
     after the data is received from Cassandra. 
     To help ERDDAP deal efficiently with Cassandra, you need to specify
@@ -2106,8 +2258,8 @@ 
              Detailed Descriptions of Dataset Types
              Single Value Partition Keys - If you want an ERDDAP dataset to work with only 
     one value of one partition key, specify partitionKeySourceName=value. 
-    
              Don't use quotes for a numeric column, e.g., deviceid=1007 
-    
              Do use quotes for a string column, e.g., stationid="Point Pinos"
+    
              Don't use quotes for a numeric column, for example, deviceid=1007 
+    
              Do use quotes for a string column, for example, stationid="Point Pinos"
 
   
            3. Dataset Default Sort Order - the order of the partition key <dataVariable>'s
     in datasets.xml determines the default sort order of the results from Cassandra.
@@ -2250,14 +2402,15 @@ 
              Detailed Descriptions of Dataset Types
              typeLists - ERDDAP's <dataType> for Cassandra dataVariables can include the regular 
+  
              typeLists - ERDDAP's 
+  <dataType> tag for Cassandra dataVariables can include the regular 
   ERDDAP dataTypes (see above) plus several special dataTypes that can be used 
   for Cassandra list columns: booleanList, 
   byteList, shortList, intList, longList, floatList, doubleList, 
   charList, StringList.
   When one of these list columns is in the results being passed to ERDDAP, 
   each row of source data will be expanded
-  to list.size() rows of data in ERDDAP; simple dataTypes (e.g., int)
+  to list.size() rows of data in ERDDAP; simple dataTypes (for example, int)
   in that source data row will be duplicated list.size() times. 
   If the results contain more than one list variable, all lists on a given 
   row of data MUST have the same size and MUST be "parallel" lists,
@@ -2307,10 +2460,10 @@ 
              Detailed Descriptions of Dataset Types
              By default, Cassandra integer nulls will be converted in ERDDAP 
   to 2147483647 for int columns,
   or 9223372036854775807 for long columns.
-  These will appear as "NaN" in some types of text output files (e.g., .csv),
-  "" in other types of text output files (e.g., .htmlTable),
+  These will appear as "NaN" in some types of text output files (for example, .csv),
+  "" in other types of text output files (for example, .htmlTable),
   and the specific number (2147483647 for missing int values) in other types of files
-  (e.g., binary files like .nc and mat).
+  (for example, binary files like .nc and mat).
   A user can search for rows of data with this type of missing value 
   by referring to "NaN", e.g, "&windSpeed=NaN".
   
              If you use some other integer value to indicate missing values in your Cassandra 
@@ -2437,7 +2590,7 @@ 
              Detailed Descriptions of Dataset Types
              "Cassandra stats" Diagnostic Messages - 
     For every ERDDAP user query to a Cassandra dataset, 
     ERDDAP will print a line in the log file, bigParentDirectory/logs/log.txt,
-    with some statistics related to the query, e.g.,
+    with some statistics related to the query, for example,
     
              * Cassandra stats: partitionKeyTable: 2/10000=2e-4 < 0.1 nCassRows=1200 nErddapRows=12000 nRowsToUser=7405
     
              Using the numbers in the example above, this means:
     
                
@@ -2471,7 +2624,7 @@ 
                Detailed Descriptions of Dataset Types
                
     The most important use of these diagnostic messages is to make sure that 
     ERDDAP is doing what you think it is doing.  
-    If it isn't (e.g., is it not reducing the number of distinct combinations as expected?), 
+    If it isn't (for example, is it not reducing the number of distinct combinations as expected?), 
     then you can use the information to try to figure out what's going wrong.
     
                 
 
@@ -2510,7 +2663,7 @@ 
                Detailed Descriptions of Dataset Types
                
 <dataset type="EDDTableFromCassandra" datasetID="..." active="..." >
   <ipAddress>...</ipAddress>
-    <!-- The Cassandra URL without the port number, e.g., 127.0.0.1   REQUIRED. -->
+    <!-- The Cassandra URL without the port number, for example, 127.0.0.1   REQUIRED. -->
   <connectionProperty name="name">value</connectionProperty>
     <!-- The names (for example, "readTimeoutMillis") and values of the 
       Cassandra properties that ERDDAP needs to change.  0 or more. --> 
@@ -2659,7 +2812,7 @@ 
                Detailed Descriptions of Dataset Types
                
     
                The <driverName> to use in datasets.xml (see below) is probably org.mariadb.jdbc.Driver .
-  
              • For MySQL, try 
+  
              • For MySQL and Amazon RDS, try 
     http://dev.mysql.com/downloads/connector/j/ (external link)Detailed Descriptions of Dataset Types<connectionProperty> names (for example, "user",
   "password", and "ssl"), and some of the connectionProperty values can be found by searching
-  the web for "JDBC connection properties databaseType" (for example, Oracle, MySQL, MariaDB, PostgreSQL).
+  the web for "JDBC connection properties databaseType" 
+  (for example, Oracle, MySQL, Amazon RDS, MariaDB, PostgreSQL).
   
                 
 
 
              • One Table - 
@@ -2710,7 +2864,7 @@ 
                Detailed Descriptions of Dataset Types
                
   
              • Change the date and timestamp fields/columns to use 
-    datatype=timestamp with time zone.
+    <dataType>=timestamp with time zone.
       
                Timestamps without time zone information don't work correctly in ERDDAP.
   
              • Make indexes for the columns that users often search. 
   
              • Be very aware of the case of the field/column names
@@ -2759,7 +2913,7 @@ 
                Detailed Descriptions of Dataset Types
                --sql-mode=ANSI_QUOTES (external link) . 
-  
              • For MySQL, you need to run the database with
+  
              • For MySQL and Amazon RDS, you need to run the database with
     --sql-mode=ANSI_QUOTES (external link) .
@@ -2777,7 +2931,7 @@ 
                Detailed Descriptions of Dataset Types
                Database DataTypes - 
+
              • Database <dataType> Tags - 
   Because there is some ambiguity about which
   database data types
@@ -2824,7 +2978,7 @@ 
                Detailed Descriptions of Dataset Types
                In ERDDAP's datasets.xml, in the <dataVariable> tag for 
     a timestamp variable, set
-    
                  <dataType>double</dataType> 
+    
                  <dataType>double</dataType> 
     
                and in <addAttributes> set
     
                  <att name="units">seconds since 1970-01-01T00:00:00Z</att> .
   
              • Suggestion: If the data is a time range, it is useful to have the timestamp values refer to 
@@ -2868,7 +3022,8 @@ 
                Detailed Descriptions of Dataset Types
                Consider replicating, on a different computer, the database and database tables 
     with the data that you want ERDDAP to serve.  
     (Yes, for commercial databases like Oracle, this involves
-    additional licensing fees. But for open source databases, like PostgreSQL, MySQL, and MariaDB,
+    additional licensing fees. But for open source databases, 
+    like PostgreSQL, MySQL, Amazon RDS, and MariaDB,
     this costs nothing.)  This gives you a high level of security and also 
     prevents ERDDAP requests from slowing down the original database.
   
              • We encourage you to set up ERDDAP to connect to the database as a database user that only has
@@ -2942,7 +3097,7 @@ 
                Detailed Descriptions of Dataset Types
                For MySQL, use
+    
                For MySQL and Amazon RDS, use
     
                <connectionProperty name="defaultFetchSize">10000</connectionProperty>
     
                For MariaDB, there is currently no way to change the fetch size. 
       But it is a requested feature, so search the web to see if this has been implemented.
@@ -2969,7 +3124,7 @@ 
                Detailed Descriptions of Dataset Types
                Vacuum the Table -
-    
                MySQL will respond much faster if you use
+    
                MySQL and Amazon RDS will respond much faster if you use
     OPTIMIZE TABLE (external link).
@@ -3001,7 +3156,7 @@ 
                Detailed Descriptions of Dataset Types
                <dataSourceName>java:comp/env/jdbc/postgres/erddap</dataSourceName>
       
                right next to <sourceUrl>, <driverName>, and <connectionProperty>.
-      
                And in tomcat/conf/context.xml, define a resource with the same information, e.g., 
+      
                And in tomcat/conf/context.xml, define a resource with the same information, for example, 
       
                <Resource
       
                  name="jdbc/postgres/erddap" auth="Container" type="javax.sql.DataSource"
       
                  driverClassName="org.postgresql.Driver"
@@ -3049,12 +3204,13 @@ 
                Detailed Descriptions of Dataset Types
                datasetID="..." active="..." >
   <sourceUrl>...</sourceUrl>
     <!-- The format varies for each type of database, but will be something like: 
-      For MariaDB:    jdbc:mariadb://xxx.xxx.xxx.xxx:3306/databaseName
-      For MySql:      jdbc:mysql://xxx.xxx.xxx.xxx:3306/databaseName
-      For Oracle:     jdbc:oracle:thin:@xxx.xxx.xxx.xxx:1521:databaseName
-      For Postgresql: jdbc:postgresql://xxx.xxx.xxx.xxx:5432/databaseName
-      where xxx.xxx.xxx.xxx is the host computer's numeric IP address 
-      followed by :PortNumber, which may be different for your database.
+      For MariaDB:    jdbc:mariadb://xxx.xxx.xxx.xxx:3306/databaseName
+      For MySql       jdbc:mysql://xxx.xxx.xxx.xxx:3306/databaseName
+      For Amazon RDS: jdbc:mysql://xxx.xxx.xxx.xxx:3306/databaseName
+      For Oracle:     jdbc:oracle:thin:@xxx.xxx.xxx.xxx:1521:databaseName
+      For Postgresql: jdbc:postgresql://xxx.xxx.xxx.xxx:5432/databaseName
+      where xxx.xxx.xxx.xxx is the host computer's numeric IP address 
+      followed by :PortNumber (4 digits), which may be different for your database.
       REQUIRED. -->
   <driverName>...</driverName>
     <!-- The high-level name of the database driver, for example, 
@@ -3217,7 +3373,7 @@ 
                Detailed Descriptions of Dataset Types
                addition columns defined by the ERDDAP administrator with information 
-    extracted from the file name (e.g., the time associated with the data in the file)
+    extracted from the file name (for example, the time associated with the data in the file)
     based on two attributes that you specify in the metadata for each additional
     column/dataVariable: 
     
                  
@@ -3274,7 +3430,8 @@ 
                  Detailed Descriptions of Dataset Types
                  <dataType>=int, 
+      yielding a data value of 3.
     
                
 
 
              • No <updateEveryNMillis> - 
@@ -3358,9 +3515,17 @@ 
                Detailed Descriptions of Dataset Types
                
 Currently, no other file types are supported. 
-But it is usually relatively easy to support other file types. Contact us if you have requests.
+But it is usually relatively easy to add support for other file types. Contact us if you have a request.
 Or, if your data is in an old file format that you would like to move away from, 
-we recommend converting the files to be NetCDF .nc files. NetCDF is a widely supported format, 
+we recommend converting the files to be NetCDF .nc files
+(and especially .nc files with the  
+CF Discrete Sampling Geometries (external link) 
+Contiguous Ragged Array data structure -- 
+ERDDAP can extract data from them very quickly). 
+NetCDF is a widely supported format, 
 allows fast random access to the data, and is already supported by ERDDAP.
 
 
                Details - The following information applies to all of the subclasses of EDDTableFromFiles.
@@ -3379,15 +3544,11 @@ 
                Detailed Descriptions of Dataset Types
                units attributes (if any). 
     ERDDAP checks, but it is an imperfect test -- if there are different values, ERDDAP
     doesn't know which is correct and therefore which files are invalid.
+    
                 
   
              
-
            4. Directories - The files can be in one directory, or in a directory 
-  and its subdirectories (recursively).
-  If there are a large number of files (for example, >1,000), the operating system
-  (and thus EDDTableFromFiles) will operate much more efficiently if you store 
-  the files in a
-  series of subdirectories (one per year, or one per month for datasets with very frequent files),
-  so that there are never a huge number of files in a given directory.
-
            5. Cached File Information - When an EDDTableFromFiles dataset is first loaded, 
+
+
            6. Cached File Information
+- When an EDDTableFromFiles dataset is first loaded, 
   EDDTableFromFiles reads information from all of the relevant files 
   and creates tables (one row for each file) with information about each valid file and each "bad" 
   (different or invalid) file. 
@@ -3423,7 +3584,9 @@ 
              Detailed Descriptions of Dataset Types
              flag system
       to force ERDDAP to update the cached file information.
+    
               
   
          
+
 
        • Handling Requests - ERDDAP tabular data requests can put constraints on any variable.
   
            
   
          • When a client's request for data is processed, EDDTableFromFiles can quickly look 
@@ -3436,7 +3599,9 @@ 
            Detailed Descriptions of Dataset Types
            airPressure!=NaN            , EDDTableFromFiles can efficiently determine which buoys 
     have air pressure data.
+    
             
   
          
+
 
        • Updating the Cached File Information - 
     Whenever the dataset is reloaded, the cached file
   information is updated.
@@ -3459,10 +3624,14 @@ 
          Detailed Descriptions of Dataset Types
           
+
 
        • Bad Files - The table of bad files and the reasons the files were declared bad (corrupted file, 
   missing variables, incorrect axis values, etc.) is emailed to the emailEverythingTo email address 
   (probably you) every time the dataset is reloaded.
   You should replace or repair these files as soon as possible.  
+  
           
+
 
        • Near Real Time Data - 
     EDDTableFromFiles treats requests for very recent data as a special case.
   The problem: If the files making up the dataset are updated frequently, it is likely that the
@@ -3472,13 +3641,9 @@ 
          Detailed Descriptions of Dataset Types
          flag system,
       but this might lead to ERDDAP reloading the dataset almost continually.
   So in most cases, we don't recommend it.)
-  Instead, EDDTableFromFiles does two things to deal with this situation:
-  
            
-  
          1. When the dataset is loaded, if the maximum value for the time variable is in the last 24 hours, 
-    ERDDAP sets the maximum time to be NaN (meaning Now).
-  
          2.  When ERDDAP gets a request for data within the last 20 hours (for example, 8 hours ago until Now), 
-    ERDDAP will search all files which have any data in the last 20 hours.
-  
          
+  Instead, EDDTableFromFiles deals with this by the following system:
+  When ERDDAP gets a request for data within the last 20 hours (for example, 8 hours ago until Now), 
+  ERDDAP will search all files which have any data in the last 20 hours.
   Thus, ERDDAP doesn't need to have perfectly up-to-date data for all of the files in order to
   find the latest data.
   You should still set <reloadEveryNMinutes> 
@@ -3500,21 +3665,75 @@ 
          Detailed Descriptions of Dataset Types
          Recommended organization of near-real-time data in the files:
   Store the data in chunks, for example, all data for one station/buoy/trajectory for
   one year (or one month).
-  Then, when a new datum arrives, you only have to read and rewrite the file with this year's
-  (or month's) data. 
-  All of the files for previous years (or months) for that station remain unchanged.
-  And when ERDDAP reloads the dataset, most files are unchanged; only a few, small files have
-  changed and need to be read.
-  
           
-
-  
        • Best: Use the technique recommended above (each file has the data
-  for one station/buoy/trajectory for a chunk of time) with .nc files which have an unlimited 
-  dimension in the files. Then,
+  Then, when a new datum arrives, only the file with this year's (or month's) data is affected.
+  
            
+  
          • Best: Use .nc files with an unlimited dimension (time). Then,
   to add new data, you can just append the new data without having to read and re-write the entire file.
-  
+  The change is made very efficiently and essentially atomically,
+  so the file isn't ever in an inconsistent state.
+  
          • Otherwise: If you don't/can't use .nc files with an unlimited dimension (time),
+  then, when you need to add new data, you have to read and rewrite the entire affected file
+  (hopefully small because it just has a year's (or month's) worth of data).
+  Fortunately, all of the files for previous years (or months) for that station remain unchanged.
+  
          
+  In both cases, when ERDDAP reloads the dataset, most files are unchanged; 
+  only a few, small files have changed and need to be read.
   
           
+
        +
      • Directories - The files can be in one directory, or in a directory + and its subdirectories (recursively). + If there are a large number of files (for example, >1,000), the operating system + (and thus EDDTableFromFiles) will operate much more efficiently if you store + the files in a + series of subdirectories (one per year, or one per month for datasets with very frequent files), + so that there are never a huge number of files in a given directory. +
          + +
      • Millions of Files - + Some datasets have millions of source files. + ERDDAP can handle this, but with mixed results. + For many requests, ERDDAP can scan the dataset's + cached file information + and figure out that only a few of the files + might have data which is relevant to the request and thus respond quickly. + But for other requests (for example, waterTemperature=18 degrees_C) + where any file might have relevant data, + ERDDAP has to open a large number of files to see if each of the files has + any data which is relevant to the request. + The files are opened sequentially. On any operating system and any file system + (other than solid state drives), this takes a long time (so ERDDAP responds slowly) + and really ties up the file system (so ERDDAP responds slowly to other requests). + +

        Fortunately, there is a solution. +

          +
        1. Set up the dataset on a non-public ERDDAP (your personal computer?). +
        2. Create and run a script which requests a series of .ncCF files, each with + a large chunk of the dataset (for example, all of the data for a given month). + (The request will fail if the resulting file would be greater than 2GB, + so choose an appropriate time span for your dataset.) + If the dataset has near-real-time data, run the script to regenerate the + file for the current month frequently (every 10 minutes? every hour?). + Requests to ERDDAP for .ncCF files create a .nc file that uses the + CF Discrete Sampling Geometries (external link) + Contiguous Ragged Array data structures). + +
        3. Set up an EDDTableFromNcCFFiles + dataset on your public ERDDAP which gets data from the .nc(CF) files. + ERDDAP can extract data from these files very quickly. + And since there are now dozens or hundreds (instead of millions) of files, + even if ERDDAP has to open all of the files, it can do so quickly. +
        + This system works very well. +
        [Bob knew this was a possibility, but it was Kevin O'Brien who first did this + and showed that it works well. Now, Bob uses this for the GTSPP dataset + which has about 6 million source files and now has about 500 .nc(CF) files.] +
          +
      • FTP Trouble/Advice - If you FTP new data files to the ERDDAP server while ERDDAP is running, there is the chance that ERDDAP will be reloading the dataset during the FTP process. It happens more often than you might think! @@ -3526,6 +3745,7 @@

        Detailed Descriptions of Dataset Types

         
      • File Name Extracts - EDDTableFromFiles has a system for extracting a String from each file name and using that to make a psuedo data variable. @@ -3587,6 +3807,7 @@

        Detailed Descriptions of Dataset Types

         
      • global: sourceNames - Global metadata attributes in each file can be promoted to be data. If a variable's <sourceName> @@ -3855,10 +4076,10 @@

        Detailed Descriptions of Dataset Types

      • Most often, the files will have column names on the first row and data - starting on the second row. + starting on the second row. (Here, the first row of the file is called row number 1.) But you can use <columnNamesRow> and <firstDataRow> in your datasets.xml file to - a specify different row number. + specify a different row number.
      • ERDDAP allows the rows of data to have different numbers of data values. ERDDAP assumes that the missing data values are the final columns in the row. ERDDAP assigns the standard missing value values for the missing data values. (added v1.56) @@ -3957,7 +4178,7 @@

        Detailed Descriptions of Dataset Types

        GenerateDatasetsXml program to make a rough draft of the datasets.xml chunk for this dataset. You can then edit that to fine tune it. -
      • In most cases, each file has multiple values for the leftmost dimension, e.g. time. +
      • In most cases, each file has multiple values for the leftmost dimension, for example time.
      • The files often (but don't have to) have a single value for the other dimensions (for example, altitude (or depth), latitude, longitude).
      • The files may have character variables with an additional dimension (for example, nCharacters). @@ -4288,7 +4509,7 @@

        Detailed Descriptions of Dataset Types

        GenerateDatasetsXml program to make a rough draft of the datasets.xml chunk for this dataset. You can then edit that to fine tune it. -
      • In most cases, each file has multiple values for the leftmost dimension, e.g. time. +
      • In most cases, each file has multiple values for the leftmost dimension, for example time.
      • The files often (but don't have to) have a single value for the other dimensions (for example, altitude (or depth), latitude, longitude).
      • The files may have character variables with an additional dimension (for example, nCharacters). @@ -4930,7 +5151,7 @@

        Details

        that specifies how often the dataset should be reloaded. For example,
        <reloadEveryNMinutes>60</reloadEveryNMinutes>
          -
        • Generally, datasets that change frequently (e.g., get new data files) +
        • Generally, datasets that change frequently (for example, get new data files) should be reloaded frequently, for example, every 60 minutes.
        • Datasets that change infrequently should be reloaded infrequently, for example, every 1440 minutes (daily) or 10080 minutes (weekly). @@ -4993,7 +5214,7 @@

          Details

          faster and with little or no lag." Yes and no. The problem is that loading more than one dataset at a time creates several hard new problems. They all need to be solved or dealt with. - The current system works well and has manageable problems (e.g., potential + The current system works well and has manageable problems (for example, potential for lag before a flag is noticed). (If you need help managing them, email bob dot simons at noaa dot gov .) Note that the related @@ -5054,13 +5275,13 @@

          Details

          file is incomplete; declare the file to be a "bad" file and remove it (temporarilly) from the dataset.
          To avoid this, we STRONGLY RECOMMEND that you copy a new file - into the directory with a temporary name (e.g., 20150226.ncTmp) + into the directory with a temporary name (for example, 20150226.ncTmp) that doesn't match the datasets fileNameRegex (*\.nc), - then rename the file to the correct name (e.g., 20150226.nc). + then rename the file to the correct name (for example, 20150226.nc). If you use this approach, ERDDAP will ignore the temporary file and only notice the correctly named file when it is complete and ready to be used. -
        • If you modify existing datafiles in place (e.g., to add a new data point), +
        • If you modify existing datafiles in place (for example, to add a new data point), <updateEveryNMillis> will work well if the changes appear atomically (in an instant) and the file is always a valid file. For example, the netcdf-java library allow for additions to the @@ -5180,9 +5401,9 @@

          Details

          Instead, you must provide a "sourceUrl" global attribute, usually in the global >addAttributes<. - If there is no actual source URL (e.g., if the data is stored in local files), + If there is no actual source URL (for example, if the data is stored in local files), this attribute often just has a placeholder value, - e.g., <att name="name">(local files)</att> . + for example, <att name="name">(local files)</att> .
        • For most datasets, this is the base of the url that is used to request data. For example, for DAP servers, this is the url to which .dods, .das, .dds, or .html could be added.
        • If the URL has a query part (after the "?"), it MUST be already @@ -5838,7 +6059,7 @@

          Details

          of metadata conventions used by this dataset.
          If a dataset uses ACDD 1.0, this attribute is a STRONGLY RECOMMENDED, for example,
          <att name="Metadata_Conventions">COARDS, CF-1.6, Unidata Dataset Discovery v1.0</att> -
          But ERDDAP how recommends ACDD-1.3. +
          But ERDDAP now recommends ACDD-1.3. If you have switched your datasets to use ACDD-1.3, use of Metadata_Conventions is STRONGLY DISCOURAGED: just use @@ -6147,7 +6368,7 @@

          Details

          <dataVariable> <sourceName>waterTemperature</sourceName> <destinationName>sea_water_temperature</destinationName> - <dataType>float</dataType> + <dataType>float</dataType> <addAttributes> <att name="ioos_category">Temperature</att> <att name="long_name">Sea Water Temperature</att> @@ -6174,7 +6395,7 @@

          Details

          use:
          <sourceName>=0</sourceName>
          <destinationName>altitude</destinationName> -
          <dataType>float</dataType> +
          <dataType>float</dataType>
        • <destinationName> - the name for the variable that will be shown to and used by ERDDAP users. @@ -6190,7 +6411,7 @@

          Details

          longitude, latitude, altitude (or depth), and time data variables are special.
        -
      • <dataType> - specifies the data type coming from the source. (In some cases, e.g., when reading data from ASCII files, +
      • <dataType> - specifies the data type coming from the source. (In some cases, for example, when reading data from ASCII files, it specifies how the data coming from the source should be stored.)
        • This is REQUIRED by some dataset types and IGNORED by others. @@ -6216,15 +6437,15 @@

          Details

          to tell ERDDAP how to interact with the data source.
      • If you want to change a data variable from the dataType in the source files - (e.g., short) into some other dataType in the dataset (e.g., int), + (for example, short) into some other dataType in the dataset (for example, int), don't use <dataType> to specify what you want. (It works for some types of datasets, but not others.) Instead:
          -
        • Use <dataType> to specify what is in the files (e.g., short). +
        • Use <dataType> to specify what is in the files (for example, short).
        • In the <addAttributes> for the variable, add a scale_factor attribute - with the new dataType (e.g., int) and a value of 1, e.g., + with the new dataType (for example, int) and a value of 1, for example,
          <att name="scale_factor" type="int">1</att>
      @@ -6709,7 +6930,7 @@

      Details


      <att name="time_precision">1970-01-01</att>
      time_precision specifies the precision to be used whenever ERDDAP formats the time values from that variable as strings on web pages, including .htmlTable responses. - In file formats where ERDDAP formats times as strings (e.g., .csv and .json), + In file formats where ERDDAP formats times as strings (for example, .csv and .json), ERDDAP only uses the time_precision-specified format if it includes fractional seconds; otherwise, ERDDAP uses the 1970-01-01T00:00:00Z format.
    19. Valid values are 1970-01, 1970-01-01, 1970-01-01T00Z, @@ -6880,8 +7101,8 @@

      Details

      ERDDAP recognizes timeStamp variables by their time-related "units" metadata, which must match this regular expression "[a-zA-Z]+ +since +[0-9].+" - (for numeric dateTimes, e.g., "seconds since 1970-01-01T00:00:00Z") - or be a dateTime format string containing "yy" or "YY" (e.g., "yyyy-MM-dd'T'HH:mm:ssZ"). + (for numeric dateTimes, for example, "seconds since 1970-01-01T00:00:00Z") + or be a dateTime format string containing "yy" or "YY" (for example, "yyyy-MM-dd'T'HH:mm:ssZ"). But please still use the destinationName "time" for the main dateTime variable.

      Always check your work to be sure that the time data that shows up

      20. ComponentBandwidth (GBytes/s)
      DDR memory~2.5
      ComponentApproximate Bandwidth (GBytes/s)
      DDR memory2.5
      SSD drive1
      SATA hard drive0.3
      Gigabit Ethernet0.1