diff --git a/WEB-INF/classes/com/cohort/array/PrimitiveArray.java b/WEB-INF/classes/com/cohort/array/PrimitiveArray.java index d8601f4b4..ff289b1e5 100644 --- a/WEB-INF/classes/com/cohort/array/PrimitiveArray.java +++ b/WEB-INF/classes/com/cohort/array/PrimitiveArray.java @@ -2813,6 +2813,42 @@ public static boolean testValueOpValue(double value1, String op, double value2) "Unknown operator=\"" + op + "\"."); } + /** + * This tests if 'value1 op value2' is true for doubles to 12 sig figures + * (e.g., for time). + * The <=, >=, and = tests are (partly) done with Math2.almostEqual12 + * so there is a little fudge factor. + * The =~ regex test must be tested with String testValueOpValue, not here, + * because value2 is a regex (not a double). + * + * @param value1 + * @param op one of EDDTable.OPERATORS + * @param value2 + * @return true if 'value1 op value2' is true. + *
Tests of "NaN = NaN" will evaluate to true. + *
Tests of "nonNaN != NaN" will evaluate to true. + *
All other tests where value1 is NaN or value2 is NaN will evaluate to false. + * @throws RuntimeException if trouble (e.g., invalid op) + */ + public static boolean testValueOpValueExtra(double value1, String op, double value2) { + //String2.log("testValueOpValue (double): " + value1 + op + value2); + //if (Double.isNaN(value2) && Double.isNaN(value1)) { //test2 first, less likely to be NaN + // return (op.equals("=") || op.equals("<=") || op.equals(">=")); //the '=' matters + //} + if (op.equals("<=")) return value1 <= value2 || Math2.almostEqual(12, value1, value2); + if (op.equals(">=")) return value1 >= value2 || Math2.almostEqual(12, value1, value2); + if (op.equals("=")) return (Double.isNaN(value1) && Double.isNaN(value2)) || + Math2.almostEqual(12, value1, value2); + if (op.equals("<")) return value1 < value2; + if (op.equals(">")) return value1 > value2; + if (op.equals("!=")) return Double.isNaN(value1) && Double.isNaN(value2)? false : + value1 != value2; + //Regex test has to be handled via String testValueOpValue + // if (op.equals(PrimitiveArray.REGEX_OP)) + throw new SimpleException("Query error: " + + "Unknown operator=\"" + op + "\"."); + } + /** * This tests if 'value1 op value2' is true. * The ops containing with < and > compare value1.toLowerCase() @@ -2855,7 +2891,8 @@ public static boolean testValueOpValue(String value1, String op, String value2) * This tests the keep=true elements to see if 'get(element) op value2' is true. * If the test is false, the keep element is set to false. *
For float and double tests, the <=, >=, and = tests are (partly) - * done with Math2.almostEqual(6) and (9), so there is a little fudge factor. + * done with Math2.almostEqual(6) and (9) and (13 for morePrecise doubles), + * so there is a little fudge factor. *
The =~ regex test is tested with String testValueOpValue, * because value2 is a regex (not a numeric type). * @@ -2871,7 +2908,7 @@ public static boolean testValueOpValue(String value1, String op, String value2) * @return nStillGood * @throws RuntimeException if trouble (e.g., invalid op or invalid keep element) */ - public int applyConstraint(BitSet keep, String op, String value2) { + public int applyConstraint(boolean morePrecise, BitSet keep, String op, String value2) { //regex if (op.equals(REGEX_OP)) { @@ -2925,6 +2962,19 @@ public int applyConstraint(BitSet keep, String op, String value2) { return nStillGood; } + //treat everything else via double tests (that should be all that is left) + if (morePrecise) { + //String2.log("applyConstraint(double)"); + int nStillGood = 0; + double value2d = String2.parseDouble(value2); + for (int row = keep.nextSetBit(0); row >= 0; row = keep.nextSetBit(row + 1)) { + if (testValueOpValueExtra(getDouble(row), op, value2d)) + nStillGood++; + else keep.clear(row); + } + return nStillGood; + } + //treat everything else via double tests (that should be all that is left) //String2.log("applyConstraint(double)"); int nStillGood = 0; @@ -3791,7 +3841,7 @@ public static void testTestValueOpValue() { keep = new BitSet(); keep.set(0, pa.size()); tTime = System.currentTimeMillis(); - pa.applyConstraint(keep, "=~", "(10|zztop)"); + pa.applyConstraint(false, keep, "=~", "(10|zztop)"); pa.justKeep(keep); Test.ensureEqual(pa.size(), 1, ""); Test.ensureEqual(pa.getDouble(0), 10, ""); @@ -3805,7 +3855,7 @@ public static void testTestValueOpValue() { keep = new BitSet(); keep.set(0, pa.size()); tTime = System.currentTimeMillis(); - pa.applyConstraint(keep, ">=", "hubert"); //>= uses case insensitive test + pa.applyConstraint(false, keep, ">=", "hubert"); //>= uses case insensitive test pa.justKeep(keep); Test.ensureEqual(pa.size(), 1, ""); Test.ensureEqual(pa.getString(0), "Nate", ""); @@ -3819,7 +3869,7 @@ public static void testTestValueOpValue() { keep = new BitSet(); keep.set(0, pa.size()); tTime = System.currentTimeMillis(); - pa.applyConstraint(keep, ">=", "9"); + pa.applyConstraint(false, keep, ">=", "9"); pa.justKeep(keep); Test.ensureEqual(pa.size(), 1, ""); Test.ensureEqual(pa.getDouble(0), 10, ""); @@ -3833,7 +3883,7 @@ public static void testTestValueOpValue() { keep = new BitSet(); keep.set(0, pa.size()); tTime = System.currentTimeMillis(); - pa.applyConstraint(keep, ">=", "9"); + pa.applyConstraint(false, keep, ">=", "9"); pa.justKeep(keep); Test.ensureEqual(pa.size(), 1, ""); Test.ensureEqual(pa.getDouble(0), 10, ""); @@ -3847,7 +3897,7 @@ public static void testTestValueOpValue() { keep = new BitSet(); keep.set(0, pa.size()); tTime = System.currentTimeMillis(); - pa.applyConstraint(keep, ">=", "9"); + pa.applyConstraint(false, keep, ">=", "9"); pa.justKeep(keep); Test.ensureEqual(pa.size(), 1, ""); Test.ensureEqual(pa.getDouble(0), 10, ""); diff --git a/WEB-INF/classes/com/cohort/util/Calendar2.java b/WEB-INF/classes/com/cohort/util/Calendar2.java index 8f4f3fbcd..2d7f59099 100644 --- a/WEB-INF/classes/com/cohort/util/Calendar2.java +++ b/WEB-INF/classes/com/cohort/util/Calendar2.java @@ -337,12 +337,11 @@ public static double factorToGetSeconds(String units) throws Exception { * The 'T' connector can be any non-digit. * This may include hours, minutes, seconds, decimal, and Z or timezone offset (default=Zulu). * - * @param isoZuluString + * @param isoZuluString (to millis precision) * @return seconds * @throws exception if trouble (e.g., input is null or invalid format) */ public static double isoStringToEpochSeconds(String isoZuluString) { - //pre 2012-05-22 was return Math2.floorDiv(isoZuluStringToMillis(isoZuluString), 1000); return isoZuluStringToMillis(isoZuluString) / 1000.0; } @@ -351,7 +350,6 @@ public static double isoStringToEpochSeconds(String isoZuluString) { */ public static double safeIsoStringToEpochSeconds(String isoZuluString) { try { - //pre 2012-05-22 was return Math2.floorDiv(isoZuluStringToMillis(isoZuluString), 1000); return isoZuluStringToMillis(isoZuluString) / 1000.0; } catch (Exception e) { return Double.NaN; @@ -507,7 +505,7 @@ public static int isoStringToEpochHours(String isoZuluString) { /** * This converts seconds since 1970-01-01T00:00:00Z * to an ISO Zulu dateTime String with 'T'. - * The doubles are rounded to the nearest second. + * The doubles are rounded to the nearest millisecond. * In many ways trunc would be better, but doubles are often bruised. * round works symmetrically with + and - numbers. * @@ -1011,7 +1009,7 @@ public static String formatAsISODateTimeT3(GregorianCalendar gc) { * limited precision string. * * @param time_precision can be "1970", "1970-01", "1970-01-01", "1970-01-01T00Z", - * "1970-01-01T00:00Z", "1970-01-01T00:00:00Z" (used if time_precision not matched), + * "1970-01-01T00:00Z", "1970-01-01T00:00:00Z" (used if time_precision is null or not matched), * "1970-01-01T00:00:00.0Z", "1970-01-01T00:00:00.00Z", "1970-01-01T00:00:00.000Z". * Versions without 'Z' are allowed. */ @@ -1020,7 +1018,7 @@ public static String limitedFormatAsISODateTimeT(String time_precision, String zString = ""; if (time_precision == null || time_precision.length() == 0) - time_precision = "Z"; + time_precision = "1970-01-01T00:00:00Z"; if (time_precision.charAt(time_precision.length() - 1) == 'Z') { time_precision = time_precision.substring(0, time_precision.length() - 1); zString = "Z"; diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/Projects.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/Projects.java index 696f76392..af306a2bc 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/Projects.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/Projects.java @@ -8491,6 +8491,62 @@ public static void makeIsaacNPH() throws Throwable { table.saveAsFlatNc("/u00/data/points/isaac/NPH_IDS.nc", "time", false); //convertToFakeMV=false } + /** + * Make simpleTest.nc + */ + public static void makeSimpleTestNc() throws Throwable { + Table table = new Table(); + + ByteArray timeStamps = new ByteArray(new byte[]{}); + ByteArray arByte2 = new ByteArray( new byte[] {5, 15, 50, 25}); + + table.addColumn(0, "days", + new ByteArray(new byte[] {1,2,3,4}), + new Attributes()); + table.addColumn(1, "hours", + new ByteArray(new byte[] {5,6,7,8}), + new Attributes()); + table.addColumn(2, "minutes", + new ByteArray(new byte[] {9,10,11,12}), + new Attributes()); + table.addColumn(3, "seconds", + new ByteArray(new byte[] {20,21,22,23}), + new Attributes()); + table.addColumn(4, "millis", + new ByteArray(new byte[] {30,31,32,33}), + new Attributes()); + table.addColumn(5, "bytes", + new ByteArray(new byte[] {40,41,42,43}), + new Attributes()); + table.addColumn(6, "shorts", + new ShortArray(new short[] {10000,10001,10002,10004}), + new Attributes()); + table.addColumn(7, "ints", + new IntArray(new int[] {1000000,1000001,1000002,1000004}), + new Attributes()); + table.addColumn(8, "longs", + new LongArray(new long[] {10000000000L,10000000001L,10000000002L,10000000004L}), + new Attributes()); + table.addColumn(9, "floats", + new FloatArray(new float[] {0,1.1f,2.2f,4.4f}), + new Attributes()); + double d = 1e12; + table.addColumn(10, "doubles", + new DoubleArray(new double[] {d, d+0.1, d+0.2, d+0.3}), + new Attributes()); + table.addColumn(11, "Strings", + new StringArray(new String[] {"0","10","20","30"}), + new Attributes()); + + table.columnAttributes(0).set("units", "days since 1970-01-01"); + table.columnAttributes(1).set("units", "hours since 1980-01-01"); + table.columnAttributes(2).set("units", "minutes since 1990-01-01"); + table.columnAttributes(3).set("units", "seconds since 2000-01-01"); + table.columnAttributes(4).set("units", "millis since 2010-01-01"); + + table.saveAsFlatNc("/erddapTest/simpleTest.nc", "row", false); //convertToFakeMV=false + } + /** * Convert Isaac's PCUI .csv into .nc (and clean up). * I removed 2014 row by hand: 2014,-9999,-9999,-9999,-9999,-9999,-9999 diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java index c65d6b99e..90540df83 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java @@ -136,7 +136,7 @@ public static void main(String args[]) throws Throwable { // String2.log("tl=" + tl + " td=" + td); // -// Table.debug = true; DasDds.main(new String[]{"erdMH1chla1day", "-verbose"}); +// Table.debug = true; DasDds.main(new String[]{"dominic2", "-verbose"}); // String2.log(DigirHelper.getObisInventoryString( // "http://iobis.marine.rutgers.edu/digir2/DiGIR.php", // "OBIS-SEAMAP", @@ -275,9 +275,9 @@ public static void main(String args[]) throws Throwable { // EDDGridFromNcFiles.testGrib2_43(true); //42 or 43 for netcdfAll 4.2- or 4.3+ // EDDGridFromNcFiles.testNc(false); // String2.log(EDDGridFromNcFiles.generateDatasetsXml( -// "c:/data/dominic/", ".*\\.nc", -// "c:/data/dominic/IT_MAG-L1b-GEOF_G16_s20140201000000_e20140201000029_c00000000000000.nc", -// -1, null)); +// "/erddapTest/", "simpleTest\\.nc", +// "/erddapTest/simpleTest.nc", +// 90, null)); /* StringBuilder sb = new StringBuilder(); //for (int i1 = 0; i1 < 4320; i1++) @@ -290,7 +290,7 @@ public static void main(String args[]) throws Throwable { // *** Daily // Projects.viirsLatLon(true); //create -// String2.log(NcHelper.dumpString("/data/scrippsGliders/sp063-20140928T065100.nc", true)); +// String2.log(NcHelper.dumpString("/erddapTest/dominic2/IT_MAG-L1b-GEOF_G16_s20140201000030_e20140201000059_c00000000000000.nc", "IB_time")); // String2.log(NcHelper.dumpString("/u00/data/points/scrippsGlidersIoos1/sp031-20140412T155500.nc", false)); // String2.log(NcHelper.dumpString("c:/u00/satellite/VH/pic/8day/V20120012012008.L3m_8D_NPP_PIC_pic_4km", false)); // String2.log(NcHelper.dumpString("/u00/data/points/tao/daily/airt0n110w_dy.cdf", "AT_21")); @@ -643,7 +643,7 @@ public static void main(String args[]) throws Throwable { // Temporarily switching off parts of McAfee : Virus Scan Console (2X speedup!) // On Access Scanner : All Processes // Scan Items: check: specified file types only (instead of usual All Files) -// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2014, 5, 2014, 8, false); //first/last year(1990..)/month(1..), testMode +// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2014, 3, 2014, 9, false); //first/last year(1990..)/month(1..), testMode // log file is c:/data/gtspp/log.txt // 2b) Email the "good" but "impossible" stations to Charles Sun // [was Melanie Hamilton, now retired] @@ -663,8 +663,8 @@ public static void main(String args[]) throws Throwable { // //one time: EDDTableFromNcFiles.bobFindGtsppDuplicateCruises(); // EDDTableFromNcFiles.testErdGtsppBest("erdGtsppBestNc"); // 6) Create ncCF files with the same date range as 2a) above: -// !!!! HIDE THE WINDOW !!! IT WILL RUN 1000X FASTER!!! -// EDDTableFromNcFiles.bobCreateGtsppNcCFFiles(2014, 5, 2014, 8); //e.g., first/last year(1990..)/month(1..) +// !!!! HIDE THE WINDOW !!! IT WILL RUN MUCH FASTER!!! takes ~2 minutes per month processed +// EDDTableFromNcFiles.bobCreateGtsppNcCFFiles(2014, 3, 2014, 9); //e.g., first/last year(1990..)/month(1..) // String2.log(NcHelper.dumpString("/u00/data/points/gtsppNcCF/201406a.nc", false)); // 7) * Load erdGtsppBest in localHost ERDDAP. (long time if lots of files changed) // * Generate .json file from @@ -921,6 +921,7 @@ public static void main(String args[]) throws Throwable { // Projects.makeCRWNcml34("2000-12-02", 3, "2000-12-05", "dhw"); //sst, anomaly, dhw, hotspot, baa // Projects.makeCRWToMatch("baa"); // Projects.makeCRWNcml34("2013-12-19", 4, "2014-12-31", "baa"); //sst, anomaly, dhw, hotspot, baa +// Projects.makeSimpleTestNc(); // Projects.makeVH1dayNcmlFiles(2012, 2035); // Projects.makeVH8dayNcmlFiles(2012, 2035); // Projects.makeVHmdayNcmlFiles(2012, 2035); @@ -1380,6 +1381,7 @@ public static void main(String args[]) throws Throwable { EDVTime edvt; EDVTimeGridAxis edvtga; EDVTimeStamp edvts; +EDVTimeStampGridAxis edvtsga; Erddap erddap; ErddapRedirect erddapRedirect; FishBase fb; diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc index 00c53255a..792d82c24 100644 Binary files a/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc and b/WEB-INF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc differ diff --git a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java index 1b41cf415..73bab83bf 100644 --- a/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java +++ b/WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java @@ -1546,8 +1546,10 @@ public void setAttributes(int lonIndex, int latIndex, int depthIndex, int timeIn trySet(globalAttributes, "cdm_data_type", cdmDataType); trySet(globalAttributes, "contributor_name", courtesy); trySet(globalAttributes, "contributor_role", "Source of data."); - trySet(globalAttributes, "Conventions", "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"); //unidata-related - trySet(globalAttributes, "Metadata_Conventions", "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"); //unidata-related + trySet(globalAttributes, "Conventions", + "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"); //unidata-related + trySet(globalAttributes, "Metadata_Conventions", + "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"); //unidata-related trySet(globalAttributes, "creator_email", creatorEmail); trySet(globalAttributes, "creator_name", creatorName); trySet(globalAttributes, "creator_url", creatorUrl); @@ -1727,8 +1729,10 @@ public void setActualRangeAndBoundingBox( globalAttributes.remove("time_coverage_start"); //unidata-related globalAttributes.remove("time_coverage_end"); } else { - globalAttributes.set("time_coverage_start", Calendar2.epochSecondsToIsoStringT(range.getDouble(0)) + "Z"); - globalAttributes.set("time_coverage_end", Calendar2.epochSecondsToIsoStringT(range.getDouble(1)) + "Z"); + globalAttributes.set("time_coverage_start", + Calendar2.epochSecondsToIsoStringT(range.getDouble(0)) + "Z"); + globalAttributes.set("time_coverage_end", + Calendar2.epochSecondsToIsoStringT(range.getDouble(1)) + "Z"); } //this doesn't set tableGlobalAttributes.set("time_coverage_resolution", "P1H"); } @@ -11084,7 +11088,7 @@ public int lowApplyConstraint(boolean requireConVar, int idCol, //test the keep=true rows for this constraint PrimitiveArray conPa = getColumn(conVarCol); - int nKeep = conPa.applyConstraint(keep, conOp, conVal); + int nKeep = conPa.applyConstraint(false, keep, conOp, conVal); if (reallyVerbose) String2.log(" applyConstraint: after " + conVar + conOp + "\"" + conVal + "\", " + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/DasDds.java b/WEB-INF/classes/gov/noaa/pfel/erddap/DasDds.java index e2be414e9..7bf94dfa4 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/DasDds.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/DasDds.java @@ -32,8 +32,10 @@ public class DasDds { private void printToBoth(String s) throws IOException { String2.log(s); + String2.flushLog(); outFile.write(s); outFile.write('\n'); + outFile.flush(); } /** This gets the i'th value from args, or prompts the user. */ @@ -104,10 +106,12 @@ public String doIt(String args[], boolean loop) throws Throwable { "\n*** DasDds ***\n" + "This generates the DAS and DDS for a dataset and puts it in\n" + outFileName + "\n" + - "Press ^C to exit at any time.\n\n" + + "Press ^D or ^C to exit at any time.\n\n" + "Which datasetID", datasetID); if (datasetID == null) { + String2.flushLog(); + outFile.flush(); outFile.close(); return String2.readFromFile(outFileName)[1]; } @@ -135,6 +139,7 @@ public String doIt(String args[], boolean loop) throws Throwable { } while (loop && args.length == 0); + outFile.flush(); outFile.close(); String ret = String2.readFromFile(outFileName)[1]; String2.returnLoggingToSystemOut(); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java index 458c4bdce..6dba95f54 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/Erddap.java @@ -2985,7 +2985,8 @@ Spec questions? Ask Jeff DLb (author of WMS spec!): Jeff.deLaBeaujardiere@noaa.g String fileTypeName = EDDTable.sosResponseFormatToFileTypeName(responseFormat); if (fileTypeName == null) //this format EDStatic.queryError + "xxx=" is parsed by Erddap section "deal with SOS error" - throw new SimpleException(EDStatic.queryError + "responseFormat=" + responseFormat + " is invalid."); + throw new SimpleException(EDStatic.queryError + + "responseFormat=" + responseFormat + " is invalid."); String responseMode = queryMap.get("responsemode"); //map keys are lowercase if (responseMode == null) @@ -4285,18 +4286,22 @@ public void doWmsGetMap(HttpServletRequest request, HttpServletResponse response //bboxCsv = "-180,-90,180,90"; //be lenient, default to full range double bbox[] = String2.toDoubleArray(String2.split(bboxCsv, ',')); if (bbox.length != 4) - throw new SimpleException(EDStatic.queryError + "BBOX length=" + bbox.length + " must be 4."); + throw new SimpleException(EDStatic.queryError + + "BBOX length=" + bbox.length + " must be 4."); double minx = bbox[0]; double miny = bbox[1]; double maxx = bbox[2]; double maxy = bbox[3]; if (!Math2.isFinite(minx) || !Math2.isFinite(miny) || !Math2.isFinite(maxx) || !Math2.isFinite(maxy)) - throw new SimpleException(EDStatic.queryError + "invalid number in BBOX=" + bboxCsv + "."); + throw new SimpleException(EDStatic.queryError + + "invalid number in BBOX=" + bboxCsv + "."); if (minx >= maxx) - throw new SimpleException(EDStatic.queryError + "BBOX minx=" + minx + " must be < maxx=" + maxx + "."); + throw new SimpleException(EDStatic.queryError + + "BBOX minx=" + minx + " must be < maxx=" + maxx + "."); if (miny >= maxy) - throw new SimpleException(EDStatic.queryError + "BBOX miny=" + miny + " must be < maxy=" + maxy + "."); + throw new SimpleException(EDStatic.queryError + + "BBOX miny=" + miny + " must be < maxy=" + maxy + "."); //if request is for JUST a transparent, non-data layer, use a _wms/... cache @@ -4418,7 +4423,8 @@ public void doWmsGetMap(HttpServletRequest request, HttpServletResponse response if (avi == eddGrid.lonIndex()) { if (maxx <= av.destinationMin() || minx >= av.destinationMax()) { - if (reallyVerbose) String2.log(" layer=" + layeri + " rejected because request is out of lon range."); + if (reallyVerbose) String2.log(" layer=" + layeri + + " rejected because request is out of lon range."); continue LAYER; } int first = av.destinationToClosestSourceIndex(minx); @@ -4432,7 +4438,8 @@ public void doWmsGetMap(HttpServletRequest request, HttpServletResponse response if (avi == eddGrid.latIndex()) { if (maxy <= av.destinationMin() || miny >= av.destinationMax()) { - if (reallyVerbose) String2.log(" layer=" + layeri + " rejected because request is out of lat range."); + if (reallyVerbose) String2.log(" layer=" + layeri + + " rejected because request is out of lat range."); continue LAYER; } int first = av.destinationToClosestSourceIndex(miny); @@ -4461,7 +4468,8 @@ public void doWmsGetMap(HttpServletRequest request, HttpServletResponse response if (Double.isNaN(tValueD) || tValueD < av.destinationCoarseMin() || tValueD > av.destinationCoarseMax()) { - if (reallyVerbose) String2.log(" layer=" + layeri + " rejected because tValueD=" + tValueD + + if (reallyVerbose) String2.log(" layer=" + layeri + + " rejected because tValueD=" + tValueD + " for " + tAvName); continue LAYER; } @@ -9827,7 +9835,8 @@ public void doAddSubscription(HttpServletRequest request, HttpServletResponse re } else if (tEmail.length() > Subscriptions.EMAIL_LENGTH) { trouble += "
  • " + EDStatic.subscriptionEmailTooLong + "\n"; tEmail = ""; //Security: if it was bad, don't show it in form (could be malicious java script) - } else if (!String2.isEmailAddress(tEmail)) { + } else if (!String2.isEmailAddress(tEmail) || + tEmail.startsWith("your.name") || tEmail.startsWith("your.email")) { trouble += "
  • " + EDStatic.subscriptionEmailInvalid + "\n"; tEmail = ""; //Security: if it was bad, don't show it in form (could be malicious java script) } @@ -9972,7 +9981,8 @@ public void doListSubscriptions(HttpServletRequest request, HttpServletResponse } else if (tEmail.length() > Subscriptions.EMAIL_LENGTH) { trouble += "
  • " + EDStatic.subscriptionEmailTooLong + "\n"; tEmail = ""; //Security: if it was bad, don't show it in form (could be malicious java script) - } else if (!String2.isEmailAddress(tEmail)) { + } else if (!String2.isEmailAddress(tEmail) || + tEmail.startsWith("your.name") || tEmail.startsWith("your.email")) { trouble += "
  • " + EDStatic.subscriptionEmailInvalid + "\n"; tEmail = ""; //Security: if it was bad, don't show it in form (could be malicious java script) } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java b/WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java index 129535622..e24873444 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/GenerateDatasetsXml.java @@ -44,8 +44,10 @@ public class GenerateDatasetsXml { private void printToBoth(String s) throws IOException { String2.log(s); + String2.flushLog(); outFile.write(s); outFile.write('\n'); + outFile.flush(); } /** This gets the i'th value from args, or prompts the user. @@ -147,7 +149,7 @@ public String doIt(String args[], boolean loop) throws Throwable { "\n*** GenerateDatasetsXml ***\n" + "Results are shown on the screen and put in\n" + outFileName + "\n" + - "Press ^C to exit at any time.\n" + + "Press ^D or ^C to exit at any time.\n" + "Type \"\" to change from a non-nothing default back to nothing.\n" + "DISCLAIMER:\n" + " The chunk of datasets.xml made by GenerateDatasetsXml isn't perfect.\n" + @@ -416,6 +418,8 @@ public String doIt(String args[], boolean loop) throws Throwable { } catch (Throwable t) { String msg = MustBe.throwableToString(t); if (msg.indexOf("ControlC") >= 0) { + String2.flushLog(); + outFile.flush(); outFile.close(); return String2.readFromFile(outFileName)[1]; } @@ -481,7 +485,7 @@ public String doIt(String args[], boolean loop) throws Throwable { outFile = new BufferedWriter(new OutputStreamWriter( new FileOutputStream(tempName), "ISO-8859-1")); //charset to match datasets.xml - //look for the beginLine + //look for the beginLine String line = inFile.readLine(); while (line != null && line.indexOf(endTag) < 0 && line.indexOf(beginLine) < 0) { @@ -490,20 +494,28 @@ public String doIt(String args[], boolean loop) throws Throwable { outFile.write(line + "\n"); line = inFile.readLine(); } + + //unexpected end of file? if (line == null) { inFile.close(); inFile = null; outFile.close(); outFile = null; File2.delete(tempName); - throw new RuntimeException(abandoningI + endTag + " not found in " + - datasetsXmlName); + throw new RuntimeException(abandoningI + "\"" + beginLine + + "\" and \"" + endTag + "\" not found in " + datasetsXmlName); } + + //found end of file if (line.indexOf(endTag) >= 0) { //found endTag . Write new stuff just before endTag. + if (reallyVerbose) + String2.log("found endTag=" + endTag + " so writing info at end of file."); outFile.write(beginLine + timeEol); outFile.write(ret); outFile.write(endLine + timeEol); - outFile.write(line + "\n"); + outFile.write(line + "\n"); //line with endTag } else { //found beginLine, so now look for the endLine (discard lines in between) + if (reallyVerbose) + String2.log("found beginLine: " + beginLine); while (line != null && line.indexOf(endTag) < 0 && line.indexOf(endLine) < 0) line = inFile.readLine(); @@ -515,6 +527,8 @@ public String doIt(String args[], boolean loop) throws Throwable { datasetsXmlName); } //found endLine. finish up. + if (reallyVerbose) + String2.log("found endLine: " + endLine); outFile.write(beginLine + timeEol); outFile.write(ret); outFile.write(endLine + timeEol); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java b/WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java index 712b3bdde..2511a04e2 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/LoadDatasets.java @@ -23,7 +23,7 @@ import gov.noaa.pfel.erddap.util.*; import gov.noaa.pfel.erddap.variable.EDV; import gov.noaa.pfel.erddap.variable.EDVGridAxis; -import gov.noaa.pfel.erddap.variable.EDVTimeGridAxis; +//import gov.noaa.pfel.erddap.variable.EDVTimeGridAxis; import java.io.FileInputStream; import java.net.HttpURLConnection; diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/AxisDataAccessor.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/AxisDataAccessor.java index 05964183f..ec1a2d385 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/AxisDataAccessor.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/AxisDataAccessor.java @@ -210,11 +210,13 @@ public AxisDataAccessor(EDDGrid tEDDGrid, String tRequestUrl, String tUserDapQue globalAttributes.set("geospatial_vertical_max", dMax); } } else if (rAxisVariables[av] instanceof EDVTimeGridAxis) { - if (Double.isNaN(dMin)) { - } else { //always iso string - globalAttributes.set("time_coverage_start", Calendar2.epochSecondsToIsoStringT(dMin) + "Z"); //unidata-related - globalAttributes.set("time_coverage_end", Calendar2.epochSecondsToIsoStringT(dMax) + "Z"); - } + String tp = rAxisVariables[av].combinedAttributes().getString( + EDV.TIME_PRECISION); + //"" unsets the attribute if dMin or dMax isNaN + globalAttributes.set("time_coverage_start", + Calendar2.epochSecondsToLimitedIsoStringT(tp, dMin, "")); + globalAttributes.set("time_coverage_end", + Calendar2.epochSecondsToLimitedIsoStringT(tp, dMax, "")); } } } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java index b3723b005..6b0ef485e 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDD.java @@ -582,16 +582,11 @@ public void ensureValid() throws Throwable { combinedGlobalAttributes.remove("featureType"); //featureType is for point types only (table 9.1) Test.ensureTrue(dataVariables != null && dataVariables.length > 0, errorInMethod + "'dataVariables' wasn't set."); - HashSet destNames = new HashSet(Math2.roundToInt(1.4 * dataVariables.length)); for (int i = 0; i < dataVariables.length; i++) { Test.ensureNotNull(dataVariables[i], errorInMethod + "'dataVariables[" + i + "]' wasn't set."); String tErrorInMethod = errorInMethod + "for dataVariable #" + i + "=" + dataVariables[i].destinationName() + ":\n"; dataVariables[i].ensureValid(tErrorInMethod); - if (!destNames.add(dataVariables[i].destinationName())) - throw new IllegalArgumentException(tErrorInMethod + - "Two variables have destinationName=" + - dataVariables[i].destinationName() + "."); } //ensure these are set in the constructor (they may be "") extendedSummary(); //ensures that extendedSummaryPartB is constructed diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java index 036238f48..392dd8c0e 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGrid.java @@ -71,6 +71,7 @@ import ucar.ma2.Array; +import ucar.ma2.DataType; import ucar.nc2.Dimension; import ucar.nc2.dataset.NetcdfDataset; @@ -533,19 +534,6 @@ else if (!lonVar.isEvenlySpaced() || //???Future: not necessary? draw map as ap EDStatic.noXxxNoLLEvenlySpaced)); //else { //NO. other axes are allowed. - // //is there an axis (with size > 0) that isn't one of LLAT? - // for (int av = 0; av < axisVariables.length; av++) { - // if (av == lonIndex || av == latIndex || - // av == altIndex || av == depthIndex || - // av == timeIndex) { - // } else if (axisVariables[av].sourceValues().size() > 1) { - // accessibleViaGeoServicesRest = String2.canonical(start + "???"); - // } - // } - - //else for (int dv = 0; dv < dataVariables.length; dv++) - // if (dataVariables[dv].hasColorBarMinMax()) - // accessibleViaGeoServicesRest = String2.canonical("???"); } @@ -610,19 +598,6 @@ else if (!lonVar.isEvenlySpaced() || //???Future: not necessary? draw map as ap EDStatic.noXxxNoLLEvenlySpaced)); //else { //NO. other axes are allowed. - // //is there an axis (with size > 0) that isn't one of LLAT? - // for (int av = 0; av < axisVariables.length; av++) { - // if (av == lonIndex || av == latIndex || - // av == altIndex || av == depthIndex || - // av == timeIndex) { - // } else if (axisVariables[av].sourceValues().size() > 1) { - // accessibleViaWCS = String2.canonical(start + "???"); - // } - // } - - //else for (int dv = 0; dv < dataVariables.length; dv++) - // if (dataVariables[dv].hasColorBarMinMax()) - // accessibleViaWCS = String2.canonical("???"); else accessibleViaWCS = String2.canonical(""); } @@ -731,6 +706,8 @@ public void ensureValid() throws Throwable { super.ensureValid(); String errorInMethod = "datasets.xml/EDDGrid.ensureValid error for datasetID=" + datasetID + ":\n "; + HashSet sourceNamesHS = new HashSet(2 * (axisVariables.length + dataVariables.length)); + HashSet destNamesHS = new HashSet(2 * (axisVariables.length + dataVariables.length)); for (int v = 0; v < axisVariables.length; v++) { Test.ensureTrue(axisVariables[v] != null, errorInMethod + "axisVariable[" + v + "] is null."); @@ -739,7 +716,38 @@ public void ensureValid() throws Throwable { Test.ensureTrue(axisVariables[v] instanceof EDVGridAxis, tErrorInMethod + "axisVariable[" + v + "] isn't an EDVGridAxis."); axisVariables[v].ensureValid(tErrorInMethod); + + //ensure unique sourceNames + String sn = axisVariables[v].sourceName(); + if (!sn.startsWith("=")) { + if (!sourceNamesHS.add(sn)) + throw new RuntimeException(errorInMethod + + "Two axisVariables have the same sourceName=" + sn + "."); + } + + //ensure unique destNames + String dn = axisVariables[v].destinationName(); + if (!destNamesHS.add(dn)) + throw new RuntimeException(errorInMethod + + "Two axisVariables have the same destinationName=" + dn + "."); + } + + for (int v = 0; v < dataVariables.length; v++) { + //ensure unique sourceNames + String sn = dataVariables[v].sourceName(); + if (!sn.startsWith("=")) { + if (!sourceNamesHS.add(sn)) + throw new RuntimeException(errorInMethod + + "Two variables have the same sourceName=" + sn + "."); + } + + //ensure unique destNames + String dn = dataVariables[v].destinationName(); + if (!destNamesHS.add(dn)) + throw new RuntimeException(errorInMethod + + "Two variables have the same destinationName=" + dn + "."); } + Test.ensureTrue(lonIndex < 0 || axisVariables[lonIndex] instanceof EDVLonGridAxis, errorInMethod + "axisVariable[lonIndex=" + lonIndex + "] isn't an EDVLonGridAxis."); Test.ensureTrue(latIndex < 0 || axisVariables[latIndex] instanceof EDVLatGridAxis, @@ -840,13 +848,14 @@ public void ensureValid() throws Throwable { if (pa != null) { //it should be; but it can be low,high or high,low, so double ttMin = Math.min(pa.getDouble(0), pa.getDouble(1)); double ttMax = Math.max(pa.getDouble(0), pa.getDouble(1)); - if (!Double.isNaN(ttMin)) //it should be - combinedGlobalAttributes.set("time_coverage_start", - Calendar2.epochSecondsToIsoStringT(ttMin) + "Z"); + String tp = axisVariables[av].combinedAttributes().getString( + EDV.TIME_PRECISION); + //"" unsets the attribute if dMin or dMax isNaN + combinedGlobalAttributes.set("time_coverage_start", + Calendar2.epochSecondsToLimitedIsoStringT(tp, ttMin, "")); //for tables (not grids) will be NaN for 'present'. Deal with this better??? - if (!Double.isNaN(ttMax)) //it should be - combinedGlobalAttributes.set("time_coverage_end", - Calendar2.epochSecondsToIsoStringT(ttMax) + "Z"); + combinedGlobalAttributes.set("time_coverage_end", + Calendar2.epochSecondsToLimitedIsoStringT(tp, ttMax, "")); } } @@ -1359,7 +1368,8 @@ protected int[] parseAxisBrackets(String deQuery, String destinationName, EDVGridAxis av = axisVariables[axis]; int nAvSourceValues = av.sourceValues().size(); - int precision = axis == timeIndex? 9 : 5; + int precision = av instanceof EDVTimeStampGridAxis? 13 : + av.destinationDataTypeClass() == double.class? 9 : 5; String diagnostic = MessageFormat.format(EDStatic.queryErrorGridDiagnostic, destinationName, "" + axis, av.destinationName()); @@ -1502,7 +1512,8 @@ else throw new SimpleException(EDStatic.queryError + throw new SimpleException(EDStatic.queryError + diagnostic + ": " + MessageFormat.format(EDStatic.queryErrorGridMissing, EDStatic.EDDGridStart)); - double startDestD = av.destinationToDouble(startS); + double startDestD = av.destinationToDouble(startS); //ISO 8601 times -> to epochSeconds w/millis precision + //String2.log("\n! startS=" + startS + " startDestD=" + startDestD + "\n"); //since closest() below makes far out values valid, need to test validity if (Double.isNaN(startDestD)) { @@ -1536,6 +1547,8 @@ else throw new SimpleException(MustBe.THERE_IS_NO_DATA + } startI = av.destinationToClosestSourceIndex(startDestD); + //String2.log("!ParseAxisBrackets startS=" + startS + " startD=" + startDestD + " startI=" + startI); + } else { //it must be a >= 0 integer index if (!startS.matches("[0-9]+")) { @@ -1547,6 +1560,7 @@ else throw new SimpleException(EDStatic.queryError + } startI = String2.parseInt(startS); + if (startI < 0 || startI > nAvSourceValues - 1) { if (repair) startI = 0; else throw new SimpleException(EDStatic.queryError + @@ -1570,7 +1584,8 @@ else throw new SimpleException(EDStatic.queryError + diagnostic + ": " + MessageFormat.format(EDStatic.queryErrorGridMissing, EDStatic.EDDGridStop)); - double stopDestD = av.destinationToDouble(stopS); + double stopDestD = av.destinationToDouble(stopS); //ISO 8601 times -> to epochSeconds w/millis precision + //String2.log("\n! stopS=" + stopS + " stopDestD=" + stopDestD + "\n"); //since closest() below makes far out values valid, need to test validity if (Double.isNaN(stopDestD)) { @@ -1604,6 +1619,8 @@ else throw new SimpleException(MustBe.THERE_IS_NO_DATA + } stopI = av.destinationToClosestSourceIndex(stopDestD); + //String2.log("!ParseAxisBrackets stopS=" + stopS + " stopD=" + stopDestD + " stopI=" + stopI); + } else { //it must be a >= 0 integer index stopS = stopS.trim(); @@ -2610,10 +2627,10 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, latStop=Double.NaN, lonStop=Double.NaN, timeStop=Double.NaN, latCenter=Double.NaN, lonCenter=Double.NaN, timeCenter=Double.NaN, latRange=Double.NaN, lonRange=Double.NaN, timeRange=Double.NaN; + String time_precision = null; int lonAscending = 0, latAscending = 0, timeAscending = 0; for (int av = 0; av < nAv; av++) { EDVGridAxis edvga = axisVariables[av]; - EDVTimeGridAxis edvtga = av == timeIndex? (EDVTimeGridAxis)edvga : null; double defStart = av == timeIndex? //note max vs first Math.max(edvga.destinationMax() - 7 * Calendar2.SECONDS_PER_DAY, edvga.destinationMin()) : edvga.firstDestinationValue(); @@ -2621,7 +2638,9 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, edvga.destinationMax(): edvga.lastDestinationValue(); sourceSize[av] = edvga.sourceValues().size(); - int precision = av == timeIndex? 10 : 7; + boolean isTimeStamp = edvga instanceof EDVTimeStampGridAxis; + int precision = isTimeStamp? 13 : + edvga.destinationDataTypeClass() == double.class? 9 : 5; showStartAndStopFields[av] = av == axisVarX || av == axisVarY; //find start and end @@ -2700,6 +2719,7 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, timeStop = dStop; timeCenter = (dStart + dStop) / 2; timeRange = dStop - dStart; + time_precision = edvga.combinedAttributes().getString(EDV.TIME_PRECISION); } } @@ -2911,7 +2931,7 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, String tLast = edvga.destinationToString(edvga.lastDestinationValue()); String edvgaTooltip = edvga.htmlRangeTooltip(); - String tUnits = av == timeIndex? "UTC" : edvga.units(); + String tUnits = edvga instanceof EDVTimeStampGridAxis? "UTC" : edvga.units(); tUnits = tUnits == null? "" : "(" + tUnits + ") "; writer.write( "\n" + @@ -3488,10 +3508,14 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, MessageFormat.format(EDStatic.magZoomOut, "8x"), "class=\"skinny\" " + (disableZoomOut? "disabled" : - "onMouseUp='f1.start" + lonIndex + ".value=\"" + String2.genEFormat6(tLonCenter - lonAscending * zoomOut8) + "\"; " + - "f1.stop" + lonIndex + ".value=\"" + String2.genEFormat6(tLonCenter + lonAscending * zoomOut8) + "\"; " + - "f1.start" + latIndex + ".value=\"" + String2.genEFormat6(tLatCenter - latAscending * zoomOut8) + "\"; " + - "f1.stop" + latIndex + ".value=\"" + String2.genEFormat6(tLatCenter + latAscending * zoomOut8) + "\"; " + + "onMouseUp='f1.start" + lonIndex + ".value=\"" + + String2.genEFormat6(tLonCenter - lonAscending * zoomOut8) + "\"; " + + "f1.stop" + lonIndex + ".value=\"" + + String2.genEFormat6(tLonCenter + lonAscending * zoomOut8) + "\"; " + + "f1.start" + latIndex + ".value=\"" + + String2.genEFormat6(tLatCenter - latAscending * zoomOut8) + "\"; " + + "f1.stop" + latIndex + ".value=\"" + + String2.genEFormat6(tLatCenter + latAscending * zoomOut8) + "\"; " + "mySubmit(true);'"))); //if zoom out, keep ranges intact by moving tCenter to safe place @@ -3505,10 +3529,14 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, MessageFormat.format(EDStatic.magZoomOut, "2x"), "class=\"skinny\" " + (disableZoomOut? "disabled" : - "onMouseUp='f1.start" + lonIndex + ".value=\"" + String2.genEFormat6(tLonCenter - lonAscending * zoomOut2) + "\"; " + - "f1.stop" + lonIndex + ".value=\"" + String2.genEFormat6(tLonCenter + lonAscending * zoomOut2) + "\"; " + - "f1.start" + latIndex + ".value=\"" + String2.genEFormat6(tLatCenter - latAscending * zoomOut2) + "\"; " + - "f1.stop" + latIndex + ".value=\"" + String2.genEFormat6(tLatCenter + latAscending * zoomOut2) + "\"; " + + "onMouseUp='f1.start" + lonIndex + ".value=\"" + + String2.genEFormat6(tLonCenter - lonAscending * zoomOut2) + "\"; " + + "f1.stop" + lonIndex + ".value=\"" + + String2.genEFormat6(tLonCenter + lonAscending * zoomOut2) + "\"; " + + "f1.start" + latIndex + ".value=\"" + + String2.genEFormat6(tLatCenter - latAscending * zoomOut2) + "\"; " + + "f1.stop" + latIndex + ".value=\"" + + String2.genEFormat6(tLatCenter + latAscending * zoomOut2) + "\"; " + "mySubmit(true);'"))); //if zoom out, keep ranges intact by moving tCenter to safe place @@ -3522,10 +3550,14 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, MessageFormat.format(EDStatic.magZoomOut, "").trim(), "class=\"skinny\" " + (disableZoomOut? "disabled" : - "onMouseUp='f1.start" + lonIndex + ".value=\"" + String2.genEFormat6(tLonCenter - lonAscending * zoomOut) + "\"; " + - "f1.stop" + lonIndex + ".value=\"" + String2.genEFormat6(tLonCenter + lonAscending * zoomOut) + "\"; " + - "f1.start" + latIndex + ".value=\"" + String2.genEFormat6(tLatCenter - latAscending * zoomOut) + "\"; " + - "f1.stop" + latIndex + ".value=\"" + String2.genEFormat6(tLatCenter + latAscending * zoomOut) + "\"; " + + "onMouseUp='f1.start" + lonIndex + ".value=\"" + + String2.genEFormat6(tLonCenter - lonAscending * zoomOut) + "\"; " + + "f1.stop" + lonIndex + ".value=\"" + + String2.genEFormat6(tLonCenter + lonAscending * zoomOut) + "\"; " + + "f1.start" + latIndex + ".value=\"" + + String2.genEFormat6(tLatCenter - latAscending * zoomOut) + "\"; " + + "f1.stop" + latIndex + ".value=\"" + + String2.genEFormat6(tLatCenter + latAscending * zoomOut) + "\"; " + "mySubmit(true);'"))); //zoom in Math.max moves rectangular maps toward square @@ -3534,30 +3566,42 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, MessageFormat.format(EDStatic.magZoomInTooltip, EDStatic.magZoomALittle), MessageFormat.format(EDStatic.magZoomIn, "").trim(), "class=\"skinny\" " + - "onMouseUp='f1.start" + lonIndex + ".value=\"" + String2.genEFormat6(lonCenter - lonAscending * zoomIn) + "\"; " + - "f1.stop" + lonIndex + ".value=\"" + String2.genEFormat6(lonCenter + lonAscending * zoomIn) + "\"; " + - "f1.start" + latIndex + ".value=\"" + String2.genEFormat6(latCenter - latAscending * zoomIn) + "\"; " + - "f1.stop" + latIndex + ".value=\"" + String2.genEFormat6(latCenter + latAscending * zoomIn) + "\"; " + + "onMouseUp='f1.start" + lonIndex + ".value=\"" + + String2.genEFormat6(lonCenter - lonAscending * zoomIn) + "\"; " + + "f1.stop" + lonIndex + ".value=\"" + + String2.genEFormat6(lonCenter + lonAscending * zoomIn) + "\"; " + + "f1.start" + latIndex + ".value=\"" + + String2.genEFormat6(latCenter - latAscending * zoomIn) + "\"; " + + "f1.stop" + latIndex + ".value=\"" + + String2.genEFormat6(latCenter + latAscending * zoomIn) + "\"; " + "mySubmit(true);'") + widgets.button("button", "", MessageFormat.format(EDStatic.magZoomInTooltip, "2x"), MessageFormat.format(EDStatic.magZoomIn, "2x"), "class=\"skinny\" " + - "onMouseUp='f1.start" + lonIndex + ".value=\"" + String2.genEFormat6(lonCenter - lonAscending * zoomIn2) + "\"; " + - "f1.stop" + lonIndex + ".value=\"" + String2.genEFormat6(lonCenter + lonAscending * zoomIn2) + "\"; " + - "f1.start" + latIndex + ".value=\"" + String2.genEFormat6(latCenter - latAscending * zoomIn2) + "\"; " + - "f1.stop" + latIndex + ".value=\"" + String2.genEFormat6(latCenter + latAscending * zoomIn2) + "\"; " + + "onMouseUp='f1.start" + lonIndex + ".value=\"" + + String2.genEFormat6(lonCenter - lonAscending * zoomIn2) + "\"; " + + "f1.stop" + lonIndex + ".value=\"" + + String2.genEFormat6(lonCenter + lonAscending * zoomIn2) + "\"; " + + "f1.start" + latIndex + ".value=\"" + + String2.genEFormat6(latCenter - latAscending * zoomIn2) + "\"; " + + "f1.stop" + latIndex + ".value=\"" + + String2.genEFormat6(latCenter + latAscending * zoomIn2) + "\"; " + "mySubmit(true);'") + widgets.button("button", "", MessageFormat.format(EDStatic.magZoomInTooltip, "8x"), MessageFormat.format(EDStatic.magZoomIn, "8x"), "class=\"skinny\" " + - "onMouseUp='f1.start" + lonIndex + ".value=\"" + String2.genEFormat6(lonCenter - lonAscending * zoomIn8) + "\"; " + - "f1.stop" + lonIndex + ".value=\"" + String2.genEFormat6(lonCenter + lonAscending * zoomIn8) + "\"; " + - "f1.start" + latIndex + ".value=\"" + String2.genEFormat6(latCenter - latAscending * zoomIn8) + "\"; " + - "f1.stop" + latIndex + ".value=\"" + String2.genEFormat6(latCenter + latAscending * zoomIn8) + "\"; " + + "onMouseUp='f1.start" + lonIndex + ".value=\"" + + String2.genEFormat6(lonCenter - lonAscending * zoomIn8) + "\"; " + + "f1.stop" + lonIndex + ".value=\"" + + String2.genEFormat6(lonCenter + lonAscending * zoomIn8) + "\"; " + + "f1.start" + latIndex + ".value=\"" + + String2.genEFormat6(latCenter - latAscending * zoomIn8) + "\"; " + + "f1.stop" + latIndex + ".value=\"" + + String2.genEFormat6(latCenter + latAscending * zoomIn8) + "\"; " + "mySubmit(true);'")); //trailing
    @@ -3568,16 +3612,17 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, if (zoomTime) { if (reallyVerbose) String2.log("zoomTime range=" + Calendar2.elapsedTimeString(timeRange * 1000) + - " center=" + Calendar2.epochSecondsToIsoStringT(timeCenter) + - "\n first=" + Calendar2.epochSecondsToIsoStringT(timeFirst) + - " start=" + Calendar2.epochSecondsToIsoStringT(timeStart) + - "\n last=" + Calendar2.epochSecondsToIsoStringT(timeLast) + - " stop=" + Calendar2.epochSecondsToIsoStringT(timeStop)); + " center=" + Calendar2.epochSecondsToLimitedIsoStringT(time_precision, timeCenter, "") + + "\n first=" + Calendar2.epochSecondsToLimitedIsoStringT(time_precision, timeFirst, "") + + " start=" + Calendar2.epochSecondsToLimitedIsoStringT(time_precision, timeStart, "") + + "\n last=" + Calendar2.epochSecondsToLimitedIsoStringT(time_precision, timeLast, "") + + " stop=" + Calendar2.epochSecondsToLimitedIsoStringT(time_precision, timeStop, "")); writer.write( "" + EDStatic.magTimeRange + "\n"); - String timeRangeString = idealTimeN + " " + Calendar2.IDEAL_UNITS_OPTIONS[idealTimeUnits]; + String timeRangeString = idealTimeN + " " + + Calendar2.IDEAL_UNITS_OPTIONS[idealTimeUnits]; String timesVary = "
    (" + EDStatic.magTimesVary + ")"; String timeRangeTip = EDStatic.magTimeRangeTooltip + EDStatic.magTimeRangeTooltip2; @@ -3602,7 +3647,8 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, //make idealized current centered time period - GregorianCalendar idMinGc = Calendar2.roundToIdealGC(timeCenter, idealTimeN, idealTimeUnits); + GregorianCalendar idMinGc = Calendar2.roundToIdealGC(timeCenter, + idealTimeN, idealTimeUnits); //if it rounded to later time period, shift to earlier time period if (idMinGc.getTimeInMillis() / 1000 > timeCenter) idMinGc.add(Calendar2.IDEAL_UNITS_FIELD[idealTimeUnits], -idealTimeN); @@ -3612,7 +3658,8 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, //time back { //make idealized beginning time - GregorianCalendar tidMinGc = Calendar2.roundToIdealGC(timeFirst, idealTimeN, idealTimeUnits); + GregorianCalendar tidMinGc = Calendar2.roundToIdealGC(timeFirst, + idealTimeN, idealTimeUnits); //if it rounded to later time period, shift to earlier time period if (tidMinGc.getTimeInMillis() / 1000 > timeFirst) tidMinGc.add(Calendar2.IDEAL_UNITS_FIELD[idealTimeUnits], -idealTimeN); @@ -3630,8 +3677,10 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, MessageFormat.format(EDStatic.magTimeRangeFirst, timeRangeString) + timesVary, "align=\"top\" " + - "onMouseUp='f1.start" + timeIndex + ".value=\"" + Calendar2.formatAsISODateTimeT(tidMinGc) + "Z\"; " + - "f1.stop" + timeIndex + ".value=\"" + Calendar2.formatAsISODateTimeT(tidMaxGc) + "Z\"; " + + "onMouseUp='f1.start" + timeIndex + ".value=\"" + + Calendar2.limitedFormatAsISODateTimeT(time_precision, tidMinGc) + "\"; " + + "f1.stop" + timeIndex + ".value=\"" + + Calendar2.limitedFormatAsISODateTimeT(time_precision, tidMaxGc) + "\"; " + "mySubmit(true);'")); } else { writer.write(timeGap); @@ -3649,8 +3698,10 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, MessageFormat.format(EDStatic.magTimeRangeBack, timeRangeString) + timesVary, "align=\"top\" " + - "onMouseUp='f1.start" + timeIndex + ".value=\"" + Calendar2.formatAsISODateTimeT(idMinGc) + "Z\"; " + - "f1.stop" + timeIndex + ".value=\"" + Calendar2.formatAsISODateTimeT(idMaxGc) + "Z\"; " + + "onMouseUp='f1.start" + timeIndex + ".value=\"" + + Calendar2.limitedFormatAsISODateTimeT(time_precision, idMinGc) + "\"; " + + "f1.stop" + timeIndex + ".value=\"" + + Calendar2.limitedFormatAsISODateTimeT(time_precision, idMaxGc) + "\"; " + "mySubmit(true);'")); idMinGc.add(Calendar2.IDEAL_UNITS_FIELD[idealTimeUnits], idealTimeN); idMaxGc.add(Calendar2.IDEAL_UNITS_FIELD[idealTimeUnits], idealTimeN); @@ -3675,8 +3726,10 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, MessageFormat.format(EDStatic.magTimeRangeForward, timeRangeString) + timesVary, "align=\"top\" " + - "onMouseUp='f1.start" + timeIndex + ".value=\"" + Calendar2.formatAsISODateTimeT(idMinGc) + "Z\"; " + - "f1.stop" + timeIndex + ".value=\"" + Calendar2.formatAsISODateTimeT(idMaxGc) + "Z\"; " + + "onMouseUp='f1.start" + timeIndex + ".value=\"" + + Calendar2.limitedFormatAsISODateTimeT(time_precision, idMinGc) + "\"; " + + "f1.stop" + timeIndex + ".value=\"" + + Calendar2.limitedFormatAsISODateTimeT(time_precision, idMaxGc) + "\"; " + "mySubmit(true);'")); idMinGc.add(Calendar2.IDEAL_UNITS_FIELD[idealTimeUnits], -idealTimeN); idMaxGc.add(Calendar2.IDEAL_UNITS_FIELD[idealTimeUnits], -idealTimeN); @@ -3704,8 +3757,10 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, MessageFormat.format(EDStatic.magTimeRangeLast, timeRangeString) + timesVary, "align=\"top\" " + - "onMouseUp='f1.start" + timeIndex + ".value=\"" + Calendar2.formatAsISODateTimeT(tidMinGc) + "Z\"; " + - "f1.stop" + timeIndex + ".value=\"" + Calendar2.formatAsISODateTimeT(tidMaxGc) + "Z\"; " + + "onMouseUp='f1.start" + timeIndex + ".value=\"" + + Calendar2.limitedFormatAsISODateTimeT(time_precision, tidMinGc) + "\"; " + + "f1.stop" + timeIndex + ".value=\"" + + Calendar2.limitedFormatAsISODateTimeT(time_precision, tidMaxGc) + "\"; " + "mySubmit(true);'")); } else { writer.write(timeGap); @@ -5207,8 +5262,10 @@ else throw new SimpleException(EDStatic.queryError + boolean isMap = vars[0] instanceof EDVLonGridAxis && vars[1] instanceof EDVLatGridAxis; - boolean xIsTimeAxis = vars[0] instanceof EDVTimeGridAxis; - boolean yIsTimeAxis = vars[1] instanceof EDVTimeGridAxis; + boolean xIsTimeAxis = vars[0] instanceof EDVTimeStampGridAxis || + vars[0] instanceof EDVTimeStamp; + boolean yIsTimeAxis = vars[1] instanceof EDVTimeStampGridAxis || + vars[1] instanceof EDVTimeStamp; int xAxisIndex = String2.indexOf(axisVariableDestinationNames(), vars[0].destinationName()); int yAxisIndex = String2.indexOf(axisVariableDestinationNames(), vars[1].destinationName()); @@ -5326,7 +5383,7 @@ else throw new SimpleException(EDStatic.queryError + otherInfo.append(td + " E"); //� didn't work else if (av == latIndex) otherInfo.append(td + " N"); //� didn't work - else if (axisVar instanceof EDVTimeGridAxis) + else if (axisVar instanceof EDVTimeStampGridAxis) otherInfo.append(Calendar2.epochSecondsToLimitedIsoStringT( axisVar.combinedAttributes().getString(EDV.TIME_PRECISION), td, "NaN")); else { @@ -6100,12 +6157,14 @@ public boolean saveAsKml( String datasetUrl = tErddapUrl + "/" + dapProtocol + "/" + datasetID; String timeString = ""; - if (nTimes >= 1) timeString += Calendar2.epochSecondsToIsoStringT(Math.min(timeStartd, timeStopd)) + "Z"; + if (nTimes >= 1) timeString += + Calendar2.epochSecondsToLimitedIsoStringT( + timeEdv.combinedAttributes().getString(EDV.TIME_PRECISION), + Math.min(timeStartd, timeStopd), ""); if (nTimes >= 2) throw new SimpleException("Error: " + "For .kml requests, the time dimension size must be 1."); - //timeString += " through " + - //Calendar2.epochSecondsToIsoStringT(Math.max(timeStartd, timeStopd)) + "Z"; + //timeString += " through " + limitedIsoStringT ... Math.max(timeStartd, timeStopd), ""); String brTimeString = timeString.length() == 0? "" : "Time: " + timeString + "
    \n"; //calculate doMax and get drawOrder @@ -6363,223 +6422,6 @@ else if (av == timeIndex) return true; } - /* - public void saveAsKml(String requestUrl, String userDapQuery, - OutputStreamSource outputStreamSource) throws Throwable { - - if (reallyVerbose) String2.log(" EDDGrid.saveAsKml"); - long time = System.currentTimeMillis(); - - //handle axis request - if (isAxisDapQuery(userDapQuery)) - throw new SimpleException("Error: " + - "The .kml format is for latitude longitude data requests only."); - - //lon and lat are required; time is not required - if (lonIndex < 0 || latIndex < 0) - throw new SimpleException("Error: " + - "The .kml format is for latitude longitude data requests only."); - - //parse the userDapQuery and get the GridDataAccessor - //this also tests for error when parsing query - GridDataAccessor gridDataAccessor = new GridDataAccessor(this, - requestUrl, userDapQuery, true, true); //rowMajor, convertToNaN - if (gridDataAccessor.dataVariables().length != 1) - throw new SimpleException("Error: " + - "The .kml format can only handle one data variable."); - StringArray tDestinationNames = new StringArray(); - tDestinationNames.add(gridDataAccessor.dataVariables()[0].destinationName()); - - //check that request meets .kml restrictions. - //.transparentPng does some of these tests, but better to catch problems - //here than in GoogleEarth. - int nTimes = 0; - double firstTime = Double.NaN, lastTime = Double.NaN, timeSpacing = Double.NaN; - PrimitiveArray lonPa = null, latPa = null, timePa = null, allTimeDestPa = null; - EDVTimeGridAxis timeEdv = null; - double lonAdjust = 0; - for (int av = 0; av < axisVariables.length; av++) { - PrimitiveArray avpa = gridDataAccessor.axisValues(av); - if (av == lonIndex) { - lonPa = avpa; - - //lon and lat axis values don't have to be evenly spaced. - //.transparentPng uses Sgt.makeCleanMap which projects data (even, e.g., Mercator) - //so resulting .png will use a geographic projection. - - //although the Google docs say lon must be +-180, lon > 180 is ok! - //if (lonPa.getDouble(0) < 180 && lonPa.getDouble(lonPa.size() - 1) > 180) - // throw new SimpleException("Error: " + - // "For .kml requests, the longitude values can't be below and above 180."); - - //But if minLon>=180, it is easy to adjust the lon value references in the kml, - //but leave the userDapQuery for the .transparentPng unchanged. - if (lonPa.getDouble(0) >= 180) - lonAdjust = -360; - } else if (av == latIndex) { - latPa = avpa; - } else if (av == timeIndex) { - timeEdv = (EDVTimeGridAxis)axisVariables[timeIndex]; - allTimeDestPa = timeEdv.destinationValues(); - timePa = avpa; - nTimes = timePa.size(); - if (nTimes > 500) //arbitrary: prevents requests that would take too long to respond to - throw new SimpleException("Error: " + - "For .kml requests, the time dimension's size must be less than 500."); - firstTime = timePa.getDouble(0); - lastTime = timePa.getDouble(nTimes - 1); - if (nTimes > 1) - timeSpacing = (lastTime - firstTime) / (nTimes - 1); - } else { - if (avpa.size() > 1) - throw new SimpleException("Error: " + - "For .kml requests, the " + - axisVariables[av].destinationName() + " dimension's size must be 1."); - } - } - if (lonPa == null || latPa == null || lonPa.size() < 2 || latPa.size() < 2) - throw new SimpleException("Error: " + - "For .kml requests, the lon and lat dimension sizes must be greater than 1."); - //request is ok and compatible with .kml request! - - //based on quirky example (but lots of useful info): - //http://161.55.17.243/cgi-bin/pydap.cgi/AG/ssta/3day/AG2006001_2006003_ssta.nc.kml?LAYERS=AGssta - //kml docs: http://earth.google.com/kml/kml_tags.html - //CDATA is necessary for url's with queries - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter( - outputStreamSource.outputStream("UTF-8"), "UTF-8")); - double dWest = lonPa.getNiceDouble(0) + lonAdjust; - double dEast = lonPa.getNiceDouble(lonPa.size() - 1) + lonAdjust; - if (dWest > dEast) { //it happens if axis is in descending order - double td = dWest; dWest = dEast; dEast = td;} - String west = String2.genEFormat10(dWest); - String east = String2.genEFormat10(dEast); - - double dSouth = latPa.getNiceDouble(0); - double dNorth = latPa.getNiceDouble(latPa.size() - 1); - if (dSouth > dNorth) { //it happens if axis is in descending order - double td = dSouth; dSouth = dNorth; dNorth = td; } - String south = String2.genEFormat10(dSouth); - String north = String2.genEFormat10(dNorth); - String datasetUrl = tErddapUrl + "/" + dapProtocol + "/" + datasetID; - String timeString = nTimes == 0? "" : - nTimes == 1? Calendar2.epochSecondsToIsoStringT(firstTime) : - Calendar2.epochSecondsToIsoStringT(firstTime) + " through " + - Calendar2.epochSecondsToIsoStringT(lastTime); - String brTimeString = timeString.length() == 0? "" : "Time: " + timeString + "
    \n"; - writer.write( - "\n" + - "\n" + - "\n" + - //human-friendly, but descriptive, - //name is used as link title -- leads to - " " + XML.encodeAsXML(title()) + "\n" + - // is what kml/description documentation recommends - " \n" + - //link to download data - "Download data from this dataset.
    \n" + - " ]]>    \n"); - - //is nTimes <= 1? - if (nTimes <= 1) { - //no timeline in Google Earth - writer.write( - //the kml link to the data - " \n" + - " " + title() + - (timeString.length() > 0? ", " + timeString : "") + - "\n" + - " \n" + - " " + - datasetUrl + ".transparentPng?" + SSR.minimalPercentEncode(userDapQuery) + //XML.encodeAsXML isn't ok - "\n" + - " \n" + - " \n" + - " " + west + "\n" + - " " + east + "\n" + - " " + south + "\n" + - " " + north + "\n" + - " \n" + - " 1\n" + - " \n"); - } else { - //nTimes >= 2, so make a timeline in Google Earth - //Problem: I don't know what time range each image represents. - // Because I don't know what the timePeriod is for the dataset (e.g., 8day). - // And I don't know if the images overlap (e.g., 8day composites, every day) - // And if the stride>1, it is further unknown. - //Solution (crummy): assume an image represents -1/2 time to previous image until 1/2 time till next image - - //get all the .dotConstraints - String parts[] = getUserQueryParts(userDapQuery); //decoded. always at least 1 part (may be "") - StringBuilder dotConstraintsSB = new StringBuilder(); - for (int i = 0; i < parts.length; i++) { - if (parts[i].startsWith(".")) { - if (dotConstraintsSB.size() > 0) - dotConstraintsSB.append("&"); - dotConstraintsSB.append(parts[i]); - } - } - String dotConstraints = dotConstraintsSB.toString(); - - IntArray tConstraints = (IntArray)gridDataAccessor.constraints().clone(); - int startTimeIndex = tConstraints.get(timeIndex * 3); - int timeStride = tConstraints.get(timeIndex * 3 + 1); - int stopTimeIndex = tConstraints.get(timeIndex * 3 + 2); - double preTime = Double.NaN; - double nextTime = allTimeDestPa.getDouble(startTimeIndex); - double currentTime = nextTime - (allTimeDestPa.getDouble(startTimeIndex + timeStride) - nextTime); - for (int tIndex = startTimeIndex; tIndex <= stopTimeIndex; tIndex += timeStride) { - preTime = currentTime; - currentTime = nextTime; - nextTime = tIndex + timeStride > stopTimeIndex? - currentTime + (currentTime - preTime) : - allTimeDestPa.getDouble(tIndex + timeStride); - //String2.log(" tIndex=" + tIndex + " preT=" + preTime + " curT=" + currentTime + " nextT=" + nextTime); - //just change the time constraints; leave all others unchanged - tConstraints.set(timeIndex * 3, tIndex); - tConstraints.set(timeIndex * 3 + 1, 1); - tConstraints.set(timeIndex * 3 + 2, tIndex); - String tDapQuery = buildDapQuery(tDestinationNames, tConstraints) + dotConstraints; - writer.write( - //the kml link to the data - " \n" + - " " + Calendar2.epochSecondsToIsoStringT(currentTime) + "Z" + "\n" + - " \n" + - " " + - datasetUrl + ".transparentPng?" + SSR.minimalPercentEncode(tDapQuery) + //XML.encodeAsXML isn't ok - "\n" + - " \n" + - " \n" + - " " + west + "\n" + - " " + east + "\n" + - " " + south + "\n" + - " " + north + "\n" + - " \n" + - " \n" + - " " + Calendar2.epochSecondsToIsoStringT((preTime + currentTime) / 2.0) + "Z\n" + - " " + Calendar2.epochSecondsToIsoStringT((currentTime + nextTime) / 2.0) + "Z\n" + - " \n" + - " 1\n" + - " \n"); - } - } - writer.write( - getKmlIconScreenOverlay() + - "    \n" + - "    \n"); - writer.flush(); //essential - - //diagnostic - if (reallyVerbose) - String2.log(" EDDGrid.saveAsKml done. TIME=" + - (System.currentTimeMillis() - time) + "\n"); - } - */ /** * This writes the grid from this dataset to the outputStream in * Matlab .mat format. @@ -6891,23 +6733,27 @@ public void saveAsNc(String requestUrl, String userDapQuery, return; } - //get gridDataAccessor first, in case of error when parsing query - //(This makes an unneccessary call to ensureMemoryAvailable with a - // possibly different nBytes than will actually be used below (via dvGda). - // But it isn't an unreasonable request. - // nBytes will still be less than partialRequestMaxBytes.) - GridDataAccessor mainGda = new GridDataAccessor(this, requestUrl, userDapQuery, + //** create gridDataAccessor first, + //to check for error when parsing query or getting data, + //and to check that file size < 2GB + GridDataAccessor gda = new GridDataAccessor(this, requestUrl, userDapQuery, true, false); //rowMajor, convertToNaN - EDV tDataVariables[] = mainGda.dataVariables(); //ensure file size < 2GB //???is there a way to allow >2GB netcdf 3 files? + //Yes: the 64-bit extension! But this code doesn't yet use that. // And even if so, what about OS limit ERDDAP is running on? and client OS? //Or, view this as protection against accidental requests for too much data (e.g., whole dataset). - if (mainGda.totalNBytes() > 2100000000) //leave some space for axis vars, etc. + if (gda.totalNBytes() > 2100000000) //leave some space for axis vars, etc. throw new SimpleException(Math2.memoryTooMuchData + " " + MessageFormat.format(EDStatic.errorMoreThan2GB, - ".nc", ((mainGda.totalNBytes() + 100000) / Math2.BytesPerMB) + " MB")); + ".nc", ((gda.totalNBytes() + 100000) / Math2.BytesPerMB) + " MB")); + + + //** Then get gridDataAllAccessor + //AllAccessor so max length of String variables will be known. + GridDataAllAccessor gdaa = new GridDataAllAccessor(gda); + EDV tDataVariables[] = gda.dataVariables(); //write the data //items determined by looking at a .nc file; items written in that order @@ -6918,7 +6764,7 @@ public void saveAsNc(String requestUrl, String userDapQuery, //find active axes IntArray activeAxes = new IntArray(); for (int av = 0; av < axisVariables.length; av++) { - if (keepUnusedAxes || mainGda.axisValues(av).size() > 1) + if (keepUnusedAxes || gda.axisValues(av).size() > 1) activeAxes.add(av); } @@ -6926,16 +6772,18 @@ public void saveAsNc(String requestUrl, String userDapQuery, int nActiveAxes = activeAxes.size(); Dimension dimensions[] = new Dimension[nActiveAxes]; Array axisArrays[] = new Array[nActiveAxes]; + int stdShape[] = new int[nActiveAxes]; for (int a = 0; a < nActiveAxes; a++) { int av = activeAxes.get(a); String avName = axisVariables[av].destinationName(); - PrimitiveArray pa = mainGda.axisValues(av); + PrimitiveArray pa = gda.axisValues(av); //if (reallyVerbose) String2.log(" create dim=" + avName + " size=" + pa.size()); + stdShape[a] = pa.size(); dimensions[a] = nc.addDimension(avName, pa.size()); if (av == lonIndex) pa.scaleAddOffset(1, lonAdjust); axisArrays[a] = Array.factory( - mainGda.axisValues(av).elementClass(), + gda.axisValues(av).elementClass(), new int[]{pa.size()}, pa.toObjectArray()); //if (reallyVerbose) String2.log(" create var=" + avName); @@ -6945,28 +6793,41 @@ public void saveAsNc(String requestUrl, String userDapQuery, } //define the data variables - Array dataArrays[] = new Array[tDataVariables.length]; for (int dv = 0; dv < tDataVariables.length; dv++) { - //if (reallyVerbose) String2.log(" create var=" + tDataVariables[dv].destinationName()); - nc.addVariable(tDataVariables[dv].destinationName(), - NcHelper.getDataType(tDataVariables[dv].destinationDataTypeClass()), - dimensions); + String destName = tDataVariables[dv].destinationName(); + Class destClass = tDataVariables[dv].destinationDataTypeClass(); + //if (reallyVerbose) String2.log(" create var=" + destName); + + //String data? need to create a strlen dimension for this variable + if (destClass == String.class) { + StringArray tsa = (StringArray)gdaa.getPrimitiveArray(dv); + Dimension tDims[] = new Dimension[nActiveAxes + 1]; + System.arraycopy(dimensions, 0, tDims, 0, nActiveAxes); + tDims[nActiveAxes] = nc.addDimension( + destName + NcHelper.StringLengthSuffix, //"_strlen" + tsa.maxStringLength()); + nc.addVariable(destName, DataType.CHAR, tDims); + + } else { + nc.addVariable(destName, NcHelper.getDataType(destClass), + dimensions); + } } //write global attributes - NcHelper.setAttributes(nc, "NC_GLOBAL", mainGda.globalAttributes); + NcHelper.setAttributes(nc, "NC_GLOBAL", gda.globalAttributes); //write axis attributes for (int a = 0; a < nActiveAxes; a++) { int av = activeAxes.get(a); NcHelper.setAttributes(nc, axisVariables[av].destinationName(), - mainGda.axisAttributes[av]); + gda.axisAttributes[av]); } //write data attributes for (int dv = 0; dv < tDataVariables.length; dv++) { NcHelper.setAttributes(nc, tDataVariables[dv].destinationName(), - mainGda.dataAttributes[dv]); + gda.dataAttributes[dv]); } //leave "define" mode @@ -6978,57 +6839,22 @@ public void saveAsNc(String requestUrl, String userDapQuery, nc.write(axisVariables[av].destinationName(), axisArrays[a]); } - //get the constraints string - String constraintsString = mainGda.constraintsString(); - //write the data variables for (int dv = 0; dv < tDataVariables.length; dv++) { - long dvTime = System.currentTimeMillis(); - - //Read/write chunks - //(I tried write data values one-by-one, but it is too slow. - //Writing takes 10X longer than reading! 9/10 of time is in nc.write(...).) - //make a GridDataAccessor for this dv EDV edv = tDataVariables[dv]; String destName = edv.destinationName(); Class edvClass = edv.destinationDataTypeClass(); - GridDataAccessor dvGda = new GridDataAccessor(this, requestUrl, - edv.destinationName() + constraintsString, - true, false); //rowMajor, convertToNaN - - int partialIndexShape[] = dvGda.partialIndex().shape(); - int totalIndexCurrent[] = dvGda.totalIndex().getCurrent(); - long rwTime = System.currentTimeMillis(); - while (dvGda.incrementChunk()) { - //make shape with just activeAxes - int ncShape[] = new int[nActiveAxes]; - int ncOffset[] = new int[nActiveAxes]; - for (int a = 0; a < nActiveAxes; a++) { - int aaa = activeAxes.get(a); - ncShape[ a] = partialIndexShape[aaa]; - ncOffset[a] = totalIndexCurrent[aaa]; - } - if (reallyVerbose)String2.log( - " ncShape=[" + String2.toCSSVString(ncShape) + "]\n" + - " ncOffset=[" + String2.toCSSVString(ncOffset) + "]"); - - Array array = Array.factory(edvClass, ncShape, - dvGda.getPartialDataValues(0).toObjectArray()); - nc.write(destName, ncOffset, array); - if (reallyVerbose) { - String2.log( - " rwTime=" + (System.currentTimeMillis() - rwTime)); - rwTime = System.currentTimeMillis(); - } - } - - if (reallyVerbose) String2.log("dv=" + dv + " done. time=" + - (System.currentTimeMillis() - dvTime)); + Array array = Array.factory(edvClass, + stdShape, gdaa.getPrimitiveArray(dv).toObjectArray()); + if (edvClass == String.class) + nc.writeStringData(destName, array); + else nc.write(destName, array); } //if close throws Throwable, it is trouble nc.close(); //it calls flush() and doesn't like flush called separately + nc = null; //rename the file to the specified name File2.rename(fullFileName + randomInt, fullFileName); @@ -7040,10 +6866,12 @@ public void saveAsNc(String requestUrl, String userDapQuery, } catch (Throwable t) { //try to close the file - try { - nc.close(); //it calls flush() and doesn't like flush called separately - } catch (Throwable t2) { - //don't care + if (nc != null) { + try { + nc.close(); //it calls flush() and doesn't like flush called separately + } catch (Throwable t2) { + //don't care + } } //delete the partial file @@ -7275,14 +7103,14 @@ public void saveAsTableWriter(AxisDataAccessor ada, TableWriter tw) throws Throwable { //make the table - //note that TableWriter expects time values as doubles, and (sometimes) displays them as ISO 8601 strings + //note that TableWriter expects time values as doubles, + // and (sometimes) displays them as ISO 8601 strings Table table = new Table(); + table.globalAttributes().add(ada.globalAttributes()); int nRAV = ada.nRequestedAxisVariables(); for (int av = 0; av < nRAV; av++) { - table.addColumn(ada.axisVariables(av).destinationName(), ada.axisValues(av)); - String tUnits = ada.axisVariables(av).units(); //ok if null - if (tUnits != null) - table.columnAttributes(av).set("units", tUnits); + table.addColumn(av, ada.axisVariables(av).destinationName(), + ada.axisValues(av), ada.axisAttributes(av)); } table.makeColumnsSameSize(); @@ -7328,7 +7156,7 @@ public void saveAsTableWriter(GridDataAccessor gridDataAccessor, avPa[av] = PrimitiveArray.factory(tClass, nBufferRows, false); //???need to remove file-specific metadata (e.g., actual_range) from Attributes clone? table.addColumn(av, edv.destinationName(), avPa[av], - (Attributes)edv.combinedAttributes().clone()); + gridDataAccessor.axisAttributes(av)); //(Attributes)edv.combinedAttributes().clone()); } for (int dv = 0; dv < nDv; dv++) { EDV edv = queryDataVariables[dv]; @@ -7339,13 +7167,13 @@ public void saveAsTableWriter(GridDataAccessor gridDataAccessor, dvPa[dv] = PrimitiveArray.factory(tClass, nBufferRows, false); //???need to remove file-specific metadata (e.g., actual_range) from Attributes clone? table.addColumn(nAv + dv, edv.destinationName(), dvPa[dv], - (Attributes)edv.combinedAttributes().clone()); + gridDataAccessor.dataAttributes(dv)); //(Attributes)edv.combinedAttributes().clone()); } //write the data int tRows = 0; while (gridDataAccessor.increment()) { - //put the data in row one of the table + //add a row of data to the table for (int av = 0; av < nAv; av++) { if (isDoubleAv[av]) avPa[av].addDouble(gridDataAccessor.getAxisValueAsDouble(av)); else if (isFloatAv[av]) avPa[av].addFloat( gridDataAccessor.getAxisValueAsFloat(av)); @@ -7417,7 +7245,6 @@ public void writeDapHtmlForm(String loggedInAs, writer.write(HtmlWidgets.ifJavaScriptDisabled + "\n"); HtmlWidgets widgets = new HtmlWidgets("", true, EDStatic.imageDirUrl(loggedInAs)); String formName = "form1"; - EDVTimeGridAxis timeVar = timeIndex >= 0? (EDVTimeGridAxis)axisVariables[timeIndex] : null; String liClickSubmit = "\n" + "
  • " + EDStatic.EDDClickOnSubmitHtml + "\n" + " \n"; @@ -7469,7 +7296,7 @@ public void writeDapHtmlForm(String loggedInAs, //get the extra info String extra = edvga.units(); - if (av == timeIndex) + if (edvga instanceof EDVTimeStampGridAxis) extra = "UTC"; //no longer true: "seconds since 1970-01-01..." if (extra == null) extra = ""; @@ -8334,14 +8161,17 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, "
    technique to convert a stride value in parentheses into a stride index value.\n" + "
    But dimension values often aren't evenly spaced. So for now, ERDDAP doesn't support the\n" + "
    parentheses notation for stride values.\n" + - "
  • griddap always stores date/time values as numbers (in seconds since 1970-01-01T00:00:00Z).\n" + + "
  • griddap always stores date/time values as double precision floating point numbers\n" + + "
    (seconds since 1970-01-01T00:00:00Z, sometimes with some number of milliseconds).\n" + "
    Here is an example of a query which includes date/time numbers:\n" + "
    " + fullValueExample + "\n" + - "
    Some fileTypes (notably, .csv, .tsv, .htmlTable, .odvTxt, and .xhtml) display date/time values as\n" + - "
    ISO 8601:2004 \"extended\" date/time strings" + + "
    The more human-oriented fileTypes (notably, .csv, .tsv, .htmlTable, .odvTxt, and .xhtml)\n" + + "
    display date/time values as " + + "     ISO 8601:2004 \"extended\" date/time strings" + EDStatic.externalLinkHtml(tErddapUrl) + "\n" + - " (e.g., 2002-08-03T12:30:00Z).\n" + + "
    (e.g., 2002-08-03T12:30:00Z, but some variables include milliseconds, e.g.,\n" + + "
    2002-08-03T12:30:00.123Z).\n" + (EDStatic.convertersActive? "
    ERDDAP has a utility to\n" + " Convert\n" + @@ -8354,10 +8184,12 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, EDStatic.externalLinkHtml(tErddapUrl) + "\n" + " in parentheses, which griddap then converts to the\n" + "
    internal number (in seconds since 1970-01-01T00:00:00Z) and then to the appropriate\n" + - "
    array index. The ISO date/time value should be in the form: YYYY-MM-DDThh:mm:ssZ,\n" + - "
    where Z is 'Z' or a ±hh:mm offset from UTC.\n" + - "
    If you omit Z (or the ±hh:mm offset), :ssZ, :mm:ssZ, or Thh:mm:ssZ from the ISO date/time\n" + - "
    that you specify, the missing fields are assumed to be 0.\n" + + "
    array index. The ISO date/time value should be in the form: YYYY-MM-DDThh:mm:ss.sssZ,\n" + + "
    where Z is 'Z' or a ±hh or ±hh:mm offset from the Zulu/GMT time zone. If you omit Z and the\n" + + "
    offset, the Zulu/GMT time zone is used. Separately, if you omit .sss, :ss.sss, :mm:ss.sss, or\n" + + "
    Thh:mm:ss.sss from the ISO date/time that you specify, the missing fields are assumed to be 0.\n" + + "
    In some places, ERDDAP accepts a comma (ss,sss) as the seconds decimal point, but ERDDAP\n" + + "
    always uses a period when formatting times as ISO 8601 strings.\n" + "
    The example below is equivalent (at least at the time of writing this) to the examples above:\n" + "
    " + fullTimeExample + "\n" + @@ -11029,8 +10861,7 @@ public void writeISO19115(Writer writer) throws Throwable { String domain = EDStatic.baseUrl; if (domain.startsWith("http://")) domain = domain.substring(7); - String eddCreationDate = String2.replaceAll( - Calendar2.millisToIsoZuluString(creationTimeMillis()), "-", "").substring(0, 8) + "Z"; + String eddCreationDate = Calendar2.millisToIsoZuluString(creationTimeMillis()).substring(0, 10); String acknowledgement = combinedGlobalAttributes.getString("acknowledgement"); String contributorName = combinedGlobalAttributes.getString("contributor_name"); @@ -11042,10 +10873,10 @@ public void writeISO19115(Writer writer) throws Throwable { //creatorUrl: use infoUrl String dateCreated = combinedGlobalAttributes.getString("date_created"); String dateIssued = combinedGlobalAttributes.getString("date_issued"); - if (dateCreated != null && dateCreated.length() >= 10 && !dateCreated.endsWith("Z")) - dateCreated = dateCreated.substring(0, 10) + "Z"; - if (dateIssued != null && dateIssued.length() >= 10 && !dateIssued.endsWith("Z")) - dateIssued = dateIssued.substring(0, 10) + "Z"; + if (dateCreated != null && dateCreated.length() > 10) + dateCreated = dateCreated.substring(0, 10); + if (dateIssued != null && dateIssued.length() > 10) + dateIssued = dateIssued.substring(0, 10); String history = combinedGlobalAttributes.getString("history"); String infoUrl = combinedGlobalAttributes.getString("infoUrl"); String institution = combinedGlobalAttributes.getString("institution"); @@ -12227,7 +12058,7 @@ public void writeISO19115(Writer writer) throws Throwable { " \n" + " \n" + //see list at https://github.com/OSGeo/Cat-Interop/blob/master/LinkPropertyLookupTable.csv from John Maurer -" template\n" + +" order\n" + " \n" + " \n" + " Data Subset Form\n" + @@ -12257,7 +12088,7 @@ public void writeISO19115(Writer writer) throws Throwable { " \n" + " \n" + //see list at https://github.com/OSGeo/Cat-Interop/blob/master/LinkPropertyLookupTable.csv from John Maurer -" template\n" + +" order\n" + " \n" + " \n" + " Make-A-Graph Form\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java index 249c28bf1..80eb137ce 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridAggregateExistingDimension.java @@ -276,11 +276,13 @@ else throw new RuntimeException(errorInMethod + else if (altIndex == 0) axisVariables[0] = new EDVAltGridAxis( sn, sa, aa, cumSV); else if (depthIndex == 0) axisVariables[0] = new EDVDepthGridAxis(sn, sa, aa, cumSV); else if (timeIndex == 0) axisVariables[0] = new EDVTimeGridAxis( sn, sa, aa, cumSV); + else if (av0 instanceof EDVTimeStampGridAxis) + axisVariables[0] = + new EDVTimeStampGridAxis(sn, av0.destinationName(), sa, aa, cumSV); else {axisVariables[0] = new EDVGridAxis(sn, av0.destinationName(), sa, aa, cumSV); axisVariables[0].setActualRangeFromDestinationMinMax(); } - int nDv = firstChild.dataVariables.length; dataVariables = new EDV[nDv]; System.arraycopy(firstChild.dataVariables, 0, dataVariables, 0, nDv); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromBinaryFile.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromBinaryFile.java index f242645fb..3e5ed5493 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromBinaryFile.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromBinaryFile.java @@ -247,6 +247,12 @@ public EDDGridFromBinaryFile(String tDatasetID, String tAccessibleTo, axisVariables[av] = new EDVTimeGridAxis(tSourceAxisName, tSourceAttributes, tAddAttributes, tSourceValues); + //is it a timestamp axis? + } else if (EDVTimeStampGridAxis.hasTimeUnits(tSourceAttributes, tAddAttributes)) { + axisVariables[av] = new EDVTimestampGridAxis( + tSourceAxisName, tDestinationName, + tSourceAttributes, tAddAttributes, tSourceValues); + //it is some other axis variable } else { axisVariables[av] = new EDVGridAxis( @@ -257,17 +263,24 @@ public EDDGridFromBinaryFile(String tDatasetID, String tAccessibleTo, } //create dataVariable - Test.ensureEqual(tDataVariables.length, 1, errorInMethod + "dataVariables.length must be 1."); + Test.ensureEqual(tDataVariables.length, 1, + errorInMethod + "dataVariables.length must be 1."); dataVariables = new EDV[1]; String tSourceName = (String)tDataVariables[0][0]; String tDestinationName = (String)tDataVariables[0][1]; + if (tDestinationName == null || tDestinationName.length() == 0) + tDestinationName = tSourceName; Attributes tSourceAttributes = new Attributes(); Attributes tAddAttributes = (Attributes)tDataVariables[0][2]; - dataVariables[0] = new EDV( - (String)tDataVariables[0][0], - (String)tDataVariables[0][1], - new Attributes(), - (Attributes)tDataVariables[0][2], + if (tDestinationName.equals(EDV.TIME_NAME)) + throw new RuntimeException(errorInMethod + + "No EDDGrid dataVariable may have destinationName=" + EDV.TIME_NAME); + else if (EDVTime.hasTimeUnits(tSourceAttributes, tAddAttributes)) + dataVariables[0] = new EDVTimeStamp(tSourceName, tDestinationName, + tSourceAttributes, tAddAttributes, + ???int.class); + else dataVariables[0] = new EDV(tSourceName, tDestinationName, + tSourceAttributes, tAddAttributes, ???int.class); dataVariables[0].setActualRangeFromDestinationMinMax(); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java index 5a4ff88ff..3b9690e3a 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromDap.java @@ -342,9 +342,11 @@ public EDDGridFromDap( for (int dv = 0; dv < tDataVariables.length; dv++) { String tDataSourceName = (String)tDataVariables[dv][0]; String tDataDestName = (String)tDataVariables[dv][1]; - Attributes tDataSourceAttributes = new Attributes(); - - OpendapHelper.getAttributes(das, tDataSourceName, tDataSourceAttributes); + if (tDataDestName == null || tDataDestName.length() == 0) + tDataDestName = tDataSourceName; + Attributes tDataSourceAtts = new Attributes(); + OpendapHelper.getAttributes(das, tDataSourceName, tDataSourceAtts); + Attributes tDataAddAtts = (Attributes)tDataVariables[dv][2]; //get the variable BaseType bt = dds.getVariable(tDataSourceName); //throws Throwable if not found @@ -461,6 +463,12 @@ else throw new RuntimeException("dataVariable=" + tDataSourceName + " must be a axisVariables[av] = new EDVTimeGridAxis(tSourceAxisName, tSourceAttributes, tAddAttributes, tSourceValues); + //is this a timestamp axis? + } else if (EDVTimeStampGridAxis.hasTimeUnits(tSourceAttributes, tAddAttributes)) { + axisVariables[av] = new EDVTimeStampGridAxis( + tSourceAxisName, tDestinationAxisName, + tSourceAttributes, tAddAttributes, tSourceValues); + //it is some other axis variable } else { axisVariables[av] = new EDVGridAxis( @@ -470,11 +478,16 @@ else throw new RuntimeException("dataVariable=" + tDataSourceName + " must be a } } - //create the EDVGridData - dataVariables[dv] = new EDV( + //create the EDV dataVariable + if (tDataDestName.equals(EDV.TIME_NAME)) + throw new RuntimeException(errorInMethod + + "No EDDGrid dataVariable may have destinationName=" + EDV.TIME_NAME); + else if (EDVTime.hasTimeUnits(tDataSourceAtts, tDataAddAtts)) + dataVariables[dv] = new EDVTimeStamp(tDataSourceName, tDataDestName, + tDataSourceAtts, tDataAddAtts, dvSourceDataType); + else dataVariables[dv] = new EDV( tDataSourceName, tDataDestName, - tDataSourceAttributes, (Attributes)tDataVariables[dv][2], - dvSourceDataType, + tDataSourceAtts, tDataAddAtts, dvSourceDataType, Double.NaN, Double.NaN); //hard to get min and max dataVariables[dv].extractAndSetActualRange(); @@ -569,7 +582,8 @@ public void update() { //has edvga[0] changed size? EDVGridAxis edvga = axisVariables[0]; - EDVTimeGridAxis edvtga = edvga instanceof EDVTimeGridAxis? (EDVTimeGridAxis)edvga : null; + EDVTimeStampGridAxis edvtsga = edvga instanceof EDVTimeStampGridAxis? + (EDVTimeStampGridAxis)edvga : null; PrimitiveArray oldValues = edvga.sourceValues(); int oldSize = oldValues.size(); @@ -639,9 +653,9 @@ public void update() { //prepare changes to update the dataset double newMin = oldValues.getDouble(0); double newMax = newValues.getDouble(newValues.size() - 1); - if (edvtga != null) { - newMin = edvtga.sourceTimeToEpochSeconds(newMin); - newMax = edvtga.sourceTimeToEpochSeconds(newMax); + if (edvtsga != null) { + newMin = edvtsga.sourceTimeToEpochSeconds(newMin); + newMax = edvtsga.sourceTimeToEpochSeconds(newMax); } else if (edvga.scaleAddOffset()) { newMin = newMin * edvga.scaleFactor() + edvga.addOffset(); newMax = newMax * edvga.scaleFactor() + edvga.addOffset(); @@ -706,9 +720,10 @@ public void update() { edvga.setIsEvenlySpaced(newIsEvenlySpaced); edvga.initializeAverageSpacingAndCoarseMinMax(); edvga.setActualRangeFromDestinationMinMax(); - if (edvtga != null) + if (edvga instanceof EDVTimeGridAxis) combinedGlobalAttributes.set("time_coverage_end", - Calendar2.epochSecondsToIsoStringT(newMax) + "Z"); + Calendar2.epochSecondsToLimitedIsoStringT( + edvga.combinedAttributes().getString(EDV.TIME_PRECISION), newMax, "")); edvga.clearSliderCsvValues(); //do last, to force recreation next time needed updateCount++; @@ -6839,6 +6854,21 @@ public static void testSliderCsv() throws Throwable { seconds.add(123456 + i); String2.log(" make DoubleArray time=" + (System.currentTimeMillis() - time)); + //test EDVTimeStampGridAxis + EDVTimeStampGridAxis edvtsga = new EDVTimeStampGridAxis("mytime", null, + (new Attributes()).add("units", Calendar2.SECONDS_SINCE_1970), + new Attributes(), seconds); + time = System.currentTimeMillis(); + results = edvtsga.sliderCsvValues(); + expected = "\"1970-01-02T10:17:36Z\", \"1970-01-02T12:00:00Z\", \"1970-01-03\", \"1970-01-03T12:00:00Z\", \"1970-01-04\", \"1970-01-04T12:00:00Z\","; + Test.ensureEqual(results.substring(0, expected.length()), expected, + "results=\n" + results); + expected = "\"1970-04-27\", \"1970-04-27T12:00:00Z\", \"1970-04-28\", \"1970-04-28T04:04:15Z\""; + Test.ensureEqual(results.substring(results.length() - expected.length()), expected, + "results=\n" + results); + String2.log(" TimeStamp sliderCsvValues time=" + (System.currentTimeMillis() - time)); + + //EDVTimeGridAxis EDVTimeGridAxis edvtga = new EDVTimeGridAxis("time", (new Attributes()).add("units", Calendar2.SECONDS_SINCE_1970), new Attributes(), seconds); @@ -6850,7 +6880,7 @@ public static void testSliderCsv() throws Throwable { expected = "\"1970-04-27\", \"1970-04-27T12:00:00Z\", \"1970-04-28\", \"1970-04-28T04:04:15Z\""; Test.ensureEqual(results.substring(results.length() - expected.length()), expected, "results=\n" + results); - String2.log(" sliderCsvValues time=" + (System.currentTimeMillis() - time)); + String2.log(" Time sliderCsvValues time=" + (System.currentTimeMillis() - time)); } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java index c36a0be60..fb0e90603 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromErddap.java @@ -300,6 +300,11 @@ public EDDGridFromErddap(String tDatasetID, String tAccessibleTo, tAxisVariables.add(new EDVTimeGridAxis(varName, tSourceAttributes, tAddAttributes, tSourceValues)); + //is this a timestamp axis? + } else if (EDVTimeStampGridAxis.hasTimeUnits(tSourceAttributes, tAddAttributes)) { + tAxisVariables.add(new EDVTimeStampGridAxis(varName, varName, + tSourceAttributes, tAddAttributes, tSourceValues)); + //it is some other axis variable } else { tAxisVariables.add(new EDVGridAxis(varName, varName, @@ -324,10 +329,15 @@ public EDDGridFromErddap(String tDatasetID, String tAccessibleTo, tAddAttributes.add("ioos_category", tAtts.getString("ioos_category")); } - EDV edv = new EDV( - varName, varName, - tSourceAttributes, tAddAttributes, - dataType, + EDV edv; + if (varName.equals(EDV.TIME_NAME)) + throw new RuntimeException(errorInMethod + + "No EDDGrid dataVariable may have destinationName=" + EDV.TIME_NAME); + else if (EDVTime.hasTimeUnits(tSourceAttributes, tAddAttributes)) + edv = new EDVTimeStamp(varName, varName, + tSourceAttributes, tAddAttributes, dataType); + else edv = new EDV(varName, varName, + tSourceAttributes, tAddAttributes, dataType, Double.NaN, Double.NaN); //hard to get min and max edv.extractAndSetActualRange(); tDataVariables.add(edv); @@ -499,7 +509,8 @@ public void update() { //has edvga[0] changed size? EDVGridAxis edvga = axisVariables[0]; - EDVTimeGridAxis edvtga = edvga instanceof EDVTimeGridAxis? (EDVTimeGridAxis)edvga : null; + EDVTimeStampGridAxis edvtsga = edvga instanceof EDVTimeStampGridAxis? + (EDVTimeStampGridAxis)edvga : null; PrimitiveArray oldValues = edvga.sourceValues(); int oldSize = oldValues.size(); @@ -569,9 +580,9 @@ public void update() { //prepare changes to update the dataset double newMin = oldValues.getDouble(0); double newMax = newValues.getDouble(newValues.size() - 1); - if (edvtga != null) { - newMin = edvtga.sourceTimeToEpochSeconds(newMin); - newMax = edvtga.sourceTimeToEpochSeconds(newMax); + if (edvtsga != null) { + newMin = edvtsga.sourceTimeToEpochSeconds(newMin); + newMax = edvtsga.sourceTimeToEpochSeconds(newMax); } else if (edvga.scaleAddOffset()) { newMin = newMin * edvga.scaleFactor() + edvga.addOffset(); newMax = newMax * edvga.scaleFactor() + edvga.addOffset(); @@ -636,9 +647,10 @@ public void update() { edvga.setIsEvenlySpaced(newIsEvenlySpaced); edvga.initializeAverageSpacingAndCoarseMinMax(); edvga.setActualRangeFromDestinationMinMax(); - if (edvtga != null) + if (edvga instanceof EDVTimeGridAxis) combinedGlobalAttributes.set("time_coverage_end", - Calendar2.epochSecondsToIsoStringT(newMax) + "Z"); + Calendar2.epochSecondsToLimitedIsoStringT( + edvga.combinedAttributes().getString(EDV.TIME_PRECISION), newMax, "")); edvga.clearSliderCsvValues(); //do last, to force recreation next time needed updateCount++; diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java index 040a940a3..9069b1eb8 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromFiles.java @@ -988,6 +988,10 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible axisVariables[av] = new EDVTimeGridAxis(tSourceName, tSourceAtt, tAddAtt, sourceAxisValues[av]); timeIndex = av; + } else if (EDVTimeStampGridAxis.hasTimeUnits(tSourceAtt, tAddAtt)) { + axisVariables[av] = new EDVTimeStampGridAxis( + tSourceName, tDestName, + tSourceAtt, tAddAtt, sourceAxisValues[av]); } else { axisVariables[av] = new EDVGridAxis(tSourceName, tDestName, tSourceAtt, tAddAtt, sourceAxisValues[av]); @@ -1009,6 +1013,8 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible for (int dv = 0; dv < ndv; dv++) { String tSourceName = sourceDataNames.get(dv); String tDestName = (String)tDataVariables[dv][1]; + if (tDestName == null || tDestName.length() == 0) + tDestName = tSourceName; Attributes tSourceAtt = sourceDataAttributes[dv]; Attributes tAddAtt = (Attributes)tDataVariables[dv][2]; //PrimitiveArray taa = tAddAtt.get("_FillValue"); @@ -1016,7 +1022,13 @@ public EDDGridFromFiles(String tClassName, String tDatasetID, String tAccessible String tSourceType = sourceDataTypes[dv]; //if (reallyVerbose) String2.log(" dv=" + dv + " sourceName=" + tSourceName + " sourceType=" + tSourceType); - dataVariables[dv] = new EDV(tSourceName, tDestName, + if (tDestName.equals(EDV.TIME_NAME)) + throw new RuntimeException(errorInMethod + + "No EDDGrid dataVariable may have destinationName=" + EDV.TIME_NAME); + else if (EDVTime.hasTimeUnits(tSourceAtt, tAddAtt)) + dataVariables[dv] = new EDVTimeStamp(tSourceName, tDestName, + tSourceAtt, tAddAtt, tSourceType); + else dataVariables[dv] = new EDV(tSourceName, tDestName, tSourceAtt, tAddAtt, tSourceType, Double.NaN, Double.NaN); dataVariables[dv].setActualRangeFromDestinationMinMax(); } @@ -1186,9 +1198,11 @@ public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraint tConstraints.get(avi*3 + 0), tConstraints.get(avi*3 + 1), tConstraints.get(avi*3 + 2)); - for (int dvi = 0; dvi < ndv; dvi++) + for (int dvi = 0; dvi < ndv; dvi++) { + //String2.log("!dvi#" + dvi + " " + tDataVariables[dvi].destinationName() + " " + tDataVariables[dvi].sourceDataTypeClass().toString()); results[nav + dvi] = PrimitiveArray.factory( - dataVariables[dvi].sourceDataTypeClass(), 64, false); + tDataVariables[dvi].sourceDataTypeClass(), 64, false); + } IntArray ttConstraints = (IntArray)tConstraints.clone(); int nFiles = ftStartIndex.size(); int axis0Start = tConstraints.get(0); @@ -1202,9 +1216,12 @@ public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraint int tNValues = ftNValues.get(ftRow); int tStart = axis0Start - ftStartIndex.get(ftRow); int tStop = tStart; - //get as many as possible from this file - while (tStop + axis0Stride < tNValues) + //get as many axis0 values as possible from this file + // (in this file, if this file had all the remaining values) + int lookMax = Math.min(tNValues - 1, axis0Stop - ftStartIndex.get(ftRow)); + while (tStop + axis0Stride <= lookMax) tStop += axis0Stride; + //String2.log("!tStart=" + tStart + " stride=" + axis0Stride + " tStop=" + tStop + " tNValues=" + tNValues); //set ttConstraints ttConstraints.set(0, tStart); @@ -1221,6 +1238,7 @@ public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraint try { tResults = getSourceDataFromFile(tFileDir, tFileName, tDataVariables, ttConstraints); + //String2.log("!tResults[0]=" + tResults[0].toString()); } catch (Throwable t) { EDStatic.rethrowClientAbortException(t); //first thing in catch{} @@ -1248,6 +1266,7 @@ public PrimitiveArray[] getSourceData(EDV tDataVariables[], IntArray tConstraint //merge dataVariables (converting to sourceDataTypeClass if needed) for (int dv = 0; dv < ndv; dv++) results[nav + dv].append(tResults[dv]); + //String2.log("!merged tResults[1stDV]=" + results[nav].toString()); //set up for next while-iteration axis0Start += (tStop - tStart) + axis0Stride; diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFiles.java index 496f15a32..b18d4c80d 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridFromNcFiles.java @@ -23,6 +23,7 @@ import gov.noaa.pfel.coastwatch.griddata.NcHelper; import gov.noaa.pfel.coastwatch.pointdata.Table; import gov.noaa.pfel.coastwatch.sgt.SgtUtil; +import gov.noaa.pfel.coastwatch.util.SSR; import gov.noaa.pfel.erddap.GenerateDatasetsXml; import gov.noaa.pfel.erddap.util.EDStatic; @@ -242,10 +243,15 @@ public PrimitiveArray[] getSourceDataFromFile(String fileDir, String fileName, throw new RuntimeException( MessageFormat.format(EDStatic.errorNotFoundIn, "dataVariableSourceName=" + tDataVariables[dvi].sourceName(), - fileName)); //don't show directory - Array array = var.read(selection); + fileName)); //don't show directory + String tSel = selection; + if (tDataVariables[dvi].sourceDataTypeClass() == String.class) + tSel += ",0:" + (var.getShape(var.getRank() - 1) - 1); + Array array = var.read(tSel); Object object = NcHelper.getArray(array); paa[dvi] = PrimitiveArray.factory(object); + //String2.log("!EDDGridFrimNcFiles.getSourceDataFromFile " + tDataVariables[dvi].sourceName() + + // "[" + selection + "]\n" + paa[dvi].toString()); } //I care about this exception @@ -2939,6 +2945,1029 @@ public static void testNcml() throws Throwable { Test.ensureEqual(results, expected, "results=\n" + results); } + /** + * This tests that ensureValid throws exception if an AxisVariable and + * a dataVariable use the same sourceName. + * + * @throws Throwable if trouble + */ + public static void testAVDVSameSource() throws Throwable { + String2.log("\n****************** EDDGridFromNcFiles.testAVDVSameSource() *****************\n"); + String error = "shouldn't happen"; + try { + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml("testAVDVSameSource"); + } catch (Throwable t) { + String2.log(MustBe.throwableToString(t)); + error = String2.split(MustBe.throwableToString(t), '\n')[1]; + } + + Test.ensureEqual(error, + "Two variables have the same sourceName=OB_time.", + "Unexpected error message:\n" + error); + } + + /** + * This tests that ensureValid throws exception if 2 + * dataVariables use the same sourceName. + * + * @throws Throwable if trouble + */ + public static void test2DVSameSource() throws Throwable { + String2.log("\n****************** EDDGridFromNcFiles.test2DVSameSource() *****************\n"); + String error = "shouldn't happen"; + try { + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml("test2DVSameSource"); + } catch (Throwable t) { + String2.log(MustBe.throwableToString(t)); + error = String2.split(MustBe.throwableToString(t), '\n')[1]; + } + + Test.ensureEqual(error, + "Two variables have the same sourceName=IB_time.", + "Unexpected error message:\n" + error); + } + + /** + * This tests that ensureValid throws exception if an AxisVariable and + * a dataVariable use the same destinationName. + * + * @throws Throwable if trouble + */ + public static void testAVDVSameDestination() throws Throwable { + String2.log("\n****************** EDDGridFromNcFiles.testAVDVSameDestination() *****************\n"); + String error = "shouldn't happen"; + try { + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml("testAVDVSameDestination"); + } catch (Throwable t) { + String2.log(MustBe.throwableToString(t)); + error = String2.split(MustBe.throwableToString(t), '\n')[1]; + } + + Test.ensureEqual(error, + "Two variables have the same destinationName=OB_time.", + "Unexpected error message:\n" + error); + } + + /** + * This tests that ensureValid throws exception if 2 + * dataVariables use the same destinationName. + * + * @throws Throwable if trouble + */ + public static void test2DVSameDestination() throws Throwable { + String2.log("\n****************** EDDGridFromNcFiles.test2DVSameDestination() *****************\n"); + String error = "shouldn't happen"; + try { + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml("test2DVSameDestination"); + } catch (Throwable t) { + String2.log(MustBe.throwableToString(t)); + error = String2.split(MustBe.throwableToString(t), '\n')[1]; + } + + Test.ensureEqual(error, + "Two variables have the same destinationName=IB_time.", + "Unexpected error message:\n" + error); + } + + /** + * This tests sub-second time_precision in all output file types. + * + * @throws Throwable if trouble + */ + public static void testTimePrecisionMillis() throws Throwable { + String2.log("\n****************** EDDGridFromNcFiles.testTimePrecisionMillis() *****************\n"); + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml("testTimePrecisionMillis"); + String tDir = EDStatic.fullTestCacheDirectory; + String aq = "[(1984-02-01T12:00:59.001Z):1:(1984-02-01T12:00:59.401Z)]"; + String userDapQuery = "ECEF_X" + aq + ",IB_time" + aq; + String fName = "testTimePrecisionMillis"; + String tName, results, ts, expected; + int po; + + //Yes. EDDGrid.parseAxisBrackets parses ISO 8601 times to millis precision. + //I checked with this query and by turning on debug messages in parseAxisBrackets. + + //.asc + // !!!!! THIS IS ALSO THE ONLY TEST OF AN IMPORTANT BUG + // In EDDGridFromFiles could cause + // intermediate results to use data type from another variable, + // which could be lesser precision. + // (only occurred if vars had different precisions and not first var was requested, + // and if less precision mattered (e.g., double->float lost info)) + // (SEE NOTES 2014-10-20) + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".asc"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"Dataset {\n" + +" GRID {\n" + +" ARRAY:\n" + +" Float32 ECEF_X[time = 5];\n" + +" MAPS:\n" + +" Float64 time[time = 5];\n" + +" } ECEF_X;\n" + +" GRID {\n" + +" ARRAY:\n" + +" Float64 IB_time[time = 5];\n" + +" MAPS:\n" + +" Float64 time[time = 5];\n" + +" } IB_time;\n" + +"} testTimePrecisionMillis;\n" + +"---------------------------------------------\n" + +"ECEF_X.ECEF_X[5]\n" + +//missing first value is where outer dimension values would be, e.g., [0] (but none here) +", 9.96921E36, 9.96921E36, 9.96921E36, 9.96921E36, 9.96921E36\n" + +"\n" + +"ECEF_X.time[5]\n" + +"4.44484859001E8, 4.44484859101E8, 4.44484859201E8, 4.44484859301E8, 4.4448485940099996E8\n" + +"IB_time.IB_time[5]\n" + +//missing first value is where outer dimension values would be, e.g., [0] (but none here) +//THESE ARE THE VALUES THAT WERE AFFECTED BY THE BUG!!! +", 7.60017659E8, 7.600176591E8, 7.600176592E8, 7.600176593E8, 7.600176594E8\n" + +"\n" + +"IB_time.time[5]\n" + +"4.44484859001E8, 4.44484859101E8, 4.44484859201E8, 4.44484859301E8, 4.4448485940099996E8\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.csv + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".csv"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"time,ECEF_X,IB_time\n" + +"UTC,m,UTC\n" + +"1984-02-01T12:00:59.001Z,9.96921E36,1994-01-31T12:00:59.000Z\n" + +"1984-02-01T12:00:59.101Z,9.96921E36,1994-01-31T12:00:59.100Z\n" + +"1984-02-01T12:00:59.201Z,9.96921E36,1994-01-31T12:00:59.200Z\n" + +"1984-02-01T12:00:59.301Z,9.96921E36,1994-01-31T12:00:59.300Z\n" + +"1984-02-01T12:00:59.401Z,9.96921E36,1994-01-31T12:00:59.400Z\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.dods hard to test + + //.htmlTable + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".htmlTable"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    time\n" + +"ECEF_X\n" + +"IB_time\n" + +"
    UTC\n" + +"m\n" + +"UTC\n" + +"
    1984-02-01T12:00:59.001Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.000Z\n" + +"
    1984-02-01T12:00:59.101Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.100Z\n" + +"
    1984-02-01T12:00:59.201Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.200Z\n" + +"
    1984-02-01T12:00:59.301Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.300Z\n" + +"
    1984-02-01T12:00:59.401Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.400Z\n" + +"
    \n"; + po = results.indexOf("\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    timeECEF_XIB_time
    UTCmUTC
    1984-02-01T12:00:59.001Z9.96921E361994-01-31T12:00:59.000Z
    1984-02-01T12:00:59.101Z9.96921E361994-01-31T12:00:59.100Z
    1984-02-01T12:00:59.201Z9.96921E361994-01-31T12:00:59.200Z
    1984-02-01T12:00:59.301Z9.96921E361994-01-31T12:00:59.300Z
    1984-02-01T12:00:59.401Z9.96921E361994-01-31T12:00:59.400Z
    \n"; + po = results.indexOf("\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    days\n" + +"hours\n" + +"minutes\n" + +"seconds\n" + +"millis\n" + +"bytes\n" + +"shorts\n" + +"ints\n" + +"floats\n" + +"doubles\n" + +"Strings\n" + +"
    UTC\n" + +"UTC\n" + +"UTC\n" + +"UTC\n" + +"UTC\n" + +" \n" + +" \n" + +" \n" + +" \n" + +" \n" + +" \n" + +"
    1970-01-03\n" + +"1980-01-01T06Z\n" + +"1990-01-01T00:10Z\n" + +"2000-01-01T00:00:21Z\n" + +"2010-01-01T00:00:00.031Z\n" + +"41\n" + +"10001\n" + +"1000001\n" + +"1.1\n" + +"1.0000000000001E12\n" + +"10\n" + +"
    1970-01-04\n" + +"1980-01-01T07Z\n" + +"1990-01-01T00:11Z\n" + +"2000-01-01T00:00:22Z\n" + +"2010-01-01T00:00:00.032Z\n" + +"42\n" + +"10002\n" + +"1000002\n" + +"2.2\n" + +"1.0000000000002E12\n" + +"20\n" + +"
    \n"; + po = results.indexOf("\n" + +"\n" + +"\n" + +"\n" + +" \n" + +" testSimpleTestNc\n" + +"\n" + +"\n" + +"\n" + +" \n" + +"
    \n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    dayshoursminutessecondsmillisbytesshortsintsfloatsdoublesStrings
    UTCUTCUTCUTCUTC
    1970-01-03T00:00:00Z1980-01-01T06:00:00Z1990-01-01T00:10:00Z2000-01-01T00:00:21Z2010-01-01T00:00:00.031Z411000110000011.11.0000000000001E1210
    1970-01-04T00:00:00Z1980-01-01T07:00:00Z1990-01-01T00:11:00Z2000-01-01T00:00:22Z2010-01-01T00:00:00.032Z421000210000022.21.0000000000002E1220
    \n" + +"\n" + +"\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //test time on x and y axis + tName = eddGrid.makeNewFileForDapQuery(null, null, + "hours[(1970-01-02):(1970-01-05)]&.draw=linesAndMarkers" + + "&.vars=days|hours|&.marker=5|5&.color=0x000000&.colorBar=|||||", + tDir, fName, ".png"); + SSR.displayInBrowser("file://" + tDir + tName); + String2.log("\n!!!! KNOWN PROBLEM: SgtGraph DOESN'T SUPPORT TWO TIME AXES. !!!!\n" + + "See SgtGraph \"yIsTimeAxis = false;\".\n"); + Math2.sleep(10000); + + } + + /** + * This tests timestamps and other things. + * + * @throws Throwable if trouble + */ + public static void testSimpleTestNc2() throws Throwable { + String2.log("\n****************** EDDGridFromNcFiles.testSimpleTestNc2() *****************\n"); + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml("testSimpleTestNc"); + String tDir = EDStatic.fullTestCacheDirectory; + String userDapQuery = "bytes[2:3],doubles[2:3],Strings[2:3]"; + String fName = "testSimpleTestNc2"; + String tName, results, ts, expected; + int po; + + String2.log(NcHelper.dumpString("/erddapTest/simpleTest.nc", true)); + + //.asc + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".asc"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"Dataset {\n" + +" GRID {\n" + +" ARRAY:\n" + +" Byte bytes[days = 2];\n" + +" MAPS:\n" + +" Float64 days[days = 2];\n" + +" } bytes;\n" + +" GRID {\n" + +" ARRAY:\n" + +" Float64 doubles[days = 2];\n" + +" MAPS:\n" + +" Float64 days[days = 2];\n" + +" } doubles;\n" + +" GRID {\n" + +" ARRAY:\n" + +" String Strings[days = 2];\n" + +" MAPS:\n" + +" Float64 days[days = 2];\n" + +" } Strings;\n" + +"} testSimpleTestNc;\n" + +"---------------------------------------------\n" + +"bytes.bytes[2]\n" + +", 42, 43\n" + +"\n" + +"bytes.days[2]\n" + +"259200.0, 345600.0\n" + +"doubles.doubles[2]\n" + +", 1.0000000000002E12, 1.0000000000003E12\n" + +"\n" + +"doubles.days[2]\n" + +"259200.0, 345600.0\n" + +"Strings.Strings[2]\n" + +", 20, 30\n" + +"\n" + +"Strings.days[2]\n" + +"259200.0, 345600.0\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.csv + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".csv"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"days,bytes,doubles,Strings\n" + +"UTC,,,\n" + +"1970-01-04T00:00:00Z,42,1.0000000000002E12,20\n" + +"1970-01-05T00:00:00Z,43,1.0000000000003E12,30\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.dods doesn't write strings + + //.htmlTable + tName = eddGrid.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".htmlTable"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    days\n" + +"bytes\n" + +"doubles\n" + +"Strings\n" + +"
    UTC\n" + +" \n" + +" \n" + +" \n" + +"
    1970-01-04\n" + +"42\n" + +"1.0000000000002E12\n" + +"20\n" + +"
    1970-01-05\n" + +"43\n" + +"1.0000000000003E12\n" + +"30\n" + +"
    \n"; + po = results.indexOf("\n" + +"\n" + +"\n" + +"\n" + +" \n" + +" testSimpleTestNc2\n" + +"\n" + +"\n" + +"\n" + +" \n" + +"
    \n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    daysbytesdoublesStrings
    UTC
    1970-01-04T00:00:00Z421.0000000000002E1220
    1970-01-05T00:00:00Z431.0000000000003E1230
    \n" + +"\n" + +"\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + } + + /** * This tests this class. * @@ -2955,6 +3984,13 @@ public static void test(boolean deleteCachedDatasetInfo) throws Throwable { testGenerateDatasetsXml(); testGenerateDatasetsXml2(); testSpeed(-1); //-1 = all + testAVDVSameSource(); + test2DVSameSource(); + testAVDVSameDestination(); + test2DVSameDestination(); + testTimePrecisionMillis(); + testSimpleTestNc(); + testSimpleTestNc2(); /* */ //one time tests diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridSideBySide.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridSideBySide.java index 9de7f8ba7..e9dd9e9ba 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridSideBySide.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDGridSideBySide.java @@ -307,6 +307,10 @@ else if (depthIndex == 0) else if (timeIndex == 0) axisVariables[0] = new EDVTimeGridAxis(fav.sourceName(), fav.sourceAttributes(), fav.addAttributes(), newAxis0Values); + else if (fav instanceof EDVTimeStampGridAxis) + axisVariables[0] = new EDVTimeStampGridAxis( + fav.sourceName(), fav.destinationName(), + fav.sourceAttributes(), fav.addAttributes(), newAxis0Values); else {axisVariables[0] = new EDVGridAxis(fav.sourceName(), fav.destinationName(), fav.sourceAttributes(), fav.addAttributes(), newAxis0Values); axisVariables[0].setActualRangeFromDestinationMinMax(); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTable.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTable.java index b8ce9d24c..540d478be 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTable.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTable.java @@ -516,6 +516,24 @@ public void ensureValid() throws Throwable { String errorInMethod = "datasets.xml/EDDTable.ensureValid error for datasetID=" + datasetID + ":\n "; + HashSet sourceNamesHS = new HashSet(2 * dataVariables.length); + HashSet destNamesHS = new HashSet(2 * dataVariables.length); + for (int v = 0; v < dataVariables.length; v++) { + //ensure unique sourceNames + String sn = dataVariables[v].sourceName(); + if (!sn.startsWith("=")) { + if (!sourceNamesHS.add(sn)) + throw new RuntimeException(errorInMethod + + "Two dataVariables have the same sourceName=" + sn + "."); + } + + //ensure unique destNames + String dn = dataVariables[v].destinationName(); + if (!destNamesHS.add(dn)) + throw new RuntimeException(errorInMethod + + "Two dataVariables have the same destinationName=" + dn + "."); + } + Test.ensureTrue(lonIndex < 0 || dataVariables[lonIndex] instanceof EDVLon, errorInMethod + "dataVariable[lonIndex=" + lonIndex + "] isn't an EDVLon."); Test.ensureTrue(latIndex < 0 || dataVariables[latIndex] instanceof EDVLat, @@ -625,14 +643,16 @@ public void ensureValid() throws Throwable { //time if (timeIndex >= 0) { - PrimitiveArray pa = dataVariables[timeIndex].combinedAttributes().get("actual_range"); + Attributes catts = dataVariables[timeIndex].combinedAttributes(); + PrimitiveArray pa = catts.get("actual_range"); if (pa != null) { - double d = pa.getDouble(0); - if (!Double.isNaN(d)) - combinedGlobalAttributes.set("time_coverage_start", Calendar2.epochSecondsToIsoStringT(d) + "Z"); - d = pa.getDouble(1); //will be NaN for 'present'. Deal with this better??? - if (!Double.isNaN(d)) - combinedGlobalAttributes.set("time_coverage_end", Calendar2.epochSecondsToIsoStringT(d) + "Z"); + String tp = catts.getString(EDV.TIME_PRECISION); + //"" unsets the attribute if min or max isNaN + combinedGlobalAttributes.set("time_coverage_start", + Calendar2.epochSecondsToLimitedIsoStringT(tp, pa.getDouble(0), "")); + //for tables (not grids) will be NaN for 'present'. Deal with this better??? + combinedGlobalAttributes.set("time_coverage_end", + Calendar2.epochSecondsToLimitedIsoStringT(tp, pa.getDouble(1), "")); } } @@ -1437,7 +1457,8 @@ public void applyConstraints(Table table, boolean applyAllConstraints, edv.destinationFillValue(), edv.destinationMissingValue()); //String2.log(" nSwitched=" + nSwitched); } - int nStillGood = dataPa.applyConstraint(keep, constraintOp, constraintValue); + int nStillGood = dataPa.applyConstraint(edv instanceof EDVTimeStamp, + keep, constraintOp, constraintValue); if (reallyVerbose) String2.log(" nStillGood=" + nStillGood); if (nStillGood == 0) break; @@ -2936,7 +2957,7 @@ else if (altCol >= 0) String columnUnits[] = new String[table.nColumns()]; boolean columnIsString[] = new boolean[table.nColumns()]; boolean columnIsTimeStamp[] = new boolean[table.nColumns()]; - String columnTimeFormat[] = new String[table.nColumns()]; + String columnTimePrecision[] = new String[table.nColumns()]; for (int col = 0; col < table.nColumns(); col++) { String units = table.columnAttributes(col).getString("units"); //test isTimeStamp before prepending " " @@ -2946,7 +2967,7 @@ else if (altCol >= 0) " " + units; columnUnits[col] = units; columnIsString[col] = table.getColumn(col) instanceof StringArray; - columnTimeFormat[col] = table.columnAttributes(col).getString(EDV.TIME_PRECISION); + columnTimePrecision[col] = table.columnAttributes(col).getString(EDV.TIME_PRECISION); } //based on kmz example from http://www.coriolis.eu.org/cdc/google_earth.htm @@ -3049,7 +3070,7 @@ else if (altCol >= 0) XML.encodeAsXML(table.getColumnName(col) + " = " + (columnIsTimeStamp[col]? Calendar2.epochSecondsToLimitedIsoStringT( - columnTimeFormat[col], td, "") : + columnTimePrecision[col], td, "") : columnIsString[col]? ts : (Double.isNaN(td)? "NaN" : ts) + columnUnits[col]))); } @@ -5062,13 +5083,13 @@ public static void saveAsODV(OutputStreamSource outputStreamSource, if (tnRows == 0) throw new SimpleException(MustBe.THERE_IS_NO_DATA + " (at start of saveAsODV)"); //the other params are all required by EDD, so it's a programming error if they are missing - if (tDatasetID == null || tDatasetID.length() == 0) + if (!String2.isSomething(tDatasetID)) throw new SimpleException(EDStatic.errorInternal + "saveAsODV error: datasetID wasn't specified."); - if (tPublicSourceUrl == null || tPublicSourceUrl.length() == 0) + if (!String2.isSomething(tPublicSourceUrl)) throw new SimpleException(EDStatic.errorInternal + "saveAsODV error: publicSourceUrl wasn't specified."); - if (tInfoUrl == null || tInfoUrl.length() == 0) + if (!String2.isSomething(tInfoUrl)) throw new SimpleException(EDStatic.errorInternal + "saveAsODV error: infoUrl wasn't specified."); @@ -5115,7 +5136,7 @@ public static void saveAsODV(OutputStreamSource outputStreamSource, //Presence of longitude, latitude, time is checked above. int cruiseCol = -1; int stationCol = -1; - int timeCol = table.findColumnNumber(EDV.TIME_NAME); //but ODV requires it. see above + int timeCol = table.findColumnNumber(EDV.TIME_NAME); //ODV requires it. see above //2010-07-07 email from Stephan Heckendorff says altitude (or similar) if present MUST be primaryVar int primaryVarCol = table.findColumnNumber(EDV.ALT_NAME); if (primaryVarCol < 0) @@ -5162,11 +5183,10 @@ public static void saveAsODV(OutputStreamSource outputStreamSource, writer.write("\tCruise:METAVAR:TEXT:2"); if (stationCol == -1) writer.write("\tStation:METAVAR:TEXT:2"); - if (timeCol == -1) - writer.write("\tyyyy-mm-ddThh:mm:ss.sss"); //columns from selected data - boolean isTimeStamp[] = new boolean[nCols]; + boolean isTimeStamp[] = new boolean[nCols]; //includes time var + String time_precision[] = new String[nCols]; //includes time var for (int col = 0; col < nCols; col++) { //write tab first (since Type column comes before first table column) @@ -5177,6 +5197,12 @@ public static void saveAsODV(OutputStreamSource outputStreamSource, String colNameLC = colName.toLowerCase(); String units = table.columnAttributes(col).getString("units"); isTimeStamp[col] = EDV.TIME_UNITS.equals(units); //units may be null + //just keep time_precision if it includes fractional seconds + String tp = table.columnAttributes(col).getString(EDV.TIME_PRECISION); + if (tp != null && !tp.startsWith("1970-01-01T00:00:00.0")) + tp = null; //default + time_precision[col] = tp; + if (units == null || units.length() == 0) { units = ""; } else { @@ -5265,9 +5291,10 @@ else throw new SimpleException(EDStatic.errorInternal + for (int col = 0; col < nCols; col++) { writer.write('\t'); //since Type and other columns were written above if (isTimeStamp[col]) { - //no Z at end; ODV ignores time zone info (see 2010-06-15 notes) - //!!! Keep as ISO 8601, not time_precision. - writer.write(Calendar2.safeEpochSecondsToIsoStringT(table.getDoubleData(col, row), "")); + //!!! use time_precision (may be greater seconds or decimal seconds). + //ODV ignores time zone info, but okay to specify, e.g., Z (see 2010-06-15 notes) + writer.write(Calendar2.epochSecondsToLimitedIsoStringT( + time_precision[col], table.getDoubleData(col, row), "")); //missing numeric will be empty cell; that's fine } else { //See comments above about ISO-8859-1. I am choosing *not* to strip high ASCII chars here. @@ -6620,15 +6647,17 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, "
    these extensions (notably \"=~\") sometimes aren't practical because tabledap\n" + "
    may need to download lots of extra data from the source (which takes time)\n" + "
    in order to test the constraint.\n" + - "
  • tabledap always stores date/time values as numbers (in seconds since 1970-01-01T00:00:00Z).\n" + + "
  • tabledap always stores date/time values as double precision floating point numbers\n" + + "
    (seconds since 1970-01-01T00:00:00Z, sometimes with some number of milliseconds).\n" + "
    Here is an example of a query which includes date/time numbers:\n" + "
    " + XML.encodeAsHTML( fullValueExample) + "\n" + - "
    Some fileTypes (notably, .csv, .tsv, .htmlTable, .odvTxt, and .xhtml) display\n" + - "
    date/time values as \n" + + "
    The more human-oriented fileTypes (notably, .csv, .tsv, .htmlTable, .odvTxt, and .xhtml)\n" + + "
    display date/time values as \n" + " ISO 8601:2004 \"extended\" date/time strings" + EDStatic.externalLinkHtml(tErddapUrl) + "\n" + - "
    (e.g., 2002-08-03T12:30:00Z).\n" + + "
    (e.g., 2002-08-03T12:30:00Z, but some variables include milliseconds, e.g.,\n" + + "
    2002-08-03T12:30:00.123Z).\n" + (EDStatic.convertersActive? "
    ERDDAP has a utility to\n" + " Convert\n" + @@ -6636,10 +6665,13 @@ public static void writeGeneralDapHtmlInstructions(String tErddapUrl, "
    See also:\n" + "     How\n" + " ERDDAP Deals with Time.\n" : "") + - "
  • tabledap extends the OPeNDAP standard to allow you to specify time values in the ISO 8601\n" + - "
    date/time format (YYYY-MM-DDThh:mm:ssZ, where Z is 'Z' or a ±hh:mm offset from UTC). \n" + - "
    If you omit Z (or the ±hh:mm offset), :ssZ, :mm:ssZ, or Thh:mm:ssZ from the ISO date/time\n" + - "
    that you specify, the missing fields are assumed to be 0.\n" + + "
  • tabledap extends the OPeNDAP standard to allow you to specify time values in the\n" + + "
    ISO 8601 date/time format (YYYY-MM-DDThh:mm:ss.sssZ, where Z is 'Z' or a ±hh\n" + + "
    or ±hh:mm offset from the Zulu/GMT time zone. If you omit Z and the offset, the\n" + + "
    Zulu/GMT time zone is used. Separately, if you omit .sss, :ss.sss, :mm:ss.sss, or\n" + + "
    Thh:mm:ss.sss from the ISO date/time that you specify, the missing fields are assumed\n" + + "
    to be 0. In some places, ERDDAP accepts a comma (ss,sss) as the seconds decimal point,\n" + + "
    but ERDDAP always uses a period when formatting times as ISO 8601 strings.\n" + "
    Here is an example of a query which includes ISO date/time values:\n" + "
    " + XML.encodeAsHTML( fullTimeExample) + " .\n" + @@ -6985,19 +7017,24 @@ public void respondToGraphQuery(HttpServletRequest request, String loggedInAs, StringArray nonAxisSA = new StringArray(); EDVTime timeVar = null; int dvTime = -1; + String time_precision[] = new String[dataVariables.length]; for (int dv = 0; dv < dataVariables.length; dv++) { - if (dataVariables[dv].destinationDataTypeClass() != String.class) { - String dn = dataVariables[dv].destinationName(); + EDV edv = dataVariables[dv]; + if (edv.destinationDataTypeClass() != String.class) { + String dn = edv.destinationName(); if (dv == lonIndex) {axisSA.add(dn); } else if (dv == latIndex) {axisSA.add(dn); } else if (dv == altIndex) {axisSA.add(dn); } else if (dv == depthIndex) {axisSA.add(dn); } else if (dv == timeIndex) {axisSA.add(dn); dvTime = sa.size(); - timeVar = (EDVTime)dataVariables[dv]; + timeVar = (EDVTime)edv; } else { nonAxisSA.add(dn); } + if (edv instanceof EDVTimeStamp) + time_precision[dv] = edv.combinedAttributes().getString(EDV.TIME_PRECISION); + if (dv != lonIndex && dv != latIndex) nonLLSA.add(dn); @@ -10457,7 +10494,8 @@ public boolean handleViaFixedOrSubsetVariables(String loggedInAs, String request //is it true? PrimitiveArray pa = PrimitiveArray.factory(edv.destinationDataTypeClass(), 1, edv.sourceName().substring(1)); - if (pa.applyConstraint(keep, constraintOps.get(cv), constraintValues.get(cv)) == 0) + if (pa.applyConstraint(edv instanceof EDVTimeStamp, + keep, constraintOps.get(cv), constraintValues.get(cv)) == 0) throw new SimpleException(MustBe.THERE_IS_NO_DATA + " (after " + constraintVariables.get(cv) + constraintOps.get(cv) + constraintValues.get(cv) + ")"); } else { @@ -15355,8 +15393,7 @@ public void writeISO19115(Writer writer) throws Throwable { String domain = EDStatic.baseUrl; if (domain.startsWith("http://")) domain = domain.substring(7); - String eddCreationDate = String2.replaceAll( - Calendar2.millisToIsoZuluString(creationTimeMillis()), "-", "").substring(0, 8) + "Z"; + String eddCreationDate = Calendar2.millisToIsoZuluString(creationTimeMillis()).substring(0, 10); String acknowledgement = combinedGlobalAttributes.getString("acknowledgement"); String contributorName = combinedGlobalAttributes.getString("contributor_name"); @@ -15368,10 +15405,10 @@ public void writeISO19115(Writer writer) throws Throwable { //creatorUrl: use infoUrl String dateCreated = combinedGlobalAttributes.getString("date_created"); String dateIssued = combinedGlobalAttributes.getString("date_issued"); - if (dateCreated != null && dateCreated.length() >= 10 && !dateCreated.endsWith("Z")) - dateCreated = dateCreated.substring(0, 10) + "Z"; - if (dateIssued != null && dateIssued.length() >= 10 && !dateIssued.endsWith("Z")) - dateIssued = dateIssued.substring(0, 10) + "Z"; + if (dateCreated != null && dateCreated.length() > 10) + dateCreated = dateCreated.substring(0, 10); + if (dateIssued != null && dateIssued.length() > 10) + dateIssued = dateIssued.substring(0, 10); String history = combinedGlobalAttributes.getString("history"); String infoUrl = combinedGlobalAttributes.getString("infoUrl"); String institution = combinedGlobalAttributes.getString("institution"); @@ -16549,7 +16586,7 @@ public void writeISO19115(Writer writer) throws Throwable { " \n" + " \n" + //see list at https://github.com/OSGeo/Cat-Interop/blob/master/LinkPropertyLookupTable.csv from John Maurer -" template\n" + +" order\n" + " \n" + " \n" + " Data Subset Form\n" + @@ -16579,7 +16616,7 @@ public void writeISO19115(Writer writer) throws Throwable { " \n" + " \n" + //see list at https://github.com/OSGeo/Cat-Interop/blob/master/LinkPropertyLookupTable.csv from John Maurer -" template\n" + +" order\n" + " \n" + " \n" + " Make-A-Graph Form\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromEDDGrid.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromEDDGrid.java index 4c172da4e..1cf8d6077 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromEDDGrid.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromEDDGrid.java @@ -321,7 +321,7 @@ public void getDataForDapQuery(String loggedInAs, String requestUrl, if (conOp.equals(PrimitiveArray.REGEX_OP)) { //FUTURE: this could be improved to find the range of matching axis values } else { - boolean avIsTime = edvga instanceof EDVTimeGridAxis; + boolean avIsTimeStamp = edvga instanceof EDVTimeStampGridAxis; double oldAvMin = avMin[av]; double oldAvMax = avMax[av]; double conValD = String2.parseDouble(conVal); @@ -336,7 +336,7 @@ public void getDataForDapQuery(String loggedInAs, String requestUrl, passed = false; else { double destVal = edvga.destinationValue(si).getDouble(0); - passed = avIsTime? + passed = avIsTimeStamp? conValD == destVal : //exact Math2.almostEqual(5, conValD, destVal); //fuzzy, biased towards passing avMin[av] = Math.max(avMin[av], conValD); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcFiles.java index c8dddcac9..0763d4e72 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromNcFiles.java @@ -4119,7 +4119,7 @@ public static void testErdGtsppBest(String tDatasetID) throws Throwable { " \\}\n" + " station_id \\{\n" + " Int32 _FillValue 2147483647;\n" + -" Int32 actual_range 1, 20668663;\n" + //changes every month //don't regex this. It's important to see the changes. +" Int32 actual_range 1, 20937550;\n" + //changes every month //don't regex this. It's important to see the changes. " String cf_role \"profile_id\";\n" + " String comment \"Identification number of the station \\(profile\\) in the GTSPP Continuously Managed Database\";\n" + " String ioos_category \"Identifier\";\n" + @@ -4165,7 +4165,7 @@ public static void testErdGtsppBest(String tDatasetID) throws Throwable { " time \\{\n" + " String _CoordinateAxisType \"Time\";\n" + " Float64 _FillValue NaN;\n" + -" Float64 actual_range 6.31152e\\+8, 1.4091408e\\+9;\n" + //2nd value changes +" Float64 actual_range 6.31152e\\+8, 1.411992e\\+9;\n" + //2nd value changes " String axis \"T\";\n" + " String ioos_category \"Time\";\n" + " String long_name \"Time\";\n" + @@ -4228,7 +4228,7 @@ public static void testErdGtsppBest(String tDatasetID) throws Throwable { " \\}\n" + " NC_GLOBAL \\{\n" + " String acknowledgment \"These data were acquired from the US NOAA National Oceanographic " + - "Data Center \\(NODC\\) on 2014-09-16 from http://www.nodc.noaa.gov/GTSPP/.\";\n" + //changes monthly + "Data Center \\(NODC\\) on 2014-10-07 from http://www.nodc.noaa.gov/GTSPP/.\";\n" + //changes monthly " String cdm_altitude_proxy \"depth\";\n" + " String cdm_data_type \"TrajectoryProfile\";\n" + " String cdm_profile_variables \"station_id, longitude, latitude, time\";\n" + @@ -4239,7 +4239,7 @@ public static void testErdGtsppBest(String tDatasetID) throws Throwable { " String creator_url \"http://www.nodc.noaa.gov/GTSPP/\";\n" + " String crs \"EPSG:4326\";\n" + (tDatasetID.equals("erdGtsppBest")? //changes - " String defaultGraphQuery \"longitude,latitude,station_id&time%3E=2014-08-25&time%3C=2014-09-01&.draw=markers&.marker=1\\|5\";\n" : + " String defaultGraphQuery \"longitude,latitude,station_id&time%3E=2014-09-24&time%3C=2014-10-01&.draw=markers&.marker=1\\|5\";\n" : "") + " Float64 Easternmost_Easting 179.999;\n" + " String featureType \"TrajectoryProfile\";\n" + @@ -4258,9 +4258,9 @@ public static void testErdGtsppBest(String tDatasetID) throws Throwable { " String gtspp_handbook_version \"GTSPP Data User's Manual 1.0\";\n" + " String gtspp_program \"writeGTSPPnc40.f90\";\n" + " String gtspp_programVersion \"1.7\";\n" + -" String history \"2014-09-01 csun writeGTSPPnc40.f90 Version 1.7\n" + //date changes +" String history \"2014-10-01 csun writeGTSPPnc40.f90 Version 1.7\n" + //date changes ".tgz files from ftp.nodc.noaa.gov /pub/gtspp/best_nc/ \\(http://www.nodc.noaa.gov/GTSPP/\\)\n" + -"2014-09-15 Most recent ingest, clean, and reformat at ERD \\(bob.simons at noaa.gov\\).\n"; //date changes +"2014-10-07 Most recent ingest, clean, and reformat at ERD \\(bob.simons at noaa.gov\\).\n"; //date changes po = results.indexOf("bob.simons at noaa.gov).\n"); String tResults = results.substring(0, po + 25); @@ -4279,7 +4279,7 @@ public static void testErdGtsppBest(String tDatasetID) throws Throwable { " String keywords_vocabulary \"NODC Data Types, CF Standard Names, GCMD Science Keywords\";\n" + " String LEXICON \"NODC_GTSPP\";\n" + //date below changes " String license \"These data are openly available to the public. Please acknowledge the use of these data with:\n" + -"These data were acquired from the US NOAA National Oceanographic Data Center \\(NODC\\) on 2014-09-16 from http://www.nodc.noaa.gov/GTSPP/.\n" + +"These data were acquired from the US NOAA National Oceanographic Data Center \\(NODC\\) on 2014-10-07 from http://www.nodc.noaa.gov/GTSPP/.\n" + "\n" + "The data may be used and redistributed for free but is not intended\n" + "for legal use, since it may contain inaccuracies. Neither the data\n" + @@ -4305,7 +4305,7 @@ public static void testErdGtsppBest(String tDatasetID) throws Throwable { "Requesting data for a specific station_id may be slow, but it works.\n" + "\n" + "\\*\\*\\* This ERDDAP dataset has data for the entire world for all available times \\(currently, " + - "up to and including the August 2014 data\\) but is a subset of the " + //month changes + "up to and including the September 2014 data\\) but is a subset of the " + //month changes "original NODC 'best-copy' data. It only includes data where the quality flags indicate the data is 1=CORRECT, 2=PROBABLY GOOD, or 5=MODIFIED. It does not include some of the metadata, any of the history data, or any of the quality flag data of the original dataset. You can always get the complete, up-to-date dataset \\(and additional, near-real-time data\\) from the source: http://www.nodc.noaa.gov/GTSPP/ . Specific differences are:\n" + "\\* Profiles with a position_quality_flag or a time_quality_flag other than 1\\|2\\|5 were removed.\n" + "\\* Rows with a depth \\(z\\) value less than -0.4 or greater than 10000 or a z_variable_quality_flag other than 1\\|2\\|5 were removed.\n" + @@ -4317,7 +4317,7 @@ public static void testErdGtsppBest(String tDatasetID) throws Throwable { "http://www.nodc.noaa.gov/GTSPP/document/qcmans/GTSPP_RT_QC_Manual_20090916.pdf .\n" + "The Quality Flag definitions are also at\n" + "http://www.nodc.noaa.gov/GTSPP/document/qcmans/qcflags.htm .\";\n" + -" String time_coverage_end \"2014-08-27T12:00:00Z\";\n" + //changes +" String time_coverage_end \"2014-09-29T12:00:00Z\";\n" + //changes " String time_coverage_start \"1990-01-01T00:00:00Z\";\n" + " String title \"Global Temperature and Salinity Profile Programme \\(GTSPP\\) Data\";\n" + " Float64 Westernmost_Easting -180.0;\n" + @@ -4720,7 +4720,7 @@ public static void testModTime() throws Throwable { expected = "time,irradiance\n" + "UTC,W/m^2\n" + -"2011-07-01T00:00:00Z,1361.3473\n"; +"2011-07-01T00:00:00.000Z,1361.3473\n"; dapQuery = "time,irradiance&time=\"2011-07-01\""; tName = eddTable.makeNewFileForDapQuery(null, null, dapQuery, @@ -9978,9 +9978,9 @@ public static void testGlobec() throws Throwable { "//GeneralField\n" + "//GeneralType\n" + "Type:METAVAR:TEXT:2\tStation:METAVAR:TEXT:2\tCruise:METAVAR:TEXT:7\tship:METAVAR:TEXT:12\tcast:SHORT\tLongitude [degrees_east]:METAVAR:FLOAT\tLatitude [degrees_north]:METAVAR:FLOAT\taltitude [m]:PRIMARYVAR:INTEGER\tyyyy-mm-ddThh:mm:ss.sss\tbottle_posn:BYTE\tchl_a_total [ug L-1]:FLOAT\tchl_a_10um [ug L-1]:FLOAT\tphaeo_total [ug L-1]:FLOAT\tphaeo_10um [ug L-1]:FLOAT\tsal00 [PSU]:FLOAT\tsal11 [PSU]:FLOAT\ttemperature0 [degree_C]:FLOAT\ttemperature1 [degree_C]:FLOAT\tfluor_v [volts]:FLOAT\txmiss_v [volts]:FLOAT\tPO4 [micromoles L-1]:FLOAT\tN_N [micromoles L-1]:FLOAT\tNO3 [micromoles L-1]:FLOAT\tSi [micromoles L-1]:FLOAT\tNO2 [micromoles L-1]:FLOAT\tNH4 [micromoles L-1]:FLOAT\toxygen [mL L-1]:FLOAT\tpar [volts]:FLOAT\n" + -"*\t\tnh0207\tNew_Horizon\t20\t-124.4\t44.0\t0\t2002-08-03T01:29:00\t1\t\t\t\t\t33.9939\t33.9908\t7.085\t7.085\t0.256\t0.518\t2.794\t35.8\t35.7\t71.11\t0.093\t0.037\t\t0.1545\n" + -"*\t\tnh0207\tNew_Horizon\t20\t-124.4\t44.0\t0\t2002-08-03T01:29:00\t2\t\t\t\t\t33.8154\t33.8111\t7.528\t7.53\t0.551\t0.518\t2.726\t35.87\t35.48\t57.59\t0.385\t0.018\t\t0.1767\n" + -"*\t\tnh0207\tNew_Horizon\t20\t-124.4\t44.0\t0\t2002-08-03T01:29:00\t3\t1.463\t\t1.074\t\t33.5858\t33.5834\t7.572\t7.573\t0.533\t0.518\t2.483\t31.92\t31.61\t48.54\t0.307\t0.504\t\t0.3875\n"; +"*\t\tnh0207\tNew_Horizon\t20\t-124.4\t44.0\t0\t2002-08-03T01:29:00Z\t1\t\t\t\t\t33.9939\t33.9908\t7.085\t7.085\t0.256\t0.518\t2.794\t35.8\t35.7\t71.11\t0.093\t0.037\t\t0.1545\n" + +"*\t\tnh0207\tNew_Horizon\t20\t-124.4\t44.0\t0\t2002-08-03T01:29:00Z\t2\t\t\t\t\t33.8154\t33.8111\t7.528\t7.53\t0.551\t0.518\t2.726\t35.87\t35.48\t57.59\t0.385\t0.018\t\t0.1767\n" + +"*\t\tnh0207\tNew_Horizon\t20\t-124.4\t44.0\t0\t2002-08-03T01:29:00Z\t3\t1.463\t\t1.074\t\t33.5858\t33.5834\t7.572\t7.573\t0.533\t0.518\t2.483\t31.92\t31.61\t48.54\t0.307\t0.504\t\t0.3875\n"; po = results.indexOf(expected.substring(0, 13)); Test.ensureEqual(results.substring(po, po + expected.length()), expected, "\nresults=\n" + String2.annotatedString(results)); @@ -11148,7 +11148,7 @@ public static void testPmelTaoAirt() throws Throwable { " String geospatial_vertical_units \"m\";\n" + //date on line below changes monthly DON'T REGEX THIS. I WANT TO SEE THE CHANGES. " String history \"This dataset has data from the TAO/TRITON, RAMA, and PIRATA projects.\n" + "This dataset is a product of the TAO Project Office at NOAA/PMEL.\n" + -"2014-09-12 Bob Simons at NOAA/NMFS/SWFSC/ERD \\(bob.simons@noaa.gov\\) fully refreshed ERD's copy of this dataset by downloading all of the .cdf files from the PMEL TAO FTP site. Since then, the dataset has been partially refreshed everyday by downloading and merging the latest version of the last 25 days worth of data\\."; +"2014-10-06 Bob Simons at NOAA/NMFS/SWFSC/ERD \\(bob.simons@noaa.gov\\) fully refreshed ERD's copy of this dataset by downloading all of the .cdf files from the PMEL TAO FTP site. Since then, the dataset has been partially refreshed everyday by downloading and merging the latest version of the last 25 days worth of data\\."; int tPo = results.indexOf("worth of data."); Test.ensureTrue(tPo >= 0, "tPo=-1 results=\n" + results); Test.ensureLinesMatch(results.substring(0, tPo + 14), expected, "\nresults=\n" + results); @@ -11541,6 +11541,1318 @@ public static void testNow() throws Throwable { //if (true) throw new RuntimeException("stop here"); } + /** + * This tests sub-second time_precision in all output file types. + * + * @throws Throwable if trouble + */ + public static void testTimePrecisionMillis() throws Throwable { + String2.log("\n****************** EDDTableFromNcFiles.testTimePrecisionMillis() *****************\n"); + EDDTable eddTable = (EDDTable)oneFromDatasetXml("testTimePrecisionMillisTable"); + String tDir = EDStatic.fullTestCacheDirectory; + String userDapQuery = "time,ECEF_X,IB_time" + + "&time>1984-02-01T12:00:59.001Z" + //value is .001, so this just barely fails + "&time<=1984-02-01T12:00:59.401Z"; //value is .401, so this just barely succeeds + String fName = "testTimePrecisionMillis"; + String tName, results, ts, expected; + int po; + + //.asc + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".asc"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"Dataset {\n" + +" Sequence {\n" + +" Float64 time;\n" + +" Float32 ECEF_X;\n" + +" Float64 IB_time;\n" + +" } s;\n" + +"} s;\n" + +"---------------------------------------------\n" + +"s.time, s.ECEF_X, s.IB_time\n" + +"4.44484859101E8, 9.96921E36, 7.600176591E8\n" + +"4.44484859201E8, 9.96921E36, 7.600176592E8\n" + +"4.44484859301E8, 9.96921E36, 7.600176593E8\n" + +"4.4448485940099996E8, 9.96921E36, 7.600176594E8\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.csv + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".csv"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"time,ECEF_X,IB_time\n" + +"UTC,m,UTC\n" + +"1984-02-01T12:00:59.101Z,9.96921E36,1994-01-31T12:00:59.100Z\n" + +"1984-02-01T12:00:59.201Z,9.96921E36,1994-01-31T12:00:59.200Z\n" + +"1984-02-01T12:00:59.301Z,9.96921E36,1994-01-31T12:00:59.300Z\n" + +"1984-02-01T12:00:59.401Z,9.96921E36,1994-01-31T12:00:59.400Z\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.dods doesn't write strings + + //.htmlTable + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".htmlTable"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    time\n" + +"ECEF_X\n" + +"IB_time\n" + +"
    UTC\n" + +"m\n" + +"UTC\n" + +"
    1984-02-01T12:00:59.101Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.100Z\n" + +"
    1984-02-01T12:00:59.201Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.200Z\n" + +"
    1984-02-01T12:00:59.301Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.300Z\n" + +"
    1984-02-01T12:00:59.401Z\n" + +"9.96921E36\n" + +"1994-01-31T12:00:59.400Z\n" + +"
    \n"; + po = results.indexOf("\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    timeECEF_XIB_time
    UTCmUTC
    1984-02-01T12:00:59.101Z9.96921E361994-01-31T12:00:59.100Z
    1984-02-01T12:00:59.201Z9.96921E361994-01-31T12:00:59.200Z
    1984-02-01T12:00:59.301Z9.96921E361994-01-31T12:00:59.300Z
    1984-02-01T12:00:59.401Z9.96921E361994-01-31T12:00:59.400Z
    \n"; + po = results.indexOf("=1970-01-02T00:00:00Z" + //should just barely succeed, 86400 + "&time<1970-01-05T00:00:00Z"; //should just barely fail, 345600 + String fName = "testSimpleTestNcTable"; + String tName, results, ts, expected; + int po; + + String2.log(NcHelper.dumpString("/erddapTest/simpleTest.nc", true)); + + //all + tName = eddTable.makeNewFileForDapQuery(null, null, "", tDir, + fName, ".csv"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"time,hours,minutes,seconds,millis,latitude,longitude,ints,floats,doubles,Strings\n" + +"UTC,UTC,UTC,UTC,UTC,degrees_north,degrees_east,,,,\n" + +"1970-01-02T00:00:00Z,1980-01-01T05:00:00Z,1990-01-01T00:09:00Z,2000-01-01T00:00:20Z,2010-01-01T00:00:00.030Z,40,10000,1000000,0.0,1.0E12,0\n" + +"1970-01-03T00:00:00Z,1980-01-01T06:00:00Z,1990-01-01T00:10:00Z,2000-01-01T00:00:21Z,2010-01-01T00:00:00.031Z,41,10001,1000001,1.1,1.0000000000001E12,10\n" + +"1970-01-04T00:00:00Z,1980-01-01T07:00:00Z,1990-01-01T00:11:00Z,2000-01-01T00:00:22Z,2010-01-01T00:00:00.032Z,42,10002,1000002,2.2,1.0000000000002E12,20\n" + +"1970-01-05T00:00:00Z,1980-01-01T08:00:00Z,1990-01-01T00:12:00Z,2000-01-01T00:00:23Z,2010-01-01T00:00:00.033Z,43,10004,1000004,4.4,1.0000000000003E12,30\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.asc + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".asc"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"Dataset {\n" + +" Sequence {\n" + +" Float64 time;\n" + +" Float64 hours;\n" + +" Float64 minutes;\n" + +" Float64 seconds;\n" + +" Float64 millis;\n" + +" Byte latitude;\n" + +" Int16 longitude;\n" + +" Int32 ints;\n" + +" Float32 floats;\n" + +" Float64 doubles;\n" + +" String Strings;\n" + +" } s;\n" + +"} s;\n" + +"---------------------------------------------\n" + +"s.time, s.hours, s.minutes, s.seconds, s.millis, s.latitude, s.longitude, s.ints, s.floats, s.doubles, s.Strings\n" + +"86400.0, 3.155508E8, 6.3115254E8, 9.4668482E8, 1.26230400003E9, 40, 10000, 1000000, 0.0, 1.0E12, \"0\"\n" + +"172800.0, 3.155544E8, 6.311526E8, 9.46684821E8, 1.262304000031E9, 41, 10001, 1000001, 1.1, 1.0000000000001E12, \"10\"\n" + +"259200.0, 3.15558E8, 6.3115266E8, 9.46684822E8, 1.262304000032E9, 42, 10002, 1000002, 2.2, 1.0000000000002E12, \"20\"\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.csv + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".csv"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"time,hours,minutes,seconds,millis,latitude,longitude,ints,floats,doubles,Strings\n" + +"UTC,UTC,UTC,UTC,UTC,degrees_north,degrees_east,,,,\n" + +"1970-01-02T00:00:00Z,1980-01-01T05:00:00Z,1990-01-01T00:09:00Z,2000-01-01T00:00:20Z,2010-01-01T00:00:00.030Z,40,10000,1000000,0.0,1.0E12,0\n" + +"1970-01-03T00:00:00Z,1980-01-01T06:00:00Z,1990-01-01T00:10:00Z,2000-01-01T00:00:21Z,2010-01-01T00:00:00.031Z,41,10001,1000001,1.1,1.0000000000001E12,10\n" + +"1970-01-04T00:00:00Z,1980-01-01T07:00:00Z,1990-01-01T00:11:00Z,2000-01-01T00:00:22Z,2010-01-01T00:00:00.032Z,42,10002,1000002,2.2,1.0000000000002E12,20\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.dods hard to test + + //.geoJson + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".geoJson"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"{\n" + +" \"type\": \"FeatureCollection\",\n" + +" \"propertyNames\": [\"time\", \"hours\", \"minutes\", \"seconds\", \"millis\", \"ints\", \"floats\", \"doubles\", \"Strings\"],\n" + +" \"propertyUnits\": [\"UTC\", \"UTC\", \"UTC\", \"UTC\", \"UTC\", null, null, null, null],\n" + +" \"features\": [\n" + +"\n" + +"{\"type\": \"Feature\",\n" + +" \"geometry\": {\n" + +" \"type\": \"Point\",\n" + +" \"coordinates\": [10000.0, 40.0] },\n" + +" \"properties\": {\n" + +" \"time\": \"1970-01-02T00:00:00Z\",\n" + +" \"hours\": \"1980-01-01T05:00:00Z\",\n" + +" \"minutes\": \"1990-01-01T00:09:00Z\",\n" + +" \"seconds\": \"2000-01-01T00:00:20Z\",\n" + +" \"millis\": \"2010-01-01T00:00:00.030Z\",\n" + +" \"ints\": 1000000,\n" + +" \"floats\": 0.0,\n" + +" \"doubles\": 1.0E12,\n" + +" \"Strings\": \"0\" }\n" + +"},\n" + +"{\"type\": \"Feature\",\n" + +" \"geometry\": {\n" + +" \"type\": \"Point\",\n" + +" \"coordinates\": [10001.0, 41.0] },\n" + +" \"properties\": {\n" + +" \"time\": \"1970-01-03T00:00:00Z\",\n" + +" \"hours\": \"1980-01-01T06:00:00Z\",\n" + +" \"minutes\": \"1990-01-01T00:10:00Z\",\n" + +" \"seconds\": \"2000-01-01T00:00:21Z\",\n" + +" \"millis\": \"2010-01-01T00:00:00.031Z\",\n" + +" \"ints\": 1000001,\n" + +" \"floats\": 1.1,\n" + +" \"doubles\": 1.0000000000001E12,\n" + +" \"Strings\": \"10\" }\n" + +"},\n" + +"{\"type\": \"Feature\",\n" + +" \"geometry\": {\n" + +" \"type\": \"Point\",\n" + +" \"coordinates\": [10002.0, 42.0] },\n" + +" \"properties\": {\n" + +" \"time\": \"1970-01-04T00:00:00Z\",\n" + +" \"hours\": \"1980-01-01T07:00:00Z\",\n" + +" \"minutes\": \"1990-01-01T00:11:00Z\",\n" + +" \"seconds\": \"2000-01-01T00:00:22Z\",\n" + +" \"millis\": \"2010-01-01T00:00:00.032Z\",\n" + +" \"ints\": 1000002,\n" + +" \"floats\": 2.2,\n" + +" \"doubles\": 1.0000000000002E12,\n" + +" \"Strings\": \"20\" }\n" + +"}\n" + +"\n" + +" ],\n" + +" \"bbox\": [10000.0, 40.0, 10002.0, 42.0]\n" + +"}\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.htmlTable + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".htmlTable"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"
    \n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    time\n" + +"hours\n" + +"minutes\n" + +"seconds\n" + +"millis\n" + +"latitude\n" + +"longitude\n" + +"ints\n" + +"floats\n" + +"doubles\n" + +"Strings\n" + +"
    UTC\n" + +"UTC\n" + +"UTC\n" + +"UTC\n" + +"UTC\n" + +"degrees_north\n" + +"degrees_east\n" + +" \n" + +" \n" + +" \n" + +" \n" + +"
    1970-01-02\n" + +"1980-01-01T05Z\n" + +"1990-01-01T00:09Z\n" + +"2000-01-01T00:00:20Z\n" + +"2010-01-01T00:00:00.030Z\n" + +"40\n" + +"10000\n" + +"1000000\n" + +"0.0\n" + +"1.0E12\n" + +"0\n" + +"
    1970-01-03\n" + +"1980-01-01T06Z\n" + +"1990-01-01T00:10Z\n" + +"2000-01-01T00:00:21Z\n" + +"2010-01-01T00:00:00.031Z\n" + +"41\n" + +"10001\n" + +"1000001\n" + +"1.1\n" + +"1.0000000000001E12\n" + +"10\n" + +"
    1970-01-04\n" + +"1980-01-01T07Z\n" + +"1990-01-01T00:11Z\n" + +"2000-01-01T00:00:22Z\n" + +"2010-01-01T00:00:00.032Z\n" + +"42\n" + +"10002\n" + +"1000002\n" + +"2.2\n" + +"1.0000000000002E12\n" + +"20\n" + +"
    \n"; + po = results.indexOf("\n" + +"\n" + +"\n" + +" My Title\n" + +" My summary.\n" + +"
    View/download more data from this dataset.\n" + +" ]]>    \n" + +" 1\n" + +" \n" + +" \n" + +" \n" + +" normal#BUOY OUT\n" + +" highlight#BUOY ON\n" + +" \n" + +" \n" + +" Lat=40, Lon=-80\n" + +" Data courtesy of NOAA NMFS SWFSC ERD\n" + +"
    time = 1970-01-02\n" + +"
    hours = 1980-01-01T05Z\n" + +"
    minutes = 1990-01-01T00:09Z\n" + +"
    seconds = 2000-01-01T00:00:20Z\n" + +"
    millis = 2010-01-01T00:00:00.030Z\n" + +"
    latitude = 40 degrees_north\n" + +"
    longitude = 10000 degrees_east\n" + +"
    ints = 1000000\n" + +"
    floats = 0.0\n" + +"
    doubles = 1.0E12\n" + +"
    Strings = 0\n" + +"
    View tabular data for this location.\n" + +"\n" + +"
    View/download more data from this dataset.\n" + +"]]>    \n" + +" #BUOY\n" + +" \n" + +" -79.99999999999972,40.0\n" + +" \n" + +"     \n" + +" \n" + +" Lat=41, Lon=-79\n" + +" Data courtesy of NOAA NMFS SWFSC ERD\n" + +"
    time = 1970-01-03\n" + +"
    hours = 1980-01-01T06Z\n" + +"
    minutes = 1990-01-01T00:10Z\n" + +"
    seconds = 2000-01-01T00:00:21Z\n" + +"
    millis = 2010-01-01T00:00:00.031Z\n" + +"
    latitude = 41 degrees_north\n" + +"
    longitude = 10001 degrees_east\n" + +"
    ints = 1000001\n" + +"
    floats = 1.1\n" + +"
    doubles = 1.0000000000001E12\n" + +"
    Strings = 10\n" + +"
    View tabular data for this location.\n" + +"\n" + +"
    View/download more data from this dataset.\n" + +"]]>    \n" + +" #BUOY\n" + +" \n" + +" -79.00000000000023,41.0\n" + +" \n" + +"     \n" + +" \n" + +" Lat=42, Lon=-78\n" + +" Data courtesy of NOAA NMFS SWFSC ERD\n" + +"
    time = 1970-01-04\n" + +"
    hours = 1980-01-01T07Z\n" + +"
    minutes = 1990-01-01T00:11Z\n" + +"
    seconds = 2000-01-01T00:00:22Z\n" + +"
    millis = 2010-01-01T00:00:00.032Z\n" + +"
    latitude = 42 degrees_north\n" + +"
    longitude = 10002 degrees_east\n" + +"
    ints = 1000002\n" + +"
    floats = 2.2\n" + +"
    doubles = 1.0000000000002E12\n" + +"
    Strings = 20\n" + +"
    View tabular data for this location.\n" + +"\n" + +"
    View/download more data from this dataset.\n" + +"]]>    \n" + +" #BUOY\n" + +" \n" + +" -77.99999999999943,42.0\n" + +" \n" + +"     \n" + +" \n" + +" -79.00000000000023\n" + +" 41.0\n" + +" 466666.6666666667\n" + +" \n" + +" \n" + +" http://127.0.0.1:8080/cwexperimental\n" + +" Logo\n" + +" http://127.0.0.1:8080/cwexperimental/images/nlogo.gif\n" + +" \n" + +" \n" + +" \n" + +" \n" + +"     \n" + +"    \n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.mat is hard to test +//!!! but need to test to ensure not rounding to the nearest second + + //.nc + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".nc"); + results = NcHelper.dumpString(tDir + tName, true); + expected = +"netcdf testSimpleTestNcTable.nc {\n" + +" dimensions:\n" + +" row = 3;\n" + +" Strings_strlen = 2;\n" + +" variables:\n" + +" double time(row=3);\n" + +" :_CoordinateAxisType = \"Time\";\n" + +" :actual_range = 86400.0, 259200.0; // double\n" + +" :axis = \"T\";\n" + +" :ioos_category = \"Time\";\n" + +" :long_name = \"Time\";\n" + +" :standard_name = \"time\";\n" + +" :time_origin = \"01-JAN-1970 00:00:00\";\n" + +" :time_precision = \"1970-01-01\";\n" + +" :units = \"seconds since 1970-01-01T00:00:00Z\";\n" + +"\n" + +" double hours(row=3);\n" + +" :actual_range = 3.155508E8, 3.15558E8; // double\n" + +" :ioos_category = \"Time\";\n" + +" :long_name = \"Hours\";\n" + +" :time_origin = \"01-JAN-1970 00:00:00\";\n" + +" :time_precision = \"1970-01-01T00Z\";\n" + +" :units = \"seconds since 1970-01-01T00:00:00Z\";\n" + +"\n" + +" double minutes(row=3);\n" + +" :actual_range = 6.3115254E8, 6.3115266E8; // double\n" + +" :ioos_category = \"Time\";\n" + +" :long_name = \"Minutes\";\n" + +" :time_origin = \"01-JAN-1970 00:00:00\";\n" + +" :time_precision = \"1970-01-01T00:00Z\";\n" + +" :units = \"seconds since 1970-01-01T00:00:00Z\";\n" + +"\n" + +" double seconds(row=3);\n" + +" :actual_range = 9.4668482E8, 9.46684822E8; // double\n" + +" :ioos_category = \"Time\";\n" + +" :long_name = \"Seconds\";\n" + +" :time_origin = \"01-JAN-1970 00:00:00\";\n" + +" :time_precision = \"not valid\";\n" + +" :units = \"seconds since 1970-01-01T00:00:00Z\";\n" + +"\n" + +" double millis(row=3);\n" + +" :actual_range = 1.26230400003E9, 1.262304000032E9; // double\n" + +" :ioos_category = \"Time\";\n" + +" :long_name = \"Millis\";\n" + +" :time_origin = \"01-JAN-1970 00:00:00\";\n" + +" :time_precision = \"1970-01-01T00:00:00.000Z\";\n" + +" :units = \"seconds since 1970-01-01T00:00:00Z\";\n" + +"\n" + +" byte latitude(row=3);\n" + +" :_CoordinateAxisType = \"Lat\";\n" + +" :actual_range = 40B, 42B; // byte\n" + +" :axis = \"Y\";\n" + +" :ioos_category = \"Location\";\n" + +" :long_name = \"Latitude\";\n" + +" :standard_name = \"latitude\";\n" + +" :units = \"degrees_north\";\n" + +"\n" + +" short longitude(row=3);\n" + +" :_CoordinateAxisType = \"Lon\";\n" + +" :actual_range = 10000S, 10002S; // short\n" + +" :axis = \"X\";\n" + +" :ioos_category = \"Location\";\n" + +" :long_name = \"Longitude\";\n" + +" :standard_name = \"longitude\";\n" + +" :units = \"degrees_east\";\n" + +"\n" + +" int ints(row=3);\n" + +" :actual_range = 1000000, 1000002; // int\n" + +" :ioos_category = \"Unknown\";\n" + +"\n" + +" float floats(row=3);\n" + +" :actual_range = 0.0f, 2.2f; // float\n" + +" :ioos_category = \"Unknown\";\n" + +"\n" + +" double doubles(row=3);\n" + +" :actual_range = 1.0E12, 1.0000000000002E12; // double\n" + +" :ioos_category = \"Unknown\";\n" + +"\n" + +" char Strings(row=3, Strings_strlen=2);\n" + +" :ioos_category = \"Unknown\";\n" + +"\n" + +" // global attributes:\n" + +" :cdm_data_type = \"Point\";\n" + +" :Conventions = \"COARDS, CF-1.6, Unidata Dataset Discovery v1.0\";\n" + +" :Easternmost_Easting = 10002.0; // double\n" + +" :featureType = \"Point\";\n" + +" :geospatial_lat_max = 42.0; // double\n" + +" :geospatial_lat_min = 40.0; // double\n" + +" :geospatial_lat_units = \"degrees_north\";\n" + +" :geospatial_lon_max = 10002.0; // double\n" + +" :geospatial_lon_min = 10000.0; // double\n" + +" :geospatial_lon_units = \"degrees_east\";\n" + +" :history = \""; //2014-10-22T16:16:21Z (local files)\n"; + ts = results.substring(0, expected.length()); + Test.ensureEqual(ts, expected, "\nresults=\n" + results); + +expected = +//"2014-10-22T16:16:21Z http://127.0.0.1:8080/cwexperimental +"/tabledap/testSimpleTestNcTable.nc?time,hours,minutes,seconds,millis,latitude,longitude,ints,floats,doubles,Strings&time>=1970-01-02T00:00:00Z&time<1970-01-05T00:00:00Z\";\n" + +" :id = \"simpleTest\";\n" + +" :infoUrl = \"???\";\n" + +" :institution = \"NOAA NMFS SWFSC ERD\";\n" + +" :keywords = \"data, local, longs, source, strings\";\n" + +" :license = \"The data may be used and redistributed for free but is not intended\n" + +"for legal use, since it may contain inaccuracies. Neither the data\n" + +"Contributor, ERD, NOAA, nor the United States Government, nor any\n" + +"of their employees or contractors, makes any warranty, express or\n" + +"implied, including warranties of merchantability and fitness for a\n" + +"particular purpose, or assumes any legal liability for the accuracy,\n" + +"completeness, or usefulness, of this information.\";\n" + +" :Metadata_Conventions = \"COARDS, CF-1.6, Unidata Dataset Discovery v1.0\";\n" + +" :Northernmost_Northing = 42.0; // double\n" + +" :sourceUrl = \"(local files)\";\n" + +" :Southernmost_Northing = 40.0; // double\n" + +" :standard_name_vocabulary = \"CF-12\";\n" + +" :summary = \"My summary.\";\n" + +" :time_coverage_end = \"1970-01-04\";\n" + +" :time_coverage_start = \"1970-01-02\";\n" + +" :title = \"My Title\";\n" + +" :Westernmost_Easting = 10000.0; // double\n" + +" data:\n" + +"time =\n" + +" {86400.0, 172800.0, 259200.0}\n" + +"hours =\n" + +" {3.155508E8, 3.155544E8, 3.15558E8}\n" + +"minutes =\n" + +" {6.3115254E8, 6.311526E8, 6.3115266E8}\n" + +"seconds =\n" + +" {9.4668482E8, 9.46684821E8, 9.46684822E8}\n" + +"millis =\n" + +" {1.26230400003E9, 1.262304000031E9, 1.262304000032E9}\n" + +"latitude =\n" + +" {40, 41, 42}\n" + +"longitude =\n" + +" {10000, 10001, 10002}\n" + +"ints =\n" + +" {1000000, 1000001, 1000002}\n" + +"floats =\n" + +" {0.0, 1.1, 2.2}\n" + +"doubles =\n" + +" {1.0E12, 1.0000000000001E12, 1.0000000000002E12}\n" + +"Strings =\"0\", \"10\", \"20\"\n" + +"}\n"; + po = results.indexOf("/tabledap/testSimpleTestNcTable.nc?"); + ts = results.substring(Math.max(0, po), Math.min(results.length(), po + expected.length())); + Test.ensureEqual(ts, expected, "\nresults=\n" + results); + + //.odvTxt + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".odvTxt"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +////??? +////2014-10-22T21:33:31 +////ERDDAP - Version 1.53 +////http://127.0.0.1:8080/cwexperimental/tabledap/testSimpleTestNcTable.html +"//ODV Spreadsheet V4.0\n" + +"//GeneralField\n" + +"//GeneralType\n" + +"Type:METAVAR:TEXT:2\tCruise:METAVAR:TEXT:2\tStation:METAVAR:TEXT:2\tyyyy-mm-ddThh:mm:ss.sss\ttime_ISO8601\ttime_ISO8601\ttime_ISO8601\ttime_ISO8601\tLatitude [degrees_north]:METAVAR:BYTE\tLongitude [degrees_east]:METAVAR:SHORT\tints:PRIMARYVAR:INTEGER\tfloats:FLOAT\tdoubles:DOUBLE\tStrings:METAVAR:TEXT:3\n" + +"*\t\t\t1970-01-02T00:00:00Z\t1980-01-01T05:00:00Z\t1990-01-01T00:09:00Z\t2000-01-01T00:00:20Z\t2010-01-01T00:00:00.030Z\t40\t10000\t1000000\t0.0\t1.0E12\t0\n" + +"*\t\t\t1970-01-03T00:00:00Z\t1980-01-01T06:00:00Z\t1990-01-01T00:10:00Z\t2000-01-01T00:00:21Z\t2010-01-01T00:00:00.031Z\t41\t10001\t1000001\t1.1\t1.0000000000001E12\t10\n" + +"*\t\t\t1970-01-04T00:00:00Z\t1980-01-01T07:00:00Z\t1990-01-01T00:11:00Z\t2000-01-01T00:00:22Z\t2010-01-01T00:00:00.032Z\t42\t10002\t1000002\t2.2\t1.0000000000002E12\t20\n"; + po = results.indexOf("//"); + ts = results.substring(Math.max(0, po), Math.min(results.length(), po + expected.length())); + Test.ensureEqual(ts, expected, "\nresults=\n" + results); + + //.xhtml + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".xhtml"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"\n" + +"\n" + +"\n" + +"\n" + +" \n" + +" testSimpleTestNcTable\n" + +"\n" + +"\n" + +"\n" + +" \n" + +"
    \n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    timehoursminutessecondsmillislatitudelongitudeintsfloatsdoublesStrings
    UTCUTCUTCUTCUTCdegrees_northdegrees_east
    1970-01-02T00:00:00Z1980-01-01T05:00:00Z1990-01-01T00:09:00Z2000-01-01T00:00:20Z2010-01-01T00:00:00.030Z401000010000000.01.0E120
    1970-01-03T00:00:00Z1980-01-01T06:00:00Z1990-01-01T00:10:00Z2000-01-01T00:00:21Z2010-01-01T00:00:00.031Z411000110000011.11.0000000000001E1210
    1970-01-04T00:00:00Z1980-01-01T07:00:00Z1990-01-01T00:11:00Z2000-01-01T00:00:22Z2010-01-01T00:00:00.032Z421000210000022.21.0000000000002E1220
    \n" + +"\n" + +"\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + + } + + /** + * This tests timestamps and other things. + * + * @throws Throwable if trouble + */ + public static void testSimpleTestNc2Table() throws Throwable { + String2.log("\n****************** EDDTableFromNcFiles.testSimpleTestNc2Table() *****************\n"); + EDDTable eddTable = (EDDTable)oneFromDatasetXml("testSimpleTestNcTable"); + String tDir = EDStatic.fullTestCacheDirectory; + String userDapQuery = "time,millis,latitude,longitude,doubles,Strings" + + "&millis>2010-01-01T00:00:00.030Z" + //should reject .030 and accept 0.031 + "&millis<=2010-01-01T00:00:00.032Z"; //should accept 0.032, but reject 0.033 + String fName = "testSimpleTestNc2Table"; + String tName, results, ts, expected; + int po; + + String2.log(NcHelper.dumpString("/erddapTest/simpleTest.nc", true)); + + //.asc + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".asc"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"Dataset {\n" + +" Sequence {\n" + +" Float64 time;\n" + +" Float64 millis;\n" + +" Byte latitude;\n" + +" Int16 longitude;\n" + +" Float64 doubles;\n" + +" String Strings;\n" + +" } s;\n" + +"} s;\n" + +"---------------------------------------------\n" + +"s.time, s.millis, s.latitude, s.longitude, s.doubles, s.Strings\n" + +"172800.0, 1.262304000031E9, 41, 10001, 1.0000000000001E12, \"10\"\n" + +"259200.0, 1.262304000032E9, 42, 10002, 1.0000000000002E12, \"20\"\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.csv + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".csv"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"time,millis,latitude,longitude,doubles,Strings\n" + +"UTC,UTC,degrees_north,degrees_east,,\n" + +"1970-01-03T00:00:00Z,2010-01-01T00:00:00.031Z,41,10001,1.0000000000001E12,10\n" + +"1970-01-04T00:00:00Z,2010-01-01T00:00:00.032Z,42,10002,1.0000000000002E12,20\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.dods hard to test + + //.geoJson + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".geoJson"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"{\n" + +" \"type\": \"FeatureCollection\",\n" + +" \"propertyNames\": [\"time\", \"millis\", \"doubles\", \"Strings\"],\n" + +" \"propertyUnits\": [\"UTC\", \"UTC\", null, null],\n" + +" \"features\": [\n" + +"\n" + +"{\"type\": \"Feature\",\n" + +" \"geometry\": {\n" + +" \"type\": \"Point\",\n" + +" \"coordinates\": [10001.0, 41.0] },\n" + +" \"properties\": {\n" + +" \"time\": \"1970-01-03T00:00:00Z\",\n" + +" \"millis\": \"2010-01-01T00:00:00.031Z\",\n" + +" \"doubles\": 1.0000000000001E12,\n" + +" \"Strings\": \"10\" }\n" + +"},\n" + +"{\"type\": \"Feature\",\n" + +" \"geometry\": {\n" + +" \"type\": \"Point\",\n" + +" \"coordinates\": [10002.0, 42.0] },\n" + +" \"properties\": {\n" + +" \"time\": \"1970-01-04T00:00:00Z\",\n" + +" \"millis\": \"2010-01-01T00:00:00.032Z\",\n" + +" \"doubles\": 1.0000000000002E12,\n" + +" \"Strings\": \"20\" }\n" + +"}\n" + +"\n" + +" ],\n" + +" \"bbox\": [10001.0, 41.0, 10002.0, 42.0]\n" + +"}\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.htmlTable + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".htmlTable"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    time\n" + +"millis\n" + +"latitude\n" + +"longitude\n" + +"doubles\n" + +"Strings\n" + +"
    UTC\n" + +"UTC\n" + +"degrees_north\n" + +"degrees_east\n" + +" \n" + +" \n" + +"
    1970-01-03\n" + +"2010-01-01T00:00:00.031Z\n" + +"41\n" + +"10001\n" + +"1.0000000000001E12\n" + +"10\n" + +"
    1970-01-04\n" + +"2010-01-01T00:00:00.032Z\n" + +"42\n" + +"10002\n" + +"1.0000000000002E12\n" + +"20\n" + +"
    \n"; + po = results.indexOf("\n" + +"\n" + +"\n" + +" My Title\n" + +" My summary.\n" + +"
    View/download more data from this dataset.\n" + +" ]]>    \n" + +" 1\n" + +" \n" + +" \n" + +" \n" + +" normal#BUOY OUT\n" + +" highlight#BUOY ON\n" + +" \n" + +" \n" + +" Lat=41, Lon=-79\n" + +" Data courtesy of NOAA NMFS SWFSC ERD\n" + +"
    time = 1970-01-03\n" + +"
    millis = 2010-01-01T00:00:00.031Z\n" + +"
    latitude = 41 degrees_north\n" + +"
    longitude = 10001 degrees_east\n" + +"
    doubles = 1.0000000000001E12\n" + +"
    Strings = 10\n" + +"
    View tabular data for this location.\n" + +"\n" + +"
    View/download more data from this dataset.\n" + +"]]>    \n" + +" #BUOY\n" + +" \n" + +" -79.00000000000023,41.0\n" + +" \n" + +"     \n" + +" \n" + +" Lat=42, Lon=-78\n" + +" Data courtesy of NOAA NMFS SWFSC ERD\n" + +"
    time = 1970-01-04\n" + +"
    millis = 2010-01-01T00:00:00.032Z\n" + +"
    latitude = 42 degrees_north\n" + +"
    longitude = 10002 degrees_east\n" + +"
    doubles = 1.0000000000002E12\n" + +"
    Strings = 20\n" + +"
    View tabular data for this location.\n" + +"\n" + +"
    View/download more data from this dataset.\n" + +"]]>    \n" + +" #BUOY\n" + +" \n" + +" -77.99999999999943,42.0\n" + +" \n" + +"     \n" + +" \n" + +" -78.49999999999977\n" + +" 41.5\n" + +" 466666.6666666667\n" + +" \n" + +" \n" + +" http://127.0.0.1:8080/cwexperimental\n" + +" Logo\n" + +" http://127.0.0.1:8080/cwexperimental/images/nlogo.gif\n" + +" \n" + +" \n" + +" \n" + +" \n" + +"     \n" + +"    \n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + + //.mat hard to test +//!!! but need to test to ensure not rounding to the nearest second + + //.nc + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".nc"); + results = NcHelper.dumpString(tDir + tName, true); + expected = +"netcdf testSimpleTestNc2Table.nc {\n" + +" dimensions:\n" + +" row = 2;\n" + +" Strings_strlen = 2;\n" + +" variables:\n" + +" double time(row=2);\n" + +" :_CoordinateAxisType = \"Time\";\n" + +" :actual_range = 172800.0, 259200.0; // double\n" + +" :axis = \"T\";\n" + +" :ioos_category = \"Time\";\n" + +" :long_name = \"Time\";\n" + +" :standard_name = \"time\";\n" + +" :time_origin = \"01-JAN-1970 00:00:00\";\n" + +" :time_precision = \"1970-01-01\";\n" + +" :units = \"seconds since 1970-01-01T00:00:00Z\";\n" + +"\n" + +" double millis(row=2);\n" + +" :actual_range = 1.262304000031E9, 1.262304000032E9; // double\n" + +" :ioos_category = \"Time\";\n" + +" :long_name = \"Millis\";\n" + +" :time_origin = \"01-JAN-1970 00:00:00\";\n" + +" :time_precision = \"1970-01-01T00:00:00.000Z\";\n" + +" :units = \"seconds since 1970-01-01T00:00:00Z\";\n" + +"\n" + +" byte latitude(row=2);\n" + +" :_CoordinateAxisType = \"Lat\";\n" + +" :actual_range = 41B, 42B; // byte\n" + +" :axis = \"Y\";\n" + +" :ioos_category = \"Location\";\n" + +" :long_name = \"Latitude\";\n" + +" :standard_name = \"latitude\";\n" + +" :units = \"degrees_north\";\n" + +"\n" + +" short longitude(row=2);\n" + +" :_CoordinateAxisType = \"Lon\";\n" + +" :actual_range = 10001S, 10002S; // short\n" + +" :axis = \"X\";\n" + +" :ioos_category = \"Location\";\n" + +" :long_name = \"Longitude\";\n" + +" :standard_name = \"longitude\";\n" + +" :units = \"degrees_east\";\n" + +"\n" + +" double doubles(row=2);\n" + +" :actual_range = 1.0000000000001E12, 1.0000000000002E12; // double\n" + +" :ioos_category = \"Unknown\";\n" + +"\n" + +" char Strings(row=2, Strings_strlen=2);\n" + +" :ioos_category = \"Unknown\";\n" + +"\n" + +" // global attributes:\n" + +" :cdm_data_type = \"Point\";\n" + +" :Conventions = \"COARDS, CF-1.6, Unidata Dataset Discovery v1.0\";\n" + +" :Easternmost_Easting = 10002.0; // double\n" + +" :featureType = \"Point\";\n" + +" :geospatial_lat_max = 42.0; // double\n" + +" :geospatial_lat_min = 41.0; // double\n" + +" :geospatial_lat_units = \"degrees_north\";\n" + +" :geospatial_lon_max = 10002.0; // double\n" + +" :geospatial_lon_min = 10001.0; // double\n" + +" :geospatial_lon_units = \"degrees_east\";\n" + +" :history = \""; //2014-10-22T16:16:21Z (local files)\n"; + ts = results.substring(0, expected.length()); + Test.ensureEqual(ts, expected, "\nresults=\n" + results); + +expected = +//"2014-10-22T16:16:21Z http://127.0.0.1:8080/cwexperimental +"/tabledap/testSimpleTestNcTable.nc?time,millis,latitude,longitude,doubles,Strings&millis>2010-01-01T00:00:00.030Z&millis<=2010-01-01T00:00:00.032Z\";\n" + +" :id = \"simpleTest\";\n" + +" :infoUrl = \"???\";\n" + +" :institution = \"NOAA NMFS SWFSC ERD\";\n" + +" :keywords = \"data, local, longs, source, strings\";\n" + +" :license = \"The data may be used and redistributed for free but is not intended\n" + +"for legal use, since it may contain inaccuracies. Neither the data\n" + +"Contributor, ERD, NOAA, nor the United States Government, nor any\n" + +"of their employees or contractors, makes any warranty, express or\n" + +"implied, including warranties of merchantability and fitness for a\n" + +"particular purpose, or assumes any legal liability for the accuracy,\n" + +"completeness, or usefulness, of this information.\";\n" + +" :Metadata_Conventions = \"COARDS, CF-1.6, Unidata Dataset Discovery v1.0\";\n" + +" :Northernmost_Northing = 42.0; // double\n" + +" :sourceUrl = \"(local files)\";\n" + +" :Southernmost_Northing = 41.0; // double\n" + +" :standard_name_vocabulary = \"CF-12\";\n" + +" :summary = \"My summary.\";\n" + +" :time_coverage_end = \"1970-01-04\";\n" + +" :time_coverage_start = \"1970-01-03\";\n" + +" :title = \"My Title\";\n" + +" :Westernmost_Easting = 10001.0; // double\n" + +" data:\n" + +"time =\n" + +" {172800.0, 259200.0}\n" + +"millis =\n" + +" {1.262304000031E9, 1.262304000032E9}\n" + +"latitude =\n" + +" {41, 42}\n" + +"longitude =\n" + +" {10001, 10002}\n" + +"doubles =\n" + +" {1.0000000000001E12, 1.0000000000002E12}\n" + +"Strings =\"10\", \"20\"\n" + +"}\n"; + po = results.indexOf("/tabledap/testSimpleTestNcTable.nc?"); + ts = results.substring(Math.max(0, po), Math.min(results.length(), po + expected.length())); + Test.ensureEqual(ts, expected, "\nresults=\n" + results); + + //.odvTxt + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".odvTxt"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +//??? +//2014-10-22T22:43:55 +//ERDDAP - Version 1.53 +//http://127.0.0.1:8080/cwexperimental/tabledap/testSimpleTestNcTable.html +"//ODV Spreadsheet V4.0\n" + +"//GeneralField\n" + +"//GeneralType\n" + +"Type:METAVAR:TEXT:2\tCruise:METAVAR:TEXT:2\tStation:METAVAR:TEXT:2\tyyyy-mm-ddThh:mm:ss.sss\ttime_ISO8601\tLatitude [degrees_north]:METAVAR:BYTE\tLongitude [degrees_east]:METAVAR:SHORT\tdoubles:PRIMARYVAR:DOUBLE\tStrings:METAVAR:TEXT:3\n" + +"*\t\t\t1970-01-03T00:00:00Z\t2010-01-01T00:00:00.031Z\t41\t10001\t1.0000000000001E12\t10\n" + +"*\t\t\t1970-01-04T00:00:00Z\t2010-01-01T00:00:00.032Z\t42\t10002\t1.0000000000002E12\t20\n"; + po = results.indexOf("//"); + ts = results.substring(Math.max(0, po), Math.min(results.length(), po + expected.length())); + Test.ensureEqual(ts, expected, "\nresults=\n" + results); + + //.xhtml + tName = eddTable.makeNewFileForDapQuery(null, null, userDapQuery, tDir, + fName, ".xhtml"); + results = new String((new ByteArray(tDir + tName)).toArray()); + expected = +"\n" + +"\n" + +"\n" + +"\n" + +" \n" + +" testSimpleTestNc2Table\n" + +"\n" + +"\n" + +"\n" + +" \n" + +"
    \n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"\n" + +"
    timemillislatitudelongitudedoublesStrings
    UTCUTCdegrees_northdegrees_east
    1970-01-03T00:00:00Z2010-01-01T00:00:00.031Z41100011.0000000000001E1210
    1970-01-04T00:00:00Z2010-01-01T00:00:00.032Z42100021.0000000000002E1220
    \n" + +"\n" + +"\n"; + Test.ensureEqual(results, expected, "\nresults=\n" + results); + } /** * This tests the methods in this class. @@ -11598,6 +12910,9 @@ public static void test(boolean doAllGraphicsTests) throws Throwable { testBigRequest(); testPmelTaoAirt(); testNow(); + testTimePrecisionMillis(); + testSimpleTestNcTable(); + testSimpleTestNc2Table(); /* */ //not usually run diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromSOS.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromSOS.java index a40f60326..6e61eee71 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromSOS.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromSOS.java @@ -7856,6 +7856,49 @@ public static void testWhoiSos() throws Throwable { } + /** + * This tests that ensureValid throws exception if 2 + * dataVariables use the same sourceName. + * These tests are in EDDTableFromSOS because EDDTableFromFiles has a separate test. + * + * @throws Throwable if trouble + */ + public static void test2DVSameSource() throws Throwable { + String2.log("\n****************** EDDTableFromNcFiles.test2DVSameSource() *****************\n"); + String error = "shouldn't happen"; + try { + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml("tabletest2DVSameSource"); + } catch (Throwable t) { + String2.log(MustBe.throwableToString(t)); + error = String2.split(MustBe.throwableToString(t), '\n')[1]; + } + + Test.ensureEqual(error, + "Two dataVariables have the same sourceName=sensor_id.", + "Unexpected error message:\n" + error); + } + + /** + * This tests that ensureValid throws exception if 2 + * dataVariables use the same destinationName. + * These tests are in EDDTableFromSOS because EDDTableFromFiles has a separate test. + * + * @throws Throwable if trouble + */ + public static void test2DVSameDestination() throws Throwable { + String2.log("\n****************** EDDTableFromNcFiles.test2DVSameDestination() *****************\n"); + String error = "shouldn't happen"; + try { + EDDGrid eddGrid = (EDDGrid)oneFromDatasetXml("tabletest2DVSameDestination"); + } catch (Throwable t) { + String2.log(MustBe.throwableToString(t)); + error = String2.split(MustBe.throwableToString(t), '\n')[1]; + } + + Test.ensureEqual(error, + "Two dataVariables have the same destinationName=sensor_id.", + "Unexpected error message:\n" + error); + } /** * This runs all of the tests for this class. @@ -7877,6 +7920,9 @@ public static void test() throws Throwable { testNdbcSosWTemp(""); testNdbcSosWaves(""); + test2DVSameSource(); + test2DVSameDestination(); + //test NDBC Wind and quickRestart String qrName = File2.forceExtension(quickRestartFullFileName("ndbcSosWind"), ".xml"); String2.log("\n\n*** deleting quickRestartFile exists=" + File2.isFile(qrName) + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromWFSFiles.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromWFSFiles.java index 4afbbf1a0..4cf719ba9 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromWFSFiles.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/EDDTableFromWFSFiles.java @@ -1071,8 +1071,8 @@ public static void testBasic() throws Throwable { " String standard_name_vocabulary \"CF-12\";\n" + " String subsetVariables \"WellName,APINo,Label,Operator,WellType,Field,County,State,FormationTD,OtherName\";\n" + " String summary \"Borehole temperature measurements in West Virginia\";\n" + -" String time_coverage_end \"2012-08-05T00:00:00Z\";\n" + -" String time_coverage_start \"1899-01-31T00:00:00Z\";\n" + +" String time_coverage_end \"2012-08-05\";\n" + +" String time_coverage_start \"1899-01-31\";\n" + " String title \"West Virginia Borehole Temperatures, AASG State Geothermal Data\";\n" + " Float64 Westernmost_Easting -82.54999;\n" + " }\n" + diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/GridDataAccessor.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/GridDataAccessor.java index c1c179e38..37dba18ed 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/GridDataAccessor.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/GridDataAccessor.java @@ -231,8 +231,13 @@ public GridDataAccessor(EDDGrid tEDDGrid, String tRequestUrl, String tUserDapQue globalAttributes.set("geospatial_vertical_max", dMax); } } else if (av == eddGrid.timeIndex) { - globalAttributes.set("time_coverage_start", Calendar2.epochSecondsToIsoStringT(dMin) + "Z"); //unidata-related - globalAttributes.set("time_coverage_end", Calendar2.epochSecondsToIsoStringT(dMax) + "Z"); + String tp = axisAttributes[av].getString(EDV.TIME_PRECISION); + //"" unsets the attribute if dMin or dMax isNaN + globalAttributes.set("time_coverage_start", + Calendar2.epochSecondsToLimitedIsoStringT(tp, dMin, "")); + //for tables (not grids) will be NaN for 'present'. Deal with this better??? + globalAttributes.set("time_coverage_end", + Calendar2.epochSecondsToLimitedIsoStringT(tp, dMax, "")); } } @@ -547,7 +552,7 @@ protected void getChunk() throws Throwable { " nElements/dv=" + partialResults[partialResults.length - 1].size() + " time=" + (System.currentTimeMillis() - time)); //for (int i = 0; i < partialResults.length; i++) - // String2.log(" pa[" + i + "]=" + partialResults[i]); + // String2.log("!pa[" + i + "]=" + partialResults[i]); } catch (WaitThenTryAgainException twwae) { throw twwae; @@ -592,7 +597,9 @@ protected void getChunk() throws Throwable { //process the results for (int dv = 0; dv < dataVariables.length; dv++) { //dv in the query //convert source values to destination values and store + //String2.log("!source dv=" + dataVariables[dv].destinationName() + " " + partialResults[nAxisVariables + dv]); partialDataValues[dv] = dataVariables[dv].toDestination(partialResults[nAxisVariables + dv]); + //String2.log("!dest dv=" + dataVariables[dv].destinationName() + " " + partialDataValues[dv]); //convert missing_value to NaN if (convertToNaN) { diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriter.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriter.java index fe72cd803..ff404a670 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriter.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriter.java @@ -83,7 +83,8 @@ protected void ensureCompatible(Table table) throws Throwable { columnTypes = tColumnTypes; columnAttributes = new Attributes[nColumns]; for (int col = 0; col < nColumns; col++) { - //no need to make copies (clones) off atts since standardizeResultsTable has made copies for the table + //no need to make copies (clones) off atts since standardizeResultsTable + //has made copies for the table columnAttributes[col] = table.columnAttributes(col); //String2.log("\nTableWriter attributes " + columnNames[col] + "\n" + columnAttributes[col]); } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterAllWithMetadata.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterAllWithMetadata.java index 3c5beb3de..94e46fd51 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterAllWithMetadata.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterAllWithMetadata.java @@ -251,8 +251,13 @@ private void finishMetadata() { globalAttributes.remove("time_coverage_start"); //unidata-related globalAttributes.remove("time_coverage_end"); } else { //always iso string - globalAttributes.set("time_coverage_start", Calendar2.epochSecondsToIsoStringT(dMin) + "Z"); //unidata-related - globalAttributes.set("time_coverage_end", Calendar2.epochSecondsToIsoStringT(dMax) + "Z"); + String tp = columnAttributes(col).getString(EDV.TIME_PRECISION); + //"" unsets the attribute if min or max isNaN + globalAttributes.set("time_coverage_start", + Calendar2.epochSecondsToLimitedIsoStringT(tp, dMin, "")); + //for tables (not grids) will be NaN for 'present'. Deal with this better??? + globalAttributes.set("time_coverage_end", + Calendar2.epochSecondsToLimitedIsoStringT(tp, dMax, "")); } } } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterGeoJson.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterGeoJson.java index d3261ad87..88bb3c8a8 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterGeoJson.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterGeoJson.java @@ -4,6 +4,7 @@ */ package gov.noaa.pfel.erddap.dataset; +import com.cohort.array.Attributes; import com.cohort.util.Calendar2; import com.cohort.util.MustBe; import com.cohort.util.SimpleException; @@ -36,8 +37,9 @@ public class TableWriterGeoJson extends TableWriter { //set by firstTime protected int lonColumn = -1, latColumn = -1, altColumn = -1; - protected boolean isTimeStamp[]; protected boolean isString[]; + protected boolean isTimeStamp[]; + protected String time_precision[]; protected BufferedWriter writer; protected double minLon = Double.MAX_VALUE, maxLon = -Double.MAX_VALUE; protected double minLat = Double.MAX_VALUE, maxLat = -Double.MAX_VALUE; @@ -102,10 +104,19 @@ public void writeSome(Table table) throws Throwable { //it is unclear to me if specification supports altitude in coordinates info... altColumn = -1; //table.findColumnNumber(EDV.ALT_NAME); isTimeStamp = new boolean[nColumns]; + time_precision = new String[nColumns]; for (int col = 0; col < nColumns; col++) { - String u = table.columnAttributes(col).getString("units"); + Attributes catts = table.columnAttributes(col); + String u = catts.getString("units"); isTimeStamp[col] = u != null && (u.equals(EDV.TIME_UNITS) || u.equals(EDV.TIME_UCUM_UNITS)); + if (isTimeStamp[col]) { + //just keep time_precision if it includes fractional seconds + String tp = catts.getString(EDV.TIME_PRECISION); + if (tp != null && !tp.startsWith("1970-01-01T00:00:00.0")) + tp = null; //default + time_precision[col] = tp; + } } //write the header @@ -235,7 +246,8 @@ public void writeSome(Table table) throws Throwable { if (isTimeStamp[col]) { double d = table.getDoubleData(col, row); s = Double.isNaN(d)? "null" : - "\"" + Calendar2.epochSecondsToIsoStringT(d) + "Z\""; + "\"" + Calendar2.epochSecondsToLimitedIsoStringT( + time_precision[col], d, "") + "\""; } else if (isString[col]) { s = String2.toJson(table.getStringData(col, row)); } else { //numeric diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterHtmlTable.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterHtmlTable.java index f49aaaafc..4b7cd7928 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterHtmlTable.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterHtmlTable.java @@ -56,10 +56,10 @@ public class TableWriterHtmlTable extends TableWriter { protected String noWrap; //set firstTime - protected boolean isTimeStamp[]; protected boolean isString[]; - protected BufferedWriter writer; + protected boolean isTimeStamp[]; protected String time_precision[]; + protected BufferedWriter writer; //set later public boolean isMBLimited = false; //ie, did htmlTableMaxMB reduce showFirstNRows? @@ -171,8 +171,13 @@ public void writeSome(Table table) throws Throwable { String u = catts.getString("units"); isTimeStamp[col] = u != null && (u.equals(EDV.TIME_UNITS) || u.equals(EDV.TIME_UCUM_UNITS)); - if (isTimeStamp[col] && !xhtmlMode) - time_precision[col] = catts.getString(EDV.TIME_PRECISION); + if (isTimeStamp[col]) { + //for xhtmlMode, just keep time_precision if it includes fractional seconds + String tp = catts.getString(EDV.TIME_PRECISION); + if (xhtmlMode && tp != null && !tp.startsWith("1970-01-01T00:00:00.0")) + tp = null; //default + time_precision[col] = tp; + } if (isTimeStamp[col]) { bytesPerRow += 20 + noWrap.length(); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterJson.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterJson.java index cca6f6206..55cee0e7e 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterJson.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterJson.java @@ -4,6 +4,7 @@ */ package gov.noaa.pfel.erddap.dataset; +import com.cohort.array.Attributes; import com.cohort.util.Calendar2; import com.cohort.util.MustBe; import com.cohort.util.SimpleException; @@ -33,8 +34,9 @@ public class TableWriterJson extends TableWriter { protected boolean writeUnits; //set by firstTime - protected boolean isTimeStamp[]; protected boolean isString[]; + protected boolean isTimeStamp[]; + protected String time_precision[]; protected BufferedWriter writer; //other @@ -92,10 +94,19 @@ public void writeSome(Table table) throws Throwable { int nColumns = table.nColumns(); if (firstTime) { isTimeStamp = new boolean[nColumns]; + time_precision = new String[nColumns]; for (int col = 0; col < nColumns; col++) { - String u = table.columnAttributes(col).getString("units"); + Attributes catts = table.columnAttributes(col); + String u = catts.getString("units"); isTimeStamp[col] = u != null && (u.equals(EDV.TIME_UNITS) || u.equals(EDV.TIME_UCUM_UNITS)); + if (isTimeStamp[col]) { + //just keep time_precision if it includes fractional seconds + String tp = catts.getString(EDV.TIME_PRECISION); + if (tp != null && !tp.startsWith("1970-01-01T00:00:00.0")) + tp = null; //default + time_precision[col] = tp; + } } //write the header @@ -157,9 +168,9 @@ public void writeSome(Table table) throws Throwable { if (col > 0) writer.write(", "); if (isTimeStamp[col]) { double d = table.getDoubleData(col, row); - String s = Double.isNaN(d)? "null" : - "\"" + Calendar2.epochSecondsToIsoStringT(d) + "Z\""; - writer.write(s); + writer.write(Double.isNaN(d)? "null" : + "\"" + Calendar2.epochSecondsToLimitedIsoStringT( + time_precision[col], d, "") + "\""); } else if (isString[col]) { String s = table.getStringData(col, row); writer.write(String2.toJson(s)); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterSeparatedValue.java b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterSeparatedValue.java index 386837cfb..dbab10e70 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterSeparatedValue.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/dataset/TableWriterSeparatedValue.java @@ -4,6 +4,7 @@ */ package gov.noaa.pfel.erddap.dataset; +import com.cohort.array.Attributes; import com.cohort.util.Calendar2; import com.cohort.util.MustBe; import com.cohort.util.SimpleException; @@ -36,8 +37,9 @@ public class TableWriterSeparatedValue extends TableWriter { protected String nanString; //set by firstTime - protected boolean isTimeStamp[]; protected boolean isString[]; + protected boolean isTimeStamp[]; + protected String time_precision[]; protected BufferedWriter writer; public long totalNRows = 0; @@ -103,10 +105,19 @@ public void writeSome(Table table) throws Throwable { int nColumns = table.nColumns(); if (firstTime) { isTimeStamp = new boolean[nColumns]; + time_precision = new String[nColumns]; for (int col = 0; col < nColumns; col++) { - String u = table.columnAttributes(col).getString("units"); + Attributes catts = table.columnAttributes(col); + String u = catts.getString("units"); isTimeStamp[col] = u != null && (u.equals(EDV.TIME_UNITS) || u.equals(EDV.TIME_UCUM_UNITS)); + if (isTimeStamp[col]) { + //just keep time_precision if it includes fractional seconds + String tp = catts.getString(EDV.TIME_PRECISION); + if (tp != null && !tp.startsWith("1970-01-01T00:00:00.0")) + tp = null; //default + time_precision[col] = tp; + } } //write the header @@ -164,12 +175,8 @@ public void writeSome(Table table) throws Throwable { for (int row = 0; row < nRows; row++) { for (int col = 0; col < nColumns; col++) { if (isTimeStamp[col]) { - double d = table.getDoubleData(col, row); - //Always using standard ISO 8601 string (seconds) ensures easy to parse. - //It would be nice to use decimal seconds if variables time_precision specifies it, - //but time_precision isn't known here - String s = Calendar2.safeEpochSecondsToIsoStringTZ(d, ""); - writer.write(s); + writer.write(Calendar2.epochSecondsToLimitedIsoStringT( + time_precision[col], table.getDoubleData(col, row), "")); } else if (isString[col]) { writer.write(String2.quoteIfNeeded(quoted, table.getStringData(col, row))); } else { diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java b/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java index 700eea8be..3bf747cc8 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/EDStatic.java @@ -130,7 +130,7 @@ public class EDStatic { *
    1.50 released on 2014-09-06 *
    1.52 released on 2014-10-03 */ - public static String erddapVersion = "1.52"; + public static String erddapVersion = "1.54"; /** * This is almost always false. @@ -1250,9 +1250,22 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { emailFromAddress = setup.getString("emailFromAddress", null); emailEverythingToCsv = setup.getString("emailEverythingTo", ""); //won't be null emailDailyReportToCsv = setup.getString("emailDailyReportTo", ""); //won't be null + String tsar[] = String2.split(emailEverythingToCsv, ','); + if (emailEverythingToCsv.length() > 0) + for (int i = 0; i < tsar.length; i++) + if (!String2.isEmailAddress(tsar[i]) || tsar[i].startsWith("your.")) //prohibit the default email addresses + throw new RuntimeException("setup.xml error: invalid email address=" + tsar[i] + + " in ."); emailSubscriptionsFrom = tsar.length > 0? tsar[0] : ""; //won't be null + tsar = String2.split(emailDailyReportToCsv, ','); + if (emailDailyReportToCsv.length() > 0) + for (int i = 0; i < tsar.length; i++) + if (!String2.isEmailAddress(tsar[i]) || tsar[i].startsWith("your.")) //prohibit the default email addresses + throw new RuntimeException("setup.xml error: invalid email address=" + tsar[i] + + " in ."); + //test of email //Test.error("This is a test of emailing an error in Erddap constructor."); @@ -1406,6 +1419,27 @@ public static int convertToPublicSourceUrlFromSlashPo(String tFrom) { adminCountry = setup.getNotNothingString("adminCountry", ""); adminEmail = setup.getNotNothingString("adminEmail", ""); + if (adminInstitution.length() == 0 || adminInstitution.startsWith("Your")) + throw new RuntimeException("setup.xml error: invalid =" + adminInstitution); + if (adminIndividualName.length() == 0 || adminIndividualName.startsWith("Your")) + throw new RuntimeException("setup.xml error: invalid =" + adminIndividualName); + if (adminPosition.length() == 0) + throw new RuntimeException("setup.xml error: invalid =" + adminPosition); + if (adminPhone.length() == 0 || adminPhone.indexOf("999-999") >= 0) + throw new RuntimeException("setup.xml error: invalid =" + adminPhone); + if (adminAddress.length() == 0 || adminAddress.equals("123 Main St.")) + throw new RuntimeException("setup.xml error: invalid =" + adminAddress); + if (adminCity.length() == 0 || adminCity.equals("Some Town")) + throw new RuntimeException("setup.xml error: invalid =" + adminCity); + if (adminStateOrProvince.length() == 0) + throw new RuntimeException("setup.xml error: invalid =" + adminStateOrProvince); + if (adminPostalCode.length() == 0 || adminPostalCode.equals("99999")) + throw new RuntimeException("setup.xml error: invalid =" + adminPostalCode); + if (adminCountry.length() == 0) + throw new RuntimeException("setup.xml error: invalid =" + adminCountry); + if (!String2.isEmailAddress(adminEmail) || adminEmail.startsWith("your.")) + throw new RuntimeException("setup.xml error: invalid =" + adminEmail); + accessConstraints = setup.getNotNothingString("accessConstraints", ""); accessRequiresAuthorization= setup.getNotNothingString("accessRequiresAuthorization",""); fees = setup.getNotNothingString("fees", ""); @@ -2713,13 +2747,24 @@ public static String email(String emailAddresses[], String subject, String conte emailSmtpHost == null || emailSmtpHost.length() == 0) return ""; + //send email String errors = ""; try { - SSR.sendEmail(emailSmtpHost, emailSmtpPort, emailUserName, - emailPassword, emailProperties, emailFromAddress, emailAddressesCSSV, - erddapUrl + " " + subject, //always non-https url - erddapUrl + " reports:\n" + content); //always non-https url + //catch common problem: sending email to one invalid address + if (emailAddresses.length == 1 && + !String2.isEmailAddress(emailAddresses[0]) || + emailAddresses[0].startsWith("your.name") || + emailAddresses[0].startsWith("your.email")) { + errors = "Error in EDStatic.email: invalid emailAddresses=" + emailAddressesCSSV; + String2.log(errors); + } + + if (errors.length() == 0) + SSR.sendEmail(emailSmtpHost, emailSmtpPort, emailUserName, + emailPassword, emailProperties, emailFromAddress, emailAddressesCSSV, + erddapUrl + " " + subject, //always non-https url + erddapUrl + " reports:\n" + content); //always non-https url } catch (Throwable t) { String msg = "Error: Sending email to " + emailAddressesCSSV + " failed"; String2.log(MustBe.throwable(msg, t)); diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/util/messages.xml b/WEB-INF/classes/gov/noaa/pfel/erddap/util/messages.xml index 352882bde..4018ad8f1 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/util/messages.xml +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/util/messages.xml @@ -472,6 +472,12 @@ time values in ERDDAP always use the Zulu (UTC, GMT) time zone.
     
  • UDUNITS Time Units - {1} +
  • Converter Precision - + When representing times as numbers, this converter always +
    leaves the numbers at their full precision. When representing times as Strings, +
    this converter formats times to the (truncated) second (i.e., omitting millis). +
      +
  • Invalid Input - ERDDAP tries to deal with improperly formatted input.
    If the input can''t be dealt with, ERDDAP will generate an error message. @@ -494,20 +500,34 @@ time values in ERDDAP always use the Zulu (UTC, GMT) time zone.
    Zulu time zone, each an instant in time).
      -
  • Time Zones - ERDDAP always uses the Zulu (UTC, GMT) time zone - and never uses -
    daylight savings times. Time data from a source with time zone information (which -
    may incorporate daylight savings time information) is converted to Zulu time. -
    Time data from a source without time zone information is assumed to be already in -
    Zulu time. +
  • Time Zones - When writing time values, ERDDAP always uses the Zulu (UTC, GMT) +
    time zone and never uses daylight savings times. Time data from a source with +
    time zone information (which may incorporate daylight savings time information) +
    is converted to Zulu time. Time data from a source without time zone information +
    is assumed to be already in Zulu time.
     
  • Precision - ERDDAP deals with time internally as "seconds since 1970-00-00T00:00:00Z", -
    ignoring leap seconds (see below), stored as double precision floating point numbers. -
    Thus, most time data can be stored very precisely (roughly to the nearest microsecond, +
    ignoring leap seconds, stored as double precision floating point numbers. Thus, +
    most time data can be stored very precisely (roughly to the nearest microsecond,
    but progressively less precisely very far in the past or future).
      +
    When doing calculations, ERDDAP often works to the nearest millisecond, to +
    avoid problems with floating point numbers that are slightly more or less +
    than you expect. +
      +
    When representing times as numbers, ERDDAP always leaves the numbers at their full +
    precision. +
      +
    By default, when representing times as Strings, ERDDAP formats times to the +
    (truncated) second (i.e., omitting millis). However, ERDDAP administrators can +
    configure specific variables in specific datasets to represent String times at +
    other precisions (e.g., from millisecond precision up to month precision; see the +
    <time_precision> variable attribute). Whenever a time field (e.g., milliseconds +
    or seconds) is not shown, the absent value is assumed to be 0 (except for +
    day-of-month, which is assumed to be 1). +
     
  • Consistent Units - Using the same units for time ("seconds since 1970-00-00T00:00:00Z") @@ -550,10 +570,12 @@ time values in ERDDAP always use the Zulu (UTC, GMT) time zone. http://xkcd.com/1179/ (external link)) -
    ERDDAP administrators can configure a given dataset to display the time data values -
    truncated to 0.001 second, 0.01 second, 0.1 second, second, minute, hour, date, or month +
    ERDDAP administrators can configure a given variable to display the time data values +
    to greater precision (truncated to 0.001 second, 0.01 second, 0.1 second) +
    or to lesser precision (truncated to second, minute, hour, date, or month)
    to indicate the precision of the time data values. -

    When reading ISO 8601 times with decimal seconds, ERDDAP accepts a period separator +
      +
    When reading ISO 8601 times with decimal seconds, ERDDAP accepts a period separator
    (e.g., 2011-04-26T14:07:12.059Z) or a comma separator (e.g., 2011-04-26T14:07:12,059Z).
    Currently, when writing ISO 8601 times ERDDAP always uses a period separator.
      @@ -734,8 +756,11 @@ For example, "seconds since 1970-01-01T00:00:00Z".
      mon, mons, month, months,
      yr, yrs, year, or years.

    "since" is required. -

    The time can be any time in the format YYYY-MM-DDThh:mm:ssZ . -
    The T, hh, :mm, :ss, and Z parts are optional. +

    The time can be any time in the format YYYY-MM-DDThh:mm:ss.sssZ, +
    where Z is 'Z' or a ±hh or ±hh:mm offset from the Zulu/GMT time zone. +
    If you omit Z and the offset, the Zulu/GMT time zone is used. +
    Separately, if you omit .sss, :ss.sss, :mm:ss.sss, or Thh:mm:ss.sss, the +
    missing fields are assumed to be 0.

    So another example is "hours since 0001-01-01".

    Technically, ERDDAP does NOT follow the CF standard when converting "years since"
    and "months since" time values to "seconds since". The CF standard defines a diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVGridAxis.java b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVGridAxis.java index e9230c7e5..79f11a5a6 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVGridAxis.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVGridAxis.java @@ -222,7 +222,7 @@ public PrimitiveArray destinationValue(int which) { /** * This returns one of this axis' source values as a nice double destination value. - * EDVTimeGridAxis subclass overrides this. + * EDVTimeStampGridAxis subclass overrides this. */ public double destinationDouble(int which) { if (scaleAddOffset) { @@ -284,7 +284,7 @@ public String sliderCsvValues() throws Throwable { try { long eTime = System.currentTimeMillis(); int nSourceValues = sourceValues.size(); - boolean isTime = this instanceof EDVTimeGridAxis; + boolean isTimeStamp = this instanceof EDVTimeStampGridAxis; IntArray sliderIndices = new IntArray(); sliderIndices.add(0); //add first index @@ -292,7 +292,7 @@ public String sliderCsvValues() throws Throwable { for (int i = 1; i < nSourceValues; i++) sliderIndices.add(i); - } else if (isTime) { + } else if (isTimeStamp) { //make evenly spaced nice numbers (like EDV.sliderCsvValues()), // then find closest actual values. //Dealing with indices (later sorted) works regardless of isAscending. @@ -330,7 +330,7 @@ public String sliderCsvValues() throws Throwable { StringBuilder sb = new StringBuilder(); for (int i = 0; i < nValues; i++) { if (i > 0) sb.append(", "); - sb.append(toSliderString(destinationString(sliderIndices.get(i)), isTime)); + sb.append(toSliderString(destinationString(sliderIndices.get(i)), isTimeStamp)); } //store in compact utf8 format @@ -474,7 +474,7 @@ public void setIsEvenlySpaced(boolean tIsEvenlySpaced) { * If there are 2 or more values, this returns the average spacing between values * (will be negative if axis is descending!). * If isEvenlySpaced, then these are evenly spaced. - * For EDVTimeGridAxis, this is in epochSeconds. + * For EDVTimeStampGridAxis, this is in epochSeconds. * * @return If there are 2 or more values, * this returns the average spacing between values (in destination units). @@ -485,10 +485,10 @@ public void setIsEvenlySpaced(boolean tIsEvenlySpaced) { * This returns a human-oriented description of the spacing of this EDVGridAxis. (May be negative.) */ public String spacingDescription() { - boolean isTime = destinationName.equals(EDV.TIME_NAME); + boolean isTimeStamp = this instanceof EDVTimeStampGridAxis; if (sourceValues.size() == 1) return "(" + EDStatic.EDDGridJustOneValue + ")"; - String s = isTime? + String s = isTimeStamp? Calendar2.elapsedTimeString(Math.rint(averageSpacing()) * 1000) : "" + Math2.floatToDouble(averageSpacing()); return s + " (" + @@ -503,14 +503,14 @@ public String spacingDescription() { */ public String htmlRangeTooltip() { String tUnits = units(); - boolean isTime = destinationName.equals(EDV.TIME_NAME); - if (tUnits == null || isTime) + boolean isTimeStamp = this instanceof EDVTimeStampGridAxis; + if (tUnits == null || isTimeStamp) tUnits = ""; if (sourceValues.size() == 1) return destinationName + " has 1 value: " + destinationToString(firstDestinationValue()) + " " + tUnits; - String tSpacing = isTime? + String tSpacing = isTimeStamp? Calendar2.elapsedTimeString(Math.rint(averageSpacing()) * 1000) : "" + Math2.floatToDouble(averageSpacing()) + " " + tUnits; return diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVLatGridAxis.java b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVLatGridAxis.java index 7140308b3..e5741f662 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVLatGridAxis.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVLatGridAxis.java @@ -56,7 +56,7 @@ public EDVLatGridAxis(String tSourceName, * @return a string representation of this EDV. */ public String toString() { - return "Lat " + super.toString(); + return "EDVLatGridAxis/" + super.toString(); } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVLonGridAxis.java b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVLonGridAxis.java index 67fb36873..9c1aa18ed 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVLonGridAxis.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVLonGridAxis.java @@ -56,7 +56,7 @@ public EDVLonGridAxis(String tSourceName, * @return a string representation of this EDV. */ public String toString() { - return "Lon " + super.toString(); + return "EDVLonGridAxis/" + super.toString(); } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTime.java b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTime.java index 704f2edf6..8e2708479 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTime.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTime.java @@ -5,14 +5,7 @@ package gov.noaa.pfel.erddap.variable; import com.cohort.array.Attributes; -import com.cohort.array.PrimitiveArray; -import com.cohort.array.StringArray; -import com.cohort.util.Calendar2; -import com.cohort.util.MustBe; import com.cohort.util.String2; -import com.cohort.util.Test; - -import java.util.GregorianCalendar; /** * This class holds information about *the* main time variable, @@ -34,4 +27,12 @@ public EDVTime(String tSourceName, tSourceDataType); } + /** + * This returns a string representation of this EDV. + * + * @return a string representation of this EDV. + */ + public String toString() { + return "EDVTime/" + super.toString(); + } } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeGridAxis.java b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeGridAxis.java index abccd2023..33edebb71 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeGridAxis.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeGridAxis.java @@ -5,25 +5,13 @@ package gov.noaa.pfel.erddap.variable; import com.cohort.array.Attributes; -import com.cohort.array.DoubleArray; import com.cohort.array.PrimitiveArray; -import com.cohort.array.StringArray; -import com.cohort.util.Calendar2; -import com.cohort.util.MustBe; import com.cohort.util.String2; -import com.cohort.util.Test; - -import java.text.ParsePosition; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.GregorianCalendar; -import java.util.TimeZone; - -import org.joda.time.*; -import org.joda.time.format.*; /** - * This class holds information about the time grid axis variable. + * This class holds information about *the* time grid axis variable, + * which is like other EDVTimeStampGridAxis variables, but has + * destinationName="time". * * [STRING TIMES NOT FINISHED: * The handling of String times in incomplete and probably not a good approach. @@ -33,161 +21,16 @@ * * @author Bob Simons (bob.simons@noaa.gov) 2007-06-04 */ -public class EDVTimeGridAxis extends EDVGridAxis { +public class EDVTimeGridAxis extends EDVTimeStampGridAxis { - /** Set by the constructor. */ - protected String sourceTimeFormat; - - /** These are set automatically. */ - protected boolean sourceTimeIsNumeric = true; - protected double sourceTimeBase = Double.NaN; //set if sourceTimeIsNumeric - protected double sourceTimeFactor = Double.NaN; - protected boolean parseISOWithCalendar2; - protected DateTimeFormatter dateTimeFormatter; //set if !sourceTimeIsNumeric - protected String time_precision; //see Calendar2.epochSecondsToLimitedIsoStringT - protected boolean superConstructorIsFinished = false; - - /** - * The constructor. - * - *

    Either tAddAttributes (read first) or tSourceAttributes must have "units" - * which is a UDUunits string (containing " since ") - * describing how to interpret source time values - * (which should always be numeric since they are a dimension of a grid) - * (e.g., "seconds since 1970-01-01T00:00:00"), - * where the base time is an - * ISO 8601 formatted date time string (YYYY-MM-DDThh:mm:ss). - * - *

    scale_factor an dadd_offset are allowed for numeric time variables. - * This constructor removes any scale_factor and add_offset attributes - * and stores the resulting information so that destination data - * has been converted to destinationDataType with scaleFactor and addOffset - * applied. - * - * @param tSourceName the name of the axis variable in the dataset source - * (usually with no spaces). - * @param tSourceAttributes are the attributes for the variable - * in the source - * @param tAddAttributes the attributes which will be added when data is - * extracted and which have precedence over sourceAttributes. - * Special case: value="null" causes that item to be removed from combinedAttributes. - * If this is null, an empty addAttributes will be created. - * @param tSourceValues has the values from the source. - * This can't be a StringArray. - * There must be at least one element. - * @throws Throwable if trouble - */ - public EDVTimeGridAxis(String tSourceName, + /** The constructor. */ + public EDVTimeGridAxis(String tSourceName, Attributes tSourceAttributes, Attributes tAddAttributes, - PrimitiveArray tSourceValues) - throws Throwable { - - super(tSourceName, TIME_NAME, tSourceAttributes, tAddAttributes, tSourceValues); - superConstructorIsFinished = true; - - //time_precision e.g., 1970-01-01T00:00:00Z - time_precision = combinedAttributes.getString(EDV.TIME_PRECISION); - if (time_precision != null) { - //ensure not just year (can't distinguish user input a year vs. epochSeconds) - if (time_precision.equals("1970")) - time_precision = null; - //ensure Z at end of time - if (time_precision.length() >= 13 && !time_precision.endsWith("Z")) - time_precision = null; - } - - //currently, EDVTimeGridAxis doesn't support String sourceValues - String errorInMethod = "datasets.xml/EDVTimeGridAxis constructor error for sourceName=" + tSourceName + ":\n"; - if (sourceValues instanceof StringArray) - throw new RuntimeException(errorInMethod + - "Currently, EDVTimeGridAxis doesn't support String source values for the time axis."); - - //read units before it is changed below - sourceTimeFormat = units(); - Test.ensureNotNothing(sourceTimeFormat, - errorInMethod + "'units' wasn't found."); //match name in datasets.xml - if (sourceTimeFormat.indexOf(" since ") > 0) { - sourceTimeIsNumeric = true; - double td[] = Calendar2.getTimeBaseAndFactor(sourceTimeFormat); - sourceTimeBase = td[0]; - sourceTimeFactor = td[1]; - } else { - sourceTimeIsNumeric = false; - throw new RuntimeException( - "Currently, the source units for the time axis must include \" since \"."); - /* If Strings are ever supported... - //ensure scale_factor=1 and add_offset=0 - if (scaleAddOffset) - throw new RuntimeException(errorInMethod + - "For String source times, scale_factor and add_offset MUST NOT be used."); - - if (sourceTimeFormat.equals(ISO8601T_FORMAT) || - sourceTimeFormat.equals(ISO8601TZ_FORMAT)) { - if (verbose) String2.log("parseISOWithCalendar2=true"); - dateTimeFormatter = ISODateTimeFormat.dateTimeNoMillis().withZone(DateTimeZone.UTC); - parseISOWithCalendar2 = true; - } else if (sourceTimeFormat.equals(ISO8601T3_FORMAT) || - sourceTimeFormat.equals(ISO8601T3Z_FORMAT)) { - if (verbose) String2.log("parseISOWithCalendar2=true"); - dateTimeFormatter = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC); - parseISOWithCalendar2 = true; - } else { - //future: support time zones - dateTimeFormatter = DateTimeFormat.forPattern(sourceTimeFormat).withZone(DateTimeZone.UTC); - parseISOWithCalendar2 = false; - } - */ - } - - //extractScaleAddOffset It sets destinationDataType - extractScaleAddOffset(); - if (scaleAddOffset) { - setDestinationMin(destinationMin * scaleFactor + addOffset); - setDestinationMax(destinationMax * scaleFactor + addOffset); - } - //test for min>max after extractScaleAddOffset, since order may have changed - if (destinationMin > destinationMax) { - double d = destinationMin; destinationMin = destinationMax; destinationMax = d; - } - - units = TIME_UNITS; - combinedAttributes.set("_CoordinateAxisType", "Time"); //unidata-related - combinedAttributes.set("axis", "T"); - combinedAttributes.set("ioos_category", TIME_CATEGORY); - combinedAttributes.set("standard_name", TIME_STANDARD_NAME); - combinedAttributes.set("time_origin", "01-JAN-1970 00:00:00"); - combinedAttributes.set("units", units); - longName = combinedAttributes.getString("long_name"); - if (longName == null || longName.toLowerCase().equals("time")) //catch nothing or alternate case - combinedAttributes.set("long_name", TIME_LONGNAME); - longName = combinedAttributes.getString("long_name"); - - //previously computed evenSpacing is fine - //since source must be numeric, isEvenlySpaced is fine. - //If "months since", it's better than recalculating since recalc will reflect - // different number of days in months. + PrimitiveArray tSourceValues) throws Throwable { - //set destinationMin max and actual_range - //(they were temporarily source values) - //(they will be destination values in epochSeconds) - //(simpler than EDVTimeStamp because always numeric and range known from axis values) - destinationDataType = "double"; - destinationDataTypeClass = double.class; - int n = sourceValues.size(); - destinationMin = sourceTimeToEpochSeconds(sourceValues.getNiceDouble(0)); - destinationMax = sourceTimeToEpochSeconds(sourceValues.getNiceDouble(n - 1)); - if (Double.isNaN(destinationMin)) - throw new RuntimeException("ERROR related to time values and/or time source units: " + - "[0]=" + sourceValues.getString(0) + " => NaN epochSeconds."); - if (Double.isNaN(destinationMax)) - throw new RuntimeException("ERROR related to time values and/or time source units: " + - "[n-1]=" + sourceValues.getString(n-1) + " => NaN epochSeconds."); - - setActualRangeFromDestinationMinMax(); - initializeAverageSpacingAndCoarseMinMax(); - if (reallyVerbose) String2.log("\nEDVTimeGridAxis created, sourceTimeFormat=" + sourceTimeFormat + - " destMin=" + destinationMin + " destMax=" + destinationMax + "\n"); + super(tSourceName, EDV.TIME_NAME, tSourceAttributes, tAddAttributes, + tSourceValues); } /** @@ -196,355 +39,7 @@ public EDVTimeGridAxis(String tSourceName, * @return a string representation of this EDV. */ public String toString() { - return - "Time " + super.toString() + - " sourceTimeFormat=" + sourceTimeFormat + "\n"; - } - - /** - * This is used by the EDD constructor to determine if this - * EDV is valid. - * - * @param errorInMethod the start string for an error message - * @throws Throwable if this EDV is not valid - */ - public void ensureValid(String errorInMethod) throws Throwable { - super.ensureValid(errorInMethod); - //sourceTimeFormat is validated in constructor - } - - /** - * sourceTimeFormat is either a udunits string - * describing how to interpret numbers - * (e.g., "seconds since 1970-01-01T00:00:00") - * or a java.text.SimpleDateFormat string describing how to interpret string times - * (see http://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html). - * Examples: - *
    Date and Time Pattern Result - *
    "yyyy.MM.dd G 'at' HH:mm:ss z" 2001.07.04 AD at 12:08:56 PDT - *
    "EEE, MMM d, ''yy" Wed, Jul 4, '01 - *
    "yyyyy.MMMMM.dd GGG hh:mm aaa" 02001.July.04 AD 12:08 PM - *
    "yyMMddHHmmssZ" 010704120856-0700 - *
    "yyyy-MM-dd'T'HH:mm:ss.SSSZ" 2001-07-04T12:08:56.235-0700 - * - * @return the source time's units - */ - public String sourceTimeFormat() {return sourceTimeFormat;} - - /** - * This returns true if the source time is numeric. - * - * @return true if the source time is numeric. - */ - public boolean sourceTimeIsNumeric() { - return sourceTimeIsNumeric; - } - - /** - * This returns true if the destinationValues equal the sourceValues - * (e.g., scaleFactor = 1 and addOffset = 0). - *
    Some subclasses overwrite this to cover other situations: - *
    EDVTimeStamp only returns true if sourceTimeIsNumeric and - * sourceTimeBase = 0 and sourceTimeFactor = 1. - * - * @return true if the destinationValues equal the sourceValues. - */ - public boolean destValuesEqualSourceValues() { - return sourceTimeIsNumeric && - sourceTimeBase == 0.0 && sourceTimeFactor == 1.0 && - !scaleAddOffset; - } - - /** - * This converts a destination double value to a string - * (time variable override this to make an iso string). - * NaN returns "". - * - * @param destD epochSeconds - * @return destination String - */ - public String destinationToString(double destD) { - return Calendar2.epochSecondsToLimitedIsoStringT(time_precision, destD, ""); - } - - /** - * This converts a destination String value to a destination double - * (time variable overrides this to catch iso 8601 strings). - * "" or null returns NaN. - * - * @param destS - * @return destination double - */ - public double destinationToDouble(String destS) { - if (destS == null || destS.length() == 0) - return Double.NaN; - if (Calendar2.isIsoDate(destS)) - return Calendar2.isoStringToEpochSeconds(destS); - return String2.parseDouble(destS); - } - - /** - * This is the destinationMin time value in the dataset (as an ISO date/time string, - * e.g., "1990-01-01T00:00:00Z"). - * - * @return the destinationMin time - */ - public String destinationMinString() { - return destinationToString(destinationMin); - } - - /** - * This is the destinationMax time value in the dataset (an ISO date/time string, - * e.g., "2005-12-31T23:59:59Z"). - * - * @return the destinationMax time - */ - public String destinationMaxString() { - return destinationToString(destinationMax); - } - - - /** - * An indication of the precision of the time values, e.g., - * "1970-01-01T00:00:00Z" (default) or null (goes to default). - * See Calendar2.epochSecondsToLimitedIsoStringT() - */ - public String time_precision() { - return time_precision; - } - - /** - * If sourceTimeIsNumeric, this converts a source time to an ISO T time. - * - * @param sourceTime a numeric sourceTime - * @return seconds since 1970-01-01T00:00:00. - * If sourceTime is NaN, this returns NaN (but there shouldn't ever be missing values). - */ - public double sourceTimeToEpochSeconds(double sourceTime) { - if (scaleAddOffset) - sourceTime = sourceTime * scaleFactor + addOffset; - double sec = Calendar2.unitsSinceToEpochSeconds(sourceTimeBase, sourceTimeFactor, sourceTime); - //if (reallyVerbose) - // String2.log(" EDVTimeGridAxis stBase=" + sourceTimeBase + - // " scale=" + scaleFactor + " addOffset=" + addOffset + - // " stFactor=" + sourceTimeFactor + " sourceTime=" + sourceTime + - // " result=" + sec + " = " + Calendar2.epochSecondsToIsoStringT(sec)); - return sec; + return "EDVTimeGridAxis/" + super.toString(); } - /** - * If sourceTimeIsNumeric or not, this converts a source time to - * seconds since 1970-01-01T00:00:00Z. - * - * @param sourceTime either a number (as a string) or a string - * @return the source time converted to seconds since 1970-01-01T00:00:00Z. - * This returns NaN if trouble (sourceMissingValue, "", or invalid format). - */ - public double sourceTimeToEpochSeconds(String sourceTime) { - //This method is called twice by the EDVGridAxis constructor (needlessly). - //Apparently, sourceTimeIsNumeric hasn't yet been set to true, - // so this throws Exception. So just avoid this error. - //String2.log(">>sttes sourceTimeIsNumeric=" + sourceTimeIsNumeric); - if (!superConstructorIsFinished) - return Double.NaN; - - //sourceTime is numeric - if (sourceTimeIsNumeric) - return sourceTimeToEpochSeconds(String2.parseDouble(sourceTime)); - - //time is a string - try { - double d = parseISOWithCalendar2? - //parse with Calendar2.parseISODateTime - Calendar2.isoStringToEpochSeconds(sourceTime) : - //parse with Joda - dateTimeFormatter.parseMillis(sourceTime) / 1000.0; //thread safe - //String2.log(" EDVTimeGridAxis sourceTime=" + sourceTime + " epSec=" + d + " Calendar2=" + Calendar2.epochSecondsToIsoStringT(d)); - return d; - } catch (Throwable t) { - if (verbose && sourceTime != null && sourceTime.length() > 0) - String2.log(" EDVTimeGridAxis.sourceTimeToEpochSeconds: Invalid sourceTime=" + - sourceTime + "\n" + - (reallyVerbose? MustBe.throwableToString(t) : t.toString())); - return Double.NaN; - } - } - - /** - * This returns a PrimitiveArray (the original if the data type wasn't changed) - * with source values converted to destinationValues. - * - *

    Time variables will return a DoubleArray. - * - * @return a PrimitiveArray (the original if the data type wasn't changed) - * with source values converted to destinationValues. - */ - public PrimitiveArray toDestination(PrimitiveArray source) { - int size = source.size(); - DoubleArray destPa = source instanceof DoubleArray? - (DoubleArray)source : - new DoubleArray(size, true); - if (sourceTimeIsNumeric) { - for (int i = 0; i < size; i++) - destPa.set(i, sourceTimeToEpochSeconds(source.getNiceDouble(i))); - } else { - for (int i = 0; i < size; i++) - destPa.set(i, sourceTimeToEpochSeconds(source.getString(i))); - } - return destPa; - } - - /** - * This returns a new StringArray - * with source values converted to String destinationValues. - * - * @return a StringArray (the original if the data type wasn't changed) - * with source values converted to destinationValues. - */ - public PrimitiveArray toDestinationStrings(PrimitiveArray source) { - //memory is an issue! always generate this on-the-fly - int n = source.size(); - StringArray sa = source instanceof StringArray? - (StringArray)source : - new StringArray(n, true); - if (sourceTimeIsNumeric) { - for (int i = 0; i < n; i++) - sa.set(i, Calendar2.epochSecondsToLimitedIsoStringT( - time_precision, sourceTimeToEpochSeconds(source.getNiceDouble(i)), "")); - } else { - for (int i = 0; i < n; i++) - sa.set(i, Calendar2.epochSecondsToLimitedIsoStringT( - time_precision, sourceTimeToEpochSeconds(source.getString(i)), "")); - } - return sa; - - } - - /** - * This returns a PrimitiveArray (the original if the data type wasn't changed) - * with destination values converted to sourceValues. - * This doesn't change the order of the values. - * - *

    This version currently doesn't support scaleAddOffset. - * - * @param destination epochSecond double values - * @return a PrimitiveArray (the original if the data type wasn't changed) - * with destination values converted to sourceValues. - */ - public PrimitiveArray toSource(PrimitiveArray destination) { - //this doesn't support scaleAddOffset - int size = destination.size(); - PrimitiveArray source = sourceDataTypeClass == destination.elementClass()? - destination : - PrimitiveArray.factory(sourceDataTypeClass, size, true); - if (source instanceof StringArray) { - for (int i = 0; i < size; i++) - source.setString(i, epochSecondsToSourceTimeString(destination.getDouble(i))); - } else { - for (int i = 0; i < size; i++) - source.setDouble(i, epochSecondsToSourceTimeDouble(destination.getNiceDouble(i))); - } - return source; - } - - /** - * This returns the PrimitiveArray with the destination values for this axis. - * Don't change these values. - * This returns the sourceValues (with scaleFactor and - * addOffset if active; alt is special; time is special). - * This doesn't change the order of the values (even if source is depth and - * dest is altitude). - */ - public PrimitiveArray destinationValues() { - //alt and time may modify the values, so use sourceValues.clone() - return toDestination((PrimitiveArray)sourceValues.clone()); - } - - /** - * This returns one of this axis' source values as epochSeconds. - */ - public double destinationDouble(int which) { - return sourceTimeIsNumeric? - sourceTimeToEpochSeconds(sourceValues.getNiceDouble(which)) : - sourceTimeToEpochSeconds(sourceValues.getString(which)); - } - - /** - * This returns one of this axis' source values as a nice String destination value. - * For most EDVGridAxis, this returns destinationValues (which equal - * the String destination values). The Time subclass overrides this. - */ - public String destinationString(int which) { - return destinationToString(destinationDouble(which)); - } - - /** This returns a PrimitiveArray with the destination values for this axis. - * Don't change these values. - * If destination=source, this may return the sourceValues PrimitiveArray. - * The alt and time subclasses override this. - * The time subclass returns these as ISO 8601 'T' strings (to facilitate displaying options to users). - * !!!For time, if lots of values (e.g., 10^6), this is SLOW (e.g., 30 seconds)!!! - */ - public PrimitiveArray destinationStringValues() { - return toDestinationStrings(sourceValues); - } - - /** - * This converts epochSeconds to a numeric sourceTime. - * - * @param epochSeconds seconds since 1970-01-01T00:00:00Z. - * @return sourceTime. - * If sourceTime is NaN, this returns sourceMissingValue (but there shouldn't ever be missing values). - */ - public double epochSecondsToSourceTimeDouble(double epochSeconds) { - if (Double.isNaN(epochSeconds)) - return sourceMissingValue; - double source = Calendar2.epochSecondsToUnitsSince(sourceTimeBase, sourceTimeFactor, epochSeconds); - if (scaleAddOffset) - source = (source - addOffset) / scaleFactor; - return source; - } - - /** - * Call this whether or not sourceTimeIsNumeric to convert epochSeconds to - * sourceTime (numeric, or via dateTimeFormatter). - * - * @param epochSeconds seconds since 1970-01-01T00:00:00. - * @return the corresponding sourceTime (numeric, or via dateTimeFormatter). - * If epochSeconds is NaN, this returns sourceMissingValue (if sourceTimeIsNumeric) - * or "". - */ - public String epochSecondsToSourceTimeString(double epochSeconds) { - if (Double.isNaN(epochSeconds)) - return sourceTimeIsNumeric? "" + sourceMissingValue : ""; - if (sourceTimeIsNumeric) - return "" + epochSecondsToSourceTimeDouble(epochSeconds); - return dateTimeFormatter.print(Math.round(epochSeconds * 1000)); //round to long - } - - /** - * This converts a source time to a (limited) destination ISO TZ time. - * - * @param sourceTime - * @return a (limited) ISO T Time (e.g., "1993-12-31T23:59:59Z"). - * If sourceTime is invalid, this returns "" (but there shouldn't ever be missing values). - */ - public String sourceTimeToIsoStringT(double sourceTime) { - double destD = sourceTimeToEpochSeconds(sourceTime); - return destinationToString(destD); - } - - /** - * This converts a destination ISO time to a source time. - * - * @param isoString an ISO T Time (e.g., "1993-12-31T23:59:59"). - * @return sourceTime - * @throws Throwable if ISO time is invalid - */ - public double isoStringToSourceTime(String isoString) { - return epochSecondsToSourceTimeDouble(Calendar2.isoStringToEpochSeconds(isoString)); - } - - - } diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeStamp.java b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeStamp.java index c3cffea1d..5e0dc81f6 100644 --- a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeStamp.java +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeStamp.java @@ -242,7 +242,7 @@ public static boolean hasTimeUnits(Attributes sourceAttributes, Attributes addAt /** * This determines if a variable is a TimeStamp variable by looking - * for " since " (used for numeric times) or + * for " since " (used for UDUNITS numeric times) or * "yy" or "YY" (a formatting string which has the year designator) in the units attribute. */ public static boolean hasTimeUnits(String tUnits) { @@ -383,7 +383,9 @@ public double sourceTimeToEpochSeconds(double sourceTime) { return Double.NaN; if (scaleAddOffset) sourceTime = sourceTime * scaleFactor + addOffset; - return Calendar2.unitsSinceToEpochSeconds(sourceTimeBase, sourceTimeFactor, sourceTime); + double d = Calendar2.unitsSinceToEpochSeconds(sourceTimeBase, sourceTimeFactor, sourceTime); + //String2.log("!sourceTimeToEp " + destinationName + " src=" + sourceTime + " ep=" + d); + return d; } /** diff --git a/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeStampGridAxis.java b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeStampGridAxis.java new file mode 100644 index 000000000..71cfbfa1f --- /dev/null +++ b/WEB-INF/classes/gov/noaa/pfel/erddap/variable/EDVTimeStampGridAxis.java @@ -0,0 +1,596 @@ +/* + * EDVTimeStampGridAxis Copyright 2014, NOAA. + * See the LICENSE.txt file in this file's directory. + */ +package gov.noaa.pfel.erddap.variable; + +import com.cohort.array.Attributes; +import com.cohort.array.DoubleArray; +import com.cohort.array.PrimitiveArray; +import com.cohort.array.StringArray; +import com.cohort.util.Calendar2; +import com.cohort.util.MustBe; +import com.cohort.util.String2; +import com.cohort.util.Test; + +import java.text.ParsePosition; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.GregorianCalendar; +import java.util.TimeZone; + +import org.joda.time.*; +import org.joda.time.format.*; + +/** + * This class holds information about a timestamp grid axis variable. + * + * [STRING TIMES NOT FINISHED: + * The handling of String times in incomplete and probably not a good approach. + * Probably better is: really encapsulate the strings, so that any users of + * this class just see/deal with numeric values (epochSoconds). + * There are just too many places where it is assumed that all axes are numeric.] + * + * @author Bob Simons (bob.simons@noaa.gov) 2014-10-07 + */ +public class EDVTimeStampGridAxis extends EDVGridAxis { + + + /** Set by the constructor. */ + protected String sourceTimeFormat; + + /** These are set automatically. */ + protected boolean sourceTimeIsNumeric = true; + protected double sourceTimeBase = Double.NaN; //set if sourceTimeIsNumeric + protected double sourceTimeFactor = Double.NaN; + protected boolean parseISOWithCalendar2; + protected DateTimeFormatter dateTimeFormatter; //set if !sourceTimeIsNumeric + protected String time_precision; //see Calendar2.epochSecondsToLimitedIsoStringT + protected boolean superConstructorIsFinished = false; + + /** + * The constructor. + * + *

    Either tAddAttributes (read first) or tSourceAttributes must have "units" + * which is a UDUunits string (containing " since ") + * describing how to interpret source time values + * (which should always be numeric since they are a dimension of a grid) + * (e.g., "seconds since 1970-01-01T00:00:00"), + * where the base time is an + * ISO 8601 formatted date time string (YYYY-MM-DDThh:mm:ss). + * + *

    scale_factor and add_offset are allowed for numeric time variables. + * This constructor removes any scale_factor and add_offset attributes + * and stores the resulting information so that destination data + * has been converted to destinationDataType with scaleFactor and addOffset + * applied. + * + * @param tSourceName the name of the axis variable in the dataset source + * (usually with no spaces). + * @param tDestinationName should be "time" for *the* destination variable + * (type=EDVTimeGridAxis), otherwise some other name. + * If null or "", tSourceName will be used. + * @param tSourceAttributes are the attributes for the variable + * in the source + * @param tAddAttributes the attributes which will be added when data is + * extracted and which have precedence over sourceAttributes. + * Special case: value="null" causes that item to be removed from combinedAttributes. + * If this is null, an empty addAttributes will be created. + * @param tSourceValues has the values from the source. + * This can't be a StringArray. + * There must be at least one element. + * @throws Throwable if trouble + */ + public EDVTimeStampGridAxis(String tSourceName, String tDestinationName, + Attributes tSourceAttributes, Attributes tAddAttributes, + PrimitiveArray tSourceValues) + throws Throwable { + + super(tSourceName, tDestinationName, tSourceAttributes, tAddAttributes, tSourceValues); + superConstructorIsFinished = true; + + //time_precision e.g., 1970-01-01T00:00:00Z + time_precision = combinedAttributes.getString(EDV.TIME_PRECISION); + if (time_precision != null) { + //ensure not just year (can't distinguish user input a year vs. epochSeconds) + if (time_precision.equals("1970")) + time_precision = null; + //ensure Z at end of time + if (time_precision.length() >= 13 && !time_precision.endsWith("Z")) + time_precision = null; + } + + //currently, EDVTimeStampGridAxis doesn't support String sourceValues + String errorInMethod = "datasets.xml/EDVTimeStampGridAxis constructor error for sourceName=" + + tSourceName + ":\n"; + if (sourceValues instanceof StringArray) + throw new RuntimeException(errorInMethod + + "Currently, EDVTimeStampGridAxis doesn't support String source " + + "values for the time axis."); + + //read units before it is changed below + sourceTimeFormat = units(); + Test.ensureNotNothing(sourceTimeFormat, + errorInMethod + "'units' wasn't found."); //match name in datasets.xml + if (sourceTimeFormat.indexOf(" since ") > 0) { + sourceTimeIsNumeric = true; + double td[] = Calendar2.getTimeBaseAndFactor(sourceTimeFormat); + sourceTimeBase = td[0]; + sourceTimeFactor = td[1]; + } else { + sourceTimeIsNumeric = false; + throw new RuntimeException( + "Currently, the source units for the time axis must include \" since \"."); + /* If Strings are ever supported... + //ensure scale_factor=1 and add_offset=0 + if (scaleAddOffset) + throw new RuntimeException(errorInMethod + + "For String source times, scale_factor and add_offset MUST NOT be used."); + + if (sourceTimeFormat.equals(ISO8601T_FORMAT) || + sourceTimeFormat.equals(ISO8601TZ_FORMAT)) { + if (verbose) String2.log("parseISOWithCalendar2=true"); + dateTimeFormatter = ISODateTimeFormat.dateTimeNoMillis().withZone(DateTimeZone.UTC); + parseISOWithCalendar2 = true; + } else if (sourceTimeFormat.equals(ISO8601T3_FORMAT) || + sourceTimeFormat.equals(ISO8601T3Z_FORMAT)) { + if (verbose) String2.log("parseISOWithCalendar2=true"); + dateTimeFormatter = ISODateTimeFormat.dateTime().withZone(DateTimeZone.UTC); + parseISOWithCalendar2 = true; + } else { + //future: support time zones + dateTimeFormatter = DateTimeFormat.forPattern(sourceTimeFormat).withZone(DateTimeZone.UTC); + parseISOWithCalendar2 = false; + } + */ + } + + //extractScaleAddOffset It sets destinationDataType + extractScaleAddOffset(); + if (scaleAddOffset) { + setDestinationMin(destinationMin * scaleFactor + addOffset); + setDestinationMax(destinationMax * scaleFactor + addOffset); + } + //test for min>max after extractScaleAddOffset, since order may have changed + if (destinationMin > destinationMax) { + double d = destinationMin; destinationMin = destinationMax; destinationMax = d; + } + + units = TIME_UNITS; + if (destinationName.equals(EDV.TIME_NAME)) { + combinedAttributes.set("_CoordinateAxisType", "Time"); //unidata-related + combinedAttributes.set("axis", "T"); + } + combinedAttributes.set("ioos_category", TIME_CATEGORY); + combinedAttributes.set("standard_name", TIME_STANDARD_NAME); + combinedAttributes.set("time_origin", "01-JAN-1970 00:00:00"); + combinedAttributes.set("units", units); + longName = combinedAttributes.getString("long_name"); + if (longName == null || longName.toLowerCase().equals("time")) //catch nothing or alternate case + combinedAttributes.set("long_name", TIME_LONGNAME); + longName = combinedAttributes.getString("long_name"); + + //previously computed evenSpacing is fine + //since source must be numeric, isEvenlySpaced is fine. + //If "months since", it's better than recalculating since recalc will reflect + // different number of days in months. + + //set destinationMin max and actual_range + //(they were temporarily source values) + //(they will be destination values in epochSeconds) + //(simpler than EDVTimeStamp because always numeric and range known from axis values) + destinationDataType = "double"; + destinationDataTypeClass = double.class; + int n = sourceValues.size(); + destinationMin = sourceTimeToEpochSeconds(sourceValues.getNiceDouble(0)); + destinationMax = sourceTimeToEpochSeconds(sourceValues.getNiceDouble(n - 1)); + if (Double.isNaN(destinationMin)) + throw new RuntimeException("ERROR related to time values and/or time source units: " + + "[0]=" + sourceValues.getString(0) + " => NaN epochSeconds."); + if (Double.isNaN(destinationMax)) + throw new RuntimeException("ERROR related to time values and/or time source units: " + + "[n-1]=" + sourceValues.getString(n-1) + " => NaN epochSeconds."); + + setActualRangeFromDestinationMinMax(); + initializeAverageSpacingAndCoarseMinMax(); + if (reallyVerbose) String2.log("\nEDVTimeStampGridAxis created, " + + "sourceTimeFormat=" + sourceTimeFormat + + " destMin=" + destinationMin + " destMax=" + destinationMax + "\n"); + } + + /** + * This returns a string representation of this EDV. + * + * @return a string representation of this EDV. + */ + public String toString() { + return + "EDVTimeStampGridAxis/" + super.toString() + + " sourceTimeFormat=" + sourceTimeFormat + "\n"; + } + + /** + * This is used by the EDD constructor to determine if this + * EDV is valid. + * + * @param errorInMethod the start string for an error message + * @throws Throwable if this EDV is not valid + */ + public void ensureValid(String errorInMethod) throws Throwable { + super.ensureValid(errorInMethod); + //sourceTimeFormat is validated in constructor + } + + /** + * sourceTimeFormat is either a udunits string + * describing how to interpret numbers + * (e.g., "seconds since 1970-01-01T00:00:00") + * or a java.text.SimpleDateFormat string describing how to interpret string times + * (see http://docs.oracle.com/javase/7/docs/api/java/text/SimpleDateFormat.html). + * Examples: + *
    Date and Time Pattern Result + *
    "yyyy.MM.dd G 'at' HH:mm:ss z" 2001.07.04 AD at 12:08:56 PDT + *
    "EEE, MMM d, ''yy" Wed, Jul 4, '01 + *
    "yyyyy.MMMMM.dd GGG hh:mm aaa" 02001.July.04 AD 12:08 PM + *
    "yyMMddHHmmssZ" 010704120856-0700 + *
    "yyyy-MM-dd'T'HH:mm:ss.SSSZ" 2001-07-04T12:08:56.235-0700 + * + * @return the source time's units + */ + public String sourceTimeFormat() {return sourceTimeFormat;} + + /** + * This returns true if the source time is numeric. + * + * @return true if the source time is numeric. + */ + public boolean sourceTimeIsNumeric() { + return sourceTimeIsNumeric; + } + + /** + * This returns true if the destinationValues equal the sourceValues + * (e.g., scaleFactor = 1 and addOffset = 0). + *
    Some subclasses overwrite this to cover other situations: + *
    EDVTimeStamp only returns true if sourceTimeIsNumeric and + * sourceTimeBase = 0 and sourceTimeFactor = 1. + * + * @return true if the destinationValues equal the sourceValues. + */ + public boolean destValuesEqualSourceValues() { + return sourceTimeIsNumeric && + sourceTimeBase == 0.0 && sourceTimeFactor == 1.0 && + !scaleAddOffset; + } + + /** + * This converts a destination double value to a string + * (time variable override this to make an iso string). + * NaN returns "". + * + * @param destD epochSeconds + * @return destination String + */ + public String destinationToString(double destD) { + return Calendar2.epochSecondsToLimitedIsoStringT(time_precision, destD, ""); + } + + /** + * This converts a destination String value to a destination double + * (time variable overrides this to catch iso 8601 strings). + * "" or null returns NaN. + * + * @param destS + * @return destination double + */ + public double destinationToDouble(String destS) { + if (destS == null || destS.length() == 0) + return Double.NaN; + if (Calendar2.isIsoDate(destS)) + return Calendar2.isoStringToEpochSeconds(destS); //to millis precision + return String2.parseDouble(destS); + } + + /** + * This is the destinationMin time value in the dataset (as an ISO date/time string, + * e.g., "1990-01-01T00:00:00Z"). + * + * @return the destinationMin time + */ + public String destinationMinString() { + return destinationToString(destinationMin); + } + + /** + * This is the destinationMax time value in the dataset (an ISO date/time string, + * e.g., "2005-12-31T23:59:59Z"). + * + * @return the destinationMax time + */ + public String destinationMaxString() { + return destinationToString(destinationMax); + } + + + /** + * An indication of the precision of the time values, e.g., + * "1970-01-01T00:00:00Z" (default) or null (goes to default). + * See Calendar2.epochSecondsToLimitedIsoStringT() + */ + public String time_precision() { + return time_precision; + } + + /** + * If sourceTimeIsNumeric, this converts a source time to an ISO T time. + * + * @param sourceTime a numeric sourceTime + * @return seconds since 1970-01-01T00:00:00. + * If sourceTime is NaN, this returns NaN (but there shouldn't ever be missing values). + */ + public double sourceTimeToEpochSeconds(double sourceTime) { + if (scaleAddOffset) + sourceTime = sourceTime * scaleFactor + addOffset; + double sec = Calendar2.unitsSinceToEpochSeconds(sourceTimeBase, + sourceTimeFactor, sourceTime); + //if (reallyVerbose) + // String2.log(" EDVTimeStampGridAxis stBase=" + sourceTimeBase + + // " scale=" + scaleFactor + " addOffset=" + addOffset + + // " stFactor=" + sourceTimeFactor + " sourceTime=" + sourceTime + + // " result=" + sec + " = " + Calendar2.epochSecondsToIsoStringT(sec)); + return sec; + } + + /** + * If sourceTimeIsNumeric or not, this converts a source time to + * seconds since 1970-01-01T00:00:00Z. + * + * @param sourceTime either a number (as a string) or a string + * @return the source time converted to seconds since 1970-01-01T00:00:00Z. + * This returns NaN if trouble (sourceMissingValue, "", or invalid format). + */ + public double sourceTimeToEpochSeconds(String sourceTime) { + //This method is called twice by the EDVGridAxis constructor (needlessly). + //Apparently, sourceTimeIsNumeric hasn't yet been set to true, + // so this throws Exception. So just avoid this error. + //String2.log(">>sttes sourceTimeIsNumeric=" + sourceTimeIsNumeric); + if (!superConstructorIsFinished) + return Double.NaN; + + //sourceTime is numeric + if (sourceTimeIsNumeric) + return sourceTimeToEpochSeconds(String2.parseDouble(sourceTime)); + + //time is a string + try { + double d = parseISOWithCalendar2? + //parse with Calendar2.parseISODateTime + Calendar2.isoStringToEpochSeconds(sourceTime) : + //parse with Joda + dateTimeFormatter.parseMillis(sourceTime) / 1000.0; //thread safe + //String2.log(" EDVTimeStampGridAxis sourceTime=" + sourceTime + + // " epSec=" + d + " Calendar2=" + Calendar2.epochSecondsToIsoStringT(d)); + return d; + } catch (Throwable t) { + if (verbose && sourceTime != null && sourceTime.length() > 0) + String2.log(" EDVTimeStampGridAxis.sourceTimeToEpochSeconds: " + + "Invalid sourceTime=" + sourceTime + "\n" + + (reallyVerbose? MustBe.throwableToString(t) : t.toString())); + return Double.NaN; + } + } + + /** + * This returns a PrimitiveArray (the original if the data type wasn't changed) + * with source values converted to destinationValues. + * + *

    Time variables will return a DoubleArray. + * + * @return a PrimitiveArray (the original if the data type wasn't changed) + * with source values converted to destinationValues. + */ + public PrimitiveArray toDestination(PrimitiveArray source) { + int size = source.size(); + DoubleArray destPa = source instanceof DoubleArray? + (DoubleArray)source : + new DoubleArray(size, true); + if (sourceTimeIsNumeric) { + for (int i = 0; i < size; i++) + destPa.set(i, sourceTimeToEpochSeconds(source.getNiceDouble(i))); + } else { + for (int i = 0; i < size; i++) + destPa.set(i, sourceTimeToEpochSeconds(source.getString(i))); + } + return destPa; + } + + /** + * This returns a new StringArray + * with source values converted to String destinationValues. + * + * @return a StringArray (the original if the data type wasn't changed) + * with source values converted to destinationValues. + */ + public PrimitiveArray toDestinationStrings(PrimitiveArray source) { + //memory is an issue! always generate this on-the-fly + int n = source.size(); + StringArray sa = source instanceof StringArray? + (StringArray)source : + new StringArray(n, true); + if (sourceTimeIsNumeric) { + for (int i = 0; i < n; i++) + sa.set(i, Calendar2.epochSecondsToLimitedIsoStringT( + time_precision, sourceTimeToEpochSeconds(source.getNiceDouble(i)), "")); + } else { + for (int i = 0; i < n; i++) + sa.set(i, Calendar2.epochSecondsToLimitedIsoStringT( + time_precision, sourceTimeToEpochSeconds(source.getString(i)), "")); + } + return sa; + + } + + /** + * This returns a PrimitiveArray (the original if the data type wasn't changed) + * with destination values converted to sourceValues. + * This doesn't change the order of the values. + * + *

    This version currently doesn't support scaleAddOffset. + * + * @param destination epochSecond double values + * @return a PrimitiveArray (the original if the data type wasn't changed) + * with destination values converted to sourceValues. + */ + public PrimitiveArray toSource(PrimitiveArray destination) { + //this doesn't support scaleAddOffset + int size = destination.size(); + PrimitiveArray source = sourceDataTypeClass == destination.elementClass()? + destination : + PrimitiveArray.factory(sourceDataTypeClass, size, true); + if (source instanceof StringArray) { + for (int i = 0; i < size; i++) + source.setString(i, epochSecondsToSourceTimeString(destination.getDouble(i))); + } else { + for (int i = 0; i < size; i++) + source.setDouble(i, epochSecondsToSourceTimeDouble(destination.getNiceDouble(i))); + } + return source; + } + + /** + * This returns the PrimitiveArray with the destination values for this axis. + * Don't change these values. + * This returns the sourceValues (with scaleFactor and + * addOffset if active; alt is special; time is special). + * This doesn't change the order of the values (even if source is depth and + * dest is altitude). + */ + public PrimitiveArray destinationValues() { + //alt and time may modify the values, so use sourceValues.clone() + return toDestination((PrimitiveArray)sourceValues.clone()); + } + + /** + * This returns one of this axis' source values as epochSeconds. + */ + public double destinationDouble(int which) { + return sourceTimeIsNumeric? + sourceTimeToEpochSeconds(sourceValues.getNiceDouble(which)) : + sourceTimeToEpochSeconds(sourceValues.getString(which)); + } + + /** + * This returns one of this axis' source values as a nice String destination value. + * For most EDVGridAxis, this returns destinationValues (which equal + * the String destination values). The Time subclass overrides this. + */ + public String destinationString(int which) { + return destinationToString(destinationDouble(which)); + } + + /** This returns a PrimitiveArray with the destination values for this axis. + * Don't change these values. + * If destination=source, this may return the sourceValues PrimitiveArray. + * The alt and time subclasses override this. + * The time subclass returns these as ISO 8601 'T' strings + * (to facilitate displaying options to users). + * !!!For time, if lots of values (e.g., 10^6), this is SLOW (e.g., 30 seconds)!!! + */ + public PrimitiveArray destinationStringValues() { + return toDestinationStrings(sourceValues); + } + + /** + * This converts epochSeconds to a numeric sourceTime. + * + * @param epochSeconds seconds since 1970-01-01T00:00:00Z. + * @return sourceTime. + * If sourceTime is NaN, this returns sourceMissingValue + * (but there shouldn't ever be missing values). + */ + public double epochSecondsToSourceTimeDouble(double epochSeconds) { + if (Double.isNaN(epochSeconds)) + return sourceMissingValue; + double source = Calendar2.epochSecondsToUnitsSince(sourceTimeBase, + sourceTimeFactor, epochSeconds); + if (scaleAddOffset) + source = (source - addOffset) / scaleFactor; + return source; + } + + /** + * Call this whether or not sourceTimeIsNumeric to convert epochSeconds to + * sourceTime (numeric, or via dateTimeFormatter). + * + * @param epochSeconds seconds since 1970-01-01T00:00:00. + * @return the corresponding sourceTime (numeric, or via dateTimeFormatter). + * If epochSeconds is NaN, this returns sourceMissingValue (if sourceTimeIsNumeric) + * or "". + */ + public String epochSecondsToSourceTimeString(double epochSeconds) { + if (Double.isNaN(epochSeconds)) + return sourceTimeIsNumeric? "" + sourceMissingValue : ""; + if (sourceTimeIsNumeric) + return "" + epochSecondsToSourceTimeDouble(epochSeconds); + return dateTimeFormatter.print(Math.round(epochSeconds * 1000)); //round to long + } + + /** + * This converts a source time to a (limited) destination ISO TZ time. + * + * @param sourceTime + * @return a (limited) ISO T Time (e.g., "1993-12-31T23:59:59Z"). + * If sourceTime is invalid, this returns "" + * (but there shouldn't ever be missing values). + */ + public String sourceTimeToIsoStringT(double sourceTime) { + double destD = sourceTimeToEpochSeconds(sourceTime); + return destinationToString(destD); + } + + /** + * This converts a destination ISO time to a source time. + * + * @param isoString an ISO T Time (e.g., "1993-12-31T23:59:59"). + * @return sourceTime + * @throws Throwable if ISO time is invalid + */ + public double isoStringToSourceTime(String isoString) { + return epochSecondsToSourceTimeDouble(Calendar2.isoStringToEpochSeconds(isoString)); + } + + /** + * This determines if a variable is a TimeStamp variable by looking + * for " since " (used for UDUNITS numeric times). + * Currently, this does not look for String time units + * ("yy" or "YY", a formatting string which has the year designator) + * in the units attribute because this class currently doesn't support String times. + */ + public static boolean hasTimeUnits(Attributes sourceAttributes, + Attributes addAttributes) { + String tUnits = null; + if (addAttributes != null) //priority + tUnits = addAttributes.getString("units"); + if (tUnits == null && sourceAttributes != null) + tUnits = sourceAttributes.getString("units"); + return hasTimeUnits(tUnits); + } + + /** + * This determines if a variable is a TimeStamp variable by looking + * for " since " (used for UDUNITS numeric times). + * Currently, this does not look for String time units + * ("yy" or "YY", a formatting string which has the year designator) + * in the units attribute because this class currently doesn't support String times. + */ + public static boolean hasTimeUnits(String tUnits) { + if (tUnits == null) + return false; + return tUnits.indexOf(" since ") > 0;// || + //tUnits.indexOf("yy") >= 0 || + //tUnits.indexOf("YY") >= 0; + } + + + +} diff --git a/WEB-INF/classes/gov/noaa/pmel/sgt/MinuteHourAxis.java b/WEB-INF/classes/gov/noaa/pmel/sgt/MinuteHourAxis.java index 35f116d95..b37d0f970 100644 --- a/WEB-INF/classes/gov/noaa/pmel/sgt/MinuteHourAxis.java +++ b/WEB-INF/classes/gov/noaa/pmel/sgt/MinuteHourAxis.java @@ -57,10 +57,10 @@ public double computeLocation(double prev,double now) { } public void computeDefaults(GeoDate delta) { long msec = delta.getTime() % GeoDate.MSECS_IN_DAY; - if(msec > 7200000) { + if(msec > 6000000) { defaultMinorLabelInterval_ = 15; defaultMajorLabelInterval_ = 2; - } else if(msec > 1800000) { + } else if(msec > 1200000) { defaultMinorLabelInterval_ = 5; defaultMajorLabelInterval_ = 1; } else { diff --git a/WEB-INF/classes/gov/noaa/pmel/sgt/SecondMinuteAxis.java b/WEB-INF/classes/gov/noaa/pmel/sgt/SecondMinuteAxis.java index ff8c57638..dbd7ff80f 100644 --- a/WEB-INF/classes/gov/noaa/pmel/sgt/SecondMinuteAxis.java +++ b/WEB-INF/classes/gov/noaa/pmel/sgt/SecondMinuteAxis.java @@ -53,10 +53,10 @@ public double computeLocation(double prev,double now) { public void computeDefaults(GeoDate delta) { long msec = delta.getTime() % GeoDate.MSECS_IN_DAY; //System.out.println(">>range msec=" + msec); - if(msec > 120000) { + if(msec > 100000) { defaultMinorLabelInterval_ = 15; defaultMajorLabelInterval_ = 2; - } else if(msec > 30000) { + } else if(msec > 20000) { defaultMinorLabelInterval_ = 5; defaultMajorLabelInterval_ = 1; } else { diff --git a/download/changes.html b/download/changes.html index 83a74ae95..aacaa2e73 100644 --- a/download/changes.html +++ b/download/changes.html @@ -42,6 +42,59 @@

    ERDDAP Changes

    + + + + +
    +

    Changes in ERDDAP version 1.54 (released 2014-10-24)

    +
      +
    • New Features (for users): +
        +
      • Some variables now work with time at the milliseconds precision, e.g., + 2014-10-24T16:41:22.485Z. + Thanks to Dominic Fuller-Rowell. +
      + +
    • Small changes/Bug Fixes: +
        +
      • Bug fix: with a certain combination of circumstances, EDDGridFromNcFile + datasets returned data at reduced precision (e.g., floats instead of doubles). + This could only affect data values with > 8 significant figures. + My apologies. (And it was a classic computer programming bug: one wrong character.) + Thanks to Dominic Fuller-Rowell. +
      • Many small changes. +
      + +
    • Things ERDDAP Administrators Need to Know and Do: +
        +
      • Griddap datasets now support timestamp axis variables and data variables + (i.e., variables with time values, but a destinationName other than "time"). + Thanks to Dominic Fuller-Rowell. +
      • ERDDAP now correctly supports milliseconds time_precision "1970-01-01T00:00:00.000Z". + One intentional quirk: + when writing times to human-oriented files (e.g., .csv, .tsv, .json, .xhtml), + ERDDAP uses the specified time_precision if it includes seconds and/or decimal seconds; + otherwise, it uses seconds time_precision "1970-01-01T00:00:00Z" (for consistency + and backwards compatibility). + Thanks to Dominic Fuller-Rowell. +
      • EDDGridFromNcFiles now supports reading String dataVariables. +
      • .nc files written by griddap can now have String dataVariables. +
      • GenerateDatasetsXml now includes more flush() calls to avoid the problem + of information not being written to the files. + Thanks to Thierry Valero. +
      • The documentation for GenerateDatasetsXml was improved, notably to point + out that the -i switch only works if you specify all the answers on the command line + (e.g., script mode). And script mode is explained. + Thanks to Thierry Valero. +
      • ERDDAP no longer allows two variables in a dataset to have the same sourceName. + (If someone did it before, it probably led to error messages.) + As before, ERDDAP doesn't allow two variables in a dataset to have the same destinationName. +
      + +
    + +
    diff --git a/download/setup.html b/download/setup.html index 5b4a775fa..634a5b70a 100644 --- a/download/setup.html +++ b/download/setup.html @@ -290,17 +290,19 @@

    How To Do the Initial Setup of ERDDAP on Your Ser
  • Set up the tomcat/content/erddap configuration files.
    On Linux, Mac, and Windows, download erddapContent.zip - (version 1.50, size=22,132 bytes, MD5=25697F19B1AE7C595EFE0E3F25DFFDE2) + + (version 1.52, size=22,132 bytes, MD5=9CFB41D3FA7F6A267B600D629FABBC17) and unzip it into tomcat, creating tomcat/content/erddap .

    [The previous version - is also available - - (version 1.46, size=22,268 bytes, MD5=FE827B9C411ECAC535F8C65392A5CDFD).] + (version 1.46, size=22,268 bytes, MD5=FE827B9C411ECAC535F8C65392A5CDFD).] +

    For Red Hat Enterprise Linux (RHEL), unzip it into ~tomcat and set the system property erddapContentDirectory=~tomcat/content/erddap @@ -338,7 +340,9 @@

    How To Do the Initial Setup of ERDDAP on Your Ser
  • Install the erddap.war file.
    On Linux, Mac, and Windows, download erddap.war - (version 1.50, size=486,076,835 bytes, MD5=917B108FDF089B6426027168A2DC2B8A) + + (version 1.52, size=486,085,688 bytes, MD5=4CDAF259D9E51792D75C6CE66570BF2C) into tomcat/webapps . The .war file is big because it contains high resolution coastline, boundary, and elevation data needed to create maps. @@ -346,12 +350,13 @@

    How To Do the Initial Setup of ERDDAP on Your Ser

    [The previous version is also available - (version 1.46, size=482,115,063 bytes, MD5=26957D4A6866F5DBAF3238E7A39BD0FE). + Rename it to erddap.war after you download it.]

  • Use ProxyPass so users don't have to put the port number, e.g., :8080, in the URL. @@ -446,8 +451,9 @@

    How To Do an Update of an Existing ERDDAP on Your Serve (version 1.42, size=477,356,261 bytes, MD5=EC29F6D9E6185BC71557CF47063E3276) (version 1.44, size=482,013,960 bytes, MD5=CDFF1E9DD3201F6A5AEB19EB3DB556FC) (version 1.46, size=482,115,063 bytes, MD5=26957D4A6866F5DBAF3238E7A39BD0FE) - (version 1.48, size=486,082,750 bytes, MD5=6DC8DA75FF9DE313B3622854A59564EC) --> - (version 1.50, size=486,076,835 bytes, MD5=917B108FDF089B6426027168A2DC2B8A) + (version 1.48, size=486,082,750 bytes, MD5=6DC8DA75FF9DE313B3622854A59564EC) + (version 1.50, size=486,076,835 bytes, MD5=917B108FDF089B6426027168A2DC2B8A) --> + (version 1.52, size=486,085,688 bytes, MD5=4CDAF259D9E51792D75C6CE66570BF2C) into a temporary directory.
      diff --git a/download/setupDatasetsXml.html b/download/setupDatasetsXml.html index 3c84a2b93..c425ac589 100644 --- a/download/setupDatasetsXml.html +++ b/download/setupDatasetsXml.html @@ -252,6 +252,17 @@

    Introduction

    to ensure that the resulting dataset appears as you want it to in ERDDAP.
  • Feel free to make small changes by hand, for example, supply a better infoUrl, summary, or title. +
  • Scripting: As an alternative to answering the questions interactively + at the keyboard and looping to generate additional datasets, + you can provide command line arguments to answer + all of the questions to generate one dataset. + GenerateDatasetsXml will process those parameters, + write the output to the output file, and exit the program. + To set this up, first use the program in interactive mode + and write down your answers. + Then generate the command line (usually in a script) with all of the arguments. + This should be useful for datasets that change frequently in a way that + necessitates re-running GenerateDatasetsXml (notably EDDGridFromThreddsCatalog).
  • GenerateDatasetsXml supports a -idatasetsXmlName#tagName command line parameter which inserts the output into the specified datasets.xml file (the default is tomcat/content/erddap/datasets.xml). @@ -260,14 +271,23 @@

    Introduction


    and
    <!-- End GenerateDatasetsXml #tagName someDatetime -->
    and replaces everything in between those lines with the new content, and changes the someDatetime. -
    If the Begin and End lines are not found, then those lines and the new content +
      +
    • The -i switch is only processed (and changes to datasets.xml are only made) + if you run GenerateDatasetsXml with command line arguments which specify all + the answers to all of the questions for one loop of the program. (See 'Scripting' above.) + (The thinking is: This parameter is for use with scripts. + If you use the program in interactive mode (typing info on the keyboard), you are + likely to generate some incorrect chunks of XML before you generate + the one you want.) +
    • If the Begin and End lines are not found, then those lines and the new content are inserted right before </erddapDatasets>. -
      There is also a -I switch for testing purposes which works the same as -i, +
    • There is also a -I switch for testing purposes which works the same as -i, but creates a file called datasets.xmlDateTime and doesn't make changes to datasets.xml. -
      Don't run GenerateDatasetsXml with -i in two processes at once. +
    • Don't run GenerateDatasetsXml with -i in two processes at once. There is a chance only one set of changes will be kept. There may be serious trouble (for example, corrupted files). +
    If you use "GenerateDatasetsXml -verbose", it will print more diagnostic messages than usual. @@ -477,10 +497,9 @@

    Notes

  • Use the destinationName "time" only for variables that include the entire date+time (or date, if that is all there is). If, for example, there are separate columns for date and timeOfDay, don't use the variable name "time". -
  • See units for more information about the units attribute for - time and timeStamp variables. +
  • See units for more information about the units attribute for time and timeStamp variables.
  • The time variable and related - timeStamp variables are unique in that they + timeStamp variables are unique in that they always convert data values from the source's time format (what ever it is) into a numeric value (seconds since 1970-01-01T00:00:00Z) or a String value (ISO 8601:2004(E) format), depending on the situation. @@ -4345,7 +4364,8 @@

    Details

    <subsetVariables>, but the columns MAY be in any order.
  • It MAY have extra columns (they'll be removed and newly redundant rows will be removed). -
  • TimeStamp columns should have ISO 8601:2004(E) formatted date+timeZ strings +
  • Time and timestamp columns should have + ISO 8601:2004(E) formatted date+timeZ strings (for example, 1985-01-31T15:31:00Z).
  • Missing values should be missing values (not fake numbers like -99).
  • .json files may be a little harder to create but deal with Unicode characters well. @@ -4634,7 +4654,8 @@

    Details

    <addAttributes> for this dataset in datasets.xml, for example,
    <att name="actual_range" type="doubleList">-180 180</att> -
  • For numeric time and timestamp variables, the values specified should be the +
  • For numeric time and timestamp variables, + the values specified should be the relevant source (not destination) numeric values. For example, if the source time values are stored as "days since 1985-01-01", then the actual_range should be specified in "days since 1985-01-01". @@ -5015,11 +5036,15 @@

    Details

  • time_precision
    • time_precision is an OPTIONAL attribute used by ERDDAP (and no metadata standards) - for time and timestamp variables. - It specifies the precision to be used when displaying time values from the variable - on web pages in ERDDAP. The only data file output format that uses - this is .htmlTable. For example, + for time and timestamp variables, + which may be in gridded datasets or tabular datasets, + and in axisVariables or dataVariables. For example,
      <att name="time_precision">1970-01-01</att> +
      time_precision specifies the precision to be used whenever ERDDAP formats the time + values from that variable as strings on web pages, including .htmlTable responses. + In file formats where ERDDAP formats times as strings (e.g., .csv and .json), + ERDDAP only uses the time_precision-specified format if it includes + fractional seconds; otherwise, ERDDAP uses the 1970-01-01T00:00:00Z format.
    • Valid values are 1970-01, 1970-01-01, 1970-01-01T00Z, 1970-01-01T00:00Z, 1970-01-01T00:00:00Z (the default), 1970-01-01T00:00:00.0Z, 1970-01-01T00:00:00.00Z, 1970-01-01T00:00:00.000Z. @@ -5037,7 +5062,7 @@

      Details

      ISO 8601:2004 "extended" Time Format Specification (external link). -
    • WARNING: You should only use a limited time_precision to the extent that +
    • WARNING: You should only use a limited time_precision if all of the data values for the variable have only the minimum value for all of the fields that are hidden.
        @@ -5048,7 +5073,7 @@

        Details

        if there are non-0 hour, minute, or seconds values, (for example 2005-03-05T12:00:00Z) because the non-default hour value wouldn't be displayed. -
        Otherwise, if a user asks for all data with time=2005-03-05, + Otherwise, if a user asks for all data with time=2005-03-05, the request will fail unexpectedly.
    @@ -5135,7 +5160,7 @@

    Details

    and does not strictly follow the CF standard.

    Ideally, the baseTime is an ISO 8601:2004(E) formatted date time string
    (yyyy-MM-dd'T'HH:mm:ssZ, for example, 1970-01-01T00:00:00Z). - ERDDAP tries to work with a wide range of variations of that ideal format, for example, "1970-1-1 0:0:0". + ERDDAP tries to work with a wide range of variations of that ideal format, for example, "1970-1-1 0:0:0" is supported. If the time zone information is missing, it is assumed to be Zulu time zone (AKA GMT). Even if another time zone is specified, ERDDAP never uses Daylight Savings Time. @@ -5177,12 +5202,17 @@

    Details

    destinationName time and their units metadata (which must be suitable). -

    For tabular datasets, other variables can be timeStamp variables. They behave - like the main time variable (converting the source's time format into - "seconds since 1970-01-01T00:00:00Z" and/or ISO 8601:2004(E) format), but have a - different destinationName. TimeStamp variables are recognized by their +

    Any other variable + (axisVariable or dataVariable, in an EDDGrid or EDDTable dataset) + can be a timeStamp variable. + Timestamp variables are variables that have time-related units and time data, + but have a <destinationName> other than time. + TimeStamp variables behave like the main time variable in that they + convert the source's time format into + "seconds since 1970-01-01T00:00:00Z" and/or ISO 8601:2004(E) format). + ERDDAP recognizes timeStamp variables by their time-related "units" - metadata, which must contain " since " (for numeric dateTimes) or "yy" or "YY" + metadata, which must contain " since " (for numeric dateTimes) or "yy" or "YY" for formatted String dateTimes. But please still use the destinationName "time" for the main dateTime variable.