Skip to content

Commit

Permalink
v1.54
Browse files Browse the repository at this point in the history
  • Loading branch information
Bob Simons committed Oct 25, 2014
1 parent 153e0d7 commit be11a89
Show file tree
Hide file tree
Showing 46 changed files with 3,936 additions and 1,154 deletions.
64 changes: 57 additions & 7 deletions WEB-INF/classes/com/cohort/array/PrimitiveArray.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2813,6 +2813,42 @@ public static boolean testValueOpValue(double value1, String op, double value2)
"Unknown operator=\"" + op + "\".");
}

/**
* This tests if 'value1 op value2' is true for doubles to 12 sig figures
* (e.g., for time).
* The <=, >=, and = tests are (partly) done with Math2.almostEqual12
* so there is a little fudge factor.
* The =~ regex test must be tested with String testValueOpValue, not here,
* because value2 is a regex (not a double).
*
* @param value1
* @param op one of EDDTable.OPERATORS
* @param value2
* @return true if 'value1 op value2' is true.
* <br>Tests of "NaN = NaN" will evaluate to true.
* <br>Tests of "nonNaN != NaN" will evaluate to true.
* <br>All other tests where value1 is NaN or value2 is NaN will evaluate to false.
* @throws RuntimeException if trouble (e.g., invalid op)
*/
public static boolean testValueOpValueExtra(double value1, String op, double value2) {
//String2.log("testValueOpValue (double): " + value1 + op + value2);
//if (Double.isNaN(value2) && Double.isNaN(value1)) { //test2 first, less likely to be NaN
// return (op.equals("=") || op.equals("<=") || op.equals(">=")); //the '=' matters
//}
if (op.equals("<=")) return value1 <= value2 || Math2.almostEqual(12, value1, value2);
if (op.equals(">=")) return value1 >= value2 || Math2.almostEqual(12, value1, value2);
if (op.equals("=")) return (Double.isNaN(value1) && Double.isNaN(value2)) ||
Math2.almostEqual(12, value1, value2);
if (op.equals("<")) return value1 < value2;
if (op.equals(">")) return value1 > value2;
if (op.equals("!=")) return Double.isNaN(value1) && Double.isNaN(value2)? false :
value1 != value2;
//Regex test has to be handled via String testValueOpValue
// if (op.equals(PrimitiveArray.REGEX_OP))
throw new SimpleException("Query error: " +
"Unknown operator=\"" + op + "\".");
}

/**
* This tests if 'value1 op value2' is true.
* The ops containing with &lt; and &gt; compare value1.toLowerCase()
Expand Down Expand Up @@ -2855,7 +2891,8 @@ public static boolean testValueOpValue(String value1, String op, String value2)
* This tests the keep=true elements to see if 'get(element) op value2' is true.
* If the test is false, the keep element is set to false.
* <br>For float and double tests, the &lt;=, &gt;=, and = tests are (partly)
* done with Math2.almostEqual(6) and (9), so there is a little fudge factor.
* done with Math2.almostEqual(6) and (9) and (13 for morePrecise doubles),
* so there is a little fudge factor.
* <br>The =~ regex test is tested with String testValueOpValue,
* because value2 is a regex (not a numeric type).
*
Expand All @@ -2871,7 +2908,7 @@ public static boolean testValueOpValue(String value1, String op, String value2)
* @return nStillGood
* @throws RuntimeException if trouble (e.g., invalid op or invalid keep element)
*/
public int applyConstraint(BitSet keep, String op, String value2) {
public int applyConstraint(boolean morePrecise, BitSet keep, String op, String value2) {

//regex
if (op.equals(REGEX_OP)) {
Expand Down Expand Up @@ -2925,6 +2962,19 @@ public int applyConstraint(BitSet keep, String op, String value2) {
return nStillGood;
}

//treat everything else via double tests (that should be all that is left)
if (morePrecise) {
//String2.log("applyConstraint(double)");
int nStillGood = 0;
double value2d = String2.parseDouble(value2);
for (int row = keep.nextSetBit(0); row >= 0; row = keep.nextSetBit(row + 1)) {
if (testValueOpValueExtra(getDouble(row), op, value2d))
nStillGood++;
else keep.clear(row);
}
return nStillGood;
}

//treat everything else via double tests (that should be all that is left)
//String2.log("applyConstraint(double)");
int nStillGood = 0;
Expand Down Expand Up @@ -3791,7 +3841,7 @@ public static void testTestValueOpValue() {
keep = new BitSet();
keep.set(0, pa.size());
tTime = System.currentTimeMillis();
pa.applyConstraint(keep, "=~", "(10|zztop)");
pa.applyConstraint(false, keep, "=~", "(10|zztop)");
pa.justKeep(keep);
Test.ensureEqual(pa.size(), 1, "");
Test.ensureEqual(pa.getDouble(0), 10, "");
Expand All @@ -3805,7 +3855,7 @@ public static void testTestValueOpValue() {
keep = new BitSet();
keep.set(0, pa.size());
tTime = System.currentTimeMillis();
pa.applyConstraint(keep, ">=", "hubert"); //>= uses case insensitive test
pa.applyConstraint(false, keep, ">=", "hubert"); //>= uses case insensitive test
pa.justKeep(keep);
Test.ensureEqual(pa.size(), 1, "");
Test.ensureEqual(pa.getString(0), "Nate", "");
Expand All @@ -3819,7 +3869,7 @@ public static void testTestValueOpValue() {
keep = new BitSet();
keep.set(0, pa.size());
tTime = System.currentTimeMillis();
pa.applyConstraint(keep, ">=", "9");
pa.applyConstraint(false, keep, ">=", "9");
pa.justKeep(keep);
Test.ensureEqual(pa.size(), 1, "");
Test.ensureEqual(pa.getDouble(0), 10, "");
Expand All @@ -3833,7 +3883,7 @@ public static void testTestValueOpValue() {
keep = new BitSet();
keep.set(0, pa.size());
tTime = System.currentTimeMillis();
pa.applyConstraint(keep, ">=", "9");
pa.applyConstraint(false, keep, ">=", "9");
pa.justKeep(keep);
Test.ensureEqual(pa.size(), 1, "");
Test.ensureEqual(pa.getDouble(0), 10, "");
Expand All @@ -3847,7 +3897,7 @@ public static void testTestValueOpValue() {
keep = new BitSet();
keep.set(0, pa.size());
tTime = System.currentTimeMillis();
pa.applyConstraint(keep, ">=", "9");
pa.applyConstraint(false, keep, ">=", "9");
pa.justKeep(keep);
Test.ensureEqual(pa.size(), 1, "");
Test.ensureEqual(pa.getDouble(0), 10, "");
Expand Down
10 changes: 4 additions & 6 deletions WEB-INF/classes/com/cohort/util/Calendar2.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -337,12 +337,11 @@ public static double factorToGetSeconds(String units) throws Exception {
* The 'T' connector can be any non-digit.
* This may include hours, minutes, seconds, decimal, and Z or timezone offset (default=Zulu).
*
* @param isoZuluString
* @param isoZuluString (to millis precision)
* @return seconds
* @throws exception if trouble (e.g., input is null or invalid format)
*/
public static double isoStringToEpochSeconds(String isoZuluString) {
//pre 2012-05-22 was return Math2.floorDiv(isoZuluStringToMillis(isoZuluString), 1000);
return isoZuluStringToMillis(isoZuluString) / 1000.0;
}

Expand All @@ -351,7 +350,6 @@ public static double isoStringToEpochSeconds(String isoZuluString) {
*/
public static double safeIsoStringToEpochSeconds(String isoZuluString) {
try {
//pre 2012-05-22 was return Math2.floorDiv(isoZuluStringToMillis(isoZuluString), 1000);
return isoZuluStringToMillis(isoZuluString) / 1000.0;
} catch (Exception e) {
return Double.NaN;
Expand Down Expand Up @@ -507,7 +505,7 @@ public static int isoStringToEpochHours(String isoZuluString) {
/**
* This converts seconds since 1970-01-01T00:00:00Z
* to an ISO Zulu dateTime String with 'T'.
* The doubles are rounded to the nearest second.
* The doubles are rounded to the nearest millisecond.
* In many ways trunc would be better, but doubles are often bruised.
* round works symmetrically with + and - numbers.
*
Expand Down Expand Up @@ -1011,7 +1009,7 @@ public static String formatAsISODateTimeT3(GregorianCalendar gc) {
* limited precision string.
*
* @param time_precision can be "1970", "1970-01", "1970-01-01", "1970-01-01T00Z",
* "1970-01-01T00:00Z", "1970-01-01T00:00:00Z" (used if time_precision not matched),
* "1970-01-01T00:00Z", "1970-01-01T00:00:00Z" (used if time_precision is null or not matched),
* "1970-01-01T00:00:00.0Z", "1970-01-01T00:00:00.00Z", "1970-01-01T00:00:00.000Z".
* Versions without 'Z' are allowed.
*/
Expand All @@ -1020,7 +1018,7 @@ public static String limitedFormatAsISODateTimeT(String time_precision,

String zString = "";
if (time_precision == null || time_precision.length() == 0)
time_precision = "Z";
time_precision = "1970-01-01T00:00:00Z";
if (time_precision.charAt(time_precision.length() - 1) == 'Z') {
time_precision = time_precision.substring(0, time_precision.length() - 1);
zString = "Z";
Expand Down
56 changes: 56 additions & 0 deletions WEB-INF/classes/gov/noaa/pfel/coastwatch/Projects.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8491,6 +8491,62 @@ public static void makeIsaacNPH() throws Throwable {
table.saveAsFlatNc("/u00/data/points/isaac/NPH_IDS.nc", "time", false); //convertToFakeMV=false
}

/**
* Make simpleTest.nc
*/
public static void makeSimpleTestNc() throws Throwable {
Table table = new Table();

ByteArray timeStamps = new ByteArray(new byte[]{});
ByteArray arByte2 = new ByteArray( new byte[] {5, 15, 50, 25});

table.addColumn(0, "days",
new ByteArray(new byte[] {1,2,3,4}),
new Attributes());
table.addColumn(1, "hours",
new ByteArray(new byte[] {5,6,7,8}),
new Attributes());
table.addColumn(2, "minutes",
new ByteArray(new byte[] {9,10,11,12}),
new Attributes());
table.addColumn(3, "seconds",
new ByteArray(new byte[] {20,21,22,23}),
new Attributes());
table.addColumn(4, "millis",
new ByteArray(new byte[] {30,31,32,33}),
new Attributes());
table.addColumn(5, "bytes",
new ByteArray(new byte[] {40,41,42,43}),
new Attributes());
table.addColumn(6, "shorts",
new ShortArray(new short[] {10000,10001,10002,10004}),
new Attributes());
table.addColumn(7, "ints",
new IntArray(new int[] {1000000,1000001,1000002,1000004}),
new Attributes());
table.addColumn(8, "longs",
new LongArray(new long[] {10000000000L,10000000001L,10000000002L,10000000004L}),
new Attributes());
table.addColumn(9, "floats",
new FloatArray(new float[] {0,1.1f,2.2f,4.4f}),
new Attributes());
double d = 1e12;
table.addColumn(10, "doubles",
new DoubleArray(new double[] {d, d+0.1, d+0.2, d+0.3}),
new Attributes());
table.addColumn(11, "Strings",
new StringArray(new String[] {"0","10","20","30"}),
new Attributes());

table.columnAttributes(0).set("units", "days since 1970-01-01");
table.columnAttributes(1).set("units", "hours since 1980-01-01");
table.columnAttributes(2).set("units", "minutes since 1990-01-01");
table.columnAttributes(3).set("units", "seconds since 2000-01-01");
table.columnAttributes(4).set("units", "millis since 2010-01-01");

table.saveAsFlatNc("/erddapTest/simpleTest.nc", "row", false); //convertToFakeMV=false
}

/**
* Convert Isaac's PCUI .csv into .nc (and clean up).
* I removed 2014 row by hand: 2014,-9999,-9999,-9999,-9999,-9999,-9999
Expand Down
18 changes: 10 additions & 8 deletions WEB-INF/classes/gov/noaa/pfel/coastwatch/TestAll.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ public static void main(String args[]) throws Throwable {
// String2.log("tl=" + tl + " td=" + td);
//

// Table.debug = true; DasDds.main(new String[]{"erdMH1chla1day", "-verbose"});
// Table.debug = true; DasDds.main(new String[]{"dominic2", "-verbose"});
// String2.log(DigirHelper.getObisInventoryString(
// "http://iobis.marine.rutgers.edu/digir2/DiGIR.php",
// "OBIS-SEAMAP",
Expand Down Expand Up @@ -275,9 +275,9 @@ public static void main(String args[]) throws Throwable {
// EDDGridFromNcFiles.testGrib2_43(true); //42 or 43 for netcdfAll 4.2- or 4.3+
// EDDGridFromNcFiles.testNc(false);
// String2.log(EDDGridFromNcFiles.generateDatasetsXml(
// "c:/data/dominic/", ".*\\.nc",
// "c:/data/dominic/IT_MAG-L1b-GEOF_G16_s20140201000000_e20140201000029_c00000000000000.nc",
// -1, null));
// "/erddapTest/", "simpleTest\\.nc",
// "/erddapTest/simpleTest.nc",
// 90, null));
/*
StringBuilder sb = new StringBuilder();
//for (int i1 = 0; i1 < 4320; i1++)
Expand All @@ -290,7 +290,7 @@ public static void main(String args[]) throws Throwable {
// *** Daily
// Projects.viirsLatLon(true); //create

// String2.log(NcHelper.dumpString("/data/scrippsGliders/sp063-20140928T065100.nc", true));
// String2.log(NcHelper.dumpString("/erddapTest/dominic2/IT_MAG-L1b-GEOF_G16_s20140201000030_e20140201000059_c00000000000000.nc", "IB_time"));
// String2.log(NcHelper.dumpString("/u00/data/points/scrippsGlidersIoos1/sp031-20140412T155500.nc", false));
// String2.log(NcHelper.dumpString("c:/u00/satellite/VH/pic/8day/V20120012012008.L3m_8D_NPP_PIC_pic_4km", false));
// String2.log(NcHelper.dumpString("/u00/data/points/tao/daily/airt0n110w_dy.cdf", "AT_21"));
Expand Down Expand Up @@ -643,7 +643,7 @@ public static void main(String args[]) throws Throwable {
// Temporarily switching off parts of McAfee : Virus Scan Console (2X speedup!)
// On Access Scanner : All Processes
// Scan Items: check: specified file types only (instead of usual All Files)
// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2014, 5, 2014, 8, false); //first/last year(1990..)/month(1..), testMode
// EDDTableFromNcFiles.bobConsolidateGtsppTgz(2014, 3, 2014, 9, false); //first/last year(1990..)/month(1..), testMode
// log file is c:/data/gtspp/log.txt
// 2b) Email the "good" but "impossible" stations to Charles Sun
// [was Melanie Hamilton, now retired]
Expand All @@ -663,8 +663,8 @@ public static void main(String args[]) throws Throwable {
// //one time: EDDTableFromNcFiles.bobFindGtsppDuplicateCruises();
// EDDTableFromNcFiles.testErdGtsppBest("erdGtsppBestNc");
// 6) Create ncCF files with the same date range as 2a) above:
// !!!! HIDE THE WINDOW !!! IT WILL RUN 1000X FASTER!!!
// EDDTableFromNcFiles.bobCreateGtsppNcCFFiles(2014, 5, 2014, 8); //e.g., first/last year(1990..)/month(1..)
// !!!! HIDE THE WINDOW !!! IT WILL RUN MUCH FASTER!!! takes ~2 minutes per month processed
// EDDTableFromNcFiles.bobCreateGtsppNcCFFiles(2014, 3, 2014, 9); //e.g., first/last year(1990..)/month(1..)
// String2.log(NcHelper.dumpString("/u00/data/points/gtsppNcCF/201406a.nc", false));
// 7) * Load erdGtsppBest in localHost ERDDAP. (long time if lots of files changed)
// * Generate .json file from
Expand Down Expand Up @@ -921,6 +921,7 @@ public static void main(String args[]) throws Throwable {
// Projects.makeCRWNcml34("2000-12-02", 3, "2000-12-05", "dhw"); //sst, anomaly, dhw, hotspot, baa
// Projects.makeCRWToMatch("baa");
// Projects.makeCRWNcml34("2013-12-19", 4, "2014-12-31", "baa"); //sst, anomaly, dhw, hotspot, baa
// Projects.makeSimpleTestNc();
// Projects.makeVH1dayNcmlFiles(2012, 2035);
// Projects.makeVH8dayNcmlFiles(2012, 2035);
// Projects.makeVHmdayNcmlFiles(2012, 2035);
Expand Down Expand Up @@ -1380,6 +1381,7 @@ public static void main(String args[]) throws Throwable {
EDVTime edvt;
EDVTimeGridAxis edvtga;
EDVTimeStamp edvts;
EDVTimeStampGridAxis edvtsga;
Erddap erddap;
ErddapRedirect erddapRedirect;
FishBase fb;
Expand Down
Binary file modified ...NF/classes/gov/noaa/pfel/coastwatch/griddata/OQNux10S1day_20050712_x-135_X-105_y22_Y50.nc
View file
Binary file not shown.
14 changes: 9 additions & 5 deletions WEB-INF/classes/gov/noaa/pfel/coastwatch/pointdata/Table.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1546,8 +1546,10 @@ public void setAttributes(int lonIndex, int latIndex, int depthIndex, int timeIn
trySet(globalAttributes, "cdm_data_type", cdmDataType);
trySet(globalAttributes, "contributor_name", courtesy);
trySet(globalAttributes, "contributor_role", "Source of data.");
trySet(globalAttributes, "Conventions", "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"); //unidata-related
trySet(globalAttributes, "Metadata_Conventions", "COARDS, CF-1.6, Unidata Dataset Discovery v1.0"); //unidata-related
trySet(globalAttributes, "Conventions",
"COARDS, CF-1.6, Unidata Dataset Discovery v1.0"); //unidata-related
trySet(globalAttributes, "Metadata_Conventions",
"COARDS, CF-1.6, Unidata Dataset Discovery v1.0"); //unidata-related
trySet(globalAttributes, "creator_email", creatorEmail);
trySet(globalAttributes, "creator_name", creatorName);
trySet(globalAttributes, "creator_url", creatorUrl);
Expand Down Expand Up @@ -1727,8 +1729,10 @@ public void setActualRangeAndBoundingBox(
globalAttributes.remove("time_coverage_start"); //unidata-related
globalAttributes.remove("time_coverage_end");
} else {
globalAttributes.set("time_coverage_start", Calendar2.epochSecondsToIsoStringT(range.getDouble(0)) + "Z");
globalAttributes.set("time_coverage_end", Calendar2.epochSecondsToIsoStringT(range.getDouble(1)) + "Z");
globalAttributes.set("time_coverage_start",
Calendar2.epochSecondsToIsoStringT(range.getDouble(0)) + "Z");
globalAttributes.set("time_coverage_end",
Calendar2.epochSecondsToIsoStringT(range.getDouble(1)) + "Z");
}
//this doesn't set tableGlobalAttributes.set("time_coverage_resolution", "P1H");
}
Expand Down Expand Up @@ -11084,7 +11088,7 @@ public int lowApplyConstraint(boolean requireConVar, int idCol,

//test the keep=true rows for this constraint
PrimitiveArray conPa = getColumn(conVarCol);
int nKeep = conPa.applyConstraint(keep, conOp, conVal);
int nKeep = conPa.applyConstraint(false, keep, conOp, conVal);

if (reallyVerbose)
String2.log(" applyConstraint: after " + conVar + conOp + "\"" + conVal + "\", " +
Expand Down
7 changes: 6 additions & 1 deletion WEB-INF/classes/gov/noaa/pfel/erddap/DasDds.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,10 @@ public class DasDds {

private void printToBoth(String s) throws IOException {
String2.log(s);
String2.flushLog();
outFile.write(s);
outFile.write('\n');
outFile.flush();
}

/** This gets the i'th value from args, or prompts the user. */
Expand Down Expand Up @@ -104,10 +106,12 @@ public String doIt(String args[], boolean loop) throws Throwable {
"\n*** DasDds ***\n" +
"This generates the DAS and DDS for a dataset and puts it in\n" +
outFileName + "\n" +
"Press ^C to exit at any time.\n\n" +
"Press ^D or ^C to exit at any time.\n\n" +
"Which datasetID",
datasetID);
if (datasetID == null) {
String2.flushLog();
outFile.flush();
outFile.close();
return String2.readFromFile(outFileName)[1];
}
Expand Down Expand Up @@ -135,6 +139,7 @@ public String doIt(String args[], boolean loop) throws Throwable {

} while (loop && args.length == 0);

outFile.flush();
outFile.close();
String ret = String2.readFromFile(outFileName)[1];
String2.returnLoggingToSystemOut();
Expand Down
Loading

0 comments on commit be11a89

Please sign in to comment.