v1.66

WEB-INF/ArchiveADataset.bat

@@ -0,0 +1,4 @@
+rem This is the Windows batch file to run ArchiveADataset.
+rem See http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html#Tools
+
+java -cp ./classes;../../../lib/servlet-api.jar;lib/activation.jar;lib/axis.jar;lib/cassandra-driver-core.jar;lib/netty-all.jar;lib/guava.jar;lib/metrics-core.jar;lib/lz4.jar;lib/snappy-java.jar;lib/commons-compress.jar;lib/commons-discovery.jar;lib/itext-1.3.1.jar;lib/jaxrpc.jar;lib/joda-time.jar;lib/joid.jar;lib/lucene-core.jar;lib/mail.jar;lib/netcdfAll-latest.jar;lib/postgresql.jdbc.jar;lib/saaj.jar;lib/tsik.jar;lib/wsdl4j.jar;lib/aws-java-sdk.jar;lib/commons-codec.jar;lib/commons-logging.jar;lib/fluent-hc.jar;lib/httpclient.jar;lib/httpclient-cache.jar;lib/httpcore.jar;lib/httpmime.jar;lib/jna.jar;lib/jna-platform.jar;lib/jackson-annotations.jar;lib/jackson-core.jar;lib/jackson-databind.jar  -Xms1500M -Xmx1500M  gov.noaa.pfel.erddap.ArchiveADataset %1 %2 %3 %4 %5 %6 %7 %8 %9

WEB-INF/ArchiveADataset.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+# This is the Unix/Linux shell script to run ArchiveADataset.
+# See http://coastwatch.pfeg.noaa.gov/erddap/download/setupDatasetsXml.html#Tools
+
+cp1="./classes:../../../lib/servlet-api.jar:lib/activation.jar:lib/axis.jar"
+cp2=":lib/cassandra-driver-core.jar:lib/netty-all.jar:lib/guava.jar:lib/metrics-core.jar:lib/lz4.jar:lib/snappy-java.jar"
+cp3=":lib/commons-compress.jar:lib/commons-discovery.jar:lib/itext-1.3.1.jar"
+cp4=":lib/jaxrpc.jar:lib/joda-time.jar:lib/joid.jar:lib/lucene-core.jar"
+cp5=":lib/mail.jar:lib/netcdfAll-latest.jar:lib/postgresql.jdbc.jar"
+cp6=":lib/saaj.jar:lib/tsik.jar:lib/wsdl4j.jar"
+cp7=":lib/aws-java-sdk.jar:lib/commons-codec.jar:lib/commons-logging.jar"
+cp8=":lib/fluent-hc.jar:lib/httpclient.jar:lib/httpclient-cache.jar:lib/httpcore.jar"
+cp9=":lib/httpmime.jar:lib/jna.jar:lib/jna-platform.jar:lib/jackson-annotations.jar"
+cp10=":lib/jackson-core.jar:lib/jackson-databind.jar"
+cp0="$cp1$cp2$cp3$cp4$cp5$cp6$cp7$cp8$cp9$cp10"
+
+java -cp $cp0 -Xms1500M -Xmx1500M gov.noaa.pfel.erddap.ArchiveADataset $1 $2 $3 $4 $5 $6 $7 $8 $9

WEB-INF/classes/com/cohort/array/Attributes.java

@@ -1,7 +1,7 @@
 /* This file is part of the EMA project and is 
- * Copyright (c) 2005 Robert Alten Simons (info@cohort.com).
+ * Copyright (c) 2005 Robert Simons (CoHortSoftware@gmail.com).
  * See the MIT/X-like license in LICENSE.txt.
- * For more information visit www.cohort.com or contact info@cohort.com.
+ * For more information visit www.cohort.com or contact CoHortSoftware@gmail.com.
  */
 package com.cohort.array;
 
@@ -16,7 +16,7 @@
  * and value is a PrimitiveArray). 
  * The backing datastructure (ConcurrentHashMap) is thread-safe.
  *
- * @author Bob Simons (info@cohort.com)
+ * @author Bob Simons (CoHortSoftware@gmail.com)
  */
 public class Attributes {
     //FUTURE: implement as ArrayString for names and ArrayList for values?
@@ -168,6 +168,49 @@ public double getDouble(String name) {
         }
     }
 
+    /**
+     * A convenience method which returns the first element of the attribute's 
+     * value PrimitiveArray as a double, regardless of the type used to store it.
+     * In this "unsigned" variant, if pa isIntegerType, then the value
+     * is interpreted as an unsigned value (two's compliment, e.g., -1B becomes 255).
+     * Note that the native cohort missing value (e.g., 127B) is treated as a value,
+     * not a missing value.
+     *
+     * @param name the name of an attribute
+     * @return the attribute as a double or Double.NaN if trouble (e.g., not found)
+     */
+    public double getUnsignedDouble(String name) {
+        try {
+            PrimitiveArray pa = get(name);
+            if (pa == null || pa.size() == 0)
+                return Double.NaN;
+            return pa.getUnsignedDouble(0);
+        } catch (Exception e) {
+            return Double.NaN;
+        }
+    }
+
+    /**
+     * A convenience method which returns the first element of the attribute's 
+     * value PrimitiveArray as a double, regardless of the type used to store it.
+     * In this "raw" variant, if pa isIntegerType, then the cohort missingValue 
+     * (e.g., ByteArray missingValue=127) is left intact 
+     * and NOT converted to new pa's missingValue (e.g., Double.NaN).
+     *
+     * @param name the name of an attribute
+     * @return the attribute as a double or Double.NaN if trouble (e.g., not found)
+     */
+    public double getRawDouble(String name) {
+        try {
+            PrimitiveArray pa = get(name);
+            if (pa == null || pa.size() == 0)
+                return Double.NaN;
+            return pa.getRawDouble(0);
+        } catch (Exception e) {
+            return Double.NaN;
+        }
+    }
+
     /**
      * A convenience method which returns the first element of the attribute's 
      * value PrimitiveArray as a double, regardless of the type used to store it.
@@ -241,6 +284,27 @@ public int getInt(String name) {
         }
     }
 
+    /**
+     * A convenience method which returns the first element of the attribute's 
+     * value PrimitiveArray as an int, regardless of the type used to store it.
+     * In this "raw" variant, if pa isIntegerType, then the cohort missingValue 
+     * (e.g., ByteArray missingValue=127) is left intact 
+     * and NOT converted to new pa's missingValue (e.g., Double.NaN).
+     *
+     * @param name the name of an attribute
+     * @return the attribute as an int or Integer.MAX_VALUE if trouble (e.g., not found)
+     */
+    public int getRawInt(String name) {
+        try {
+            PrimitiveArray pa = get(name);
+            if (pa == null || pa.size() == 0)
+                return Integer.MAX_VALUE;
+            return pa.getRawInt(0);
+        } catch (Exception e) {
+            return Integer.MAX_VALUE;
+        }
+    }
+
     /**
      * This removes a specific attribute.
      * 

WEB-INF/classes/com/cohort/array/ByteArray.java

@@ -1,7 +1,7 @@
 /* This file is part of the EMA project and is 
- * Copyright (c) 2005 Robert Alten Simons (info@cohort.com).
+ * Copyright (c) 2005 Robert Simons (CoHortSoftware@gmail.com).
  * See the MIT/X-like license in LICENSE.txt.
- * For more information visit www.cohort.com or contact info@cohort.com.
+ * For more information visit www.cohort.com or contact CoHortSoftware@gmail.com.
  */
 package com.cohort.array;
 
@@ -36,6 +36,21 @@ public class ByteArray extends PrimitiveArray {
      */
     public byte[] array;
 
+    /** This indicates if this class' type (e.g., short.class) can be contained in a long. 
+     * The integer type classes override this.
+     */
+    public boolean isIntegerType() {
+        return true;
+    }
+
+    /** 
+     * This returns for cohort missing value for this class (e.g., Integer.MAX_VALUE), 
+     * expressed as a double. FloatArray and StringArray return Double.NaN. 
+     */
+    public double missingValue() {
+        return Byte.MAX_VALUE;
+    }
+
     /**
      * A constructor for a capacity of 8 elements. The initial 'size' will be 0.
      */
@@ -389,11 +404,30 @@ public void addLong(long value) {
     /**
      * This adds an element from another PrimitiveArray.
      *
-     * @param otherPA
-     * @param otherIndex
-     */
-    public void addFromPA(PrimitiveArray otherPA, int otherIndex) {
-        addInt(otherPA.getInt(otherIndex));
+     * @param otherPA the source PA
+     * @param otherIndex the start index in otherPA
+     * @param nValues the number of values to be added
+     * @return 'this' for convenience
+     */
+    public PrimitiveArray addFromPA(PrimitiveArray otherPA, int otherIndex, int nValues) {
+
+        //add from same type
+        if (otherPA.elementClass() == elementClass()) {
+            if (otherIndex + nValues > otherPA.size)
+                throw new IllegalArgumentException(String2.ERROR + 
+                    " in CharArray.addFromPA: otherIndex=" + otherIndex + 
+                    " + nValues=" + nValues + 
+                    " > otherPA.size=" + otherPA.size);
+            ensureCapacity(size + nValues);            
+            System.arraycopy(((ByteArray)otherPA).array, otherIndex, array, size, nValues);
+            size += nValues;
+            return this;
+        }
+
+        //add from different type
+        for (int i = 0; i < nValues; i++)
+            addInt(otherPA.getInt(otherIndex++)); //does error checking
+        return this;
     }
 
     /**
@@ -631,6 +665,20 @@ public int getInt(int index) {
         return i == Byte.MAX_VALUE? Integer.MAX_VALUE : i;
     }
 
+    /**
+     * Return a value from the array as an int.
+     * This "raw" variant leaves missingValue from smaller data types 
+     * (e.g., ByteArray missingValue=127) AS IS.
+     * Floating point values are rounded.
+     * 
+     * @param index the index number 0 ... size-1
+     * @return the value as an int. String values are parsed
+     *   with String2.parseInt and so may return Integer.MAX_VALUE.
+     */
+    public int getRawInt(int index) {
+        return get(index);
+    }
+
     /**
      * Set a value in the array as an int.
      * 
@@ -702,6 +750,35 @@ public double getDouble(int index) {
         return b == Byte.MAX_VALUE? Double.NaN : b;
     }
 
+    /**
+     * Return a value from the array as a double.
+     * FloatArray converts float to double in a simplistic way.
+     * For this variant: Integer source values will be treated as unsigned.
+     * 
+     * @param index the index number 0 ... size-1
+     * @return the value as a double. String values are parsed
+     *   with String2.parseDouble and so may return Double.NaN.
+     */
+    public double getUnsignedDouble(int index) {
+        //or see http://www.unidata.ucar.edu/software/thredds/current/netcdf-java/reference/faq.html#Unsigned
+        return Byte.toUnsignedInt(get(index));
+    }
+
+    /**
+     * Return a value from the array as a double.
+     * This "raw" variant leaves missingValue from integer data types 
+     * (e.g., ByteArray missingValue=127) AS IS.
+     *
+     * <p>All integerTypes override this.
+     * 
+     * @param index the index number 0 ... size-1
+     * @return the value as a double. String values are parsed
+     *   with String2.parseDouble and so may return Double.NaN.
+     */
+    public double getRawDouble(int index) {
+        return get(index);
+    }
+
     /**
      * Set a value in the array as a double.
      * 
@@ -717,13 +794,30 @@ public void setDouble(int index, double d) {
      * Return a value from the array as a String.
      * 
      * @param index the index number 0 .. 
-     * @return For numeric types, this returns (String.valueOf(ar[index])), or "" for NaN or infinity.
+     * @return For numeric types, this returns (String.valueOf(ar[index])), 
+     *    or "" for NaN or infinity.
      */
     public String getString(int index) {
         byte b = get(index);
         return b == Byte.MAX_VALUE? "" : String.valueOf(b);
     }
 
+    /**
+     * Return a value from the array as a String.
+     * This "raw" variant leaves missingValue from integer data types 
+     * (e.g., ByteArray missingValue=127) AS IS.
+     * FloatArray and DoubleArray return "" if the stored value is NaN. 
+     *
+     * <p>All integerTypes override this.
+     * 
+     * @param index the index number 0 ... size-1
+     * @return the value as a double. String values are parsed
+     *   with String2.parseDouble and so may return Double.NaN.
+     */
+    public String getRawString(int index) {
+        return String.valueOf(get(index));
+    }
+
     /**
      * Set a value in the array as a String.
      * 
@@ -1050,26 +1144,41 @@ public static void rafWriteDouble(RandomAccessFile raf, long start, long index,
     }
 
     /**
-     * This appends the data in another primitiveArray to the current data.
-     * WARNING: information may be lost from the incoming primitiveArray if this
+     * This appends the data in another pa to the current data.
+     * WARNING: information may be lost from the incoming pa if this
      * primitiveArray is of a simpler type.
      *
-     * @param primitiveArray primitiveArray must be the same or a narrower 
+     * @param pa pa must be the same or a narrower 
      *  data type, or the data will be narrowed with Math2.narrowToByte.
      */
-    public void append(PrimitiveArray primitiveArray) {
-        int otherSize = primitiveArray.size(); //this avoids infinite loop if primitiveArray == this
+    public void append(PrimitiveArray pa) {
+        int otherSize = pa.size(); 
         ensureCapacity(size + (long)otherSize);
-        if (primitiveArray instanceof ByteArray) {
-            System.arraycopy(((ByteArray)primitiveArray).array, 0, array, size, otherSize);
+        if (pa instanceof ByteArray) {
+            System.arraycopy(((ByteArray)pa).array, 0, array, size, otherSize);
         } else {
             for (int i = 0; i < otherSize; i++) {
-                array[size + i] = Math2.narrowToByte(primitiveArray.getInt(i)); //this converts mv's
+                array[size + i] = Math2.narrowToByte(pa.getInt(i)); //this converts mv's
             }
         }
         size += otherSize; //do last to minimize concurrency problems
     }    
 
+    /**
+     * This appends the data in another pa to the current data.
+     * This "raw" variant leaves missingValue from smaller data types
+     * (e.g., ByteArray missingValue=127) AS IS.
+     * WARNING: information may be lost from the incoming pa if this
+     * primitiveArray is of a simpler type.
+     *
+     * @param pa pa must be the same or a narrower 
+     *  data type, or the data will be narrowed with Math2.narrowToByte.
+     */
+    public void rawAppend(PrimitiveArray pa) {
+        //since there are no smaller data types than byte, rawAppend() = append()
+        append(pa);
+    }    
+
     /**
      * This populates 'indices' with the indices (ranks) of the values in this ByteArray
      * (ties get the same index). For example, 10,10,25,3 returns 1,1,2,0.
@@ -1283,6 +1392,35 @@ public static void test() throws Throwable {
 
         //** test default constructor and many of the methods
         ByteArray anArray = new ByteArray();
+        Test.ensureEqual(anArray.isIntegerType(), true, "");
+        Test.ensureEqual(anArray.missingValue(), Byte.MAX_VALUE, "");
+        anArray.addString("");
+        Test.ensureEqual(anArray.get(0),               Byte.MAX_VALUE, "");
+        Test.ensureEqual(anArray.getRawInt(0),         Byte.MAX_VALUE, "");
+        Test.ensureEqual(anArray.getRawDouble(0),      Byte.MAX_VALUE, "");
+        Test.ensureEqual(anArray.getUnsignedDouble(0), Byte.MAX_VALUE, "");
+        Test.ensureEqual(anArray.getRawString(0), "" + Byte.MAX_VALUE, "");
+        Test.ensureEqual(anArray.getRawNiceDouble(0),  Byte.MAX_VALUE, "");
+        Test.ensureEqual(anArray.getInt(0),            Integer.MAX_VALUE, "");
+        Test.ensureEqual(anArray.getDouble(0),         Double.NaN, "");
+        Test.ensureEqual(anArray.getString(0), "", "");
+
+        anArray.set(0, (byte)-128); Test.ensureEqual(anArray.getUnsignedDouble(0), 128, "");
+        anArray.set(0, (byte)-127); Test.ensureEqual(anArray.getUnsignedDouble(0), 129, "");
+        anArray.set(0, (byte)  -1); Test.ensureEqual(anArray.getUnsignedDouble(0), 255, "");
+        anArray.clear();
+
+        //unsignedFactory, which uses unsignedAppend
+        anArray = (ByteArray)unsignedFactory(byte.class, 
+            new ByteArray(new byte[] {0, 1, Byte.MAX_VALUE, Byte.MIN_VALUE, -1}));
+        Test.ensureEqual(anArray.toString(), "0, 1, 127, 127, 127", ""); // -> mv
+        anArray.clear();        
+
+        anArray = (ByteArray)unsignedFactory(byte.class, 
+            new ShortArray(new short[] {0, 1, Short.MAX_VALUE, Short.MIN_VALUE, -1}));
+        Test.ensureEqual(anArray.toString(), "0, 1, 127, 127, 127", ""); // -> mv
+        anArray.clear();        
+
         Test.ensureEqual(anArray.size(), 0, "");
         anArray.add((byte)120);
         Test.ensureEqual(anArray.size(), 1, "");