UniversalDataBinFile more strict limits to filter out erroneous times…

…tamps; config, api, web ui added distinction of real and derived sensors; bale sensor config update

config/bale/sensor_translation.ini

@@ -35,15 +35,15 @@ in_bytes_str = NaN
 Altitude = NaN
 Latitude = NaN
 Longitude = NaN
-BattV = UB_status
+BattV = NaN
 BattV_Avg = UB
-WS_ms = WV_status
-AirTC = Ta_200_status
-WindDir = WD_status
+WS_ms = NaN
+AirTC = NaN
+WindDir = NaN
 ETos = evaporation_calculated
-SlrW_Avg = SWDR_300
-SlrW = SWDR_300_status
-SlrMJ_Tot = SWDR_300_MJ
-SlrMJ = SWDR_300_MJ_status
-Rso = SWDR_300_clear_sky_MJ
-Rso24 = SWDR_300_clear_sky_MJ_status
+SlrW_Avg = SWDR_200
+SlrW = NaN
+SlrMJ_Tot = SWDR_200_MJ
+SlrMJ = NaN
+Rso = SWDR_200_clear_sky_MJ
+Rso24 = NaN

config/sensors.yaml

@@ -299,6 +299,7 @@ WD:
   aggregation: average_wind_direction
   physical_range: [0.0, 360.0]   
   step_range: [0.0, 1000.0]
+  derived: false # dependency set for aggregation only
 
 WV:
   description: Wind velocity
@@ -341,7 +342,8 @@ P_RT_NRT:
   physical_range: [0.0, 500.0]
   step_range: [0.0, 1000.0]
   interpolation_mse: 4
-  category: water 
+  category: water
+  derived: false # optional dependency
 
 P_container_RT:
   description: Precipitation, actual unfiltered amount in container
@@ -364,7 +366,7 @@ P_container_NRT:
 SWDR:
   description: Shortwave Downward Radiation
   unit: kWh/m^2 (kilo watt hour per square meter) 
-  raw_source: [SWDR_300, SWDR_3700, SWDR_4400]
+  raw_source: [SWDR_300, SWDR_3700, SWDR_4400, SWDR_200]
   aggregation: sum_radiation
   physical_range: [-100.0, 1500.0]
   step_range: [0.0, 1000.0]
@@ -950,7 +952,6 @@ SD:
   unit: h (hour)
   dependency: SWDR
   aggregation: sum_sunshine
-  visibility: internal  # set virtual sensor temporarily to internal
 
 SD_Olivieri:
   description: sunshine duration (based on SWDR with Oliviéri algorithm and solar elevation angle)
@@ -1386,15 +1387,65 @@ evaporation_calculated:
   description: water lost due to evapotranspiration
   unit: mm/h (mm per hour)
   aggregation: average
+  derived: true
+
+SWDR_200:
+  description: Shortwave Downward Radiation at 2 meters height
+  unit: kWh/m^2 (kilo watt hour per square meter) 
+  aggregation: sum_radiation
+  physical_range: [-100.0, 1500.0]
+  step_range: [0.0, 1000.0]
+  empirical_diff: 157.2
+  interpolation_mse: 10000 
 
-SWDR_300_MJ:
-  description: (internal) Shortwave Downward Radiation at 3 meters height
+SWDR_200_MJ:
+  description: (internal) Shortwave Downward Radiation at 2 meters height
   unit: MJ/m^2 (megajoule per square meter)
   aggregation: sum
   visibility: internal
 
-SWDR_300_clear_sky_MJ:
+SWDR_200_clear_sky_MJ:
   description: (internal) calculated Shortwave Downward Radiation for clear sky
   unit: MJ/m^2 (megajoule per square meter)
   aggregation: sum
-  visibility: internal   
+  visibility: internal
+
+diag:
+  description: (internal)
+  visibility: internal
+
+ET24Cnt:
+  description: (internal)
+  visibility: internal 
+
+ETsz24:
+  description: (internal)
+  visibility: internal
+
+LowPower:
+  description: (internal)
+  visibility: internal  
+
+ModemReset:
+  description: (internal)
+  visibility: internal  
+
+nmbr_bytes_rtrnd:
+  description: (internal)
+  visibility: internal  
+
+Rain_mm:
+  description: (internal)
+  visibility: internal  
+
+ETsz24:
+  description: (internal)
+  visibility: internal  
+
+ETsz24:
+  description: (internal)
+  visibility: internal  
+
+ETsz24:
+  description: (internal)
+  visibility: internal    

src/tsdb/TsDB.java

@@ -33,7 +33,7 @@
 public class TsDB implements AutoCloseable {
 	private static final Logger log = LogManager.getLogger();
 
-	public static final String tubedb_version = "1.1.0";
+	public static final String tubedb_version = "1.2.0";
 
 	/**
 	 * map regionName -> Region

src/tsdb/component/Sensor.java

@@ -73,12 +73,14 @@ public class Sensor implements Serializable {
 	public boolean internal;
 
 	public double maxInterpolationMSE;
-	
+
 	public String[] raw_source = null;
 	public String[] dependency = null;
 	public String post_hour_func = null;
 	public String post_day_func = null;
 
+	private Boolean derived = null; //nullable
+
 	public Sensor(String name) {
 		this.name = name;
 		physicalMin = -Float.MAX_VALUE;
@@ -208,15 +210,17 @@ public static Sensor ofYaml(String sensorName, YamlMap yamlMap) {
 				mse -> {useInterpolation[0] = true; interpolation_mse[0] = mse;}, 
 				e -> log.warn("could not read interpolation_mse of "+sensorName+"   "+e)
 				);
-		
+
 		SensorCategory category = SensorCategory.parse(yamlMap.optString("category", "other"));
 		boolean internal = yamlMap.optString("visibility", "public").equals("internal");
-				
+
 		String[] raw_source = yamlMap.optList("raw_source").asStringArray();
 		String[] dependency = yamlMap.optList("dependency").asStringArray();
-		
+
 		String post_hour_func = yamlMap.optString("post_hour_func", null);
 		String post_day_func = yamlMap.optString("post_day_func", null);
+
+		Boolean derived = yamlMap.optBoolean("derived", null);
 
 		Sensor sensor = new Sensor(sensorName);
 		sensor.description = description;
@@ -227,7 +231,7 @@ public static Sensor ofYaml(String sensorName, YamlMap yamlMap) {
 		yamlMap.optFunStringConv("aggregation_week", AggregationType::parse, sensor::setAggregationWeek);
 		yamlMap.optFunStringConv("aggregation_month", AggregationType::parse, sensor::setAggregationMonth);
 		yamlMap.optFunStringConv("aggregation_year", AggregationType::parse, sensor::setAggregationYear);
-		
+
 		sensor.physicalMin = physicalMin;
 		sensor.physicalMax = physicalMax;
 		sensor.stepMin = stepMin;
@@ -241,10 +245,11 @@ public static Sensor ofYaml(String sensorName, YamlMap yamlMap) {
 		sensor.dependency = dependency.length == 0 ? null : dependency;
 		sensor.post_hour_func = post_hour_func;
 		sensor.post_day_func = post_day_func;
+		sensor.derived = derived;
 
 		return sensor;
 	}
-	
+
 	public AggregationType getAggregationHour() {
 		return aggregationHour;	
 	}
@@ -264,31 +269,31 @@ public AggregationType getAggregationMonth() {
 	public AggregationType getAggregationYear() {
 		return aggregationYear;	
 	}
-	
+
 	public void setAllAggregations(AggregationType agg) {		
 		aggregationHour = agg;
 		aggregationDay = agg;
 		aggregationWeek = agg;
 		aggregationMonth = agg;
 		aggregationYear = agg;
 	}
-	
+
 	public void setAggregationHour(AggregationType agg) {		
 		aggregationHour = agg;
 	}
-	
+
 	public void setAggregationDay(AggregationType agg) {		
 		aggregationDay = agg;
 	}
-	
+
 	public void setAggregationWeek(AggregationType agg) {		
 		aggregationWeek = agg;
 	}
-	
+
 	public void setAggregationMonth(AggregationType agg) {
 		aggregationMonth = agg;
 	}
-	
+
 	public void setAggregationYear(AggregationType agg) {
 		aggregationYear = agg;
 	}
@@ -317,4 +322,12 @@ public boolean equals(Object obj) {
 			return false;
 		return true;
 	}
+
+	public boolean isDerived() {
+		if(derived == null) {
+			return raw_source != null || dependency != null;
+		} else {
+			return derived;
+		}
+	}
 }

src/tsdb/loader/bale/TOA5Loader.java

@@ -26,7 +26,7 @@ public TOA5Loader(TsDB tsdb) {
 	}
 
 	public void loadDirectoryRecursive(Path path) {
-		//log.info("import "+path);
+		log.info("TOA5 import "+path);
 		try(DirectoryStream<Path> rootStream = Files.newDirectoryStream(path)) {
 			for(Path sub:rootStream) {
 				if(!Files.isDirectory(sub)) {
@@ -47,6 +47,7 @@ public void loadDirectoryRecursive(Path path) {
 	}
 
 	public void loadFile(Path filename) throws FileNotFoundException, IOException {
+		log.info("TOA5 load " + filename);
 		TOA5Table table = new TOA5Table(filename.toString());
 
 		if(table.rows.length==0) {