Merge pull request #228 from NEONScience/loc.filt.optn

Loc.filt.optn
NEONScience · Nov 22, 2023 · 6a40ccf · 6a40ccf
2 parents fab3bba + c064ff7
commit 6a40ccf
Show file tree

Hide file tree

Showing 454 changed files with 1,557 additions and 900 deletions.
diff --git a/flow/flow.cal.asgn/Dockerfile b/flow/flow.cal.asgn/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Calibration assignment
 
 # Start with the calibration package image. 
-FROM quay.io/battelleecology/neon-is-pack-cal-r:v1.2.2
+FROM quay.io/battelleecology/neon-is-pack-cal-r:v1.2.5
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.cal.conv/Dockerfile b/flow/flow.cal.conv/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Calibration Conversion
 
 # Start with the calibration package image. 
-FROM quay.io/battelleecology/neon-is-pack-cal-r:v1.2.2
+FROM quay.io/battelleecology/neon-is-pack-cal-r:v1.2.5
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.cal.conv/renv.lock b/flow/flow.cal.conv/renv.lock
@@ -19,10 +19,10 @@
     },
     "arrow": {
       "Package": "arrow",
-      "Version": "13.0.0.1",
+      "Version": "14.0.0",
       "Source": "Repository",
       "Repository": "CRAN",
-      "Hash": "5f5124b727a451be32d23dcd2c41ebe8",
+      "Hash": "daa5319da4e6fa5595647054dcbe7e8d",
       "Requirements": [
         "R6",
         "assertthat",

diff --git a/flow/flow.data.comb.ts/Dockerfile b/flow/flow.data.comb.ts/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Combine Timeseries Data
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.kfka.comb/Dockerfile b/flow/flow.kfka.comb/Dockerfile
@@ -3,7 +3,7 @@
 # docker build --no-cache -t neon-is-kfka-comb-r ~/R/NEON-IS-data-processing/flow/flow.kfka.comb/
 
 # Start with the pub package image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.loc.data.trnc.comb/Dockerfile b/flow/flow.loc.data.trnc.comb/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Truncate and merge data files by location
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.loc.grp.asgn/Dockerfile b/flow/flow.loc.grp.asgn/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Calibration assignment
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.loc.grp.asgn/flow.loc.grp.asgn.R b/flow/flow.loc.grp.asgn/flow.loc.grp.asgn.R
@@ -61,6 +61,11 @@
 #' the dates over which it was active (should have been producing data). 'group' corresponds to a group
 #' file specific to a group member, including what groups the member is in and properties of the group.
 #' 
+#' 5. "Prop=value" (optional), where value contains any number of properties in the location or group 
+#' file to retain, separated by pipes (|). The meaning of this input changes according to TypeFile. 
+#' See the filtering functions for each type for details. Defaults to 'all', which results in no filtering.
+#'  Currently only relevant for 'TypeFile=asset', with filtering function NEONprocIS.base::def.loc.filt.
+
 #' Note: This script implements optional parallelization as well as logging (described in 
 #' \code{\link[NEONprocIS.base]{def.log.init}}), both of which use system environment variables if available. 
 
@@ -79,6 +84,7 @@
 #' Rscript flow.loc.grp.asgn.R "DirIn=/pfs/proc_group/2019/01/01/prt/27134" "DirOut=/pfs/out" "TypeFile=asset"
 
 #' @seealso \code{\link[NEONprocIS.base]{def.log.init}}
+#' @seealso \code{\link[NEONprocIS.base]{def.loc.filt}}
 
 # changelog and author contributions / copyrights
 #   Cove Sturtevant (2021-03-15)
@@ -87,6 +93,8 @@
 #     Add datum error routing
 #   Cove Sturtevant (2022-11-22)
 #     Add option for group files
+#   Cove Sturtevant (2023-11-16)
+#     Add option for retaining only particular properties
 ##############################################################################################
 library(foreach)
 library(doParallel)
@@ -119,13 +127,16 @@ Para <-
   NEONprocIS.base::def.arg.pars(
     arg = arg,
     NameParaReqd = c("DirIn", "DirOut","DirErr","FileYear","TypeFile"),
+    NameParaOpt = c('Prop'),
+    ValuParaOpt = list(Prop = 'all'),
     log = log
   )
 
 # Echo arguments
 log$debug(base::paste0('Input directory: ', Para$DirIn))
 log$debug(base::paste0('Output directory: ', Para$DirOut))
 log$debug(base::paste0('Error directory: ', Para$DirErr))
+log$debug(base::paste0('Properties to retain: ', base::paste0(Para$Prop,collapse=',')))
 
 # Parse the file containing the years to populate
 log$debug(base::paste0('File containing data years to populate: ', Para$FileYear))
@@ -163,6 +174,7 @@ foreach::foreach(idxDirIn = DirIn) %dopar% {
                     TimeBgn=timeBgn,
                     TimeEnd=timeEnd,
                     TypeFile=Para$TypeFile,
+                    Prop=Para$Prop,
                     log=log
       ),
       error = function(err) {

diff --git a/flow/flow.loc.grp.asgn/wrap.loc.grp.asgn.R b/flow/flow.loc.grp.asgn/wrap.loc.grp.asgn.R
@@ -31,6 +31,9 @@
 #' location file specific to a named location, including the properties of that named location and
 #' the dates over which it was active (should have been producing data). 'group' corresponds to a group
 #' file specific to a group member, including what groups the member is in and properties of the group.
+#' @param Prop character vector of the properties in the file to retain. The meaning of this input changes
+#' according to TypeFile. See the filtering functions for each type for details. Defaults to 'all'. Currently
+#' relevant for TypeFile='asset' and TypeFile='namedLocation'.
 #' @param log A logger object as produced by NEONprocIS.base::def.log.init to produce structured log
 #' output. Defaults to NULL, in which the logger will be created and used within the function.
 #' 
@@ -68,6 +71,7 @@ wrap.loc.grp.asgn <- function(DirIn,
                           TimeBgn,
                           TimeEnd,
                           TypeFile=c('asset','namedLocation','group')[0],
+                          Prop='all',
                           log=NULL
                           ){
 
@@ -241,13 +245,15 @@ wrap.loc.grp.asgn <- function(DirIn,
                                                   NameFileOut=nameFileOut,
                                                   TimeBgn=ts[idxDay],
                                                   TimeEnd=ts[idxDay]+timeDiffDay,
+                                                  Prop=Prop,
                                                   log=log
         )
       } else if (TypeFile == 'namedLocation'){
         listFile <- NEONprocIS.base::def.loc.trnc.actv(NameFileIn=nameFile,
                                                        NameFileOut=nameFileOut,
                                                        TimeBgn=ts[idxDay],
                                                        TimeEnd=ts[idxDay]+timeDiffDay,
+                                                       Prop=Prop,
                                                        log=log
         )
       } else if (TypeFile == 'group'){

diff --git a/flow/flow.loc.repo.strc/Dockerfile b/flow/flow.loc.repo.strc/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Restructure repo by location
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.pub.tabl.srf/Dockerfile b/flow/flow.pub.tabl.srf/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Create pub tables and apply srf
 
 # Start with the pub package image. 
-FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.pub.tabl.srf/renv.lock b/flow/flow.pub.tabl.srf/renv.lock
@@ -19,10 +19,10 @@
     },
     "arrow": {
       "Package": "arrow",
-      "Version": "13.0.0.1",
+      "Version": "14.0.0",
       "Source": "Repository",
       "Repository": "CRAN",
-      "Hash": "5f5124b727a451be32d23dcd2c41ebe8",
+      "Hash": "daa5319da4e6fa5595647054dcbe7e8d",
       "Requirements": [
         "R6",
         "assertthat",

diff --git a/flow/flow.qaqc.data.comb/Dockerfile b/flow/flow.qaqc.data.comb/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Merge QC-filtered data files
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.qaqc.plau/Dockerfile b/flow/flow.qaqc.plau/Dockerfile
@@ -3,7 +3,7 @@
 # docker build --no-cache -t neon-is-qaqc-plau-r <path to this directory>
 
 # Start with the neon-is-pack-qaqc-r image. 
-FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.4
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.qaqc.qm.dp0p/Dockerfile b/flow/flow.qaqc.qm.dp0p/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Instantaneous quality metrics and final quality flag module
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.4
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.qaqc.qm/Dockerfile b/flow/flow.qaqc.qm/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Quality metrics and final quality flag module
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.4
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.qaqc.temp.air.aspi/Dockerfile b/flow/flow.qaqc.temp.air.aspi/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Sensor-specific QA/QC module for aspirated air temperature
 
 # Start with the neon-is-pack-qaqc-r image. 
-FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.4
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.qaqc.wq.sens.na/Dockerfile b/flow/flow.qaqc.wq.sens.na/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Water Quality Sensor NA test 
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.rglr/Dockerfile b/flow/flow.rglr/Dockerfile
@@ -2,8 +2,8 @@
 
 # Start with the neon-is-base-r image. 
 # Use a multi-stage build to obscure the value of GITHUB_PAT used to install eddy4R.base
-#FROM quay.io/battelleecology/neon-is-base-r:v1.3.0 #as intermediate
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+#FROM quay.io/battelleecology/neon-is-base-r:v1.4.2 #as intermediate
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]
@@ -22,7 +22,7 @@ MAINTAINER "Cove Sturtevant" [email protected]
 #RUN git clone -b deve https://$GITHUB_PAT:[email protected]/NEONScience/NEON-FIU-algorithm.git
 #
 # Now make the final build, which will leave behind our access token
-#FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+#FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 #
 # copy the eddy4R.base package from the previous image
 #COPY --from=intermediate /NEON-FIU-algorithm/ext/eddy4R/pack/eddy4R.base/ /pack/eddy4R.base/

diff --git a/flow/flow.srf.asgn/Dockerfile b/flow/flow.srf.asgn/Dockerfile
@@ -3,7 +3,7 @@
 # docker build -t neon-is-srf-asgn-r .
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.stat.basc/Dockerfile b/flow/flow.stat.basc/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Basic statistics module
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-pack-stat-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-stat-r:v1.1.5
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.thsh.slct/Dockerfile b/flow/flow.thsh.slct/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Threshold selection by date, source-id, term and context
 
 # Start with the neon-is-pack-qaqc-r image. 
-FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-qaqc-r:v1.1.4
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.troll.cond.conv/Dockerfile b/flow/flow.troll.cond.conv/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Missing Temp Flag and Conductivity Conversion
 
 # Start with the neon-is-base-r image.
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Nora Catolico" [email protected]
@@ -22,4 +22,4 @@ COPY ./wrap.troll.cond.conv.R .
 
 # Run as app user
 RUN chown app:app -R /home/app
-USER app
+USER app
diff --git a/flow/flow.troll.flags/Dockerfile b/flow/flow.troll.flags/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Below Zero Pressure flags
 
 # Start with the neon-is-base-r image.
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Nora Catolico" [email protected]
@@ -22,4 +22,4 @@ COPY ./wrap.troll.flags.R .
 
 # Run as app user
 RUN chown app:app -R /home/app
-USER app
+USER app
diff --git a/flow/flow.troll.uncertainty/Dockerfile b/flow/flow.troll.uncertainty/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Troll Elevation Conversion and Uncertainty Calculations
 
 # Start with the NEON IS stats package image 
-FROM quay.io/battelleecology/neon-is-pack-stat-r:v1.1.2
+FROM quay.io/battelleecology/neon-is-pack-stat-r:v1.1.5
 
 # maintainer handle
 MAINTAINER "Nora Catolico" [email protected]

diff --git a/flow/flow.tsdl.comb.splt/Dockerfile b/flow/flow.tsdl.comb.splt/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Combine Temperature Specific Depth Lakes data and split by HOR.VER
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Guy Litt" [email protected], "Cove Sturtevant" [email protected]

diff --git a/flow/flow.wq.fdom.corr/Dockerfile b/flow/flow.wq.fdom.corr/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - fDOM Temperature and Absorbance Corrections for Water Quality
 
 # Start with the neon-is-base-r image. 
-FROM quay.io/battelleecology/neon-is-pack-wq-r:v1.1.0
+FROM quay.io/battelleecology/neon-is-pack-wq-r:v1.1.2
 
 # maintainer handle
 MAINTAINER "Kaelin Cawley" [email protected]

diff --git a/modules/padded_timeseries_analyzer/padded_timeseries_analyzer/padded_timeseries_analyzer.py b/modules/padded_timeseries_analyzer/padded_timeseries_analyzer/padded_timeseries_analyzer.py
@@ -25,7 +25,7 @@ def __init__(self, data_path: Path, out_path: Path, err_path: Path, relative_pat
         self.data_path = data_path
         self.out_path = out_path
         # DirErrBase: the user specified error directory, i.e., /tmp/out/errored
-        self.DirErrBase = Path(self.out_path, err_path)
+        self.DirErrBase = Path(err_path)
         self.relative_path_index = relative_path_index
 
     def analyze(self) -> None:

diff --git a/modules_combined/calibration_group_and_convert/Dockerfile b/modules_combined/calibration_group_and_convert/Dockerfile
@@ -3,7 +3,7 @@
 # docker build -t neon-is-cal-grp-conv -f ./modules_combined/calibration_group_and_convert/Dockerfile .
 
 # Start with the calibration package image. 
-FROM quay.io/battelleecology/neon-is-pack-cal-r:v1.2.2
+FROM quay.io/battelleecology/neon-is-pack-cal-r:v1.2.5
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/modules_combined/data_source_trino/Dockerfile b/modules_combined/data_source_trino/Dockerfile
@@ -5,7 +5,7 @@
 # docker build -t data_source_trino:latest -f ./modules_combined/data_source_trino/Dockerfile .
 #
 ###
-FROM quay.io/battelleecology/genavro:v1.1.1
+FROM quay.io/battelleecology/genavro:v1.3.1
 
 ARG MODULE_DIR="modules"
 ARG APP_DIR="parquet_linkmerge"
@@ -22,4 +22,6 @@ RUN python3 -mpip install --no-cache-dir -r ${CONTAINER_APP_DIR}/${APP_DIR}/requ
 COPY ${MODULE_DIR}/${APP_DIR} ${CONTAINER_APP_DIR}/${APP_DIR}
 COPY ${MODULE_DIR}/${COMMON_DIR} ${CONTAINER_APP_DIR}/${COMMON_DIR}
 
+
+
 USER appuser
diff --git a/modules_combined/date_gap_fill_and_regularize/Dockerfile b/modules_combined/date_gap_fill_and_regularize/Dockerfile
@@ -4,7 +4,7 @@
 # docker build --no-cache -t neon-is-gf-rglr -f <path to this file> <path to parent directory of flow/ and modules/>
 
 # Start with the base R image. 
-FROM quay.io/battelleecology/neon-is-base-r:v1.3.0
+FROM quay.io/battelleecology/neon-is-base-r:v1.4.2
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/modules_combined/level1_group_consolidate_srf/Dockerfile b/modules_combined/level1_group_consolidate_srf/Dockerfile
@@ -6,7 +6,7 @@
 #
 ###
 # Start with the pub package R image. 
-FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.1
+FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]