fix minor bug in srf filtering and application, sometimes overflaggin…

…g by 1 data point at the edge of the flag period.
NEONScience · Nov 27, 2023 · 02414fb · 02414fb
1 parent a2479ff
commit 02414fb
Show file tree

Hide file tree

Showing 23 changed files with 61 additions and 27 deletions.
diff --git a/flow/flow.pub.tabl.srf/Dockerfile b/flow/flow.pub.tabl.srf/Dockerfile
@@ -1,7 +1,7 @@
 # Dockerfile for NEON IS Data Processing - Create pub tables and apply srf
 
 # Start with the pub package image. 
-FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4
+FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.5
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/flow/flow.srf.asgn/Dockerfile b/flow/flow.srf.asgn/Dockerfile
@@ -3,7 +3,7 @@
 # docker build -t neon-is-srf-asgn-r .
 
 # Start with the base image. 
-FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4
+FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.5
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/modules_combined/level1_group_consolidate_srf/Dockerfile b/modules_combined/level1_group_consolidate_srf/Dockerfile
@@ -6,7 +6,7 @@
 #
 ###
 # Start with the pub package R image. 
-FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4
+FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.5
 
 # maintainer handle
 MAINTAINER "Cove Sturtevant" [email protected]

diff --git a/pack/NEONprocIS.pub/DESCRIPTION b/pack/NEONprocIS.pub/DESCRIPTION
@@ -1,6 +1,6 @@
 Package: NEONprocIS.pub
 Title: Publication functions for NEON IS data processing
-Version: 1.0.1
+Version: 1.0.2
 Authors@R: 
     person(given = "Cove",
            family = "Sturtevant",

diff --git a/pack/NEONprocIS.pub/R/def.srf.aply.R b/pack/NEONprocIS.pub/R/def.srf.aply.R
@@ -90,7 +90,7 @@ def.srf.aply <- function(srf,
 
     # Populate the SRF for the applicable time period
     nameVarSrf <- pubWbTabl$fieldName[pubWbTabl$DPNumber == idDp[idxSrf]] # SRF term name
-    setQf <- dataTabl[[NameVarTimeEnd]] >= srf$start_date[idxSrf] & dataTabl[[NameVarTimeBgn]] < srf$end_date[idxSrf] # Rows to flag
+    setQf <- dataTabl[[NameVarTimeEnd]] > srf$start_date[idxSrf] & dataTabl[[NameVarTimeBgn]] < srf$end_date[idxSrf] # Rows to flag
     numRcd <- base::sum(setQf)
     dataTabl[setQf,nameVarSrf] <- srf$srf[idxSrf] # Populate flag
     log$debug(base::paste0('Populated ',numRcd, ' timestamps for SRF term ',nameVarSrf, ' with SRF value ', srf$srf[idxSrf]))

diff --git a/pack/NEONprocIS.pub/R/def.srf.filt.R b/pack/NEONprocIS.pub/R/def.srf.filt.R
@@ -72,7 +72,7 @@ def.srf.filt <- function(srf,
     }
 
     # Filter for relevant records based on date range
-    srf <- srf[srf$end_date > TimeBgn & srf$start_date <= TimeEnd,]
+    srf <- srf[srf$end_date > TimeBgn & srf$start_date < TimeEnd,]
 
     # End early if no remaining srfs
     if(base::nrow(srf) == 0){

diff --git a/...procIS.pub/tests/testthat/pfs/pubWb/par-quantum-line_CPER001000_science_review_flags.json b/...procIS.pub/tests/testthat/pfs/pubWb/par-quantum-line_CPER001000_science_review_flags.json
@@ -27,11 +27,23 @@
         {
             "group_name": "par-quantum-line_CPER001000",
             "id": 7724,
-            "start_date": "2017-09-20T00:00:00Z",
-            "end_date": "2017-11-01T20:00:00Z",
+            "start_date": "2023-02-03T01:30:00Z",
+            "end_date": "2023-02-03T03:00:00Z",
             "measurement_stream_name": "NEON.D10.CPER.DP1.00066.001.03985.001.000.001",
             "srf_term_name": "finalQFSciRvw",
             "srf": 0,
+            "user_comment": "Incorrect calibration applied.",
+            "create_date": "2019-07-23T20:43:59Z",
+            "last_update_date": "2020-06-10T18:43:43Z"
+        },
+        {
+            "group_name": "par-quantum-line_CPER001000",
+            "id": 7724,
+            "start_date": "2023-02-03T05:30:00Z",
+            "end_date": "2023-02-03T10:00:00Z",
+            "measurement_stream_name": "NEON.D10.CPER.DP1.00066.001.03985.001.000.001",
+            "srf_term_name": "finalQFSciRvw",
+            "srf": 1,
             "user_comment": "Incorrect calibration applied.\n: Issue resolved by reprocessing - flag removed.",
             "create_date": "2019-07-23T20:43:59Z",
             "last_update_date": "2020-06-10T18:43:43Z"

diff --git a/pack/NEONprocIS.pub/tests/testthat/test-srf-aply.R b/pack/NEONprocIS.pub/tests/testthat/test-srf-aply.R
@@ -57,7 +57,7 @@
 #   Mija Choi (2023-03-21)
 #      Original Creation
 ##############################################################################################
-test_that("   Testing def.read.srf.R, definition function. Read science review file",
+test_that("   Testing def.srf.aply.R, definition function. Apply science review flag",
           {
             wk_dir <- getwd()
 
@@ -78,10 +78,9 @@ test_that("   Testing def.read.srf.R, definition function. Read science review f
             FileData <-
               c('pfs/pubWb/par-quantum-line_CPER001000_2023-02-03_PARQL_1min_001.parquet') 
             # Files must have same # of rows
-            data <- base::lapply(FileData, arrow::open_dataset)
             dataTabl <- NEONprocIS.base::def.read.parq(NameFile = FileData)
 
-            TimeBgn = 'startDTime'
+            TimeBgn = 'startDateTime'
             TimeEnd = 'endDateTime'
 
             returnedDataTabl = NEONprocIS.pub::def.srf.aply(
@@ -98,7 +97,12 @@ test_that("   Testing def.read.srf.R, definition function. Read science review f
             testthat::expect_true((ncol(returnedDataTabl) >= ncol(dataTabl)) == TRUE)
             # A data frame with applicable SRF actions applied.
             testthat::expect_true(any(srf$qfFinl %in% colnames(returnedDataTabl)))
-            # srf$srf[2] = 2
+            # Data are appropriately flagged
+            testthat::expect_true(sum(returnedDataTabl$finalQFSciRvw == 0,na.rm=TRUE) == 90)
+            testthat::expect_true(sum(returnedDataTabl$finalQFSciRvw == 1,na.rm=TRUE) == 270)
+
+
+            # Test redaction
             srf$srf[3] = 2
             returnedDataTabl = NEONprocIS.pub::def.srf.aply(
               srf = srf,
@@ -107,6 +111,11 @@ test_that("   Testing def.read.srf.R, definition function. Read science review f
               NameVarTimeBgn = TimeBgn,
               NameVarTimeEnd = TimeEnd
             )
+            redacted=which(returnedDataTabl$finalQFSciRvw == 2)
+            notRedacted = which(returnedDataTabl$finalQFSciRvw == 1)
+            testthat::expect_true(all(is.na(returnedDataTabl$linePARNumPts[redacted])))
+            testthat::expect_true(sum(returnedDataTabl$finalQFSciRvw == 1,na.rm=TRUE) == 270)
+            testthat::expect_true(all(!is.na(returnedDataTabl$linePARNumPts[notRedacted])))
 
 
           })
diff --git a/pack/NEONprocIS.pub/tests/testthat/test-srf-filt.R b/pack/NEONprocIS.pub/tests/testthat/test-srf-filt.R
@@ -89,4 +89,17 @@ test_that("   Testing def.srf.filt.R, Filter Science Review Flag file ", {
   srfFilt <-NEONprocIS.pub::def.srf.filt(srf = srf,TimeBgn = TimeBgn,TimeEnd = TimeEnd)
   testthat::expect_true(is.data.frame(srfFilt) == TRUE)
   testthat::expect_true(nrow(srfFilt) == 0)
+
+  #4. no row in the output srf (both data and srf end dates are exclusive)
+
+  if (file.exists(NameFileOut)) {
+    file.remove(NameFileOut)
+  }
+
+  TimeBgn <- base::as.POSIXct('2020-01-03', tz = 'GMT')
+  TimeEnd <- base::as.POSIXct('2020-01-10 10:00:00', tz = 'GMT')
+  srfFilt <-NEONprocIS.pub::def.srf.filt(srf = srf,TimeBgn = TimeBgn,TimeEnd = TimeEnd)
+  testthat::expect_true(is.data.frame(srfFilt) == TRUE)
+  testthat::expect_true(nrow(srfFilt) == 0)
+
 })
diff --git a/pipe/groundwaterPhysical/groundwaterPhysical_level1_group_consolidate_srf.yaml b/pipe/groundwaterPhysical/groundwaterPhysical_level1_group_consolidate_srf.yaml
@@ -4,7 +4,7 @@ pipeline:
 transform:
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8
+  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3
   cmd:
   - sh
   - "-c"

diff --git a/pipe/groundwaterPhysical/groundwaterPhysical_srf_assignment.yaml b/pipe/groundwaterPhysical/groundwaterPhysical_srf_assignment.yaml
@@ -12,7 +12,7 @@ transform:
       DirOut=/pfs/out
       DirErr=$ERR_PATH
       FileYear=$FILE_YEAR
-  image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8
+  image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
   env:

diff --git a/pipe/parQuantumLine/parQuantumLine_level1_group_consolidate_srf.yaml b/pipe/parQuantumLine/parQuantumLine_level1_group_consolidate_srf.yaml
@@ -4,7 +4,7 @@ pipeline:
 transform:
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8
+  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3
   cmd:
   - sh
   - "-c"

diff --git a/pipe/parQuantumLine/parQuantumLine_pre_stats_qm.yaml b/pipe/parQuantumLine/parQuantumLine_pre_stats_qm.yaml
@@ -31,8 +31,8 @@ resource_limits:
   memory: 1.5G
   cpu: 1.2
 sidecar_resource_requests:
-  memory: 3G
-  cpu: 1
+  memory: 4G
+  cpu: 1.5
 datum_set_spec:
   number: 1
 scheduling_spec:

diff --git a/pipe/parQuantumLine/parQuantumLine_srf_assignment.yaml b/pipe/parQuantumLine/parQuantumLine_srf_assignment.yaml
@@ -12,7 +12,7 @@ transform:
       DirOut=/pfs/out
       DirErr=$ERR_PATH
       FileYear=$FILE_YEAR
-  image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8
+  image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
   env:

diff --git a/pipe/parWaterSurface/parWaterSurface_level1_group_consolidate_srf.yaml b/pipe/parWaterSurface/parWaterSurface_level1_group_consolidate_srf.yaml
@@ -4,7 +4,7 @@ pipeline:
 transform:
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8
+  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3
   cmd:
   - sh
   - "-c"

diff --git a/pipe/parWaterSurface/parWaterSurface_srf_assignment.yaml b/pipe/parWaterSurface/parWaterSurface_srf_assignment.yaml
@@ -12,7 +12,7 @@ transform:
       DirOut=/pfs/out
       DirErr=$ERR_PATH
       FileYear=$FILE_YEAR
-  image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8
+  image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
   env:

diff --git a/pipe/relHumidity/relHumidity_level1_group_consolidate_srf.yaml b/pipe/relHumidity/relHumidity_level1_group_consolidate_srf.yaml
@@ -4,7 +4,7 @@ pipeline:
 transform:
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8
+  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3
   cmd:
   - sh
   - "-c"

diff --git a/pipe/relHumidity/relHumidity_srf_assignment.yaml b/pipe/relHumidity/relHumidity_srf_assignment.yaml
@@ -12,7 +12,7 @@ transform:
       DirOut=/pfs/out
       DirErr=$ERR_PATH
       FileYear=$FILE_YEAR
-  image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8
+  image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
   env:

diff --git a/pipe/sandbox/test_srf_assignment.yaml b/pipe/sandbox/test_srf_assignment.yaml
@@ -12,7 +12,7 @@ transform:
       DirOut=/pfs/out
       DirErr=$ERR_PATH
       FileYear=$FILE_YEAR
-  image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8
+  image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
   env:

diff --git a/pipe/surfacewaterPhysical/surfacewaterPhysical_level1_group_consolidate_srf.yaml b/pipe/surfacewaterPhysical/surfacewaterPhysical_level1_group_consolidate_srf.yaml
@@ -4,7 +4,7 @@ pipeline:
 transform:
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8
+  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3
   cmd:
   - sh
   - "-c"

diff --git a/pipe/surfacewaterPhysical/surfacewaterPhysical_srf_assignment.yaml b/pipe/surfacewaterPhysical/surfacewaterPhysical_srf_assignment.yaml
@@ -12,7 +12,7 @@ transform:
       DirOut=/pfs/out
       DirErr=$ERR_PATH
       FileYear=$FILE_YEAR
-  image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8
+  image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
   env:

diff --git a/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml b/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml
@@ -4,7 +4,7 @@ pipeline:
 transform:
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
-  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8
+  image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3
   cmd:
   - sh
   - "-c"

diff --git a/pipe/tempSoil/tempSoil_srf_assignment.yaml b/pipe/tempSoil/tempSoil_srf_assignment.yaml
@@ -12,7 +12,7 @@ transform:
       DirOut=/pfs/out
       DirErr=$ERR_PATH
       FileYear=$FILE_YEAR
-  image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8
+  image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3
   image_pull_secrets:
   - battelleecology-quay-read-all-pull-secret
   env: