diff --git a/flow/flow.pub.tabl.srf/Dockerfile b/flow/flow.pub.tabl.srf/Dockerfile index 29c13ae14..06773f733 100644 --- a/flow/flow.pub.tabl.srf/Dockerfile +++ b/flow/flow.pub.tabl.srf/Dockerfile @@ -1,7 +1,7 @@ # Dockerfile for NEON IS Data Processing - Create pub tables and apply srf # Start with the pub package image. -FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4 +FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.5 # maintainer handle MAINTAINER "Cove Sturtevant" csturtevant@battelleecology.org diff --git a/flow/flow.srf.asgn/Dockerfile b/flow/flow.srf.asgn/Dockerfile index 9109e36dd..b7172ca2b 100644 --- a/flow/flow.srf.asgn/Dockerfile +++ b/flow/flow.srf.asgn/Dockerfile @@ -3,7 +3,7 @@ # docker build -t neon-is-srf-asgn-r . # Start with the base image. -FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4 +FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.5 # maintainer handle MAINTAINER "Cove Sturtevant" csturtevant@battelleecology.org diff --git a/modules_combined/level1_group_consolidate_srf/Dockerfile b/modules_combined/level1_group_consolidate_srf/Dockerfile index 7f53e908c..941ec8e5a 100644 --- a/modules_combined/level1_group_consolidate_srf/Dockerfile +++ b/modules_combined/level1_group_consolidate_srf/Dockerfile @@ -6,7 +6,7 @@ # ### # Start with the pub package R image. -FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.4 +FROM quay.io/battelleecology/neon-is-pack-pub-r:v1.1.5 # maintainer handle MAINTAINER "Cove Sturtevant" csturtevant@battelleecology.org diff --git a/pack/NEONprocIS.pub/DESCRIPTION b/pack/NEONprocIS.pub/DESCRIPTION index f3e8cec48..61fa88a18 100644 --- a/pack/NEONprocIS.pub/DESCRIPTION +++ b/pack/NEONprocIS.pub/DESCRIPTION @@ -1,6 +1,6 @@ Package: NEONprocIS.pub Title: Publication functions for NEON IS data processing -Version: 1.0.1 +Version: 1.0.2 Authors@R: person(given = "Cove", family = "Sturtevant", diff --git a/pack/NEONprocIS.pub/R/def.srf.aply.R b/pack/NEONprocIS.pub/R/def.srf.aply.R index aceada016..88f768f50 100644 --- a/pack/NEONprocIS.pub/R/def.srf.aply.R +++ b/pack/NEONprocIS.pub/R/def.srf.aply.R @@ -90,7 +90,7 @@ def.srf.aply <- function(srf, # Populate the SRF for the applicable time period nameVarSrf <- pubWbTabl$fieldName[pubWbTabl$DPNumber == idDp[idxSrf]] # SRF term name - setQf <- dataTabl[[NameVarTimeEnd]] >= srf$start_date[idxSrf] & dataTabl[[NameVarTimeBgn]] < srf$end_date[idxSrf] # Rows to flag + setQf <- dataTabl[[NameVarTimeEnd]] > srf$start_date[idxSrf] & dataTabl[[NameVarTimeBgn]] < srf$end_date[idxSrf] # Rows to flag numRcd <- base::sum(setQf) dataTabl[setQf,nameVarSrf] <- srf$srf[idxSrf] # Populate flag log$debug(base::paste0('Populated ',numRcd, ' timestamps for SRF term ',nameVarSrf, ' with SRF value ', srf$srf[idxSrf])) diff --git a/pack/NEONprocIS.pub/R/def.srf.filt.R b/pack/NEONprocIS.pub/R/def.srf.filt.R index 5e73fcfbb..c7262e577 100644 --- a/pack/NEONprocIS.pub/R/def.srf.filt.R +++ b/pack/NEONprocIS.pub/R/def.srf.filt.R @@ -72,7 +72,7 @@ def.srf.filt <- function(srf, } # Filter for relevant records based on date range - srf <- srf[srf$end_date > TimeBgn & srf$start_date <= TimeEnd,] + srf <- srf[srf$end_date > TimeBgn & srf$start_date < TimeEnd,] # End early if no remaining srfs if(base::nrow(srf) == 0){ diff --git a/pack/NEONprocIS.pub/tests/testthat/pfs/pubWb/par-quantum-line_CPER001000_science_review_flags.json b/pack/NEONprocIS.pub/tests/testthat/pfs/pubWb/par-quantum-line_CPER001000_science_review_flags.json index 5ae73af83..ad532e3e4 100755 --- a/pack/NEONprocIS.pub/tests/testthat/pfs/pubWb/par-quantum-line_CPER001000_science_review_flags.json +++ b/pack/NEONprocIS.pub/tests/testthat/pfs/pubWb/par-quantum-line_CPER001000_science_review_flags.json @@ -27,11 +27,23 @@ { "group_name": "par-quantum-line_CPER001000", "id": 7724, - "start_date": "2017-09-20T00:00:00Z", - "end_date": "2017-11-01T20:00:00Z", + "start_date": "2023-02-03T01:30:00Z", + "end_date": "2023-02-03T03:00:00Z", "measurement_stream_name": "NEON.D10.CPER.DP1.00066.001.03985.001.000.001", "srf_term_name": "finalQFSciRvw", "srf": 0, + "user_comment": "Incorrect calibration applied.", + "create_date": "2019-07-23T20:43:59Z", + "last_update_date": "2020-06-10T18:43:43Z" + }, + { + "group_name": "par-quantum-line_CPER001000", + "id": 7724, + "start_date": "2023-02-03T05:30:00Z", + "end_date": "2023-02-03T10:00:00Z", + "measurement_stream_name": "NEON.D10.CPER.DP1.00066.001.03985.001.000.001", + "srf_term_name": "finalQFSciRvw", + "srf": 1, "user_comment": "Incorrect calibration applied.\n: Issue resolved by reprocessing - flag removed.", "create_date": "2019-07-23T20:43:59Z", "last_update_date": "2020-06-10T18:43:43Z" diff --git a/pack/NEONprocIS.pub/tests/testthat/test-srf-aply.R b/pack/NEONprocIS.pub/tests/testthat/test-srf-aply.R index fbb017631..9026abb90 100644 --- a/pack/NEONprocIS.pub/tests/testthat/test-srf-aply.R +++ b/pack/NEONprocIS.pub/tests/testthat/test-srf-aply.R @@ -57,7 +57,7 @@ # Mija Choi (2023-03-21) # Original Creation ############################################################################################## -test_that(" Testing def.read.srf.R, definition function. Read science review file", +test_that(" Testing def.srf.aply.R, definition function. Apply science review flag", { wk_dir <- getwd() @@ -78,10 +78,9 @@ test_that(" Testing def.read.srf.R, definition function. Read science review f FileData <- c('pfs/pubWb/par-quantum-line_CPER001000_2023-02-03_PARQL_1min_001.parquet') # Files must have same # of rows - data <- base::lapply(FileData, arrow::open_dataset) dataTabl <- NEONprocIS.base::def.read.parq(NameFile = FileData) - TimeBgn = 'startDTime' + TimeBgn = 'startDateTime' TimeEnd = 'endDateTime' returnedDataTabl = NEONprocIS.pub::def.srf.aply( @@ -98,7 +97,12 @@ test_that(" Testing def.read.srf.R, definition function. Read science review f testthat::expect_true((ncol(returnedDataTabl) >= ncol(dataTabl)) == TRUE) # A data frame with applicable SRF actions applied. testthat::expect_true(any(srf$qfFinl %in% colnames(returnedDataTabl))) - # srf$srf[2] = 2 + # Data are appropriately flagged + testthat::expect_true(sum(returnedDataTabl$finalQFSciRvw == 0,na.rm=TRUE) == 90) + testthat::expect_true(sum(returnedDataTabl$finalQFSciRvw == 1,na.rm=TRUE) == 270) + + + # Test redaction srf$srf[3] = 2 returnedDataTabl = NEONprocIS.pub::def.srf.aply( srf = srf, @@ -107,6 +111,11 @@ test_that(" Testing def.read.srf.R, definition function. Read science review f NameVarTimeBgn = TimeBgn, NameVarTimeEnd = TimeEnd ) + redacted=which(returnedDataTabl$finalQFSciRvw == 2) + notRedacted = which(returnedDataTabl$finalQFSciRvw == 1) + testthat::expect_true(all(is.na(returnedDataTabl$linePARNumPts[redacted]))) + testthat::expect_true(sum(returnedDataTabl$finalQFSciRvw == 1,na.rm=TRUE) == 270) + testthat::expect_true(all(!is.na(returnedDataTabl$linePARNumPts[notRedacted]))) }) diff --git a/pack/NEONprocIS.pub/tests/testthat/test-srf-filt.R b/pack/NEONprocIS.pub/tests/testthat/test-srf-filt.R index c273eb276..a29621402 100644 --- a/pack/NEONprocIS.pub/tests/testthat/test-srf-filt.R +++ b/pack/NEONprocIS.pub/tests/testthat/test-srf-filt.R @@ -89,4 +89,17 @@ test_that(" Testing def.srf.filt.R, Filter Science Review Flag file ", { srfFilt <-NEONprocIS.pub::def.srf.filt(srf = srf,TimeBgn = TimeBgn,TimeEnd = TimeEnd) testthat::expect_true(is.data.frame(srfFilt) == TRUE) testthat::expect_true(nrow(srfFilt) == 0) + + #4. no row in the output srf (both data and srf end dates are exclusive) + + if (file.exists(NameFileOut)) { + file.remove(NameFileOut) + } + + TimeBgn <- base::as.POSIXct('2020-01-03', tz = 'GMT') + TimeEnd <- base::as.POSIXct('2020-01-10 10:00:00', tz = 'GMT') + srfFilt <-NEONprocIS.pub::def.srf.filt(srf = srf,TimeBgn = TimeBgn,TimeEnd = TimeEnd) + testthat::expect_true(is.data.frame(srfFilt) == TRUE) + testthat::expect_true(nrow(srfFilt) == 0) + }) diff --git a/pipe/groundwaterPhysical/groundwaterPhysical_level1_group_consolidate_srf.yaml b/pipe/groundwaterPhysical/groundwaterPhysical_level1_group_consolidate_srf.yaml index 3d7256804..177a9b79b 100644 --- a/pipe/groundwaterPhysical/groundwaterPhysical_level1_group_consolidate_srf.yaml +++ b/pipe/groundwaterPhysical/groundwaterPhysical_level1_group_consolidate_srf.yaml @@ -4,7 +4,7 @@ pipeline: transform: image_pull_secrets: - battelleecology-quay-read-all-pull-secret - image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8 + image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3 cmd: - sh - "-c" diff --git a/pipe/groundwaterPhysical/groundwaterPhysical_srf_assignment.yaml b/pipe/groundwaterPhysical/groundwaterPhysical_srf_assignment.yaml index deb13e071..b64a6288e 100644 --- a/pipe/groundwaterPhysical/groundwaterPhysical_srf_assignment.yaml +++ b/pipe/groundwaterPhysical/groundwaterPhysical_srf_assignment.yaml @@ -12,7 +12,7 @@ transform: DirOut=/pfs/out DirErr=$ERR_PATH FileYear=$FILE_YEAR - image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8 + image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/parQuantumLine/parQuantumLine_level1_group_consolidate_srf.yaml b/pipe/parQuantumLine/parQuantumLine_level1_group_consolidate_srf.yaml index 4d1436fa2..c73b7fb85 100644 --- a/pipe/parQuantumLine/parQuantumLine_level1_group_consolidate_srf.yaml +++ b/pipe/parQuantumLine/parQuantumLine_level1_group_consolidate_srf.yaml @@ -4,7 +4,7 @@ pipeline: transform: image_pull_secrets: - battelleecology-quay-read-all-pull-secret - image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8 + image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3 cmd: - sh - "-c" diff --git a/pipe/parQuantumLine/parQuantumLine_pre_stats_qm.yaml b/pipe/parQuantumLine/parQuantumLine_pre_stats_qm.yaml index 64db28ecc..4ba488df4 100644 --- a/pipe/parQuantumLine/parQuantumLine_pre_stats_qm.yaml +++ b/pipe/parQuantumLine/parQuantumLine_pre_stats_qm.yaml @@ -31,8 +31,8 @@ resource_limits: memory: 1.5G cpu: 1.2 sidecar_resource_requests: - memory: 3G - cpu: 1 + memory: 4G + cpu: 1.5 datum_set_spec: number: 1 scheduling_spec: diff --git a/pipe/parQuantumLine/parQuantumLine_srf_assignment.yaml b/pipe/parQuantumLine/parQuantumLine_srf_assignment.yaml index 84ba63055..7afbaf786 100644 --- a/pipe/parQuantumLine/parQuantumLine_srf_assignment.yaml +++ b/pipe/parQuantumLine/parQuantumLine_srf_assignment.yaml @@ -12,7 +12,7 @@ transform: DirOut=/pfs/out DirErr=$ERR_PATH FileYear=$FILE_YEAR - image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8 + image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/parWaterSurface/parWaterSurface_level1_group_consolidate_srf.yaml b/pipe/parWaterSurface/parWaterSurface_level1_group_consolidate_srf.yaml index d6ef64cf2..cbd900adb 100644 --- a/pipe/parWaterSurface/parWaterSurface_level1_group_consolidate_srf.yaml +++ b/pipe/parWaterSurface/parWaterSurface_level1_group_consolidate_srf.yaml @@ -4,7 +4,7 @@ pipeline: transform: image_pull_secrets: - battelleecology-quay-read-all-pull-secret - image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8 + image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3 cmd: - sh - "-c" diff --git a/pipe/parWaterSurface/parWaterSurface_srf_assignment.yaml b/pipe/parWaterSurface/parWaterSurface_srf_assignment.yaml index 2b78442cd..3281c3fe2 100644 --- a/pipe/parWaterSurface/parWaterSurface_srf_assignment.yaml +++ b/pipe/parWaterSurface/parWaterSurface_srf_assignment.yaml @@ -12,7 +12,7 @@ transform: DirOut=/pfs/out DirErr=$ERR_PATH FileYear=$FILE_YEAR - image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8 + image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/relHumidity/relHumidity_level1_group_consolidate_srf.yaml b/pipe/relHumidity/relHumidity_level1_group_consolidate_srf.yaml index aeac82516..eec986a43 100644 --- a/pipe/relHumidity/relHumidity_level1_group_consolidate_srf.yaml +++ b/pipe/relHumidity/relHumidity_level1_group_consolidate_srf.yaml @@ -4,7 +4,7 @@ pipeline: transform: image_pull_secrets: - battelleecology-quay-read-all-pull-secret - image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8 + image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3 cmd: - sh - "-c" diff --git a/pipe/relHumidity/relHumidity_srf_assignment.yaml b/pipe/relHumidity/relHumidity_srf_assignment.yaml index 164540003..76a2d7e81 100644 --- a/pipe/relHumidity/relHumidity_srf_assignment.yaml +++ b/pipe/relHumidity/relHumidity_srf_assignment.yaml @@ -12,7 +12,7 @@ transform: DirOut=/pfs/out DirErr=$ERR_PATH FileYear=$FILE_YEAR - image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8 + image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/sandbox/test_srf_assignment.yaml b/pipe/sandbox/test_srf_assignment.yaml index cd33d83fe..ba98dabc6 100644 --- a/pipe/sandbox/test_srf_assignment.yaml +++ b/pipe/sandbox/test_srf_assignment.yaml @@ -12,7 +12,7 @@ transform: DirOut=/pfs/out DirErr=$ERR_PATH FileYear=$FILE_YEAR - image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8 + image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/surfacewaterPhysical/surfacewaterPhysical_level1_group_consolidate_srf.yaml b/pipe/surfacewaterPhysical/surfacewaterPhysical_level1_group_consolidate_srf.yaml index 5570bcd3a..48081d143 100644 --- a/pipe/surfacewaterPhysical/surfacewaterPhysical_level1_group_consolidate_srf.yaml +++ b/pipe/surfacewaterPhysical/surfacewaterPhysical_level1_group_consolidate_srf.yaml @@ -4,7 +4,7 @@ pipeline: transform: image_pull_secrets: - battelleecology-quay-read-all-pull-secret - image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8 + image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3 cmd: - sh - "-c" diff --git a/pipe/surfacewaterPhysical/surfacewaterPhysical_srf_assignment.yaml b/pipe/surfacewaterPhysical/surfacewaterPhysical_srf_assignment.yaml index 21edaf7d4..31aaf16ec 100644 --- a/pipe/surfacewaterPhysical/surfacewaterPhysical_srf_assignment.yaml +++ b/pipe/surfacewaterPhysical/surfacewaterPhysical_srf_assignment.yaml @@ -12,7 +12,7 @@ transform: DirOut=/pfs/out DirErr=$ERR_PATH FileYear=$FILE_YEAR - image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8 + image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: diff --git a/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml b/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml index 1e59f094e..d8ff2be8c 100644 --- a/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml +++ b/pipe/tempSoil/tempSoil_level1_group_consolidate_srf.yaml @@ -4,7 +4,7 @@ pipeline: transform: image_pull_secrets: - battelleecology-quay-read-all-pull-secret - image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:4178f7b8 + image: quay.io/battelleecology/neon-is-levl1-grp-cons-srf:a2479ff3 cmd: - sh - "-c" diff --git a/pipe/tempSoil/tempSoil_srf_assignment.yaml b/pipe/tempSoil/tempSoil_srf_assignment.yaml index 961c831b2..c37386a7f 100644 --- a/pipe/tempSoil/tempSoil_srf_assignment.yaml +++ b/pipe/tempSoil/tempSoil_srf_assignment.yaml @@ -12,7 +12,7 @@ transform: DirOut=/pfs/out DirErr=$ERR_PATH FileYear=$FILE_YEAR - image: quay.io/battelleecology/neon-is-srf-asgn-r:4178f7b8 + image: quay.io/battelleecology/neon-is-srf-asgn-r:a2479ff3 image_pull_secrets: - battelleecology-quay-read-all-pull-secret env: