From b870fbbe53f37c37adefff1295297e242903a1f4 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Mon, 4 Sep 2023 17:25:29 +0200 Subject: [PATCH 1/9] Fixes #894 --- R/g.part5.R | 2 +- R/g.part5.addfirstwake.R | 3 +++ R/g.part5.addsib.R | 1 + R/g.part5.savetimeseries.R | 15 ++++++++++++--- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/R/g.part5.R b/R/g.part5.R index 0f5432a82..5e09c2d8f 100644 --- a/R/g.part5.R +++ b/R/g.part5.R @@ -163,7 +163,7 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(), di = 1 fi = 1 SPTE_end = c() # if it is not loaded from part3 milestone data then this will be the default - if (length(idindex) > 0 & nrow(summarysleep) > 1) { #only attempt to load file if it was processed for sleep + if (length(idindex) > 0 & nrow(summarysleep) > 0) { #only attempt to load file if it was processed for sleep summarysleep_tmp = summarysleep #====================================================================== # load output g.part1 diff --git a/R/g.part5.addfirstwake.R b/R/g.part5.addfirstwake.R index c7ad99bb0..1fd0337f5 100644 --- a/R/g.part5.addfirstwake.R +++ b/R/g.part5.addfirstwake.R @@ -21,6 +21,9 @@ g.part5.addfirstwake =function(ts, summarysleep, nightsi, sleeplog, ID, } # test whether wake for second day is missing # if the full sleep period happens before midnights + if (length(nightsi) < 2) { + return(ts) + } if (firstwake > nightsi[2] | (summarysleep$sleeponset[1] < 18 & summarysleep$wakeup[1] < 18 & firstwake < nightsi[2])) { wake_night1_index = c() diff --git a/R/g.part5.addsib.R b/R/g.part5.addsib.R index ba9501cd9..8f5a861c6 100644 --- a/R/g.part5.addsib.R +++ b/R/g.part5.addsib.R @@ -61,6 +61,7 @@ g.part5.addsib = function(ts, epochSize, part3_output, desiredtz, sibDefinition, redo1 = nightsi[1] - ((60/epochSize)*60) # 1 hour before first midnight if (redo1 < 1) redo1 = 1 redo2 = nightsi[1] + (14*(60/epochSize)*60) # 14 hours after first midngiht + if (redo2 > Nts) redo2 = Nts # Specify defintion of sustained inactivity bout anglethreshold = as.numeric(unlist(strsplit(sibDefinition,"A"))[2]) tempi = unlist(strsplit(unlist(strsplit(sibDefinition,"A"))[1],"T")) diff --git a/R/g.part5.savetimeseries.R b/R/g.part5.savetimeseries.R index 560cb013c..f44c8ab3e 100644 --- a/R/g.part5.savetimeseries.R +++ b/R/g.part5.savetimeseries.R @@ -26,9 +26,18 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname, # Add invalid day indicator mdat$invalid_wakinghours = mdat$invalid_sleepperiod = mdat$invalid_fullwindow = 100 wakeup = which(diff(c(mdat$SleepPeriodTime,0)) == -1) + 1 # first epoch of each day - if (length(wakeup) > 1) { - for (di in 1:(length(wakeup) - 1)) { - dayindices = wakeup[di]:(wakeup[di + 1] - 1) + if (length(wakeup) > 0) { + if (length(wakeup) > 1) { + for (di in 1:(length(wakeup) - 1)) { + dayindices = wakeup[di]:(wakeup[di + 1] - 1) + wake = which(mdat$SleepPeriodTime[dayindices] == 0) + sleep = which(mdat$SleepPeriodTime[dayindices] == 1) + mdat$invalid_wakinghours[dayindices] = round(mean(mdat$invalidepoch[dayindices[wake]]) * 100, digits = 2) + mdat$invalid_sleepperiod[dayindices] = round(mean(mdat$invalidepoch[dayindices[sleep]]) * 100, digits = 2) + mdat$invalid_fullwindow[dayindices] = round(mean(mdat$invalidepoch[dayindices]) * 100, digits = 2) + } + } else { + dayindices = 1:nrow(mdat) wake = which(mdat$SleepPeriodTime[dayindices] == 0) sleep = which(mdat$SleepPeriodTime[dayindices] == 1) mdat$invalid_wakinghours[dayindices] = round(mean(mdat$invalidepoch[dayindices[wake]]) * 100, digits = 2) From 58c1e86f0ed1171462352b40d72e15674745f3d1 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 5 Sep 2023 16:12:05 +0200 Subject: [PATCH 2/9] #894 add lux to part5 time series output --- R/g.part5.R | 18 +++++++++++------- R/g.part5.savetimeseries.R | 2 ++ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/R/g.part5.R b/R/g.part5.R index 5e09c2d8f..a77758172 100644 --- a/R/g.part5.R +++ b/R/g.part5.R @@ -471,7 +471,6 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(), ################################################## # Analysis per segment: - # Group categories of objects together # to reduce number of individual objects that need to be # passed on to analyseSegment @@ -546,7 +545,9 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(), } else { napNonwear_col = c() } - g.part5.savetimeseries(ts = ts[, c("time", "ACC", "diur", "nonwear", "guider", "window", napNonwear_col)], + + g.part5.savetimeseries(ts = ts[, c("time", "ACC", "diur", "nonwear", "guider", "window", napNonwear_col, + ifelse(lightpeak_available, yes = "lightpeak", no= NULL))], LEVELS = LEVELS, desiredtz = params_general[["desiredtz"]], rawlevels_fname = rawlevels_fname, @@ -652,13 +653,16 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(), } else { # pass on functions functions2passon = c("is.ISO8601", "iso8601chartime2POSIX", "identify_levels", "g.getbout", + "g.sibreport", "extract_params", "load_params", "check_params", + "correctOlderMilestoneData", "g.part5.addfirstwake", "g.part5.addsib", + "g.part5.classifyNaps.R", "g.part5.definedays", "g.part5.fixmissingnight", - "g.part5.onsetwaketiming", "g.part5.wakesleepwindows", - "g.part5.savetimeseries", "g.fragmentation", "g.intensitygradient", - "g.part5.handle_lux_extremes", "g.part5.lux_persegment", "g.sibreport", - "extract_params", "load_params", "check_params", - "correctOlderMilestoneData", "g.part5.classifyNaps") + "g.part5.handle_lux_extremes", "g.part5.lux_persegment", + "g.part5.savetimeseries", "g.part5.wakesleepwindows", + "g.part5.onsetwaketiming", "g.part5_analyseSegment", + "g.part5_initialise_ts", + "g.fragmentation", "g.intensitygradient") errhand = 'stop' } i = 0 # declare i because foreach uses it, without declaring it diff --git a/R/g.part5.savetimeseries.R b/R/g.part5.savetimeseries.R index f44c8ab3e..ace3afb6a 100644 --- a/R/g.part5.savetimeseries.R +++ b/R/g.part5.savetimeseries.R @@ -46,6 +46,8 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname, } # round acceleration values to 3 digits to reduce storage space mdat$ACC = round(mdat$ACC, digits = 3) + # round light data to 0 digits to reduce storage space + if ("lightpeak" %in% names(mdat)) mdat$lightpeak = round(mdat$lightpeak) if (save_ms5raw_without_invalid == TRUE) { # Remove days based on data_cleaning_file if (length(DaCleanFile) > 0) { From 504b83740453f29898d2ae7f16595eb8d8fbc4f0 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 5 Sep 2023 16:15:53 +0200 Subject: [PATCH 3/9] updating changelog and release date --- DESCRIPTION | 4 ++-- NEWS.md | 4 ++++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 6c7f1e03d..3c0c2b9b1 100755 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,8 +1,8 @@ Package: GGIR Type: Package Title: Raw Accelerometer Data Analysis -Version: 2.10-2 -Date: 2023-09-01 +Version: 2.10-3 +Date: 2023-09-30 Authors@R: c(person("Vincent T","van Hees",role=c("aut","cre"), email="v.vanhees@accelting.com"), person("Jairo H","Migueles",role="aut", diff --git a/NEWS.md b/NEWS.md index 1a2b78442..11cd705f3 100644 --- a/NEWS.md +++ b/NEWS.md @@ -1,3 +1,7 @@ +# CHANGES IN GGIR VERSION 2.10-3 + +- Part 5: Time series now also exported if recording only includes one night, even though this is not sufficient for the main part 5 analyses. #894 Further, the time series now also come with lightpeak (LUX). + # CHANGES IN GGIR VERSION 2.10-2 - Part 1: Revision to readability of code (credits: Lena Kushleyeva) From 8400e35f9eedca008eb1410fa2969d62e7d28a26 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 5 Sep 2023 17:19:37 +0200 Subject: [PATCH 4/9] fix issue with previous commit and directly updated unnecessary POSIXlt to POSIXct in unit test --- R/g.part5.R | 11 ++++++++--- tests/testthat/test_read.myacc.csv.R | 12 ++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/R/g.part5.R b/R/g.part5.R index a77758172..c1ed92bbf 100644 --- a/R/g.part5.R +++ b/R/g.part5.R @@ -545,9 +545,14 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(), } else { napNonwear_col = c() } - - g.part5.savetimeseries(ts = ts[, c("time", "ACC", "diur", "nonwear", "guider", "window", napNonwear_col, - ifelse(lightpeak_available, yes = "lightpeak", no= NULL))], + if (lightpeak_available == TRUE) { + lightpeak_col = "lightpeak" + } else { + lightpeak_col = NULL + } + g.part5.savetimeseries(ts = ts[, c("time", "ACC", "diur", "nonwear", + "guider", "window", napNonwear_col, + lightpeak_col)], LEVELS = LEVELS, desiredtz = params_general[["desiredtz"]], rawlevels_fname = rawlevels_fname, diff --git a/tests/testthat/test_read.myacc.csv.R b/tests/testthat/test_read.myacc.csv.R index d60b2f895..c59133fd8 100644 --- a/tests/testthat/test_read.myacc.csv.R +++ b/tests/testthat/test_read.myacc.csv.R @@ -7,9 +7,9 @@ test_that("read.myacc.csv can handle files without header, no decimal places in N = 30 sf = 30 options(digits.secs = 4) - t0 = as.POSIXlt(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") + t0 = as.POSIXct(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") timeseq = t0 + ((0:(N - 1))/sf) - time = as.POSIXlt(timeseq, origin = "1970-1-1", tz = "Europe/London") + time = as.POSIXct(timeseq, origin = "1970-1-1", tz = "Europe/London") testfile = matrix("", 4, 1) set.seed(100) accx = rnorm(N) @@ -189,9 +189,9 @@ test_that("read.myacc.csv can handle header and bit-value acceleration", { N = 30 sf = 30 options(digits.secs = 4) - t0 = as.POSIXlt(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") + t0 = as.POSIXct(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") timeseq = t0 + ((0:(N - 1))/sf) - time = as.POSIXlt(timeseq, origin = "1970-1-1", tz = "Europe/London") + time = as.POSIXct(timeseq, origin = "1970-1-1", tz = "Europe/London") testfile = matrix("", 3, 1) set.seed(100) accx = rnorm(N) @@ -330,9 +330,9 @@ test_that("read.myacc.csv can handle gaps in time and irregular sample rate", { N = 30 sf = 30 options(digits.secs = 4) - t0 = as.POSIXlt(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") + t0 = as.POSIXct(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") timeseq = t0 + ((0:(N - 1))/sf) - time = as.POSIXlt(timeseq, origin = "1970-1-1", tz = "Europe/London") + time = as.POSIXct(timeseq, origin = "1970-1-1", tz = "Europe/London") testfile = matrix("", 1, 1) set.seed(100) accx = rnorm(N) From 2bfb7cfb24c26e0c0f7e02a91ecd35bee7ba7a8a Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 5 Sep 2023 17:41:20 +0200 Subject: [PATCH 5/9] attempt to fix GitHub Actions issue for unbuntu oldrel --- tests/testthat/test_read.myacc.csv.R | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/testthat/test_read.myacc.csv.R b/tests/testthat/test_read.myacc.csv.R index c59133fd8..cd9f1203b 100644 --- a/tests/testthat/test_read.myacc.csv.R +++ b/tests/testthat/test_read.myacc.csv.R @@ -112,7 +112,7 @@ test_that("read.myacc.csv can handle files without header, no decimal places in # Evaluate with decimal places in seconds expect_equal(nrow(D1$data), 20) expect_equal(ncol(D1$data), 5) - expect_equal(strftime(D1$data$time[1:5], format = '%Y-%m-%d %H:%M:%OS2', + expect_equal(format(D1$data$time[1:5], format = '%Y-%m-%d %H:%M:%OS2', tz = "Europe/London"), c("2022-11-02 13:01:16.00", "2022-11-02 13:01:16.03", @@ -135,7 +135,7 @@ test_that("read.myacc.csv can handle files without header, no decimal places in rmc.headername.recordingid = "ID") expect_equal(nrow(D2$data), 20) expect_equal(ncol(D2$data), 5) - expect_equal(strftime(D2$data$time[1:5], format = '%Y-%m-%d %H:%M:%OS2', + expect_equal(format(D2$data$time[1:5], format = '%Y-%m-%d %H:%M:%OS2', tz = "Europe/London"), c("2022-11-02 18:01:16.50", "2022-11-02 18:01:16.53", "2022-11-02 18:01:16.56", "2022-11-02 18:01:16.59", From 174ace82b6be9dc60c1c0cd3589dd878c207babe Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 5 Sep 2023 18:03:55 +0200 Subject: [PATCH 6/9] Update read.myacc.csv.R --- R/read.myacc.csv.R | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/R/read.myacc.csv.R b/R/read.myacc.csv.R index 597b7bcf2..4642f212f 100644 --- a/R/read.myacc.csv.R +++ b/R/read.myacc.csv.R @@ -184,7 +184,7 @@ read.myacc.csv = function(rmc.file=c(), rmc.nrow=Inf, rmc.skip=c(), rmc.dec=".", # Convert timestamps if (length(rmc.col.time) > 0) { if (rmc.unit.time == "POSIX") { - P$timestamp = as.POSIXlt(format(P$timestamp), origin = rmc.origin, tz = configtz, format = rmc.format.time) + P$timestamp = as.POSIXct(format(P$timestamp), origin = rmc.origin, tz = configtz, format = rmc.format.time) checkdec = function(x) { # function to check whether timestamp has decimal places return(length(unlist(strsplit(as.character(x), "[.]|[,]"))) == 1) From 8e5fb2be58a0b80fed0b5bf966198cd8c9243170 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 5 Sep 2023 18:12:54 +0200 Subject: [PATCH 7/9] undo changes to POSIX as it is causing new issues --- R/read.myacc.csv.R | 2 +- tests/testthat/test_read.myacc.csv.R | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/R/read.myacc.csv.R b/R/read.myacc.csv.R index 4642f212f..597b7bcf2 100644 --- a/R/read.myacc.csv.R +++ b/R/read.myacc.csv.R @@ -184,7 +184,7 @@ read.myacc.csv = function(rmc.file=c(), rmc.nrow=Inf, rmc.skip=c(), rmc.dec=".", # Convert timestamps if (length(rmc.col.time) > 0) { if (rmc.unit.time == "POSIX") { - P$timestamp = as.POSIXct(format(P$timestamp), origin = rmc.origin, tz = configtz, format = rmc.format.time) + P$timestamp = as.POSIXlt(format(P$timestamp), origin = rmc.origin, tz = configtz, format = rmc.format.time) checkdec = function(x) { # function to check whether timestamp has decimal places return(length(unlist(strsplit(as.character(x), "[.]|[,]"))) == 1) diff --git a/tests/testthat/test_read.myacc.csv.R b/tests/testthat/test_read.myacc.csv.R index cd9f1203b..d60b2f895 100644 --- a/tests/testthat/test_read.myacc.csv.R +++ b/tests/testthat/test_read.myacc.csv.R @@ -7,9 +7,9 @@ test_that("read.myacc.csv can handle files without header, no decimal places in N = 30 sf = 30 options(digits.secs = 4) - t0 = as.POSIXct(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") + t0 = as.POSIXlt(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") timeseq = t0 + ((0:(N - 1))/sf) - time = as.POSIXct(timeseq, origin = "1970-1-1", tz = "Europe/London") + time = as.POSIXlt(timeseq, origin = "1970-1-1", tz = "Europe/London") testfile = matrix("", 4, 1) set.seed(100) accx = rnorm(N) @@ -112,7 +112,7 @@ test_that("read.myacc.csv can handle files without header, no decimal places in # Evaluate with decimal places in seconds expect_equal(nrow(D1$data), 20) expect_equal(ncol(D1$data), 5) - expect_equal(format(D1$data$time[1:5], format = '%Y-%m-%d %H:%M:%OS2', + expect_equal(strftime(D1$data$time[1:5], format = '%Y-%m-%d %H:%M:%OS2', tz = "Europe/London"), c("2022-11-02 13:01:16.00", "2022-11-02 13:01:16.03", @@ -135,7 +135,7 @@ test_that("read.myacc.csv can handle files without header, no decimal places in rmc.headername.recordingid = "ID") expect_equal(nrow(D2$data), 20) expect_equal(ncol(D2$data), 5) - expect_equal(format(D2$data$time[1:5], format = '%Y-%m-%d %H:%M:%OS2', + expect_equal(strftime(D2$data$time[1:5], format = '%Y-%m-%d %H:%M:%OS2', tz = "Europe/London"), c("2022-11-02 18:01:16.50", "2022-11-02 18:01:16.53", "2022-11-02 18:01:16.56", "2022-11-02 18:01:16.59", @@ -189,9 +189,9 @@ test_that("read.myacc.csv can handle header and bit-value acceleration", { N = 30 sf = 30 options(digits.secs = 4) - t0 = as.POSIXct(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") + t0 = as.POSIXlt(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") timeseq = t0 + ((0:(N - 1))/sf) - time = as.POSIXct(timeseq, origin = "1970-1-1", tz = "Europe/London") + time = as.POSIXlt(timeseq, origin = "1970-1-1", tz = "Europe/London") testfile = matrix("", 3, 1) set.seed(100) accx = rnorm(N) @@ -330,9 +330,9 @@ test_that("read.myacc.csv can handle gaps in time and irregular sample rate", { N = 30 sf = 30 options(digits.secs = 4) - t0 = as.POSIXct(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") + t0 = as.POSIXlt(x = "2022-11-02 14:01:16.00", tz = "Europe/Amsterdam") timeseq = t0 + ((0:(N - 1))/sf) - time = as.POSIXct(timeseq, origin = "1970-1-1", tz = "Europe/London") + time = as.POSIXlt(timeseq, origin = "1970-1-1", tz = "Europe/London") testfile = matrix("", 1, 1) set.seed(100) accx = rnorm(N) From f46342589e641d4332edbb43cbc1366fa9703fc0 Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Tue, 12 Sep 2023 13:44:48 +0200 Subject: [PATCH 8/9] rephrase comment to clarify that at least 1 night is needed #895 --- R/g.part5.R | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/R/g.part5.R b/R/g.part5.R index c1ed92bbf..78b1fb8d0 100644 --- a/R/g.part5.R +++ b/R/g.part5.R @@ -163,7 +163,8 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(), di = 1 fi = 1 SPTE_end = c() # if it is not loaded from part3 milestone data then this will be the default - if (length(idindex) > 0 & nrow(summarysleep) > 0) { #only attempt to load file if it was processed for sleep + # Only attempt to load file if it has at least 1 night of data + if (length(idindex) > 0 & nrow(summarysleep) > 0) { summarysleep_tmp = summarysleep #====================================================================== # load output g.part1 From 72efd6d89b2aff5aa42a39ae30695d977ba1272b Mon Sep 17 00:00:00 2001 From: Vincent van Hees Date: Wed, 13 Sep 2023 09:44:22 +0200 Subject: [PATCH 9/9] prevent storing zero row part5_daysummary_cleaned.csv --- R/g.report.part5.R | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/R/g.report.part5.R b/R/g.report.part5.R index 446418226..c93c8b60e 100644 --- a/R/g.report.part5.R +++ b/R/g.report.part5.R @@ -234,12 +234,14 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c # store all summaries in csv files with cleaning criteria validdaysi = getValidDayIndices(x = OF3, window = uwi[j], params_cleaning = params_cleaning) - data.table::fwrite( - OF3_clean[validdaysi, ], - paste(metadatadir, "/results/part5_daysummary_", - uwi[j], "_L", uTRLi[h1], "M", uTRMi[h2], "V", uTRVi[h3], - "_", usleepparam[h4], ".csv", sep = ""), row.names = FALSE, na = "", - sep = sep_reports) + if (length(validdaysi) > 0) { + data.table::fwrite( + OF3_clean[validdaysi, ], + paste(metadatadir, "/results/part5_daysummary_", + uwi[j], "_L", uTRLi[h1], "M", uTRMi[h2], "V", uTRVi[h3], + "_", usleepparam[h4], ".csv", sep = ""), row.names = FALSE, na = "", + sep = sep_reports) + } #------------------------------------------------------------------------------------ #also compute summary per person agg_plainNweighted = function(df,