Merge pull request #895 from wadpac/issue894_export_short_ts

Allow part 5 to export time series even if there is only one night of data.
wadpac · Sep 13, 2023 · c7bfa8c · c7bfa8c
2 parents a990e0f + 72efd6d
commit c7bfa8c
Show file tree

Hide file tree

Showing 7 changed files with 48 additions and 19 deletions.
diff --git a/DESCRIPTION b/DESCRIPTION
@@ -1,8 +1,8 @@
 Package: GGIR
 Type: Package
 Title: Raw Accelerometer Data Analysis
-Version: 2.10-2
-Date: 2023-09-01
+Version: 2.10-3
+Date: 2023-09-30
 Authors@R: c(person("Vincent T","van Hees",role=c("aut","cre"),
                     email="[email protected]"),
              person("Jairo H","Migueles",role="aut",

diff --git a/NEWS.md b/NEWS.md
@@ -1,5 +1,7 @@
 # CHANGES IN GGIR VERSION 2.10-3
 
+- Part 5: Time series now also exported if recording only includes one night, even though this is not sufficient for the main part 5 analyses. #894 Further, the time series now also come with lightpeak (LUX).
+
 - Visualreport: Bug fixed extracting the numeric value of the days to exclude in g.plot5 #879
 
 - Part 1: Fixed minor bug to keep calibration data in the data quality report after re-running parts 1 and 2 with overwrite = TRUE and backup.cal.coef = "retrieve" #896

diff --git a/R/g.part5.R b/R/g.part5.R
@@ -163,7 +163,8 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
       di = 1
       fi = 1
       SPTE_end = c() # if it is not loaded from part3 milestone data then this will be the default
-      if (length(idindex) > 0 & nrow(summarysleep) > 1) { #only attempt to load file if it was processed for sleep
+      # Only attempt to load file if it has at least 1 night of data
+      if (length(idindex) > 0 & nrow(summarysleep) > 0) { 
         summarysleep_tmp = summarysleep
         #======================================================================
         # load output g.part1
@@ -471,7 +472,6 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
 
                             ##################################################
                             # Analysis per segment:
-
                             # Group categories of objects together
                             # to reduce number of individual objects that need to be
                             # passed on to analyseSegment
@@ -546,7 +546,14 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
                     } else {
                       napNonwear_col = c()
                     }
-                    g.part5.savetimeseries(ts = ts[, c("time", "ACC", "diur", "nonwear", "guider", "window", napNonwear_col)],
+                    if (lightpeak_available == TRUE) {
+                      lightpeak_col = "lightpeak"
+                    } else {
+                      lightpeak_col = NULL
+                    }
+                    g.part5.savetimeseries(ts = ts[, c("time", "ACC", "diur", "nonwear",
+                                                       "guider", "window", napNonwear_col,
+                                                       lightpeak_col)],
                                            LEVELS = LEVELS,
                                            desiredtz = params_general[["desiredtz"]],
                                            rawlevels_fname = rawlevels_fname,
@@ -652,13 +659,16 @@ g.part5 = function(datadir = c(), metadatadir = c(), f0=c(), f1=c(),
     } else {
       # pass on functions
       functions2passon = c("is.ISO8601", "iso8601chartime2POSIX", "identify_levels", "g.getbout",
+                           "g.sibreport", "extract_params", "load_params", "check_params",
+                           "correctOlderMilestoneData",
                            "g.part5.addfirstwake", "g.part5.addsib",
+                           "g.part5.classifyNaps.R",
                            "g.part5.definedays", "g.part5.fixmissingnight",
-                           "g.part5.onsetwaketiming", "g.part5.wakesleepwindows",
-                           "g.part5.savetimeseries", "g.fragmentation", "g.intensitygradient",
-                           "g.part5.handle_lux_extremes", "g.part5.lux_persegment", "g.sibreport",
-                           "extract_params", "load_params", "check_params",
-                           "correctOlderMilestoneData", "g.part5.classifyNaps")
+                           "g.part5.handle_lux_extremes", "g.part5.lux_persegment",
+                           "g.part5.savetimeseries", "g.part5.wakesleepwindows",
+                           "g.part5.onsetwaketiming", "g.part5_analyseSegment",
+                           "g.part5_initialise_ts",
+                           "g.fragmentation", "g.intensitygradient")
       errhand = 'stop'
     }
     i = 0 # declare i because foreach uses it, without declaring it

diff --git a/R/g.part5.addfirstwake.R b/R/g.part5.addfirstwake.R
@@ -21,6 +21,9 @@ g.part5.addfirstwake =function(ts, summarysleep, nightsi, sleeplog, ID,
   }
   # test whether wake for second day is missing
   # if the full sleep period happens before midnights
+  if (length(nightsi) < 2) {
+    return(ts)
+  }
   if (firstwake > nightsi[2] | (summarysleep$sleeponset[1] < 18 &
                                 summarysleep$wakeup[1] < 18 & firstwake < nightsi[2])) { 
     wake_night1_index = c()

diff --git a/R/g.part5.addsib.R b/R/g.part5.addsib.R
@@ -61,6 +61,7 @@ g.part5.addsib = function(ts, epochSize, part3_output, desiredtz, sibDefinition,
     redo1 = nightsi[1] - ((60/epochSize)*60) # 1 hour before first midnight
     if (redo1 < 1) redo1 = 1
     redo2 = nightsi[1] + (14*(60/epochSize)*60) # 14 hours after first midngiht
+    if (redo2 > Nts) redo2 = Nts
     # Specify defintion of sustained inactivity bout
     anglethreshold = as.numeric(unlist(strsplit(sibDefinition,"A"))[2])
     tempi = unlist(strsplit(unlist(strsplit(sibDefinition,"A"))[1],"T"))

diff --git a/R/g.part5.savetimeseries.R b/R/g.part5.savetimeseries.R
@@ -26,9 +26,18 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
   # Add invalid day indicator
   mdat$invalid_wakinghours = mdat$invalid_sleepperiod =  mdat$invalid_fullwindow = 100
   wakeup = which(diff(c(mdat$SleepPeriodTime,0)) == -1) + 1 # first epoch of each day
-  if (length(wakeup) > 1) {
-    for (di in 1:(length(wakeup) - 1)) {
-      dayindices = wakeup[di]:(wakeup[di + 1] - 1)
+  if (length(wakeup) > 0) {
+    if (length(wakeup) > 1) {
+      for (di in 1:(length(wakeup) - 1)) {
+        dayindices = wakeup[di]:(wakeup[di + 1] - 1)
+        wake = which(mdat$SleepPeriodTime[dayindices] == 0)
+        sleep = which(mdat$SleepPeriodTime[dayindices] == 1)
+        mdat$invalid_wakinghours[dayindices] = round(mean(mdat$invalidepoch[dayindices[wake]]) * 100, digits = 2)
+        mdat$invalid_sleepperiod[dayindices] = round(mean(mdat$invalidepoch[dayindices[sleep]]) * 100, digits = 2)
+        mdat$invalid_fullwindow[dayindices] = round(mean(mdat$invalidepoch[dayindices]) * 100, digits = 2)
+      }
+    } else {
+      dayindices = 1:nrow(mdat)
       wake = which(mdat$SleepPeriodTime[dayindices] == 0)
       sleep = which(mdat$SleepPeriodTime[dayindices] == 1)
       mdat$invalid_wakinghours[dayindices] = round(mean(mdat$invalidepoch[dayindices[wake]]) * 100, digits = 2)
@@ -37,6 +46,8 @@ g.part5.savetimeseries = function(ts, LEVELS, desiredtz, rawlevels_fname,
     }
     # round acceleration values to 3 digits to reduce storage space
     mdat$ACC = round(mdat$ACC, digits = 3)
+    # round light data to 0 digits to reduce storage space
+    if ("lightpeak" %in% names(mdat)) mdat$lightpeak = round(mdat$lightpeak)
     if (save_ms5raw_without_invalid == TRUE) {
       # Remove days based on data_cleaning_file
       if (length(DaCleanFile) > 0) { 

diff --git a/R/g.report.part5.R b/R/g.report.part5.R
@@ -234,12 +234,14 @@ g.report.part5 = function(metadatadir = c(), f0 = c(), f1 = c(), loglocation = c
                 # store all summaries in csv files with cleaning criteria
                 validdaysi = getValidDayIndices(x = OF3, window = uwi[j],
                                                 params_cleaning = params_cleaning)
-                data.table::fwrite(
-                  OF3_clean[validdaysi, ],
-                  paste(metadatadir, "/results/part5_daysummary_",
-                        uwi[j], "_L", uTRLi[h1], "M", uTRMi[h2], "V", uTRVi[h3],
-                        "_", usleepparam[h4], ".csv", sep = ""), row.names = FALSE, na = "",
-                  sep = sep_reports)
+                if (length(validdaysi) > 0) {
+                  data.table::fwrite(
+                    OF3_clean[validdaysi, ],
+                    paste(metadatadir, "/results/part5_daysummary_",
+                          uwi[j], "_L", uTRLi[h1], "M", uTRMi[h2], "V", uTRVi[h3],
+                          "_", usleepparam[h4], ".csv", sep = ""), row.names = FALSE, na = "",
+                    sep = sep_reports)
+                }
                 #------------------------------------------------------------------------------------
                 #also compute summary per person
                 agg_plainNweighted = function(df,