Merge pull request #1275 from tonygardella/met.bug

Fixes to AmerifluxLBL and met2CF.csv, and cleanup

modules/data.atmosphere/NAMESPACE

@@ -66,3 +66,6 @@ importFrom(ncdf4,ncvar_add)
 importFrom(ncdf4,ncvar_def)
 importFrom(ncdf4,ncvar_get)
 importFrom(ncdf4,ncvar_put)
+importFrom(udunits2,ud.are.convertible)
+importFrom(udunits2,ud.convert)
+importFrom(udunits2,ud.is.parseable)

modules/data.atmosphere/R/download.AmerifluxLBL.R

@@ -39,7 +39,7 @@ download.AmerifluxLBL <- function(sitename, outfolder, start_date, end_date,
   # need to query to get full file name #this is Ameriflux version
   url <- "http://wile.lbl.gov:8080/AmeriFlux/DataDownload.svc/datafileURLs"
   json_query <- paste0("{\"username\":\"", username, "\",\"siteList\":[\"", site, "\"],\"intendedUse\":\"Research - Land model/Earth system model\",\"description\":\"PEcAn download\"}")
-  result <- httr::POST(url, body = json_query, encode = "json", add_headers(`Content-Type` = "application/json"))
+  result <- httr::POST(url, body = json_query, encode = "json", httr::add_headers(`Content-Type` = "application/json"))
   link <- httr::content(result)
 
   ftplink <- NULL
@@ -164,5 +164,6 @@ download.AmerifluxLBL <- function(sitename, outfolder, start_date, end_date,
   results$formatname[rows] <- "AMERIFLUX_BASE_HH"
 
   # return list of files downloaded
-  return(invisible(results))
+
+  return(results)
 } # download.AmerifluxLBL

modules/data.atmosphere/R/met.process.R

@@ -172,7 +172,8 @@ met.process <- function(site, input_met, start_date, end_date, model,
                             start_date = start_date, end_date = end_date, 
                             con = con, host = host, 
                             overwrite = overwrite$met2cf, 
-                            format.vars = format.vars)
+                            format.vars = format.vars,
+                            bety = bety)
   }
 
   #--------------------------------------------------------------------------------------------------#

modules/data.atmosphere/R/met2CF.csv.R

@@ -35,15 +35,16 @@
 ##' in.prefix <- 'FLX_US-WCr_FLUXNET2015_SUBSET_HH_1999-2014_1-1'
 ##' outfolder <- '~/'
 ##' input.id <- 5000000005
-##' format <- query.format.vars(input.id=input.id,con)
+##' format <- query.format.vars(input.id=input.id,bety = bety)
 ##' start_date <- ymd_hm('200401010000')
 ##' end_date <- ymd_hm('200412312330')
 ##' PEcAn.data.atmosphere::met2CF.csv(in.path,in.prefix,outfolder,start_date,end_date,format,overwrite=TRUE)
 ##' }
 ##' @importFrom ncdf4 ncvar_get ncdim_def ncvar_add ncvar_put
+##' @importFrom udunits2 ud.is.parseable ud.convert ud.are.convertible
 met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, format, lat = NULL, lon = NULL, 
                        nc_verbose = FALSE, overwrite = FALSE, ...) {
-  library(PEcAn.utils)
+
 
   start_year <- lubridate::year(start_date)
   end_year   <- lubridate::year(end_date)
@@ -66,7 +67,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
   files <- files[grep("*.csv", files)]
   if (length(files) == 0) {
     return(NULL)
-    logger.warn("No met files named ", in.prefix, "found in ", in.path)
+    PEcAn.utils::logger.warn("No met files named ", in.prefix, "found in ", in.path)
   }
   files <- files[1]
 
@@ -95,19 +96,19 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
 
   # If all the files already exist, then skip the conversion unless overwrite=TRUE
   if (!overwrite && all(file.exists(all_files))) {
-    logger.debug("File '", all_files, "' already exist, skipping.")
+    PEcAn.utils::logger.debug("File '", all_files, "' already exist, skipping.")
   } else {
     # If some of the files already exist, skip those, but still need to read file
     if (!overwrite && any(file.exists(all_files))) {
-      logger.debug("Files ", all_files[which(file.exists(all_files))], " already exist, skipping those")
+      PEcAn.utils::logger.debug("Files ", all_files[which(file.exists(all_files))], " already exist, skipping those")
       all_years <- all_years[which(!file.exists(all_files))]
       all_files <- all_files[which(!file.exists(all_files))]
     }
 
     ## Read the CSV file some files have a line under the header that lists variable units
     ## search for NA's after conversion to numeric
     if (is.null(format$header)) {
-      logger.warn("please specify number of header rows in file")
+      PEcAn.utils::logger.warn("please specify number of header rows in file")
       alldat <- read.csv(files, 
                          skip = format$skip, 
                          na.strings = format$na.strings,
@@ -156,7 +157,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
         alldatetime <- as.POSIXct(yyddhhmm)
       } else {
         ## Does not match any of the known date formats, add new ones here!
-        logger.error("datetime column is not specified in format")
+        PEcAn.utils::logger.error("datetime column is not specified in format")
       }
     } else {
       datetime_units <- format$vars$input_units[datetime_index]  #lubridate function to call such as ymd_hms
@@ -175,7 +176,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
     if (!missing(start_date) && !missing(end_date)) {
       availdat <- which(years >= lubridate::year(start_date) & years <= lubridate::year(end_date))
       if (length(availdat) == 0) {
-        logger.error("data does not contain output after start_date or before end_date")
+        PEcAn.utils::logger.error("data does not contain output after start_date or before end_date")
       }
       alldat <- alldat[availdat, ]
       alldatetime <- alldatetime[availdat]
@@ -195,7 +196,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
       this.year <- all_years[i]
       availdat.year <- which(years == this.year)
       if (length(availdat.year) == 0) {
-        logger.debug("File ", all_files, " has no data for year ", this.year)
+        PEcAn.utils::logger.debug("File ", all_files, " has no data for year ", this.year)
         next
       }
       new.file <- all_files[i]
@@ -208,9 +209,9 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
       results$enddate[this.year - start_year + 1]   <- as.character(datetime[length(datetime)])
 
       ### create time dimension
-      days_since_1700 <- datetime - ymd_hm("1700-01-01 00:00")
+      days_since_1700 <- datetime - lubridate::ymd_hm("1700-01-01 00:00")
       t <- ncdim_def("time", "days since 1700-01-01", as.numeric(days_since_1700))  #define netCDF dimensions for variables
-      timestep <- as.numeric(mean(udunits2::ud.convert(diff(days_since_1700), "d", "s")))
+      timestep <- as.numeric(mean(ud.convert(diff(days_since_1700), "d", "s")))
 
       ## create lat lon dimensions
       x <- ncdim_def("longitude", "degrees_east", lon)  # define netCDF dimensions for variables
@@ -228,13 +229,13 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for airT by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for airT by name or column number")
           }
         }
         ncvar_put(nc, varid = airT.var, 
                   vals = met.conv(dat[, arrloc], format$vars$input_units[k], "celsius", "K"))
       } else {
-        logger.error("No air temperature found in met file")
+        PEcAn.utils::logger.error("No air temperature found in met file")
       }
 
       ## air_pressure (Pa) => air_pressure (Pa)
@@ -248,7 +249,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for air_pressure by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for air_pressure by name or column number")
           }
         }
         ncvar_put(nc, varid = Psurf.var, 
@@ -268,7 +269,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for co2atm by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for co2atm by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -289,7 +290,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for soilM by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for soilM by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -308,7 +309,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for soilT by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for soilT by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -327,7 +328,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for relative_humidity by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for relative_humidity by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -346,7 +347,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for specific_humidity by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for specific_humidity by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -376,7 +377,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for VPD by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for VPD by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -398,7 +399,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for surface_downwelling_longwave_flux_in_air by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for surface_downwelling_longwave_flux_in_air by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -419,7 +420,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for solar_radiation by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for solar_radiation by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -440,7 +441,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for PAR by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for PAR by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -461,7 +462,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for precipitation_flux by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for precipitation_flux by name or column number")
           }
         }
         rain <- dat[, arrloc]
@@ -473,10 +474,10 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           rain <- rain / timestep
           "Mg m-2 s-1"
         }, `in` = {
-          rain <- udunits2::ud.convert(rain / timestep, "in", "mm")
+          rain <- ud.convert(rain / timestep, "in", "mm")
           "kg m-2 s-1"
         }, `mm h-1` = {
-          rain <- udunits2::ud.convert(rain / timestep, "h", "s")
+          rain <- ud.convert(rain / timestep, "h", "s")
           "kg m-2 s-1"
         })
         ncvar_put(nc, varid = precip.var, 
@@ -495,7 +496,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for eastward_wind by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for eastward_wind by name or column number")
           }
         }
         ncvar_put(nc, 
@@ -511,7 +512,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
           if (any(colnames(format$vars) == "column_number")) {
             arrloc <- format$vars$column_number[k]
           } else {
-            logger.error("Cannot find column location for northward_wind by name or column number")
+            PEcAn.utils::logger.error("Cannot find column location for northward_wind by name or column number")
           }
           ncvar_put(nc, 
                     varid = Ewind.var, 
@@ -529,15 +530,15 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
             if (any(colnames(format$vars) == "column_number")) {
               arrloc_wd <- format$vars$column_number[k_wd]
             } else {
-              logger.error("Cannot find column location for wind_direction by name or column number")
+              PEcAn.utils::logger.error("Cannot find column location for wind_direction by name or column number")
             }
           }
           arrloc_ws <- as.character(format$vars$input_name[k_ws])
           if (arrloc_ws == "") {
             if (any(colnames(format$vars) == "column_number")) {
               arrloc_ws <- format$vars$column_number[k_ws]
             } else {
-              logger.error("Cannot find column location for wind_speed by name or column number")
+              PEcAn.utils::logger.error("Cannot find column location for wind_speed by name or column number")
             }
           }
           wind <- met.conv(dat[, arrloc_ws], format$vars$input_units[k_ws], "m s-1", "m s-1")
@@ -564,7 +565,7 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
               if (any(colnames(format$vars) == "column_number")) {
                 arrloc <- format$vars$column_number[k]
               } else {
-                logger.error("Cannot find column location for Wspd by name or column number")
+                PEcAn.utils::logger.error("Cannot find column location for Wspd by name or column number")
               }
             }
             ncvar_put(nc, 
@@ -595,14 +596,14 @@ met2CF.csv <- function(in.path, in.prefix, outfolder, start_date, end_date, form
       ncdf4::nc_close(nc)
     }  ## end loop over years
   }  ## end else file found
-  return(invisible(results))
+  return(results)
 } # met2CF.csv
 
 
 datetime <- function(list) {
   date_string <- sapply(list, as.character)
   datetime <- paste(list, "00")
-  return(ymd_hms(datetime))
+  return(lubridate::ymd_hms(datetime))
 } # datetime
 
 met.conv <- function(x, orig, bety, CF) {
@@ -615,11 +616,11 @@ met.conv <- function(x, orig, bety, CF) {
   }
   if (ud.is.parseable(orig)) {
     if (ud.are.convertible(orig, bety)) {
-      return(udunits2::ud.convert(udunits2::ud.convert(x, orig, bety), bety, CF))
+      return(ud.convert(ud.convert(x, orig, bety), bety, CF))
     } else {
-      logger.error(paste("met.conv could not convert", orig, bety, CF))
+      PEcAn.utils::logger.error(paste("met.conv could not convert", orig, bety, CF))
     }
   } else {
-    logger.error(paste("met.conv could not parse units:", orig), "Please check if these units conform to udunits")
+    PEcAn.utils::logger.error(paste("met.conv could not parse units:", orig), "Please check if these units conform to udunits")
   }
 } # met.conv

modules/data.atmosphere/R/met2cf.module.R

@@ -1,5 +1,5 @@
 .met2cf.module <- function(raw.id, register, met, str_ns, dir, machine, site.id, lat, lon, start_date, end_date, 
-                           con, host, overwrite = FALSE, format.vars) {
+                           con, host, overwrite = FALSE, format.vars, bety) {
 
   logger.info("Begin change to CF Standards")
 
@@ -75,7 +75,7 @@
                              exact.dates = FALSE)
     } else if (exists(fcn2)) {
       fcn <- fcn2
-      format <- query.format.vars(input.id, con)
+      format <- query.format.vars(input.id = input.id, bety = bety)
       cf.id <- convert.input(input.id = input.id,
                              outfolder = outfolder,
                              formatname = formatname,