parent b442c86

author meetagrawal09 <[email protected]> 1694104254 +0530 committer meetagrawal09 <[email protected]> 1700486393 +0530 added manual trigger, updated date
PecanProject · Nov 20, 2023 · b817573 · b817573
1 parent b442c86
commit b817573
Show file tree

Hide file tree

Showing 21 changed files with 232 additions and 212 deletions.
diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml
@@ -1,21 +1,49 @@
 name : Integration Tests
-on : 
+on :
+  # allow manual triggering 
+  workflow_dispatch:
+
+  # for debugging
+  push:
+  pull_request:
+
   schedule:
-  - cron: '30 4 * * 1'    
+    # run Thursday 4:30 AM UTC
+  - cron: '30 4 * * 4'    
 jobs:
   test:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
 
     env:
       GITHUB_PAT: ${{ secrets.GITHUB_TOKEN }}
 
     container: 
-      image: pecan/depends:R4.1
+      image: pecan/base:develop
 
     steps:
     - name: Checkout source code
       uses: actions/checkout@v3
+    - name: CDS API file setup
+      run: |
+        echo "url: https://cds.climate.copernicus.eu/api/v2" >> .cdsapirc
+        echo "key: ${{ secrets.ERA5_API_KEY }}" >> .cdsapirc
+    
+    - name: Print Output to Console
+      run: |
+        cat .cdsapirc
+     
+    - name: Set up Python
+      uses: actions/setup-python@v2
+
+    - name: Install dependencies
+      run: |
+        sudo apt-get update
+        sudo apt-get install python3-pip -y
+        sudo -H python3 -m pip install --upgrade pip
+        sudo -H pip install --user cdsapi
+
     - name: Run tests
       run: |
-        for FILE in modules/data.atmosphere/inst/integrationTests/*; Rscript $FILE;
-    
+        for FILE in modules/data.atmosphere/inst/integrationTests/*; do
+          Rscript $FILE;
+        done
diff --git a/book_source/_bookdown.yml b/book_source/_bookdown.yml
@@ -1,2 +1,2 @@
 rmd_subdir: true
-edit: https://github.com/tonygardella/pecan/edit/release/vtonydoc/book_source/%s
+edit: https://github.com/PecanProject/pecan/edit/develop/book_source/%s
diff --git a/docker/depends/pecan.depends.R b/docker/depends/pecan.depends.R
@@ -9,6 +9,7 @@ Sys.setenv(RLIB = rlib)
 
 # install remotes first in case packages are references in dependencies
 remotes::install_github(c(
+'adokter/suntools',
 'araiho/linkages_package',
 'chuhousen/amerifluxr',
 'ebimodeling/[email protected]',
@@ -76,7 +77,6 @@ wanted <- c(
 'magic',
 'magrittr',
 'maps',
-'maptools',
 'markdown',
 'MASS',
 'Matrix',
@@ -113,7 +113,6 @@ wanted <- c(
 'reshape',
 'reshape2',
 'reticulate',
-'rgdal',
 'rjags',
 'rjson',
 'rlang',
@@ -130,6 +129,7 @@ wanted <- c(
 'stats',
 'stringi',
 'stringr',
+'suntools',
 'swfscMisc',
 'terra',
 'testthat',

diff --git a/modules/data.atmosphere/DESCRIPTION b/modules/data.atmosphere/DESCRIPTION
@@ -36,7 +36,6 @@ Imports:
     jsonlite,
     lubridate (>= 1.6.0),
     magrittr,
-    maptools,
     MASS,
     mgcv,
     ncdf4 (>= 1.15),
@@ -49,10 +48,12 @@ Imports:
     raster,
     REddyProc,
     reshape2,
-    rgdal,
     rlang (>= 0.2.0),
+    sf,
     sp,
     stringr (>= 1.1.0),
+    suntools,
+    terra,
     testthat (>= 2.0.0),
     tibble,
     tidyr,
@@ -69,6 +70,7 @@ Suggests:
     progress,
     reticulate
 Remotes:
+    github::adokter/suntools,
     github::chuhousen/amerifluxr,
     github::ropensci/geonames,
     github::ropensci/nneo

diff --git a/modules/data.atmosphere/NAMESPACE b/modules/data.atmosphere/NAMESPACE
@@ -113,5 +113,5 @@ export(write_noaa_gefs_netcdf)
 import(dplyr)
 import(tidyselect)
 importFrom(magrittr,"%>%")
-importFrom(rgdal,checkCRSArgs)
 importFrom(rlang,.data)
+importFrom(sf,st_crs)
diff --git a/modules/data.atmosphere/R/GEFS_helper_functions.R b/modules/data.atmosphere/R/GEFS_helper_functions.R
@@ -243,25 +243,30 @@ process_gridded_noaa_download <- function(lat_list,
 
     for(hr in 1:length(curr_hours)){
       file_name <- paste0(base_filename2, curr_hours[hr])
+      grib_file_name <- paste0(working_directory,"/", file_name,".grib")
 
-      if(file.exists(paste0(working_directory,"/", file_name,".grib"))){
-        grib <- rgdal::readGDAL(paste0(working_directory,"/", file_name,".grib"), silent = TRUE)
-        lat_lon <- sp::coordinates(grib)
+      if(file.exists(grib_file_name)){
+        grib_data <- terra::rast(grib_file_name)
+
+        ## Convert to data frame
+        grib_data_df <- terra::as.data.frame(grib_data, xy=TRUE)
+        lat_lon <- grib_data_df[, c("x", "y")]
+
         for(s in 1:length(site_id)){
 
           index <- which(lat_lon[,2] == lats[s] & lat_lon[,1] == lons[s])
 
-          pressfc[s, hr]  <- grib$band1[index]
-          tmp2m[s, hr] <- grib$band2[index]
-          rh2m[s, hr]  <- grib$band3[index]
-          ugrd10m[s, hr]  <- grib$band4[index]
-          vgrd10m[s, hr]  <- grib$band5[index]
+          pressfc[s, hr]  <- grib_data_df$`SFC=Ground or water surface; Pressure [Pa]`[index]
+          tmp2m[s, hr]    <- grib_data_df$`2[m] HTGL=Specified height level above ground; Temperature [C]`[index]
+          rh2m[s, hr]     <- grib_data_df$`2[m] HTGL=Specified height level above ground; Relative humidity [%]`[index]
+          ugrd10m[s, hr]  <- grib_data_df$`10[m] HTGL=Specified height level above ground; u-component of wind [m/s]`[index]
+          vgrd10m[s, hr]  <- grib_data_df$`10[m] HTGL=Specified height level above ground; v-component of wind [m/s]`[index]
 
           if(curr_hours[hr] != "000"){
-            apcpsfc[s, hr]  <- grib$band6[index]
-            tcdcclm[s, hr]  <-  grib$band7[index]
-            dswrfsfc[s, hr]  <- grib$band8[index]
-            dlwrfsfc[s, hr]  <- grib$band9[index]
+            apcpsfc[s, hr]  <- grib_data_df$`SFC=Ground or water surface; 03 hr Total precipitation [kg/(m^2)]`[index]
+            tcdcclm[s, hr]  <- grib_data_df$`RESERVED(10) (Reserved); Total cloud cover [%]`[index]
+            dswrfsfc[s, hr] <- grib_data_df$`SFC=Ground or water surface; Downward Short-Wave Rad. Flux [W/(m^2)]`[index]
+            dlwrfsfc[s, hr] <- grib_data_df$`SFC=Ground or water surface; Downward Long-Wave Rad. Flux [W/(m^2)]`[index]
           }
         }
       }

diff --git a/modules/data.atmosphere/R/download.NARR_site.R b/modules/data.atmosphere/R/download.NARR_site.R
@@ -465,9 +465,9 @@ latlon2narr <- function(nc, lat.in, lon.in) {
 #' @inheritParams get_NARR_thredds
 #' @return `sp::SpatialPoints` object containing transformed x and y
 #' coordinates, in km, which should match NARR coordinates
-#' @importFrom rgdal checkCRSArgs
+#' @importFrom sf st_crs
   # ^not used directly here, but needed by sp::CRS.
-  # sp lists rgdal in Suggests rather than Imports,
+  # sp lists sf in Suggests rather than Imports,
   # so importing it here to ensure it's available at run time
 #' @author Alexey Shiklomanov
 #' @export

diff --git a/modules/data.atmosphere/R/met_temporal_downscale.Gaussian_ensemble.R b/modules/data.atmosphere/R/met_temporal_downscale.Gaussian_ensemble.R
@@ -216,11 +216,11 @@ met_temporal_downscale.Gaussian_ensemble <- function(in.path, in.prefix, outfold
       day <- as.POSIXct(sprintf("%s 12:00:00", date), tz = tz)
       sequence <- seq(from = day, length.out = span, by = "days")
 
-      sunrise <- maptools::sunriset(lon.lat, sequence, direction = "sunrise", 
+      sunrise <- suntools::sunriset(lon.lat, sequence, direction = "sunrise", 
                                     POSIXct.out = TRUE)
-      sunset <- maptools::sunriset(lon.lat, sequence, direction = "sunset", 
+      sunset <- suntools::sunriset(lon.lat, sequence, direction = "sunset", 
                                    POSIXct.out = TRUE)
-      solar_noon <- maptools::solarnoon(lon.lat, sequence, POSIXct.out = TRUE)
+      solar_noon <- suntools::solarnoon(lon.lat, sequence, POSIXct.out = TRUE)
 
       data.frame(date = as.Date(sunrise$time), sunrise = as.numeric(format(sunrise$time, "%H%M")), 
                  solarnoon = as.numeric(format(solar_noon$time, "%H%M")), 

diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R b/modules/data.atmosphere/inst/integrationTests/test.download.AmerifluxLBL.R
@@ -1,4 +1,5 @@
 library(testthat)
+library(PEcAn.DB)
 
 test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, lon.in) {
   # putting logger to debug mode
@@ -7,13 +8,13 @@ test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, l
   PEcAn.logger::logger.setLevel("DEBUG")
 
   # mocking functions
-  mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame())
-  mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1))
+  mockery::stub(convert_input, 'dbfile.input.check', data.frame())
+  mockery::stub(convert_input, 'db.query', data.frame(id = 1))
 
   withr::with_dir(tempdir(), {
     tmpdir <- getwd()
     # calling download function
-    res <- PEcAn.DB::convert_input(
+    res <- convert_input(
       input.id = NA,
       outfolder = tmpdir,
       formatname = NULL,
@@ -31,23 +32,23 @@ test_download_AmerifluxLBL <- function(start_date, end_date, sitename, lat.in, l
       lon.in = lon.in,
       sitename = sitename
     )
-  })
-
-  # checking if the file is downloaded
-  test_that("Downloaded files are present at the desired location", {
-    expect_true(file.exists(paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")))
-  })
 
-  test_that("Downloaded data files have the right format", {
-    firstline <- system(paste0("head -4 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
-    lastline <- system(paste0("tail -1 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
-
-    # checking if first line of CSV has the sitename
-    expect_true(grepl(sitename, firstline[1]))
-
-    # fourth and last row checked to contain non-alphabetical data since these are used to verify start and end dates
-    expect_false(grepl("[A-Za-z]", firstline[4]))
-    expect_false(grepl("[A-Za-z]", lastline[1]))
+    # checking if the file is downloaded
+    test_that("Downloaded files are present at the desired location", {
+      expect_true(file.exists(paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")))
+    })
+
+    test_that("Downloaded data files have the right format", {
+      firstline <- system(paste0("head -4 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
+      lastline <- system(paste0("tail -1 ", paste0(tmpdir, "/AMF_US-Akn_BASE_HH_6-5.csv")), intern = TRUE)
+
+      # checking if first line of CSV has the sitename
+      expect_true(grepl(sitename, firstline[1]))
+
+      # fourth and last row checked to contain non-alphabetical data since these are used to verify start and end dates
+      expect_false(grepl("[A-Za-z]", firstline[4]))
+      expect_false(grepl("[A-Za-z]", lastline[1]))
+    })
   })
 }
 

diff --git a/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R b/modules/data.atmosphere/inst/integrationTests/test.download.CRUNCEP.R
@@ -1,5 +1,6 @@
 library(testthat)
 library(ncdf4)
+library(PEcAn.DB)
 
 test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method, maxErrors, sleep) {
   # putting logger to debug mode
@@ -8,12 +9,12 @@ test_download_CRUNCEP <- function(start_date, end_date, lat.in, lon.in, method,
   PEcAn.logger::logger.setLevel("DEBUG")
 
   # mocking functions
-  mockery::stub(PEcAn.DB::convert_input, 'dbfile.input.check', data.frame())
-  mockery::stub(PEcAn.DB::convert_input, 'db.query', data.frame(id = 1))
+  mockery::stub(convert_input, 'dbfile.input.check', data.frame())
+  mockery::stub(convert_input, 'db.query', data.frame(id = 1))
 
   withr::with_dir(tempdir(), {
     tmpdir <- getwd()
-    PEcAn.DB::convert_input(
+    convert_input(
       input.id = NA,
       outfolder = tmpdir,
       formatname = NULL,