Fix annoying modis bundle_request dupe bug

ecohealthalliance · Dec 8, 2024 · 56b185f · 56b185f
1 parent e6d3141
commit 56b185f
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 48 deletions.
diff --git a/R/submit_modis_ndvi_bundle_request.R b/R/submit_modis_ndvi_bundle_request.R
@@ -56,12 +56,17 @@ submit_modis_ndvi_bundle_request <- function(modis_ndvi_token,
   }
   message(" ")
 
-  bundle_response <- httr::GET(paste("https://appeears.earthdatacloud.nasa.gov/api/bundle/", task_id, sep = ""), httr::add_headers(Authorization = modis_ndvi_token))
+  bundle_response <- httr::GET(paste("https://appeears.earthdatacloud.nasa.gov/api/bundle/", task_id, sep = ""), 
+                               httr::add_headers(Authorization = modis_ndvi_token))
+
   bundle_response <- jsonlite::fromJSON(jsonlite::toJSON(httr::content(bundle_response))) |> 
     bind_cols() |> 
     filter(file_type == "tif", grepl("NDVI", file_name)) |>
     mutate(year_doy = sub(".*doy(\\d+).*", "\\1", file_name),
-           start_date = as.Date(year_doy, format = "%Y%j")) # confirmed this is start date through manual download test
+           start_date = as.Date(year_doy, format = "%Y%j"),
+           year = year(start_date)) |> # confirmed this is start date through manual download test
+    arrange(start_date) |> 
+    filter(year == modis_ndvi_task_id_continent$year) # Ensure we're not getting stuff outside the requested year which might lead to dupes
 
   # Return bundle response files
   return(bundle_response)

diff --git a/_targets.R b/_targets.R
@@ -443,10 +443,15 @@ dynamic_targets <- tar_plan(
   # when we go to forecast. Right now we're just using step function 
   # interpolation where the NDVI value is constant for the entire 16-day period, 
   # then it steps up or down to the next interval's NDVI value.
+
+  # NCL: 
   tarchetypes::tar_group_size(name = modis_ndvi_requests, 
                               size = 10,
                               command = modis_ndvi_bundle_request |>
                                 arrange(start_date) |>
+                                group_by(sha256) |> # Remove duplicate file requests
+                                slice_max(created, n = 1) |> 
+                                ungroup() |>
                                 mutate(end_date = lead(start_date) - days(1),
                                        end_date = case_when(
                                          is.na(end_date) ~ start_date + 15,