Feat/cmr support

app/scripts/components/common/map/style-generators/zarr-timeseries.tsx

@@ -16,6 +16,7 @@ export interface ZarrTimeseriesProps extends BaseGeneratorParams {
   sourceParams?: Record<string, any>;
   stacApiEndpoint?: string;
   tileApiEndpoint?: string;
+  assetUrlReplacements?: [string, string][];
   zoomExtent?: number[];
   onStatusChange?: (result: { status: ActionStatus; id: string }) => void;
 }

app/scripts/components/common/mapbox/index.tsx

@@ -418,6 +418,7 @@ function MapboxMapComponent(
                 id={`base-${baseLayerResolvedData.id}`}
                 stacApiEndpoint={baseLayerResolvedData.stacApiEndpoint}
                 tileApiEndpoint={baseLayerResolvedData.tileApiEndpoint}
+                assetUrlReplacements={baseLayerResolvedData.assetUrlReplacements}
                 stacCol={baseLayerResolvedData.stacCol}
                 mapInstance={mapRef.current}
                 isPositionSet={!!initialPosition}

app/scripts/components/common/mapbox/layers/zarr-timeseries.tsx

@@ -4,6 +4,7 @@ import { Map as MapboxMap, RasterSource, RasterLayer } from 'mapbox-gl';
 
 import { requestQuickCache } from './utils';
 import { useMapStyle } from './styles';
+import { hasNestedKey } from '$utils/utils';
 
 import { ActionStatus, S_FAILED, S_LOADING, S_SUCCEEDED } from '$utils/status';
 
@@ -15,6 +16,7 @@ export interface MapLayerZarrTimeseriesProps {
   sourceParams?: Record<string, any>;
   stacApiEndpoint?: string;
   tileApiEndpoint?: string;
+  assetUrlReplacements?: [string, string][];
   zoomExtent?: number[];
   onStatusChange?: (result: { status: ActionStatus; id: string }) => void;
   isHidden?: boolean;
@@ -27,6 +29,7 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) {
     stacCol,
     stacApiEndpoint,
     tileApiEndpoint,
+    assetUrlReplacements,
     date,
     mapInstance,
     sourceParams,
@@ -45,22 +48,40 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) {
 
   const generatorId = 'zarr-timeseries' + idSuffix;
 
+  const replaceInAssetUrl = (url: string, replacements: ReplacementTuples[]) => {
+    for (const replacement of replacements) {
+      const [toReplace, replaceWith] = replacement;
+      url = url.replace(toReplace, replaceWith);
+    }
+    return url;
+  };
+
   //
   // Get the asset url
   //
   useEffect(() => {
     const controller = new AbortController();
 
-    async function load() {
+    const load = async () => {
       try {
         onStatusChange?.({ status: S_LOADING, id });
+
+        // Zarr collections in _VEDA_ should have a single entrypoint (zarr or virtual zarr / reference)
+        // CMR endpoints will be using individual items' assets, so we query for the asset url
+        let stacApiEndpointToUse = `${process.env.API_STAC_ENDPOINT}/collections/${stacCol}`;
+        // TODO: need a better way to configure this to search for items OR return a collections
+        if (stacApiEndpoint) {
+          stacApiEndpointToUse = `${stacApiEndpoint}/search?collections=${stacCol}&datetime=${date?.toISOString()}`;
+        }
+
         const data = await requestQuickCache({
-          url: `${stacApiEndpointToUse}/collections/${stacCol}`,
+          url: stacApiEndpointToUse,
           method: 'GET',
           controller
         });
 
-        setAssetUrl(data.assets.zarr.href);
+        const assetUrl = hasNestedKey(data, 'assets', 'zarr') ? data.assets.zarr.href : replaceInAssetUrl(data.features[0].assets.data.href, assetUrlReplacements);
+        setAssetUrl(assetUrl);
         onStatusChange?.({ status: S_SUCCEEDED, id });
       } catch (error) {
         if (!controller.signal.aborted) {
@@ -69,7 +90,7 @@ export function MapLayerZarrTimeseries(props: MapLayerZarrTimeseriesProps) {
         }
         return;
       }
-    }
+    };
 
     load();
 

app/scripts/components/exploration/components/map/layer.tsx

@@ -69,6 +69,7 @@ export function Layer(props: LayerProps) {
           stacCol={dataset.data.stacCol}
           stacApiEndpoint={dataset.data.stacApiEndpoint}
           tileApiEndpoint={dataset.data.tileApiEndpoint}
+          assetUrlReplacements={dataset.data.assetUrlReplacements}
           date={relevantDate}
           zoomExtent={params.zoomExtent}
           sourceParams={params.sourceParams}

app/scripts/context/layer-data.tsx

@@ -27,16 +27,16 @@ interface STACLayerData {
 const fetchLayerById = async (
   layer: DatasetLayer | DatasetLayerCompareNormalized
 ): Promise<STACLayerData | Error> => {
-  const { type, stacApiEndpoint, stacCol } = layer;
+  const { type, stacApiEndpoint, stacCol, time_density } = layer;
   const stacApiEndpointToUse = stacApiEndpoint ?? process.env.API_STAC_ENDPOINT;
 
   const { data } = await axios.get(
     `${stacApiEndpointToUse}/collections/${stacCol}`
   );
 
   const commonTimeseriesParams = {
-    isPeriodic: data['dashboard:is_periodic'],
-    timeDensity: data['dashboard:time_density']
+    isPeriodic: time_density != null || data['dashboard:is_periodic'],
+    timeDensity: time_density || data['dashboard:time_density']
   };
 
   if (type === 'vector') {
@@ -56,10 +56,14 @@ const fetchLayerById = async (
       ? data.summaries.datetime
       : data.extent.temporal.interval[0];
 
+    // CMR STAC returns datetimes with `null` as the last value to indicate ongoing data.
+    const lastDatetime = domain[domain.length - 1];
+    if (lastDatetime == null) {
+      domain[domain.length - 1] = new Date().toISOString();
+    }
     if (domain.some((d) => !d)) {
       throw new Error('Invalid datetime domain');
     }
-
     return {
       timeseries: {
         ...commonTimeseriesParams,

app/scripts/utils/utils.ts

@@ -96,3 +96,10 @@ export function composeVisuallyDisabled(
 export function checkEnvFlag(value?: string) {
   return (value ?? '').toLowerCase() === 'true';
 }
+
+export function hasNestedKey(obj, parentKey, nestedKey) {
+  if (obj && typeof obj[parentKey] === 'object' && obj[parentKey] !== null) {
+      return nestedKey in obj[parentKey];
+  }
+  return false;
+}

mock/datasets/GPM_3IMERGDF.data.mdx

@@ -0,0 +1,59 @@
+---
+id: GPM_3IMERGDF.v07
+name: 'GPM IMERG Daily Precipitation'
+description: "GPM IMERG Final Precipitation L3 1 day 0.1 degree x 0.1 degree"
+media:
+  src: ::file ./gpmimergdaily.png
+  alt: CMIP6 Near-Surface Air Temperature Screenshot
+  author:
+    name: NASA
+    url:   
+taxonomy:
+  - name: Topics
+    values:
+      - Climate
+layers:
+  - id: GPM_3IMERGDF.v07
+    stacApiEndpoint: 'https://cmr.earthdata.nasa.gov/cloudstac/GES_DISC'
+    tileApiEndpoint: 'https://prod-titiler-xarray.delta-backend.com/tilejson.json'
+    stacCol: GPM_3IMERGDF.v07
+    assetUrlReplacements:
+      - [https://data.gesdisc.earthdata.nasa.gov/data, s3://gesdisc-cumulus-prod-protected]
+    name: GPM IMERG Final Precipitation L3 1 day 0.1 degree x 0.1 degree
+    type: zarr
+    description: "GPM Level 3 IMERG Final Daily 10 x 10 km (GPM_3IMERGDF) accumulated precipitation"
+    time_density: day
+    zoomExtent:
+      - 0
+      - 20
+    sourceParams:
+      resampling_method: bilinear
+      variable: precipitation
+      colormap_name: gnbu
+      rescale: 0,46
+      maxzoom: 12
+    legend:
+      unit: 
+        label: 
+      type: gradient
+      min: "0 mm/hr"
+      max: "46 mm/hr"
+      stops: ['#f7fcf0', '#e6f5e1', '#d7efd1', '#c5e8c2', '#abdeb6', '#8bd2bf', '#6bc3c9', '#4bafd1', '#3193c2', '#1878b4', '#085da0', '#084081']
+---
+
+<Block>
+<Prose>
+# GPM IMERG Final Precipitation L3 1 day 0.1 degree x 0.1 degree V06 (GPM_3IMERGDF)
+
+## Dataset Description
+
+This dataset is the GPM Level 3 IMERG *Final* Daily 10 x 10 km (GPM_3IMERGDF) derived from the half-hourly GPM_3IMERGHH. The derived result represents the Final estimate of the daily mean precipitation rate in mm/day. The dataset is produced by first computing the mean precipitation rate in (mm/hour) in every grid cell, and then multiplying the result by 24.
+
+Source: [https://disc.gsfc.nasa.gov/datasets/GPM_3IMERGDF_07/summary](https://disc.gsfc.nasa.gov/datasets/GPM_3IMERGDF_07/summary)
+
+## Data Source
+
+The files represented are NetCDF files in GES DISC's Earthdata Cloud bucket and discovered from the CMR STAC entries for this collection: https://cmr.earthdata.nasa.gov/cloudstac/GES_DISC/collections/GPM_3IMERGDF.v07. The daily product is not yet available in CMR's CLOUD STAC.
+
+</Prose>
+</Block>

mock/datasets/TRMM_3B42_Daily.data.mdx

@@ -0,0 +1,59 @@
+---
+id: TRMM_3B42_Daily
+name: 'Daily accumulated precipitation from the TRMM Multi-Satellite'
+description: "Daily accumulated precipitation product generated from the research-quality 3-hourly TRMM Multi-Satellite Precipitation Analysis TMPA (3B42)"
+media:
+  src: ::file ./trmm.png
+  alt: CMIP6 Near-Surface Air Temperature Screenshot
+  author:
+    name: NASA
+    url:   
+taxonomy:
+  - name: Topics
+    values:
+      - Climate
+layers:
+  - id: TRMM_3B42_Daily.v7
+    stacApiEndpoint: 'https://cmr.earthdata.nasa.gov/cloudstac/GES_DISC'
+    tileApiEndpoint: 'https://prod-titiler-xarray.delta-backend.com/tilejson.json'
+    stacCol: TRMM_3B42_Daily.v7
+    name: TRMM_3B42_Daily
+    type: zarr
+    time_density: day
+    assetUrlReplacements:
+      - [https://data.gesdisc.earthdata.nasa.gov/data, s3://gesdisc-cumulus-prod-protected]
+    description: "Daily accumulated precipitation product generated from the research-quality 3-hourly TRMM Multi-Satellite Precipitation Analysis TMPA (3B42)"
+    zoomExtent:
+      - 0
+      - 20
+    sourceParams:
+      resampling_method: bilinear
+      variable: precipitation
+      colormap_name: gnbu
+      rescale: 0,46
+      maxzoom: 12
+    legend:
+      unit: 
+        label: 
+      type: gradient
+      min: "0 mm/hr"
+      max: "46 mm/hr"
+      stops: ['#f7fcf0', '#e6f5e1', '#d7efd1', '#c5e8c2', '#abdeb6', '#8bd2bf', '#6bc3c9', '#4bafd1', '#3193c2', '#1878b4', '#085da0', '#084081']
+---
+
+<Block>
+<Prose>
+# TRMM (TMPA) Precipitation L3 1 day 0.25 degree x 0.25 degree V7 (TRMM_3B42_Daily)
+
+## Dataset Description
+
+This daily accumulated precipitation product is generated from the research-quality 3-hourly TRMM Multi-Satellite Precipitation Analysis TMPA (3B42). It is produced at the NASA GES DISC, as a value added product. Simple summation of valid retrievals in a grid cell is applied for the data day. The result is given in (mm). The beginning and ending time for every daily granule are listed in the file global attributes, and are taken correspondingly from the first and the last 3-hourly granules participating in the aggregation. Thus the time period covered by one daily granule amounts to 24 hours, which can be inspected in the file global attributes. 
+
+Source: [https://disc.gsfc.nasa.gov/datasets/TRMM_3B42_Daily_7/summary](https://disc.gsfc.nasa.gov/datasets/TRMM_3B42_Daily_7/summary)
+
+## Data Source
+
+The files represented are NetCDF files in GES DISC's Earthdata Cloud bucket and discovered from the CMR STAC entry for this dataset: https://cmr.earthdata.nasa.gov/cloudstac/GES_DISC/collections/TRMM_3B42_Daily.v7. 
+
+</Prose>
+</Block>

mock/datasets/sandbox.data.mdx

@@ -22,7 +22,7 @@ layers:
     stacCol: combined_CMIP6_daily_GISS-E2-1-G_tas_kerchunk_DEMO
     name: CMIP6 Daily GISS-E2-1-G Near-Surface Air Temperature (demo subset)
     type: zarr
-    tileApiEndpoint: https://dev-titiler-xarray.delta-backend.com/tilejson.json
+    tileApiEndpoint: https://prod-titiler-xarray.delta-backend.com/tilejson.json
     description: "Historical (1950-2014) daily-mean near-surface (usually, 2 meter) air temperature in Kelvin."
     zoomExtent:
       - 0

parcel-resolver-veda/index.d.ts

@@ -51,11 +51,14 @@ declare module 'veda' {
     layerId: string;
   }
 
+
   export interface DatasetLayer extends DatasetLayerCommonProps {
     id: string;
     stacCol: string;
     stacApiEndpoint?: string;
     tileApiEndpoint?: string;
+    assetUrlReplacements?: [string, string][];
+    time_density?: 'day' | 'month' | 'year';
     name: string;
     description: string;
     initialDatetime?: 'newest' | 'oldest' | string;
@@ -88,6 +91,7 @@ declare module 'veda' {
     description: string;
     stacApiEndpoint?: string;
     tileApiEndpoint?: string;
+    time_density?: 'day' | 'month' | 'year';
     stacCol: string;
     type: DatasetLayerType;
     legend?: LayerLegendCategorical | LayerLegendGradient;