added support for s3 files

kerchunk/grib2.py

@@ -811,7 +811,10 @@ def extract_datatree_chunk_index(
 
 
 def _map_grib_file_by_group(
-    fname: str, mapper: Optional[Callable] = None, storage_options=None
+    fname: str,
+    mapper: Optional[Callable] = None,
+    storage_options=None,
+    remote_options: Dict = None,
 ) -> "pd.DataFrame":
     """
     Helper method used to read the cfgrib metadata associated with each message (group) in the grib file
@@ -831,6 +834,7 @@ def _map_grib_file_by_group(
     import pandas as pd
 
     mapper = (lambda x: x) if mapper is None else mapper
+    references = scan_grib(fname, storage_options=storage_options)
 
     with warnings.catch_warnings():
         warnings.simplefilter("ignore")
@@ -840,23 +844,22 @@ def _map_grib_file_by_group(
                 filter(
                     lambda item: item is not None,
                     [
-                        _extract_single_group(mapper(group), i)
-                        for i, group in enumerate(
-                            scan_grib(fname, storage_options=storage_options), start=1
-                        )
+                        _extract_single_group(mapper(group), i, remote_options)
+                        for i, group in enumerate(references, start=1)
                     ],
                 )
             )
         ).set_index("idx")
 
 
-def _extract_single_group(grib_group: dict, idx: int):
+def _extract_single_group(grib_group: dict, idx: int, remote_options: Dict):
     import datatree
 
     grib_tree_store = grib_tree(
         [
             grib_group,
-        ]
+        ],
+        remote_options,
     )
 
     if len(grib_tree_store["refs"]) <= 1:
@@ -887,6 +890,7 @@ def build_idx_grib_mapping(
     suffix: str = "idx",
     mapper: Optional[Callable] = None,
     validate: bool = True,
+    remote_options: Dict = None,
 ) -> "pd.DataFrame":
     """
     Mapping method combines the idx and grib metadata to make a mapping from
@@ -905,6 +909,8 @@ def build_idx_grib_mapping(
         the mapper if any to apply (used for hrrr subhf)
     validate : bool
         to assert the mapping is correct or fail before returning
+    remote_options: Dict
+        remote options to pass to MultiZarrToZarr
 
     Returns
     -------
@@ -914,7 +920,10 @@ def build_idx_grib_mapping(
     import pandas as pd
 
     grib_file_index = _map_grib_file_by_group(
-        fname=basename, mapper=mapper, storage_options=storage_options
+        fname=basename,
+        mapper=mapper,
+        storage_options=storage_options,
+        remote_options=remote_options,
     )
     idx_file_index = parse_grib_idx(
         basename=basename, suffix=suffix, storage_options=storage_options