Allow VectorDataset to accept list of files

tests/datasets/test_cbf.py

@@ -61,7 +61,7 @@ def test_or(self, dataset: CanadianBuildingFootprints) -> None:
         assert isinstance(ds, UnionDataset)
 
     def test_already_downloaded(self, dataset: CanadianBuildingFootprints) -> None:
-        CanadianBuildingFootprints(root=dataset.root, download=True)
+        CanadianBuildingFootprints(dataset.paths, download=True)
 
     def test_plot(self, dataset: CanadianBuildingFootprints) -> None:
         query = dataset.bounds

tests/datasets/test_openbuildings.py

@@ -37,7 +37,7 @@ def dataset(self, monkeypatch: MonkeyPatch, tmp_path: Path) -> OpenBuildings:
 
         monkeypatch.setattr(OpenBuildings, "md5s", md5s)
         transforms = nn.Identity()
-        return OpenBuildings(root=root, transforms=transforms)
+        return OpenBuildings(root, transforms=transforms)
 
     def test_no_shapes_to_rasterize(
         self, dataset: OpenBuildings, tmp_path: Path
@@ -61,19 +61,19 @@ def test_no_building_data_found(self, tmp_path: Path) -> None:
         with pytest.raises(
             RuntimeError, match="have manually downloaded the dataset as suggested "
         ):
-            OpenBuildings(root=false_root)
+            OpenBuildings(false_root)
 
     def test_corrupted(self, dataset: OpenBuildings, tmp_path: Path) -> None:
         with open(os.path.join(tmp_path, "000_buildings.csv.gz"), "w") as f:
             f.write("bad")
         with pytest.raises(RuntimeError, match="Dataset found, but corrupted."):
-            OpenBuildings(dataset.root, checksum=True)
+            OpenBuildings(dataset.paths, checksum=True)
 
     def test_no_meta_data_found(self, tmp_path: Path) -> None:
         false_root = os.path.join(tmp_path, "empty")
         os.makedirs(false_root)
         with pytest.raises(FileNotFoundError, match="Meta data file"):
-            OpenBuildings(root=false_root)
+            OpenBuildings(false_root)
 
     def test_nothing_in_index(self, dataset: OpenBuildings, tmp_path: Path) -> None:
         # change meta data to another 'title_url' so that there is no match found
@@ -85,7 +85,7 @@ def test_nothing_in_index(self, dataset: OpenBuildings, tmp_path: Path) -> None:
             json.dump(content, f)
 
         with pytest.raises(FileNotFoundError, match="data was found in"):
-            OpenBuildings(dataset.root)
+            OpenBuildings(dataset.paths)
 
     def test_getitem(self, dataset: OpenBuildings) -> None:
         x = dataset[dataset.bounds]

torchgeo/datasets/cbf.py

@@ -4,7 +4,8 @@
 """Canadian Building Footprints dataset."""
 
 import os
-from typing import Any, Callable, Optional
+from collections.abc import Iterable
+from typing import Any, Callable, Optional, Union
 
 import matplotlib.pyplot as plt
 from matplotlib.figure import Figure
@@ -60,7 +61,7 @@ class CanadianBuildingFootprints(VectorDataset):
 
     def __init__(
         self,
-        root: str = "data",
+        paths: Union[str, Iterable[str]] = "data",
         crs: Optional[CRS] = None,
         res: float = 0.00001,
         transforms: Optional[Callable[[dict[str, Any]], dict[str, Any]]] = None,
@@ -70,7 +71,7 @@ def __init__(
         """Initialize a new Dataset instance.
 
         Args:
-            root: root directory where dataset can be found
+            paths: one or more root directories to search or files to load
             crs: :term:`coordinate reference system (CRS)` to warp to
                 (defaults to the CRS of the first file found)
             res: resolution of the dataset in units of CRS
@@ -83,8 +84,11 @@ def __init__(
             FileNotFoundError: if no files are found in ``root``
             RuntimeError: if ``download=False`` and data is not found, or
                 ``checksum=True`` and checksums don't match
+
+        .. versionchanged:: 0.5
+           *root* was renamed to *paths*.
         """
-        self.root = root
+        self.paths = paths
         self.checksum = checksum
 
         if download:
@@ -96,16 +100,17 @@ def __init__(
                 + "You can use download=True to download it"
             )
 
-        super().__init__(root, crs, res, transforms)
+        super().__init__(paths, crs, res, transforms)
 
     def _check_integrity(self) -> bool:
         """Check integrity of dataset.
 
         Returns:
             True if dataset files are found and/or MD5s match, else False
         """
+        assert isinstance(self.paths, str)
         for prov_terr, md5 in zip(self.provinces_territories, self.md5s):
-            filepath = os.path.join(self.root, prov_terr + ".zip")
+            filepath = os.path.join(self.paths, prov_terr + ".zip")
             if not check_integrity(filepath, md5 if self.checksum else None):
                 return False
         return True
@@ -115,11 +120,11 @@ def _download(self) -> None:
         if self._check_integrity():
             print("Files already downloaded and verified")
             return
-
+        assert isinstance(self.paths, str)
         for prov_terr, md5 in zip(self.provinces_territories, self.md5s):
             download_and_extract_archive(
                 self.url + prov_terr + ".zip",
-                self.root,
+                self.paths,
                 md5=md5 if self.checksum else None,
             )
 

torchgeo/datasets/chesapeake.py

@@ -495,7 +495,7 @@ class ChesapeakeCVPR(GeoDataset):
     )
 
     # these are used to check the integrity of the dataset
-    files = [
+    _files = [
         "de_1m_2013_extended-debuffered-test_tiles",
         "de_1m_2013_extended-debuffered-train_tiles",
         "de_1m_2013_extended-debuffered-val_tiles",
@@ -704,7 +704,7 @@ def exists(filename: str) -> bool:
             return os.path.exists(os.path.join(self.root, filename))
 
         # Check if the extracted files already exist
-        if all(map(exists, self.files)):
+        if all(map(exists, self._files)):
             return
 
         # Check if the zip files have already been downloaded

torchgeo/datasets/enviroatlas.py

@@ -80,7 +80,7 @@ class EnviroAtlas(GeoDataset):
     )
 
     # these are used to check the integrity of the dataset
-    files = [
+    _files = [
         "austin_tx-2012_1m-test_tiles-debuffered",
         "austin_tx-2012_1m-val5_tiles-debuffered",
         "durham_nc-2012_1m-test_tiles-debuffered",
@@ -422,7 +422,7 @@ def exists(filename: str) -> bool:
             return os.path.exists(os.path.join(self.root, "enviroatlas_lotp", filename))
 
         # Check if the extracted files already exist
-        if all(map(exists, self.files)):
+        if all(map(exists, self._files)):
             return
 
         # Check if the zip files have already been downloaded

torchgeo/datasets/geo.py

@@ -75,6 +75,13 @@ class GeoDataset(Dataset[dict[str, Any]], abc.ABC):
     _crs = CRS.from_epsg(4326)
     _res = 0.0
 
+    #: Glob expression used to search for files.
+    #:
+    #: This expression should be specific enough that it will not pick up files from
+    #: other datasets. It should not include a file extension, as the dataset may be in
+    #: a different file format than what it was originally downloaded as.
+    filename_glob = "*"
+
     # NOTE: according to the Python docs:
     #
     # * https://docs.python.org/3/library/exceptions.html#NotImplementedError
@@ -98,6 +105,9 @@ def __init__(
                 and returns a transformed version
         """
         self.transforms = transforms
+        self.paths: Union[
+            str, Iterable[str]
+        ] = "data"  # TODO: temp fix when refactoring
 
         # Create an R-tree to index the dataset
         self.index = Index(interleaved=False, properties=Property(dimension=3))
@@ -269,6 +279,32 @@ def res(self, new_res: float) -> None:
         print(f"Converting {self.__class__.__name__} res from {self.res} to {new_res}")
         self._res = new_res
 
+    @property
+    def files(self) -> set[str]:
+        """A list of all files in the dataset.
+
+        Returns:
+            All files in the dataset.
+
+        .. versionadded:: 0.5
+        """
+        # Make iterable
+        if isinstance(self.paths, str):
+            paths: Iterable[str] = [self.paths]
+        else:
+            paths = self.paths
+
+        # Using set to remove any duplicates if directories are overlapping
+        files: set[str] = set()
+        for path in paths:
+            if os.path.isdir(path):
+                pathname = os.path.join(path, "**", self.filename_glob)
+                files |= set(glob.iglob(pathname, recursive=True))
+            else:
+                files.add(path)
+
+        return files
+
 
 class RasterDataset(GeoDataset):
     """Abstract base class for :class:`GeoDataset` stored as raster files."""
@@ -580,7 +616,7 @@ class VectorDataset(GeoDataset):
 
     def __init__(
         self,
-        root: str = "data",
+        paths: Union[str, Iterable[str]] = "data",
         crs: Optional[CRS] = None,
         res: float = 0.0001,
         transforms: Optional[Callable[[dict[str, Any]], dict[str, Any]]] = None,
@@ -589,7 +625,7 @@ def __init__(
         """Initialize a new Dataset instance.
 
         Args:
-            root: root directory where dataset can be found
+            paths: one or more root directories to search or files to load
             crs: :term:`coordinate reference system (CRS)` to warp to
                 (defaults to the CRS of the first file found)
             res: resolution of the dataset in units of CRS
@@ -606,13 +642,12 @@ def __init__(
         """
         super().__init__(transforms)
 
-        self.root = root
+        self.paths = paths
         self.label_name = label_name
 
         # Populate the dataset index
         i = 0
-        pathname = os.path.join(root, "**", self.filename_glob)
-        for filepath in glob.iglob(pathname, recursive=True):
+        for filepath in self.files:
             try:
                 with fiona.open(filepath) as src:
                     if crs is None:
@@ -633,7 +668,7 @@ def __init__(
                 i += 1
 
         if i == 0:
-            msg = f"No {self.__class__.__name__} data was found in `root='{root}'`"
+            msg = f"No {self.__class__.__name__} data was found in `root='{paths}'`"
             raise FileNotFoundError(msg)
 
         self._crs = crs

torchgeo/datasets/landcoverai.py

@@ -7,7 +7,7 @@
 import hashlib
 import os
 from functools import lru_cache
-from typing import Any, Callable, Optional, cast
+from typing import Any, Callable, Optional, Union, cast
 
 import matplotlib.pyplot as plt
 import numpy as np
@@ -71,7 +71,10 @@ class LandCoverAIBase(Dataset[dict[str, Any]], abc.ABC):
     }
 
     def __init__(
-        self, root: str = "data", download: bool = False, checksum: bool = False
+        self,
+        root: Union[str, list[str]] = "data",
+        download: bool = False,
+        checksum: bool = False,
     ) -> None:
         """Initialize a new LandCover.ai dataset instance.
 
@@ -87,6 +90,10 @@ def __init__(
             RuntimeError: if ``download=False`` and data is not found, or checksums
                 don't match
         """
+        if isinstance(root, list):
+            # TODO: Workaround until list are fully implemented
+            root = root[0]
+
         self.root = root
         self.download = download
         self.checksum = checksum
@@ -211,7 +218,7 @@ class LandCoverAIGeo(LandCoverAIBase, RasterDataset):
 
     def __init__(
         self,
-        root: str = "data",
+        paths: Union[str, list[str]] = "data",
         crs: Optional[CRS] = None,
         res: Optional[float] = None,
         transforms: Optional[Callable[[dict[str, Any]], dict[str, Any]]] = None,
@@ -222,7 +229,7 @@ def __init__(
         """Initialize a new LandCover.ai NonGeo dataset instance.
 
         Args:
-            root: root directory where dataset can be found
+            paths: one or more root directories to search or files to load
             crs: :term:`coordinate reference system (CRS)` to warp to
                 (defaults to the CRS of the first file found)
             res: resolution of the dataset in units of CRS
@@ -236,9 +243,14 @@ def __init__(
         Raises:
             RuntimeError: if ``download=False`` and data is not found, or checksums
                 don't match
+
+        .. versionchanged:: 0.5
+           *root* was renamed to *paths*.
         """
-        LandCoverAIBase.__init__(self, root, download, checksum)
-        RasterDataset.__init__(self, root, crs, res, transforms=transforms, cache=cache)
+        LandCoverAIBase.__init__(self, paths, download, checksum)
+        RasterDataset.__init__(
+            self, paths, crs, res, transforms=transforms, cache=cache
+        )
 
     def _verify_data(self) -> bool:
         """Verify if the images and masks are present."""