Skip to content

Commit

Permalink
Convert list_files to a property
Browse files Browse the repository at this point in the history
  • Loading branch information
adamjstewart committed Sep 29, 2023
1 parent 2bc82c8 commit d391079
Show file tree
Hide file tree
Showing 13 changed files with 28 additions and 29 deletions.
2 changes: 1 addition & 1 deletion tests/datasets/test_geo.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def test_init_with_list_of_files(self) -> None:
s2 = Sentinel2(
paths, bands=["B04", "B03", "B02"], transforms=transforms, cache=False
)
assert s2.list_files(), "No files were found"
assert s2.files, "No files were found"

def test_getitem_single_file(self, naip: NAIP) -> None:
x = naip[naip.bounds]
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/agb_live_woody_density.py
Original file line number Diff line number Diff line change
Expand Up @@ -99,7 +99,7 @@ def _verify(self) -> None:
RuntimeError: if dataset is missing
"""
# Check if the extracted files already exist
if self.list_files():
if self.files:
return

# Check if the user requested to download the dataset
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/astergdem.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ def _verify(self) -> None:
RuntimeError: if dataset is missing
"""
# Check if the extracted files already exists
if self.list_files():
if self.files:
return

raise RuntimeError(
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/cdl.py
Original file line number Diff line number Diff line change
Expand Up @@ -292,7 +292,7 @@ def _verify(self) -> None:
RuntimeError: if ``download=False`` but dataset is missing or checksum fails
"""
# Check if the extracted files already exist
if self.list_files():
if self.files:
return

# Check if the zip files have already been downloaded
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/chesapeake.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,7 +144,7 @@ def _verify(self) -> None:
RuntimeError: if ``download=False`` but dataset is missing or checksum fails
"""
# Check if the extracted file already exists
if self.list_files():
if self.files:
return

# Check if the zip file has already been downloaded
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/cms_mangrove_canopy.py
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def _verify(self) -> None:
RuntimeError: if dataset is missing or checksum fails
"""
# Check if the extracted files already exist
if self.list_files():
if self.files:
return

# Check if the zip file has already been downloaded
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/esri2020.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,7 +112,7 @@ def _verify(self) -> None:
RuntimeError: if ``download=False`` but dataset is missing or checksum fails
"""
# Check if the extracted file already exists
if self.list_files():
if self.files:
return

# Check if the zip files have already been downloaded
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/eudem.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,7 +124,7 @@ def _verify(self) -> None:
RuntimeError: if dataset is missing or checksum fails
"""
# Check if the extracted file already exists
if self.list_files():
if self.files:
return

# Check if the zip files have already been downloaded
Expand Down
33 changes: 16 additions & 17 deletions torchgeo/datasets/geo.py
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ def __init__(
FileNotFoundError: if no files are found in ``paths``
.. versionchanged:: 0.5
*root* was renamed to *paths*
*root* was renamed to *paths*.
"""
super().__init__(transforms)

Expand All @@ -364,7 +364,7 @@ def __init__(
# Populate the dataset index
i = 0
filename_regex = re.compile(self.filename_regex, re.VERBOSE)
for filepath in self.list_files():
for filepath in self.files:
match = re.match(filename_regex, os.path.basename(filepath))
if match is not None:
try:
Expand Down Expand Up @@ -423,32 +423,31 @@ def __init__(
self._crs = cast(CRS, crs)
self._res = cast(float, res)

def list_files(self, filename_glob: Optional[str] = None) -> list[str]:
"""Get list of files matching filename_glob.
@property
def files(self) -> set[str]:
"""A list of all files in the dataset.
Args:
filename_glob: Defaults to self.filename_glob
Returns:
All files in the dataset.
.. versionadded:: 0.5
"""
# Make iterable
if isinstance(self.paths, str):
paths: Iterable[str] = [self.paths]
paths = [self.paths]
else:
paths = self.paths

filename_glob = filename_glob or self.filename_glob

# using set to remove any duplicates if directories are overlapping
filepaths: set[str] = set()
for dir_or_file in paths:
if os.path.isfile(dir_or_file):
filepaths.add(dir_or_file)
# Using set to remove any duplicates if directories are overlapping
files: set[str] = set()
for path in paths:
if os.path.isdir(path):
pathname = os.path.join(path, "**", self.filename_glob)
files |= set(glob.iglob(pathname, recursive=True))
else:
pathname = os.path.join(dir_or_file, "**", filename_glob)
filepaths |= set(glob.iglob(pathname, recursive=True))
files.add(path)

return list(filepaths)
return files

def __getitem__(self, query: BoundingBox) -> dict[str, Any]:
"""Retrieve image/mask and metadata indexed by query.
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/globbiomass.py
Original file line number Diff line number Diff line change
Expand Up @@ -210,7 +210,7 @@ def _verify(self) -> None:
RuntimeError: if dataset is missing or checksum fails
"""
# Check if the extracted file already exists
if self.list_files():
if self.files:
return

# Check if the zip files have already been downloaded
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/l7irish.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@ def _verify(self) -> None:
RuntimeError: if ``download=False`` but dataset is missing or checksum fails
"""
# Check if the extracted files already exist
if self.list_files():
if self.files:
return

# Check if the tar.gz files have already been downloaded
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/l8biome.py
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,7 @@ def _verify(self) -> None:
RuntimeError: if ``download=False`` but dataset is missing or checksum fails
"""
# Check if the extracted files already exist
if self.list_files():
if self.files:
return

# Check if the tar.gz files have already been downloaded
Expand Down
2 changes: 1 addition & 1 deletion torchgeo/datasets/nlcd.py
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ def _verify(self) -> None:
RuntimeError: if ``download=False`` but dataset is missing or checksum fails
"""
# Check if the extracted files already exist
if self.list_files():
if self.files:
return

# Check if the zip files have already been downloaded
Expand Down

0 comments on commit d391079

Please sign in to comment.