Skip to content

Commit

Permalink
Merge branch 'microsoft:main' into samplers/gridgeosampler_bounds
Browse files Browse the repository at this point in the history
  • Loading branch information
remtav authored Aug 25, 2022
2 parents b0ab3fa + 0ddccaa commit 45b3490
Show file tree
Hide file tree
Showing 266 changed files with 2,893 additions and 749 deletions.
5 changes: 3 additions & 2 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@ updates:
# Allow up to 2 open pull requests at a time
open-pull-requests-limit: 2
ignore:
# torch, tensorboard require protobuf < 4
- dependency-name: "protobuf"
# radiant-mlhub 0.5+ changed download behavior:
# https://github.com/radiantearth/radiant-mlhub/pull/104
- dependency-name: "radiant-mlhub"
# segmentation-models-pytorch requires older timm, can't update
- dependency-name: "timm"
3 changes: 0 additions & 3 deletions .github/workflows/release.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ jobs:
python-version: '3.10'
- name: Install pip dependencies
run: |
pip install --pre 'rasterio>=1.0.16'
pip install .[tests]
pip list
- name: Run pytest checks
Expand All @@ -36,7 +35,6 @@ jobs:
python-version: '3.10'
- name: Install pip dependencies
run: |
pip install --pre 'rasterio>=1.0.16'
pip install .[datasets,tests]
pip list
- name: Run integration checks
Expand All @@ -53,7 +51,6 @@ jobs:
python-version: '3.10'
- name: Install pip dependencies
run: |
pip install --pre 'rasterio>=1.0.16'
pip install .[datasets,docs,tests]
pip list
- name: Run notebook checks
Expand Down
16 changes: 11 additions & 5 deletions .github/workflows/tests.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,23 @@ jobs:
python-version: ${{ matrix.python-version }}
channels: conda-forge
channel-priority: strict
if: ${{ runner.os == 'Windows' }}
if: ${{ runner.os == 'Windows' && matrix.python-version == '3.7' }}
- name: Install conda dependencies (Windows)
run: |
conda install 'fiona==1.8.21' 'rasterio==1.2.10'
conda install 'rasterio==1.2.10' 'geos=3.10.3'
conda list
conda info
if: ${{ runner.os == 'Windows' }}
- name: Install pip dependencies
if: ${{ runner.os == 'Windows' && matrix.python-version == '3.7' }}
- name: Install pip dependencies (3.8+)
run: |
pip install -r requirements/required.txt -r requirements/datasets.txt -r requirements/tests.txt
pip list
if: ${{ matrix.python-version != '3.7' }}
- name: Install pip dependencies (3.7)
run: |
pip install -r requirements/required.old -r requirements/datasets.old -r requirements/tests.txt
pip list
if: ${{ matrix.python-version == '3.7' }}
- name: Run pytest checks
run: pytest --cov=torchgeo --cov-report=xml
- name: Report coverage
Expand All @@ -85,7 +91,7 @@ jobs:
- name: Install pip dependencies
run: |
pip install cython numpy==1.17.2 # needed by pycocotools
pip install -r .github/requirements-min.txt
pip install -r requirements/min.old
pip list
- name: Run pytest checks
run: pytest --cov=torchgeo --cov-report=xml
Expand Down
8 changes: 4 additions & 4 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
repos:
- repo: https://github.com/asottile/pyupgrade
rev: v2.32.1
rev: v2.37.3
hooks:
- id: pyupgrade
args: [--py37-plus]
Expand All @@ -12,7 +12,7 @@ repos:
additional_dependencies: ["colorama>=0.4.3"]

- repo: https://github.com/psf/black
rev: 22.3.0
rev: 22.6.0
hooks:
- id: black
args: [--skip-magic-trailing-comma]
Expand All @@ -30,9 +30,9 @@ repos:
additional_dependencies: ["toml"]

- repo: https://github.com/pre-commit/mirrors-mypy
rev: v0.960
rev: v0.971
hooks:
- id: mypy
args: [--strict, --ignore-missing-imports, --show-error-codes]
additional_dependencies: [torch>=1.11, torchmetrics>=0.7, pytorch-lightning>=1.6, pytest>=6, omegaconf>=2.1, kornia>=0.6, numpy>=1.22.0]
additional_dependencies: [torch>=1.11, torchmetrics>=0.7, pytorch-lightning>=1.7, pytest>=6, omegaconf>=2.1, kornia>=0.6, numpy>=1.22.0]
exclude: (build|data|dist|logo|logs|output)/
2 changes: 1 addition & 1 deletion CITATION.cff
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,6 @@ preferred-citation:
given-names: "Arindam"
journal: "arXiv preprint arXiv:2111.08872"
month: 11
title: "TorchGeo: deep learning with geospatial data"
title: "TorchGeo: Deep Learning With Geospatial Data"
url: "https://github.com/microsoft/torchgeo"
year: 2021
127 changes: 73 additions & 54 deletions README.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion conf/chesapeake_cvpr.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ experiment:
in_channels: 4
num_classes: 7
num_filters: 256
ignore_zeros: False
ignore_index: null
imagenet_pretraining: True
datamodule:
root_dir: "data/chesapeake/cvpr"
Expand Down
2 changes: 1 addition & 1 deletion conf/cowc_counting.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -12,5 +12,5 @@ experiment:
datamodule:
root_dir: "data/cowc_counting"
seed: 0
batch_size: 32
batch_size: 64
num_workers: 4
4 changes: 2 additions & 2 deletions conf/etci2021.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@ experiment:
encoder_weights: "imagenet"
learning_rate: 1e-3
learning_rate_schedule_patience: 6
in_channels: 7
in_channels: 6
num_classes: 2
ignore_zeros: True
ignore_index: 0
datamodule:
root_dir: "data/etci2021"
batch_size: 32
Expand Down
2 changes: 1 addition & 1 deletion conf/inria.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ experiment:
learning_rate_schedule_patience: 6
in_channels: 3
num_classes: 2
ignore_zeros: True # class 0 not used for scoring
ignore_index: 0 # class 0 not used for scoring
datamodule:
root_dir: "data/inria"
batch_size: 2
Expand Down
2 changes: 1 addition & 1 deletion conf/landcoverai.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ experiment:
in_channels: 3
num_classes: 6
num_filters: 256
ignore_zeros: False
ignore_index: null
datamodule:
root_dir: "data/landcoverai"
batch_size: 32
Expand Down
2 changes: 1 addition & 1 deletion conf/naipchesapeake.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ experiment:
in_channels: 4
num_classes: 14
num_filters: 64
ignore_zeros: False
ignore_index: null
datamodule:
naip_root_dir: "data/naip"
chesapeake_root_dir: "data/chesapeake/BAYWIDE"
Expand Down
2 changes: 1 addition & 1 deletion conf/oscd.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ experiment:
in_channels: 26
num_classes: 2
num_filters: 256
ignore_zeros: True
ignore_index: 0
datamodule:
root_dir: "data/oscd"
batch_size: 32
Expand Down
2 changes: 1 addition & 1 deletion conf/sen12ms.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ experiment:
learning_rate_schedule_patience: 2
in_channels: 15
num_classes: 11
ignore_zeros: False
ignore_index: null
datamodule:
root_dir: "data/sen12ms"
band_set: "all"
Expand Down
5 changes: 5 additions & 0 deletions docs/api/datamodules.rst
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,11 @@ COWC

.. autoclass:: COWCCountingDataModule

Deep Globe Land Cover Challenge
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. autoclass:: DeepGlobeLandCoverDataModule

ETCI2021 Flood Detection
^^^^^^^^^^^^^^^^^^^^^^^^

Expand Down
25 changes: 20 additions & 5 deletions docs/api/datasets.rst
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ Sentinel
Non-geospatial Datasets
-----------------------

:class:`VisionDataset` is designed for datasets that lack geospatial information. These datasets can still be combined using :class:`ConcatDataset <torch.utils.data.ConcatDataset>`.
:class:`NonGeoDataset` is designed for datasets that lack geospatial information. These datasets can still be combined using :class:`ConcatDataset <torch.utils.data.ConcatDataset>`.

.. csv-table:: C = classification, R = regression, S = semantic segmentation, I = instance segmentation, T = time series, CD = change detection, OD = object detection
:widths: 15 7 15 12 11 12 15 13
Expand Down Expand Up @@ -164,6 +164,11 @@ Kenya Crop Type

.. autoclass:: CV4AKenyaCropType

Deep Globe Land Cover
^^^^^^^^^^^^^^^^^^^^^

.. autoclass:: DeepGlobeLandCover

DFC2022
^^^^^^^

Expand Down Expand Up @@ -219,6 +224,11 @@ LoveDA

.. autoclass:: LoveDA

Million-AID
^^^^^^^^^^^

.. autoclass:: MillionAID

NASA Marine Debris
^^^^^^^^^^^^^^^^^^

Expand All @@ -239,6 +249,11 @@ Potsdam

.. autoclass:: Potsdam2D

ReforesTree
^^^^^^^^^^^

.. autoclass:: ReforesTree

RESISC45
^^^^^^^^

Expand Down Expand Up @@ -327,15 +342,15 @@ VectorDataset

.. autoclass:: VectorDataset

VisionDataset
NonGeoDataset
^^^^^^^^^^^^^

.. autoclass:: VisionDataset
.. autoclass:: NonGeoDataset

VisionClassificationDataset
NonGeoClassificationDataset
^^^^^^^^^^^^^^^^^^^^^^^^^^^

.. autoclass:: VisionClassificationDataset
.. autoclass:: NonGeoClassificationDataset

IntersectionDataset
^^^^^^^^^^^^^^^^^^^
Expand Down
33 changes: 17 additions & 16 deletions docs/api/geo_datasets.csv
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
Dataset,Type,Source,Size (px),Resolution (m)
`Aboveground Woody Biomass`_,Mask,"Landsat, LiDAR","~40,000x40,000",~30
`Aster Global DEM`_,Mask,Aster,"3,601x3,601",30
`Canadian Building Footprints`_,Labels,Generated,,
`Chesapeake Land Cover`_,"Imagery, Labels",,,1
`Global Mangrove Distribution`_,Mask,Generated,,3
`Cropland Data Layer`_,Labels,Aerial,,
`EDDMapS`_,Labels,Vector,,
`EnviroAtlas`_,"Imagery, Labels",Aerial,,1
`Esri2020`_,Labels,Sentinel-2,,10
`EU-DEM`_,Labels,"Aster, SRTM, Russian Topomaps",,25
`GBIF`_,Labels,Vector,,
`GlobBiomass`_,Labels,Landsat,"45,000x45,000",~100
`Landsat`_,Imagery,Landsat,,30
`NAIP`_,Imagery,Aerial,,1
`Open Buildings`_,Labels,Generated,,
`Sentinel`_,Imagery,Sentinel,,10
`Aboveground Woody Biomass`_,Masks,"Landsat, LiDAR","40,000x40,000",30
`Aster Global DEM`_,Masks,Aster,"3,601x3,601",30
`Canadian Building Footprints`_,Geometries,Bing Imagery,-,-
`Chesapeake Land Cover`_,"Imagery, Masks",NAIP,-,1
`Global Mangrove Distribution`_,Masks,"Remote Sensing, In Situ Measurements",-,3
`Cropland Data Layer`_,Masks,Aerial,-,30
`EDDMapS`_,Points,Citizen Scientists,-,-
`EnviroAtlas`_,"Imagery, Masks","NAIP, NLCD, OpenStreetMap",-,1
`Esri2020`_,Masks,Sentinel-2,-,10
`EU-DEM`_,Masks,"Aster, SRTM, Russian Topomaps",-,25
`GBIF`_,Points,Citizen Scientists,-,-
`GlobBiomass`_,Masks,Landsat,"45,000x45,000",100
`iNaturalist`_,Points,Citizen Scientists,-,-
`Landsat`_,Imagery,Landsat,"8,900x8,900",30
`NAIP`_,Imagery,Aerial,"6,100x7,600",1
`Open Buildings`_,Geometries,"Maxar, CNES/Airbus",-,-
`Sentinel`_,Imagery,Sentinel,"10,000x10,000",10
17 changes: 10 additions & 7 deletions docs/api/non_geo_datasets.csv
Original file line number Diff line number Diff line change
@@ -1,33 +1,36 @@
Dataset,Task,Source,# Samples,# Classes,Size (px),Resolution (m),Bands
`ADVANCE`_,C,"Google Earth, Freesound","5,075",13,512x512,0.5,RGB
`Benin Cashew Plantations`_,S,Airbus Pléiades,70,6,"1,186x1,122",0.5,MSI
`Benin Cashew Plantations`_,S,Airbus Pléiades,70,6,"1,122x1,186",10,MSI
`BigEarthNet`_,C,Sentinel-1/2,"590,326",19--43,120x120,10,"SAR, MSI"
`COWC`_,"C, R","CSUAV AFRL, ISPRS, LINZ, AGRC","388,435",2,256x256,0.15,RGB
`Kenya Crop Type`_,S,Sentinel-2,"4,688",7,"3,035x2,016",10,MSI
`DFC2022`_,S,Aerial,,15,"2,000x2,000",0.5,RGB
`Deep Globe Land Cover`_,S,DigitalGlobe +Vivid,803,7,"2,448x2,448",0.5,RGB
`DFC2022`_,S,Aerial,"3,981",15,"2,000x2,000",0.5,RGB
`ETCI2021 Flood Detection`_,S,Sentinel-1,"66,810",2,256x256,5--20,SAR
`EuroSAT`_,C,Sentinel-2,"27,000",10,64x64,10,MSI
`FAIR1M`_,OD,Gaofen/Google Earth,"15,000",37,"1,024x1,024",0.3--0.8,RGB
`Forest Damage`_,OD,Drone imagery,"1,543",4,"1,500x1,500",,RGB
`GID-15`_,S,Gaofen-2,150,15,"6,800x7,200",3,RGB
`IDTReeS`_,"OD,C",Aerial,591,33,200x200,0.1--1,RGB
`Inria Aerial Image Labeling`_,S,Aerial,360,,"5,000x5,000",0.3,RGB
`Inria Aerial Image Labeling`_,S,Aerial,360,2,"5,000x5,000",0.3,RGB
`LandCover.ai`_,S,Aerial,"10,674",5,512x512,0.25--0.5,RGB
`LEVIR-CD+`_,CD,Google Earth,985,2,"1,024x1,024",0.5,RGB
`LoveDA`_,S,Google Earth,"5,987",7,"1,024x1,024",0.3,RGB
`Million-AID`_,C,Google Earth,1M,51--73,,0.5--153,RGB
`NASA Marine Debris`_,OD,PlanetScope,707,1,256x256,3,RGB
`NWPU VHR-10`_,I,"Google Earth, Vaihingen",800,10,"358--1,728",0.08--2,RGB
`OSCD`_,CD,Sentinel-2,24,2,"40--1,180",60,MSI
`PatternNet`_,C,Google Earth,"30,400",38,256x256,0.06--5,RGB
`Potsdam`_,S,Aerial,38,6,"6,000x6,000",0.05,MSI
`ReforesTree`_,"OD, R",Aerial,100,"4,000x4,000",0.02,RGB
`RESISC45`_,C,Google Earth,"31,500",45,256x256,0.2--30,RGB
`Seasonal Contrast`_,T,Sentinel-2,100K--1M,,264x264,10,MSI
`Seasonal Contrast`_,T,Sentinel-2,100K--1M,-,264x264,10,MSI
`SEN12MS`_,S,"Sentinel-1/2, MODIS","180,662",33,256x256,10,"SAR, MSI"
`So2Sat`_,C,Sentinel-1/2,"400,673",17,32x32,10,"SAR, MSI"
`SpaceNet`_,I,WorldView-2/3 Planet Lab Dove,"1,889--28,728",2,102--900,0.5--4,MSI
`Tropical Cyclone`_,R,GOES 8--16,"108,110",,256x256,4K--8K,MSI
`Tropical Cyclone`_,R,GOES 8--16,"108,110",-,256x256,4K--8K,MSI
`UC Merced`_,C,USGS National Map,"21,000",21,256x256,0.3,RGB
`USAVars`_,S,NAIP Aerial,~100K,,,4,"RGB, NIR"
`USAVars`_,R,NAIP Aerial,100K,-,-,4,"RGB, NIR"
`Vaihingen`_,S,Aerial,33,6,"1,281--3,816",0.09,RGB
`NWPU VHR-10`_,I,"Google Earth, Vaihingen",800,10,"358--1,728",0.08--2,RGB
`xView2`_,CD,Maxar,"3,732",4,"1,024x1,024",0.8,RGB
`ZueriCrop`_,"I, T",Sentinel-2,116K,48,24x24,10,MSI
2 changes: 1 addition & 1 deletion docs/api/samplers.rst
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ torchgeo.samplers
Samplers
--------

Samplers are used to index a dataset, retrieving a single query at a time. For :class:`~torchgeo.datasets.VisionDataset`, dataset objects can be indexed with integers, and PyTorch's builtin samplers are sufficient. For :class:`~torchgeo.datasets.GeoDataset`, dataset objects require a bounding box for indexing. For this reason, we define our own :class:`GeoSampler` implementations below. These can be used like so:
Samplers are used to index a dataset, retrieving a single query at a time. For :class:`~torchgeo.datasets.NonGeoDataset`, dataset objects can be indexed with integers, and PyTorch's builtin samplers are sufficient. For :class:`~torchgeo.datasets.GeoDataset`, dataset objects require a bounding box for indexing. For this reason, we define our own :class:`GeoSampler` implementations below. These can be used like so:

.. code-block:: python
Expand Down
1 change: 1 addition & 0 deletions docs/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,7 @@
("py:class", ".."),
# TODO: can't figure out why this isn't found
("py:class", "LightningDataModule"),
("py:class", "pytorch_lightning.core.module.LightningModule"),
# Undocumented class
("py:class", "torchvision.models.resnet.ResNet"),
("py:class", "segmentation_models_pytorch.base.model.SegmentationModel"),
Expand Down
6 changes: 3 additions & 3 deletions docs/tutorials/benchmarking.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -212,7 +212,7 @@
" chesapeake = ChesapeakeDE(chesapeake_root, cache=cache)\n",
" naip = NAIP(naip_root, crs=chesapeake.crs, res=chesapeake.res, cache=cache)\n",
" dataset = chesapeake & naip\n",
" sampler = RandomGeoSampler(naip, size=1000, length=888)\n",
" sampler = RandomGeoSampler(dataset, size=1000, length=888)\n",
" dataloader = DataLoader(\n",
" dataset, batch_size=12, sampler=sampler, collate_fn=stack_samples\n",
" )\n",
Expand Down Expand Up @@ -265,7 +265,7 @@
" chesapeake = ChesapeakeDE(chesapeake_root, cache=cache)\n",
" naip = NAIP(naip_root, crs=chesapeake.crs, res=chesapeake.res, cache=cache)\n",
" dataset = chesapeake & naip\n",
" sampler = GridGeoSampler(naip, size=1000, stride=500)\n",
" sampler = GridGeoSampler(dataset, size=1000, stride=500)\n",
" dataloader = DataLoader(\n",
" dataset, batch_size=12, sampler=sampler, collate_fn=stack_samples\n",
" )\n",
Expand Down Expand Up @@ -318,7 +318,7 @@
" chesapeake = ChesapeakeDE(chesapeake_root, cache=cache)\n",
" naip = NAIP(naip_root, crs=chesapeake.crs, res=chesapeake.res, cache=cache)\n",
" dataset = chesapeake & naip\n",
" sampler = RandomBatchGeoSampler(naip, size=1000, batch_size=12, length=888)\n",
" sampler = RandomBatchGeoSampler(dataset, size=1000, batch_size=12, length=888)\n",
" dataloader = DataLoader(dataset, batch_sampler=sampler, collate_fn=stack_samples)\n",
" duration, count = time_epoch(dataloader)\n",
" print(duration, count)"
Expand Down
2 changes: 1 addition & 1 deletion docs/tutorials/getting_started.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -330,7 +330,7 @@
},
"outputs": [],
"source": [
"sampler = RandomGeoSampler(naip, size=1000, length=10)"
"sampler = RandomGeoSampler(dataset, size=1000, length=10)"
]
},
{
Expand Down
Loading

0 comments on commit 45b3490

Please sign in to comment.