From 71b049bdb91f19742423cb52f5bda356de410a51 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Fri, 29 Sep 2023 13:42:19 -0400 Subject: [PATCH 01/15] account for bbox when saving predictions (#1931) Co-authored-by: Adeel Hassan --- .../data/label/chip_classification_labels.py | 12 +++++++++--- .../core/data/label/object_detection_labels.py | 12 +++++++++--- .../data/label/semantic_segmentation_labels.py | 8 ++++++++ .../chip_classification_geojson_store.py | 6 ++++-- .../object_detection_geojson_store.py | 3 ++- .../semantic_segmentation_label_store.py | 18 +++++++++++++++--- .../core/data/label_store/utils.py | 11 ++++++++--- 7 files changed, 55 insertions(+), 15 deletions(-) diff --git a/rastervision_core/rastervision/core/data/label/chip_classification_labels.py b/rastervision_core/rastervision/core/data/label/chip_classification_labels.py index be7fe7921..0f49bb2c5 100644 --- a/rastervision_core/rastervision/core/data/label/chip_classification_labels.py +++ b/rastervision_core/rastervision/core/data/label/chip_classification_labels.py @@ -155,8 +155,11 @@ def extend(self, labels: 'ChipClassificationLabels') -> None: for cell in labels.get_cells(): self.set_cell(cell, *labels[cell]) - def save(self, uri: str, class_config: 'ClassConfig', - crs_transformer: 'CRSTransformer') -> None: + def save(self, + uri: str, + class_config: 'ClassConfig', + crs_transformer: 'CRSTransformer', + bbox: Optional[Box] = None) -> None: """Save labels as a GeoJSON file. Args: @@ -164,11 +167,14 @@ def save(self, uri: str, class_config: 'ClassConfig', class_config (ClassConfig): ClassConfig to map class IDs to names. crs_transformer (CRSTransformer): CRSTransformer to convert from pixel-coords to map-coords before saving. + bbox (Optional[Box]): User-specified crop of the extent. Must be + provided if the corresponding RasterSource has bbox != extent. """ from rastervision.core.data import ChipClassificationGeoJSONStore label_store = ChipClassificationGeoJSONStore( uri=uri, class_config=class_config, - crs_transformer=crs_transformer) + crs_transformer=crs_transformer, + bbox=bbox) label_store.save(self) diff --git a/rastervision_core/rastervision/core/data/label/object_detection_labels.py b/rastervision_core/rastervision/core/data/label/object_detection_labels.py index fc4772fb1..52568697b 100644 --- a/rastervision_core/rastervision/core/data/label/object_detection_labels.py +++ b/rastervision_core/rastervision/core/data/label/object_detection_labels.py @@ -294,8 +294,11 @@ def prune_duplicates( score_threshold=score_thresh) return ObjectDetectionLabels.from_boxlist(pruned_boxlist) - def save(self, uri: str, class_config: 'ClassConfig', - crs_transformer: 'CRSTransformer') -> None: + def save(self, + uri: str, + class_config: 'ClassConfig', + crs_transformer: 'CRSTransformer', + bbox: Optional[Box] = None) -> None: """Save labels as a GeoJSON file. Args: @@ -303,11 +306,14 @@ def save(self, uri: str, class_config: 'ClassConfig', class_config (ClassConfig): ClassConfig to map class IDs to names. crs_transformer (CRSTransformer): CRSTransformer to convert from pixel-coords to map-coords before saving. + bbox (Optional[Box]): User-specified crop of the extent. Must be + provided if the corresponding RasterSource has bbox != extent. """ from rastervision.core.data import ObjectDetectionGeoJSONStore label_store = ObjectDetectionGeoJSONStore( uri=uri, class_config=class_config, - crs_transformer=crs_transformer) + crs_transformer=crs_transformer, + bbox=bbox) label_store.save(self) diff --git a/rastervision_core/rastervision/core/data/label/semantic_segmentation_labels.py b/rastervision_core/rastervision/core/data/label/semantic_segmentation_labels.py index 29e071755..64d488809 100644 --- a/rastervision_core/rastervision/core/data/label/semantic_segmentation_labels.py +++ b/rastervision_core/rastervision/core/data/label/semantic_segmentation_labels.py @@ -357,6 +357,7 @@ def save(self, uri: str, crs_transformer: 'CRSTransformer', class_config: 'ClassConfig', + bbox: Optional[Box] = None, tmp_dir: Optional[str] = None, save_as_rgb: bool = False, raster_output: bool = True, @@ -373,6 +374,8 @@ def save(self, crs_transformer (CRSTransformer): CRSTransformer to configure CRS and affine transform of the output GeoTiff. class_config (ClassConfig): The ClassConfig. + bbox (Optional[Box]): User-specified crop of the extent. Must be + provided if the corresponding RasterSource has bbox != extent. tmp_dir (Optional[str], optional): Temporary directory to use. If None, will be auto-generated. Defaults to None. save_as_rgb (bool, optional): If True, Saves labels as an RGB @@ -397,6 +400,7 @@ def save(self, uri=uri, crs_transformer=crs_transformer, class_config=class_config, + bbox=bbox, tmp_dir=tmp_dir, save_as_rgb=save_as_rgb, discrete_output=raster_output, @@ -529,6 +533,7 @@ def save(self, uri: str, crs_transformer: 'CRSTransformer', class_config: 'ClassConfig', + bbox: Optional[Box] = None, tmp_dir: Optional[str] = None, save_as_rgb: bool = False, discrete_output: bool = True, @@ -547,6 +552,8 @@ def save(self, crs_transformer (CRSTransformer): CRSTransformer to configure CRS and affine transform of the output GeoTiff(s). class_config (ClassConfig): The ClassConfig. + bbox (Optional[Box]): User-specified crop of the extent. Must be + provided if the corresponding RasterSource has bbox != extent. tmp_dir (Optional[str], optional): Temporary directory to use. If None, will be auto-generated. Defaults to None. save_as_rgb (bool, optional): If True, saves labels as an RGB @@ -577,6 +584,7 @@ def save(self, uri=uri, crs_transformer=crs_transformer, class_config=class_config, + bbox=bbox, tmp_dir=tmp_dir, save_as_rgb=save_as_rgb, discrete_output=discrete_output, diff --git a/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store.py b/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store.py index 9ccfc07bb..f2b319f89 100644 --- a/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store.py +++ b/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store.py @@ -30,7 +30,8 @@ def __init__(self, in GeoJSON file to pixel coords. bbox (Optional[Box], optional): User-specified crop of the extent. If provided, only labels falling inside it are returned by - :meth:`.ChipClassificationGeoJSONStore.get_labels`. + :meth:`.ChipClassificationGeoJSONStore.get_labels`. Must be + provided if the corresponding RasterSource has bbox != extent. """ self.uri = uri self.class_config = class_config @@ -51,7 +52,8 @@ def save(self, labels: ChipClassificationLabels) -> None: class_ids, self.crs_transformer, self.class_config, - scores=scores) + scores=scores, + bbox=self.bbox) json_to_file(geojson, self.uri) def get_labels(self) -> ChipClassificationLabels: diff --git a/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store.py b/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store.py index d46bb929b..8c2dae032 100644 --- a/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store.py +++ b/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store.py @@ -32,7 +32,8 @@ def __init__(self, in GeoJSON file to pixel coords. bbox (Optional[Box], optional): User-specified crop of the extent. If provided, only labels falling inside it are returned by - :meth:`.ObjectDetectionGeoJSONStore.get_labels`. + :meth:`.ObjectDetectionGeoJSONStore.get_labels`. Must be + provided if the corresponding RasterSource has bbox != extent. """ self.uri = uri self.class_config = class_config diff --git a/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store.py b/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store.py index 4ece4c8fa..aedbb0795 100644 --- a/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store.py +++ b/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store.py @@ -4,6 +4,7 @@ import numpy as np import rasterio as rio +import rasterio.windows as rio_windows from tqdm.auto import tqdm from rastervision.pipeline.file_system import ( @@ -58,7 +59,8 @@ def __init__( class_config (ClassConfig): Class config. bbox (Optional[Box], optional): User-specified crop of the extent. If provided, only labels falling inside it are returned by - :meth:`.SemanticSegmentationLabelStore.get_labels`. + :meth:`.SemanticSegmentationLabelStore.get_labels`. Must be + provided if the corresponding RasterSource has bbox != extent. tmp_dir (Optional[str], optional): Temporary directory to use. If None, will be auto-generated. Defaults to None. vector_outputs (Optional[Sequence[VectorOutputConfig]], optional): @@ -207,11 +209,17 @@ def save(self, make_dir(local_root) height, width = labels.extent.size + if self.bbox is not None: + bbox_rio_window = self.bbox.rasterio_format() + transform = rio_windows.transform(bbox_rio_window, + self.crs_transformer.transform) + else: + transform = self.crs_transformer.transform out_profile = dict( driver='GTiff', height=height, width=width, - transform=self.crs_transformer.transform, + transform=transform, crs=self.crs_transformer.image_crs, blockxsize=min(self.rasterio_block_size, width), blockysize=min(self.rasterio_block_size, height)) @@ -257,6 +265,7 @@ def write_smooth_raster_output( out_profile.update(dict(count=num_bands, dtype=dtype)) extent = labels.extent + with rio.open(scores_path, 'w', **out_profile) as ds: windows = [Box.from_rasterio(w) for _, w in ds.block_windows(1)] with tqdm(windows, desc='Saving pixel scores') as bar: @@ -310,7 +319,10 @@ def write_vector_outputs(self, labels: SemanticSegmentationLabels, bar.set_postfix(vo.dict()) class_mask = (label_arr == vo.class_id).astype(np.uint8) polys = vo.vectorize(class_mask) - polys = [self.crs_transformer.pixel_to_map(p) for p in polys] + polys = [ + self.crs_transformer.pixel_to_map(p, bbox=self.bbox) + for p in polys + ] geojson = geoms_to_geojson(polys) out_uri = vo.get_uri(vector_output_dir, self.class_config) json_to_file(geojson, out_uri) diff --git a/rastervision_core/rastervision/core/data/label_store/utils.py b/rastervision_core/rastervision/core/data/label_store/utils.py index cdd6b484f..74911f057 100644 --- a/rastervision_core/rastervision/core/data/label_store/utils.py +++ b/rastervision_core/rastervision/core/data/label_store/utils.py @@ -16,8 +16,8 @@ def boxes_to_geojson( class_ids: Sequence[int], crs_transformer: 'CRSTransformer', class_config: 'ClassConfig', - scores: Optional[Sequence[Union[float, Sequence[float]]]] = None -) -> dict: + scores: Optional[Sequence[Union[float, Sequence[float]]]] = None, + bbox: Optional['Box'] = None) -> dict: """Convert boxes and associated data into a GeoJSON dict. Args: @@ -30,6 +30,8 @@ def boxes_to_geojson( Optional list of score or scores. If floats (one for each box), property name will be "score". If lists of floats, property name will be "scores". Defaults to None. + bbox (Optional[Box]): User-specified crop of the extent. Must be + provided if the corresponding RasterSource has bbox != extent. Returns: dict: Serialized GeoJSON. @@ -46,7 +48,10 @@ def boxes_to_geojson( boxes, desc='Transforming boxes to map coords', delay=PROGRESSBAR_DELAY_SEC) as bar: - geoms = [crs_transformer.pixel_to_map(box.to_shapely()) for box in bar] + geoms = [ + crs_transformer.pixel_to_map(box.to_shapely(), bbox=bbox) + for box in bar + ] # add box properties (ID and name of predicted class) with tqdm( From c5bbc9cc82608c5b7e5d587eda5a440d710794a1 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Fri, 29 Sep 2023 12:00:10 -0400 Subject: [PATCH 02/15] use StatsTransformer in the spacenet vegas examples --- .../examples/semantic_segmentation/spacenet_vegas.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/spacenet_vegas.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/spacenet_vegas.py index 05b829159..fb679488d 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/spacenet_vegas.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/spacenet_vegas.py @@ -98,7 +98,9 @@ def build_scene(spacenet_cfg: SpacenetConfig, label_uri = spacenet_cfg.get_geojson_uri(id) raster_source = RasterioSourceConfig( - uris=[image_uri], channel_order=channel_order) + uris=[image_uri], + channel_order=channel_order, + transformers=[StatsTransformerConfig()]) # Set a line buffer to convert line strings to polygons. vector_source = GeoJSONVectorSourceConfig( From 97b621641d40345b3d01f0c73ce52720ac4eb19b Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Fri, 29 Sep 2023 12:04:49 -0400 Subject: [PATCH 03/15] fix bug in bundle command when using analyze stage --- .../rastervision/core/rv_pipeline/rv_pipeline.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline.py b/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline.py index 9e972bd95..bff0a9a67 100644 --- a/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline.py +++ b/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline.py @@ -17,7 +17,7 @@ from rastervision.core.rv_pipeline import TRAIN, VALIDATION from rastervision.pipeline.file_system.utils import ( download_if_needed, zipdir, get_local_path, upload_or_copy, make_dir, - sync_to_dir, file_exists) + sync_from_dir, file_exists) log = logging.getLogger(__name__) @@ -252,8 +252,8 @@ def bundle(self): shutil.copy(path, join(bundle_dir, fn)) if file_exists(self.config.analyze_uri, include_dir=True): - sync_to_dir(self.config.analyze_uri, join( - bundle_dir, 'analyze')) + analyze_dst = join(bundle_dir, 'analyze') + sync_from_dir(self.config.analyze_uri, analyze_dst) path = download_if_needed(self.config.get_config_uri(), tmp_dir) shutil.copy(path, join(bundle_dir, 'pipeline-config.json')) From 56b97bf59a4c0947175ec911ad5f76fbb7dff531 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Fri, 29 Sep 2023 14:42:27 -0400 Subject: [PATCH 04/15] Read TIFF chips using rasterio instead of PIL to allow reading non-RGB TIFF chips (#1932) * read TIFF chips using rasterio instead of PIL To allow reading non-rgb TIFFs. * add unit test --------- Co-authored-by: Adeel Hassan --- .../pytorch_learner/dataset/utils/utils.py | 5 +++++ tests/pytorch_learner/dataset/utils/test_utils.py | 10 ++++++++++ 2 files changed, 15 insertions(+) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/utils/utils.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/utils/utils.py index 25953ae5e..c46420804 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/utils/utils.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/utils/utils.py @@ -7,6 +7,7 @@ import numpy as np from torchvision.datasets.folder import (IMG_EXTENSIONS, DatasetFolder) from PIL import Image +import rasterio as rio IMG_EXTENSIONS = tuple([*IMG_EXTENSIONS, '.npy']) @@ -37,6 +38,10 @@ def load_image(path: PathLike) -> np.ndarray: ext = splitext(path)[-1] if ext == '.npy': img = np.load(path) + elif ext == '.tif' or ext == '.tiff': + with rio.open(path, 'r') as f: + img = f.read() + img = img.transpose(1, 2, 0) else: img = np.array(Image.open(path)) diff --git a/tests/pytorch_learner/dataset/utils/test_utils.py b/tests/pytorch_learner/dataset/utils/test_utils.py index c71594958..310c55ace 100644 --- a/tests/pytorch_learner/dataset/utils/test_utils.py +++ b/tests/pytorch_learner/dataset/utils/test_utils.py @@ -4,9 +4,11 @@ from tempfile import TemporaryDirectory import numpy as np +import rasterio as rio from torchvision.datasets.folder import DatasetFolder from rastervision.pipeline.file_system import get_tmp_dir +from rastervision.core.data.utils import write_window from rastervision.pytorch_learner.dataset import (discover_images, load_image, make_image_folder_dataset) from rastervision.pytorch_backend.pytorch_learner_backend import write_chip @@ -53,6 +55,14 @@ def test_load_image(self): write_chip(chip, path) np.testing.assert_array_equal(load_image(path), chip) + chip = np.random.randint( + 0, 256, size=(100, 100, 8), dtype=np.uint8) + path = join(tmp_dir, '4.tif') + profile = dict(height=100, width=100, count=8, dtype=np.uint8) + with rio.open(path, 'w', **profile) as ds: + write_window(ds, chip) + np.testing.assert_array_equal(load_image(path), chip) + def test_make_image_folder_dataset(self): with get_tmp_dir() as tmp_dir: with TemporaryDirectory(dir=tmp_dir) as dir_a, TemporaryDirectory( From 24780ad3ea258416b0b2286c363d84eea5f3b3c1 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Fri, 29 Sep 2023 12:20:25 -0400 Subject: [PATCH 05/15] allow specifying chip_sz in StatsTransform.from_raster_sources() --- .../core/data/raster_transformer/stats_transformer.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py b/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py index 0560b61a5..458bcb08c 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py @@ -97,7 +97,8 @@ def transform(self, def from_raster_sources(cls, raster_sources: List['RasterSource'], sample_prob: Optional[float] = 0.1, - max_stds: float = 3.) -> 'StatsTransformer': + max_stds: float = 3., + chip_sz: int = 300) -> 'StatsTransformer': """Build with stats from the given raster sources. Args: @@ -113,7 +114,10 @@ def from_raster_sources(cls, StatsTransformer: A StatsTransformer. """ stats = RasterStats() - stats.compute(raster_sources=raster_sources, sample_prob=sample_prob) + stats.compute( + raster_sources=raster_sources, + sample_prob=sample_prob, + chip_sz=chip_sz) stats_transformer = StatsTransformer.from_raster_stats( stats, max_stds=max_stds) return stats_transformer From 6f61fc136de31040043ec1f829016a8c430fadff Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Fri, 29 Sep 2023 12:19:45 -0400 Subject: [PATCH 06/15] fix typo in StatsTransformer.__repr__() --- .../core/data/raster_transformer/stats_transformer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py b/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py index 458bcb08c..e8e81a4b5 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py @@ -166,4 +166,4 @@ def stats(self) -> RasterStats: def __repr__(self) -> str: return repr_with_args( - self, means=self.means, std=self.stds, max_stds=self.max_stds) + self, means=self.means, stds=self.stds, max_stds=self.max_stds) From 969b5897c3a8e414fdb088c7c6ed8636f0acb030 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Fri, 29 Sep 2023 11:59:47 -0400 Subject: [PATCH 07/15] rename variable in test.py --- .../rastervision/pytorch_backend/examples/test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py index 4a41312fc..18c0590be 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/test.py @@ -11,13 +11,13 @@ file_to_json, sync_from_dir, upload_or_copy, download_or_copy, file_exists, sync_to_dir, NotReadableError, download_if_needed) -NEW_VERSION = '0.21' +NEW_VERSION_FULL = '0.21.2' NEW_VERSION_MAJOR_MINOR = '0.21' EXAMPLES_MODULE_ROOT = 'rastervision.pytorch_backend.examples' EXAMPLES_PATH_ROOT = '/opt/src/rastervision_pytorch_backend/rastervision/pytorch_backend/examples' # noqa -REMOTE_PROCESSED_ROOT = f's3://raster-vision/examples/{NEW_VERSION}/processed-data' -REMOTE_OUTPUT_ROOT = f's3://raster-vision/examples/{NEW_VERSION}/output' +REMOTE_PROCESSED_ROOT = f's3://raster-vision/examples/{NEW_VERSION_FULL}/processed-data' +REMOTE_OUTPUT_ROOT = f's3://raster-vision/examples/{NEW_VERSION_FULL}/output' LOCAL_RAW_ROOT = '/opt/data/raw-data' LOCAL_PROCESSED_ROOT = '/opt/data/examples/processed-data' LOCAL_OUTPUT_ROOT = '/opt/data/examples/output' From 5dc855f18ca3bc660a4479109a5f9342ff623098 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Fri, 29 Sep 2023 14:55:40 -0400 Subject: [PATCH 08/15] update release instructions to simplify patch release process --- docs/release.rst | 27 +++++++++++++++++++++------ 1 file changed, 21 insertions(+), 6 deletions(-) diff --git a/docs/release.rst b/docs/release.rst index 2214011ff..07e023df4 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -60,6 +60,13 @@ Minor or Major Version Release docker push quay.io/azavea/raster-vision:pytorch- #. Make a GitHub `tag `_ and `release `_ using the previous release as a template. +#. Remove artifacts from previous builds. From the repo root: + + .. code-block:: console + + rm -rf build/ dist/ *.egg-info + rm -rf rastervision_*/build rastervision_*/dist rastervision_*/*.egg-info + #. Publish all packages to PyPI. This step requires `twine `__ which you can install with .. code-block:: console @@ -105,12 +112,20 @@ Minor or Major Version Release #. Announce the new release in our `forum `_, and with a blog post if it's a big release. #. Make a PR to the master branch that updates the version number to the next development version, ``X.Y.Z-dev``. For example, if the last release was ``0.20.1``, update the version to ``0.20.2-dev``. -Bug Fix Release +Patch Release ----------------- -This describes how to create a new bug fix release, using incrementing from 0.8.0 to 0.8.1 as an example. This assumes that there is already a branch for a minor release called ``0.8``. +This describes how to create a new patch release (AKA a bug-fix release), using an increment from 0.8.0 to 0.8.1 as an example. This assumes that there is already a branch for a minor release called ``0.8``. + +#. Backport changes to the ``0.8`` branch. To create a patch release (version 0.8.1), we need to backport all the commits on the ``master`` branch that have been added since the last patch release onto the ``0.8`` branch. To do this: + + #. Create a new branch from the ``0.8`` branch. Let's call it ``backport``. + #. Cherry-pick each commit that we want to include from the ``master`` branch onto the ``backport`` branch. + #. Make a PR against the ``0.8`` branch from the ``backport`` branch. The title of the PR should start with ``[BACKPORT]``. +#. Update changelog and version on the ``0.8`` branch. Make and merge a PR against ``0.8`` (but not ``master``) that adds a changelog for the new release and increments the version to ``0.8.1`` throughout the repo. Wait for the ``0.8`` branch to be built by GitHub Actions and the ``0.8`` Docker images to be published to Quay. If that is successful, we can proceed to the next steps of actually publishing a release. +#. Publish the new version to PyPI. Follow the same instructions for PyPI as listed above for minor/major version releases. +#. Using the GitHub UI, make a new release. Use ``v0.8.1`` as the tag, and the ``0.8`` branch as the target. +#. Update changelog and version on the ``master`` branch. Make and merge a PR against ``master`` that -#. To create a bug fix release (version 0.8.1), we need to backport all the bug fix commits on the ``master`` branch that have been added since the last bug fix release onto the ``0.8`` branch. For each bug fix PR on ``master``, we need to create a PR against the ``0.8`` branch based on a branch of ``0.8`` that has cherry-picked the commits from the original PR. The title of the PR should start with [BACKPORT]. -#. Make and merge a PR against ``0.8`` (but not ``master``) that increments the version in each ``setup.py`` file to ``0.8.1``. Then wait for the ``0.8`` branch to be built by GitHub Actions and the ``0.8`` Docker images to be published to Quay. If that is successful, we can proceed to the next steps of actually publishing a release. -#. Using the GitHub UI, make a new release. Use ``0.8.1`` as the tag, and the ``0.8`` branch as the target. -#. Publish the new version to PyPI. Follow the same instructions for PyPI that are listed above for minor/major version releases. + * includes the cherry-picked commit that updates the changelog for ``0.8.1`` and + * increments the version to ``0.8.2-dev`` throughout the repo. From a85990ef8f285c8fbed778d39e5b8c8620dd003e Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Mon, 2 Oct 2023 13:06:46 -0400 Subject: [PATCH 09/15] fix PyPI capitalization --- scripts/pypi_publish | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pypi_publish b/scripts/pypi_publish index 46c9839d3..7c70babf9 100755 --- a/scripts/pypi_publish +++ b/scripts/pypi_publish @@ -77,12 +77,12 @@ fi if [[ "$1" == "-y" ]]; then response="y" else - read -r -p "Actually publish to PyPi? (y/N): " response + read -r -p "Actually publish to PyPI? (y/N): " response fi case "$response" in [yY][eE][sS]|[yY]) - echo "Publishing to PyPi..." + echo "Publishing to PyPI..." publish_all ;; *) From 45646ccf8f2565aff6a04f6a829c10fc07c99160 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Wed, 4 Oct 2023 16:24:14 -0400 Subject: [PATCH 10/15] free up disk space on CI machine --- .github/workflows/continuous_integration.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/continuous_integration.yml b/.github/workflows/continuous_integration.yml index 35a253671..63d13fd40 100644 --- a/.github/workflows/continuous_integration.yml +++ b/.github/workflows/continuous_integration.yml @@ -18,6 +18,10 @@ jobs: - run: df -hT + - run: rm -rf /opt/hostedtoolcache + + - run: df -hT + - run: ./scripts/cibuild - run: df -hT From 7ebfcb9f30a621e65170a5a94f89cb0785d0ba9a Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Wed, 4 Oct 2023 16:25:16 -0400 Subject: [PATCH 11/15] fix indentation --- scripts/cibuild | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/cibuild b/scripts/cibuild index 413465741..4e39a5f09 100755 --- a/scripts/cibuild +++ b/scripts/cibuild @@ -18,12 +18,12 @@ if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then usage else DOCKER_BUILDKIT=1 docker build \ - --build-arg BUILDKIT_INLINE_CACHE=1 \ - --build-arg BUILD_TYPE=fullbuild \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --build-arg BUILD_TYPE=fullbuild \ --platform linux/amd64 \ --build-arg CUDA_VERSION="12.1.1" \ --build-arg UBUNTU_VERSION="22.04" \ - --cache-from=quay.io/azavea/raster-vision:pytorch-latest \ + --cache-from=quay.io/azavea/raster-vision:pytorch-latest \ -t "raster-vision-${IMAGE_TYPE}" \ -f Dockerfile . fi From bc40af48a1a4269bf6b2ba77019ca17d01f65219 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Thu, 5 Oct 2023 10:10:43 -0400 Subject: [PATCH 12/15] Bump pillow to address CVE-2023-4863 (#1952) * Bump pillow from 9.3.0 to 10.0.1 in /rastervision_core Bumps [pillow](https://github.com/python-pillow/Pillow) from 9.3.0 to 10.0.1. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/9.3.0...10.0.1) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production ... Signed-off-by: dependabot[bot] * Bump pillow from 9.3.0 to 10.0.1 in /rastervision_pytorch_learner Bumps [pillow](https://github.com/python-pillow/Pillow) from 9.3.0 to 10.0.1. - [Release notes](https://github.com/python-pillow/Pillow/releases) - [Changelog](https://github.com/python-pillow/Pillow/blob/main/CHANGES.rst) - [Commits](https://github.com/python-pillow/Pillow/compare/9.3.0...10.0.1) --- updated-dependencies: - dependency-name: pillow dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- rastervision_core/requirements.txt | 2 +- rastervision_pytorch_learner/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rastervision_core/requirements.txt b/rastervision_core/requirements.txt index 99c4fcff2..4a7970c11 100644 --- a/rastervision_core/requirements.txt +++ b/rastervision_core/requirements.txt @@ -2,7 +2,7 @@ rastervision_pipeline==0.21.2 shapely==2.0.1 geopandas==0.13.2 numpy==1.25.0 -pillow==9.3.0 +pillow==10.0.1 pyproj==3.4.0 rasterio==1.3.7 pystac==1.6.1 diff --git a/rastervision_pytorch_learner/requirements.txt b/rastervision_pytorch_learner/requirements.txt index 3cf0aff75..4af6e8413 100644 --- a/rastervision_pytorch_learner/requirements.txt +++ b/rastervision_pytorch_learner/requirements.txt @@ -1,7 +1,7 @@ rastervision_pipeline==0.21.2 rastervision_core==0.21.2 numpy==1.25.0 -pillow==9.3.0 +pillow==10.0.1 torch==2.0.1 torchvision==0.15.2 tensorboard==2.13.0 From fe570999c15a498197008645d23053bdd98cc63e Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Thu, 5 Oct 2023 09:54:05 -0400 Subject: [PATCH 13/15] fix: ensure SS datasets always return label array with correct dtype --- .../pytorch_learner/dataset/transform.py | 3 ++- .../pytorch_learner/dataset/test_transform.py | 26 ++++++++++++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/transform.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/transform.py index baf09b4ed..82f6c05b6 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/transform.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/transform.py @@ -227,7 +227,8 @@ def semantic_segmentation_transformer( y = np.array(y) out = apply_transform(transform, image=x, mask=y) x, y = out['image'], out['mask'] - y = y.astype(int) + if y is not None: + y = y.astype(int) return x, y diff --git a/tests/pytorch_learner/dataset/test_transform.py b/tests/pytorch_learner/dataset/test_transform.py index 653b9ed42..73c3eb9c8 100644 --- a/tests/pytorch_learner/dataset/test_transform.py +++ b/tests/pytorch_learner/dataset/test_transform.py @@ -4,7 +4,8 @@ import albumentations as A from rastervision.pytorch_learner.dataset.transform import ( - yxyx_to_albu, albu_to_yxyx, xywh_to_albu, apply_transform) + yxyx_to_albu, albu_to_yxyx, xywh_to_albu, apply_transform, + semantic_segmentation_transformer) class TestTransforms(unittest.TestCase): @@ -65,6 +66,29 @@ def test_box_format_conversions_xywh(self): boxes_albu = xywh_to_albu(boxes, (10, 10)) np.testing.assert_allclose(boxes_albu, boxes_albu_gt) + def test_semantic_segmentation_transformer(self): + # w/ y, w/o transform + x_in, y_in = np.zeros((10, 10, 3), dtype=np.uint8), np.zeros((10, 10)) + x_out, y_out = semantic_segmentation_transformer((x_in, y_in), None) + np.issubdtype(y_out.dtype, int) + + # w/ y, w/ transform + x_out, y_out = semantic_segmentation_transformer((x_in, y_in), + A.Resize(20, 20)) + self.assertEqual(x_out.shape, (20, 20, 3)) + self.assertEqual(y_out.shape, (20, 20)) + np.issubdtype(y_out.dtype, int) + + # w/o y, w/o transform + x_out, y_out = semantic_segmentation_transformer((x_in, None), None) + self.assertIsNone(y_out) + + # w/o y, w/ transform + x_out, y_out = semantic_segmentation_transformer((x_in, None), + A.Resize(20, 20)) + self.assertEqual(x_out.shape, (20, 20, 3)) + self.assertIsNone(y_out) + if __name__ == '__main__': unittest.main() From 518c7d6e8200bbfb54d4c5a0f796c0a1776a33a6 Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Thu, 12 Oct 2023 09:29:09 -0400 Subject: [PATCH 14/15] fix bug in visualizer when plotting temporal data w/ batch size 1 --- .../pytorch_learner/dataset/visualizer/visualizer.py | 3 ++- .../visualizer/test_semantic_segmentation_visualizer.py | 4 ++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/visualizer.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/visualizer.py index 1a33bf1c4..16546bdaa 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/visualizer.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/visualizer.py @@ -119,7 +119,8 @@ def plot_batch(self, batch_sz, T, *_ = x.shape params['fig_args']['figsize'][1] *= T fig = plt.figure(**params['fig_args']) - subfigs = fig.subfigures(nrows=batch_sz, ncols=1, hspace=0.0) + subfigs = fig.subfigures( + nrows=batch_sz, ncols=1, hspace=0.0, squeeze=False) subfig_axs = [ subfig.subplots( nrows=T, ncols=params['subplot_args']['ncols']) diff --git a/tests/pytorch_learner/dataset/visualizer/test_semantic_segmentation_visualizer.py b/tests/pytorch_learner/dataset/visualizer/test_semantic_segmentation_visualizer.py index e7e44272e..edd73fa81 100644 --- a/tests/pytorch_learner/dataset/visualizer/test_semantic_segmentation_visualizer.py +++ b/tests/pytorch_learner/dataset/visualizer/test_semantic_segmentation_visualizer.py @@ -40,6 +40,8 @@ def test_plot_batch_temporal(self): x = torch.randn(size=(2, 3, 4, 256, 256)) y = (torch.randn(size=(2, 256, 256)) > 0).long() self.assertNoError(lambda: viz.plot_batch(x, y)) + # w/o z, batch size = 1 + self.assertNoError(lambda: viz.plot_batch(x[[0]], y[[0]])) # w/ z viz = SemanticSegmentationVisualizer( @@ -50,3 +52,5 @@ def test_plot_batch_temporal(self): y = (torch.randn(size=(2, 256, 256)) > 0).long() z = torch.randn(size=(2, num_classes, 256, 256)).softmax(dim=-3) self.assertNoError(lambda: viz.plot_batch(x, y, z=z)) + # w/ z, batch size = 1 + self.assertNoError(lambda: viz.plot_batch(x[[0]], y[[0]])) From 781d9d2a5b065a73e3fdb86b78bc1026206c1e9c Mon Sep 17 00:00:00 2001 From: Adeel Hassan Date: Thu, 12 Oct 2023 11:25:47 -0400 Subject: [PATCH 15/15] free up disk space in release workflow --- .github/workflows/release.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2cad782f4..c62b8f190 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,8 @@ jobs: steps: - uses: actions/checkout@v4 + - run: rm -rf /opt/hostedtoolcache + - run: ./scripts/cibuild - run: docker system prune -f