diff --git a/cookiecutter_template/{{cookiecutter.project_name}}/rastervision_{{cookiecutter.project_name}}/rastervision/{{cookiecutter.project_name}}/test_pipeline.py b/cookiecutter_template/{{cookiecutter.project_name}}/rastervision_{{cookiecutter.project_name}}/rastervision/{{cookiecutter.project_name}}/test_pipeline.py index 731cd5241..946592672 100644 --- a/cookiecutter_template/{{cookiecutter.project_name}}/rastervision_{{cookiecutter.project_name}}/rastervision/{{cookiecutter.project_name}}/test_pipeline.py +++ b/cookiecutter_template/{{cookiecutter.project_name}}/rastervision_{{cookiecutter.project_name}}/rastervision/{{cookiecutter.project_name}}/test_pipeline.py @@ -1,10 +1,8 @@ -from typing import List - from rastervision.pipeline.pipeline import Pipeline class TestPipeline(Pipeline): - commands: List[str] = ['print_msg'] + commands: list[str] = ['print_msg'] def print_msg(self): print(self.config.message) diff --git a/docs/conf.py b/docs/conf.py index 9ea4cd0c7..f9dce39a3 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -6,7 +6,7 @@ # full list see the documentation: # http://www.sphinx-doc.org/en/stable/config -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING import sys from unittest.mock import MagicMock @@ -21,13 +21,13 @@ def __getattr__(cls, name): return MagicMock() -MOCK_MODULES = ['pyproj', 'h5py', 'osgeo'] +MOCK_MODULES = ['h5py', 'osgeo'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) # -- Allow Jinja templates in non-template .rst files ------------------------- -def rstjinja(app: 'Sphinx', docname: str, source: List[str]) -> None: +def rstjinja(app: 'Sphinx', docname: str, source: list[str]) -> None: """Allow use of jinja templating in all doc pages. Adapted from: @@ -124,7 +124,7 @@ def setup(app: 'Sphinx') -> None: autodoc_typehints = 'both' autodoc_class_signature = 'separated' autodoc_member_order = 'groupwise' -autodoc_mock_imports = ['torch', 'torchvision', 'pycocotools', 'geopandas'] +autodoc_mock_imports = ['pycocotools'] ######################### ######################### diff --git a/integration_tests/integration_tests.py b/integration_tests/integration_tests.py index 5633c59cd..5cd1cfd28 100755 --- a/integration_tests/integration_tests.py +++ b/integration_tests/integration_tests.py @@ -1,6 +1,5 @@ #!/usr/bin/env python -from typing import List from os.path import join, dirname, abspath, isfile import math import traceback @@ -108,7 +107,7 @@ def get_actual_eval_path(test_id: str, tmp_dir: str) -> str: def check_eval_item(test_id: str, test_cfg: dict, expected_item: dict, - actual_item: dict) -> List[TestError]: + actual_item: dict) -> list[TestError]: errors = [] f1_threshold = 0.05 class_name = expected_item['class_name'] @@ -125,7 +124,7 @@ def check_eval_item(test_id: str, test_cfg: dict, expected_item: dict, return errors -def check_eval(test_id: str, test_cfg: dict, tmp_dir: str) -> List[TestError]: +def check_eval(test_id: str, test_cfg: dict, tmp_dir: str) -> list[TestError]: errors = [] actual_eval_path = get_actual_eval_path(test_id, tmp_dir) @@ -152,7 +151,7 @@ def check_eval(test_id: str, test_cfg: dict, tmp_dir: str) -> List[TestError]: def test_model_bundle_validation(pipeline, test_id: str, test_cfg: dict, tmp_dir: str, - image_uri: str) -> List[TestError]: + image_uri: str) -> list[TestError]: console_info('Checking predict command validation...') errors = [] model_bundle_uri = pipeline.get_model_bundle_uri() @@ -171,7 +170,7 @@ def test_model_bundle_validation(pipeline, test_id: str, test_cfg: dict, def test_model_bundle_results(pipeline, test_id: str, test_cfg: dict, tmp_dir: str, scenes: list, - scenes_to_uris: dict) -> List[TestError]: + scenes_to_uris: dict) -> list[TestError]: console_info('Checking model bundle produces same results...') errors = [] model_bundle_uri = pipeline.get_model_bundle_uri() @@ -213,7 +212,7 @@ def test_model_bundle(pipeline, test_id: str, test_cfg: dict, tmp_dir: str, - check_channel_order: bool = False) -> List[TestError]: + check_channel_order: bool = False) -> list[TestError]: # Check the model bundle. # This will only work with raster_sources that # have a single URI. @@ -250,7 +249,7 @@ def test_model_bundle(pipeline, return errors -def run_test(test_id: str, test_cfg: dict, tmp_dir: str) -> List[TestError]: +def run_test(test_id: str, test_cfg: dict, tmp_dir: str) -> list[TestError]: msg = f'\nRunning test: {test_id}' console_info(msg, bold=True) console_info('With params:') diff --git a/rastervision_aws_batch/rastervision/aws_batch/aws_batch_runner.py b/rastervision_aws_batch/rastervision/aws_batch/aws_batch_runner.py index 800dbcf7f..2f2909888 100644 --- a/rastervision_aws_batch/rastervision/aws_batch/aws_batch_runner.py +++ b/rastervision_aws_batch/rastervision/aws_batch/aws_batch_runner.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +from typing import TYPE_CHECKING, Any import logging import os import uuid @@ -33,7 +33,7 @@ class AWSBatchRunner(Runner): def run(self, cfg_json_uri: str, pipeline: 'Pipeline', - commands: List[str], + commands: list[str], num_splits: int = 1, pipeline_run_name: str = 'raster-vision'): # pragma: no cover parent_job_ids = [] @@ -65,7 +65,7 @@ def build_cmd(self, pipeline: 'Pipeline', num_splits: int = 1, pipeline_run_name: str = 'raster-vision' - ) -> Tuple[List[str], Dict[str, Any]]: + ) -> tuple[list[str], dict[str, Any]]: verbosity = rv_config.get_verbosity_cli_opt() @@ -105,15 +105,15 @@ def get_split_ind(self) -> int: return int(os.environ.get('AWS_BATCH_JOB_ARRAY_INDEX', 0)) def run_command(self, - cmd: List[str], - job_name: Optional[str] = None, + cmd: list[str], + job_name: str | None = None, debug: bool = False, attempts: int = 1, - parent_job_ids: Optional[List[str]] = None, - num_array_jobs: Optional[int] = None, + parent_job_ids: list[str] | None = None, + num_array_jobs: int | None = None, use_gpu: bool = False, - job_queue: Optional[str] = None, - job_def: Optional[str] = None, + job_queue: str | None = None, + job_def: str | None = None, **kwargs) -> str: # pragma: no cover """Submit a command as a job to AWS Batch. diff --git a/rastervision_aws_s3/rastervision/aws_s3/s3_file_system.py b/rastervision_aws_s3/rastervision/aws_s3/s3_file_system.py index 4c33deb78..12cacfa6c 100644 --- a/rastervision_aws_s3/rastervision/aws_s3/s3_file_system.py +++ b/rastervision_aws_s3/rastervision/aws_s3/s3_file_system.py @@ -1,4 +1,4 @@ -from typing import Any, Iterator, Tuple +from typing import Any, Iterator import io import os import subprocess @@ -132,7 +132,7 @@ def matches_uri(uri: str, mode: str) -> bool: return parsed_uri.scheme == 's3' @staticmethod - def parse_uri(uri: str) -> Tuple[str, str]: + def parse_uri(uri: str) -> tuple[str, str]: """Parse bucket name and key from an S3 URI.""" parsed_uri = urlparse(uri) bucket, key = parsed_uri.netloc, parsed_uri.path[1:] diff --git a/rastervision_aws_sagemaker/rastervision/aws_sagemaker/aws_sagemaker_runner.py b/rastervision_aws_sagemaker/rastervision/aws_sagemaker/aws_sagemaker_runner.py index 88ab7c958..198e8252b 100644 --- a/rastervision_aws_sagemaker/rastervision/aws_sagemaker/aws_sagemaker_runner.py +++ b/rastervision_aws_sagemaker/rastervision/aws_sagemaker/aws_sagemaker_runner.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING from os.path import join, basename import logging from pprint import pprint @@ -67,9 +67,9 @@ class AWSSageMakerRunner(Runner): def run(self, cfg_json_uri: str, pipeline: 'Pipeline', - commands: List[str], + commands: list[str], num_splits: int = 1, - cmd_prefix: List[str] = [ + cmd_prefix: list[str] = [ 'python', '-m', 'rastervision.pipeline.cli' ], pipeline_run_name: str = 'rv'): @@ -95,9 +95,9 @@ def run(self, def build_pipeline(self, cfg_json_uri: str, pipeline: 'Pipeline', - commands: List[str], + commands: list[str], num_splits: int = 1, - cmd_prefix: List[str] = [ + cmd_prefix: list[str] = [ 'python', '-m', 'rastervision.pipeline.cli' ], pipeline_run_name: str = 'rv') -> 'SageMakerPipeline': @@ -213,7 +213,7 @@ def build_step(self, pipeline: 'RVPipeline', step_name: str, job_name: str, - cmd: List[str], + cmd: list[str], role: str, image_uri: str, instance_type: str, @@ -222,7 +222,7 @@ def build_step(self, instance_count: int = 1, max_wait: int = DEFAULT_MAX_RUN_TIME, max_run: int = DEFAULT_MAX_RUN_TIME, - **kwargs) -> Union['TrainingStep', 'ProcessingStep']: + **kwargs) -> 'TrainingStep | ProcessingStep': """Build appropriate SageMaker pipeline step. If ``step_name=='train'``, builds a :class:`.TrainingStep`. Otherwise, @@ -247,8 +247,7 @@ def build_step(self, max_run=max_run, **kwargs, ) - step_args: Optional['_JobStepArguments'] = estimator.fit( - wait=False) + step_args: '_JobStepArguments | None' = estimator.fit(wait=False) step = TrainingStep(job_name, step_args=step_args) else: from sagemaker.processing import Processor @@ -263,38 +262,38 @@ def build_step(self, entrypoint=cmd, **kwargs, ) - step_args: Optional['_JobStepArguments'] = step_processor.run( + step_args: '_JobStepArguments | None' = step_processor.run( wait=False) step = ProcessingStep(job_name, step_args=step_args) return step def run_command(self, - cmd: List[str], + cmd: list[str], use_gpu: bool = False, - image_uri: Optional[str] = None, - instance_type: Optional[str] = None, - role: Optional[str] = None, - job_name: Optional[str] = None, - sagemaker_session: Optional['Session'] = None) -> None: + image_uri: str | None = None, + instance_type: str | None = None, + role: str | None = None, + job_name: str | None = None, + sagemaker_session: 'Session | None' = None) -> None: """Run a single command as a SageMaker processing job. Args: - cmd (List[str]): The command to run. + cmd (list[str]): The command to run. use_gpu (bool): Use the GPU instance type and image from the Everett config. This is ignored if image_uri and instance_type are provided. Defaults to False. - image_uri (Optional[str]): URI of docker image to use. If not + image_uri (str | None): URI of docker image to use. If not provided, will be picked up from Everett config. Defaults to None. - instance_type (Optional[str]): AWS instance type to use. If not + instance_type (str | None): AWS instance type to use. If not provided, will be picked up from Everett config. Defaults to None. - role (Optional[str]): AWS IAM role with SageMaker permissions. If + role (str | None): AWS IAM role with SageMaker permissions. If not provided, will be picked up from Everett config. Defaults to None. - job_name (Optional[str]): Optional job name. Defaults to None. - sagemaker_session (Optional[Session]): SageMaker session. + job_name (str | None): Optional job name. Defaults to None. + sagemaker_session (Session | None): SageMaker session. Defaults to None. """ from sagemaker.processing import Processor @@ -331,8 +330,8 @@ def _build_pytorch_estimator(self, sagemaker_session: 'PipelineSession', use_spot_instances: bool = False, instance_count: int = 1, - distribution: Optional[dict] = None, - job_name: Optional[str] = None, + distribution: dict | None = None, + job_name: str | None = None, **kwargs): from sagemaker.pytorch import PyTorch from rastervision.aws_s3.s3_file_system import S3FileSystem diff --git a/rastervision_core/rastervision/core/analyzer/analyzer.py b/rastervision_core/rastervision/core/analyzer/analyzer.py index aabd8b697..fea75dc5f 100644 --- a/rastervision_core/rastervision/core/analyzer/analyzer.py +++ b/rastervision_core/rastervision/core/analyzer/analyzer.py @@ -1,4 +1,3 @@ -from typing import List from abc import (ABC, abstractmethod) from rastervision.core.data import Scene @@ -11,5 +10,5 @@ class Analyzer(ABC): """ @abstractmethod - def process(self, scenes: List[Scene], tmp_dir: str): + def process(self, scenes: list[Scene], tmp_dir: str): """Process scenes and save result.""" diff --git a/rastervision_core/rastervision/core/analyzer/analyzer_config.py b/rastervision_core/rastervision/core/analyzer/analyzer_config.py index 40291938b..09ad0ae93 100644 --- a/rastervision_core/rastervision/core/analyzer/analyzer_config.py +++ b/rastervision_core/rastervision/core/analyzer/analyzer_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Iterable from rastervision.pipeline.config import register_config, Config @@ -10,11 +10,11 @@ class AnalyzerConfig(Config): """Configure an :class:`.Analyzer`.""" - def build(self, scene_group: Optional[Tuple[str, Iterable[str]]] = None + def build(self, scene_group: tuple[str, Iterable[str]] | None = None ) -> 'Analyzer': pass - def get_bundle_filenames(self) -> List[str]: + def get_bundle_filenames(self) -> list[str]: """Returns the names of files that should be included in a model bundle. The files are assumed to be in the analyze/ directory generated by the analyze diff --git a/rastervision_core/rastervision/core/analyzer/stats_analyzer.py b/rastervision_core/rastervision/core/analyzer/stats_analyzer.py index 7f24623d6..9846badcc 100644 --- a/rastervision_core/rastervision/core/analyzer/stats_analyzer.py +++ b/rastervision_core/rastervision/core/analyzer/stats_analyzer.py @@ -1,4 +1,4 @@ -from typing import Iterable, Optional +from typing import Iterable from rastervision.core.analyzer import Analyzer from rastervision.core.raster_stats import RasterStats @@ -9,10 +9,10 @@ class StatsAnalyzer(Analyzer): """Compute imagery statistics of scenes.""" def __init__(self, - stats_uri: Optional[str] = None, + stats_uri: str | None = None, sample_prob: float = 0.1, chip_sz: int = 300, - nodata_value: Optional[float] = 0): + nodata_value: float | None = 0): self.stats_uri = stats_uri self.sample_prob = sample_prob self.chip_sz = chip_sz diff --git a/rastervision_core/rastervision/core/analyzer/stats_analyzer_config.py b/rastervision_core/rastervision/core/analyzer/stats_analyzer_config.py index efdc2a5ee..2a1f47001 100644 --- a/rastervision_core/rastervision/core/analyzer/stats_analyzer_config.py +++ b/rastervision_core/rastervision/core/analyzer/stats_analyzer_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Iterable from os.path import join from rastervision.pipeline.config import register_config, ConfigError, Field @@ -16,13 +16,13 @@ class StatsAnalyzerConfig(AnalyzerConfig): be used to normalize chips read from them. """ - output_uri: Optional[str] = Field( + output_uri: str | None = Field( None, description='URI of directory where stats will be saved. ' 'Stats for a scene-group will be save in a JSON file at ' '//stats.json. If None, and this Config ' 'is part of an RVPipeline, this field will be auto-generated.') - sample_prob: Optional[float] = Field( + sample_prob: float | None = Field( 0.1, description=( 'The probability of using a random window for computing statistics. ' @@ -31,12 +31,12 @@ class StatsAnalyzerConfig(AnalyzerConfig): 300, description='Chip size to use when sampling chips to compute stats ' 'from.') - nodata_value: Optional[float] = Field( + nodata_value: float | None = Field( 0, description='NODATA value. If set, these pixels will be ignored when ' 'computing stats.') - def update(self, pipeline: Optional['RVPipelineConfig'] = None) -> None: + def update(self, pipeline: 'RVPipelineConfig | None' = None) -> None: if pipeline is not None and self.output_uri is None: self.output_uri = join(pipeline.analyze_uri, 'stats') @@ -44,7 +44,7 @@ def validate_config(self): if self.sample_prob > 1 or self.sample_prob <= 0: raise ConfigError('sample_prob must be <= 1 and > 0') - def build(self, scene_group: Optional[Tuple[str, Iterable[str]]] = None + def build(self, scene_group: tuple[str, Iterable[str]] | None = None ) -> StatsAnalyzer: if scene_group is None: output_uri = join(self.output_uri, f'stats.json') diff --git a/rastervision_core/rastervision/core/backend/backend.py b/rastervision_core/rastervision/core/backend/backend.py index e2339e696..c40dbf6b3 100644 --- a/rastervision_core/rastervision/core/backend/backend.py +++ b/rastervision_core/rastervision/core/backend/backend.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from abc import ABC, abstractmethod from contextlib import AbstractContextManager @@ -40,7 +40,7 @@ def train(self): """ @abstractmethod - def load_model(self, uri: Optional[str] = None): + def load_model(self, uri: str | None = None): """Load the model in preparation for one or more prediction calls. Args: diff --git a/rastervision_core/rastervision/core/backend/backend_config.py b/rastervision_core/rastervision/core/backend/backend_config.py index 1770b3f73..edaedb2d3 100644 --- a/rastervision_core/rastervision/core/backend/backend_config.py +++ b/rastervision_core/rastervision/core/backend/backend_config.py @@ -1,4 +1,4 @@ -from typing import Optional, List, TYPE_CHECKING +from typing import TYPE_CHECKING from rastervision.pipeline.config import Config, register_config @@ -14,7 +14,7 @@ class BackendConfig(Config): def build(self, pipeline: 'RVPipeline', tmp_dir: str) -> 'Backend': raise NotImplementedError() - def get_bundle_filenames(self) -> List[str]: + def get_bundle_filenames(self) -> list[str]: """Returns the names of files that should be included in a model bundle. The files are assumed to be in the train/ directory generated by the train @@ -22,9 +22,9 @@ def get_bundle_filenames(self) -> List[str]: """ raise NotImplementedError() - def update(self, pipeline: Optional['RVPipeline'] = None): # noqa + def update(self, pipeline: 'RVPipeline | None' = None): # noqa pass - def filter_commands(self, commands: List[str]) -> List[str]: + def filter_commands(self, commands: list[str]) -> list[str]: """Filter out any commands that are not needed or supported.""" return commands diff --git a/rastervision_core/rastervision/core/box.py b/rastervision_core/rastervision/core/box.py index 5c5edd0e4..31c21aab1 100644 --- a/rastervision_core/rastervision/core/box.py +++ b/rastervision_core/rastervision/core/box.py @@ -1,5 +1,5 @@ -from typing import (TYPE_CHECKING, Callable, Dict, List, Literal, Optional, - Sequence, Tuple, Union) +from typing import (TYPE_CHECKING, Literal, Self) +from collections.abc import Callable from pydantic import NonNegativeInt as NonNegInt, PositiveInt as PosInt import math import random @@ -15,21 +15,23 @@ if TYPE_CHECKING: from shapely.geometry import MultiPolygon + from shapely.geometry.base import BaseGeometry class BoxSizeError(ValueError): pass -class Box(): +class Box: """A multi-purpose box (ie. rectangle) representation.""" - def __init__(self, ymin, xmin, ymax, xmax): - """Construct a bounding box. + def __init__(self, ymin: int, xmin: int, ymax: int, xmax: int): + """Constructor. - Unless otherwise stated, the convention is that these coordinates are - in pixel coordinates and represent boxes that lie within a - RasterSource. + Although primarily intended for representing integer pixel coordinates + in a scene, this class can also be used to represent floating point + map coordinates though not all methods might be compatible with that + interpretation. Args: ymin: minimum y value (y is row) @@ -42,31 +44,32 @@ def __init__(self, ymin, xmin, ymax, xmax): self.ymax = ymax self.xmax = xmax - def __eq__(self, other: 'Box') -> bool: + def __eq__(self, other: Self) -> bool: """Return true if other has same coordinates.""" return self.tuple_format() == other.tuple_format() - def __ne__(self, other: 'Box'): + def __ne__(self, other: Self): """Return true if other has different coordinates.""" return self.tuple_format() != other.tuple_format() @property def height(self) -> int: - """Return height of Box.""" + """Height of the Box.""" return self.ymax - self.ymin @property def width(self) -> int: - """Return width of Box.""" + """Width of the Box.""" return self.xmax - self.xmin @property - def extent(self) -> 'Box': - """Return a (0, 0, h, w) Box representing the size of this Box.""" + def extent(self) -> Self: + """Return a Box(0, 0, h, w) representing the size of this Box.""" return Box(0, 0, self.height, self.width) @property - def size(self) -> Tuple[int, int]: + def size(self) -> tuple[int, int]: + """(height, width) tuple.""" return self.height, self.width @property @@ -74,37 +77,40 @@ def area(self) -> int: """Return area of Box.""" return self.height * self.width - def normalize(self) -> 'Box': + def normalize(self) -> Self: """Ensure ymin <= ymax and xmin <= xmax.""" ymin, ymax = sorted((self.ymin, self.ymax)) xmin, xmax = sorted((self.xmin, self.xmax)) return Box(ymin, xmin, ymax, xmax) - def rasterio_format(self) -> Tuple[Tuple[int, int], Tuple[int, int]]: + def rasterio_format(self) -> tuple[tuple[int, int], tuple[int, int]]: """Return Box in Rasterio format: ((ymin, ymax), (xmin, xmax)).""" return ((self.ymin, self.ymax), (self.xmin, self.xmax)) - def tuple_format(self) -> Tuple[int, int, int, int]: + def tuple_format(self) -> tuple[int, int, int, int]: + """Return a (ymin, xmin, ymax, xmax) tuple.""" return (self.ymin, self.xmin, self.ymax, self.xmax) - def shapely_format(self) -> Tuple[int, int, int, int]: + def shapely_format(self) -> tuple[int, int, int, int]: return self.to_xyxy() def to_int(self): - return Box( - int(self.ymin), int(self.xmin), int(self.ymax), int(self.xmax)) + """Return a new Box with all coordinates cast to ints.""" + ymin, xmin, ymax, xmax = self + ymin, xmin, ymax, xmax = int(ymin), int(xmin), int(ymax), int(xmax) + out = Box(ymin, xmin, ymax, xmax) + return out - def npbox_format(self): + def npbox_format(self) -> np.ndarray: """Return Box in npbox format used by TF Object Detection API. Returns: Numpy array of form [ymin, xmin, ymax, xmax] with float type """ - return np.array( - [self.ymin, self.xmin, self.ymax, self.xmax], dtype=float) + return np.array(self.tuple_format(), dtype=float) @staticmethod - def to_npboxes(boxes): + def to_npboxes(boxes: list[Self]) -> np.ndarray: """Return nx4 numpy array from list of Box.""" nb_boxes = len(boxes) npboxes = np.empty((nb_boxes, 4)) @@ -115,7 +121,7 @@ def to_npboxes(boxes): def __iter__(self): return iter(self.tuple_format()) - def __getitem__(self, i): + def __getitem__(self, i: NonNegInt): return self.tuple_format()[i] def __repr__(self) -> str: @@ -124,7 +130,7 @@ def __repr__(self) -> str: def __hash__(self) -> int: return hash(self.tuple_format()) - def geojson_coordinates(self) -> List[Tuple[int, int]]: + def geojson_coordinates(self) -> list[tuple[int, int]]: """Return Box as GeoJSON coordinates.""" # Compass directions: nw = [self.xmin, self.ymin] @@ -133,7 +139,7 @@ def geojson_coordinates(self) -> List[Tuple[int, int]]: sw = [self.xmax, self.ymin] return [nw, ne, se, sw, nw] - def make_random_square_container(self, size: int) -> 'Box': + def make_random_square_container(self, size: int) -> Self: """Return a new square Box that contains this Box. Args: @@ -141,7 +147,7 @@ def make_random_square_container(self, size: int) -> 'Box': """ return self.make_random_box_container(size, size) - def make_random_box_container(self, out_h: int, out_w: int) -> 'Box': + def make_random_box_container(self, out_h: int, out_w: int) -> Self: """Return a new rectangular Box that contains this Box. Args: @@ -167,7 +173,7 @@ def make_random_box_container(self, out_h: int, out_w: int) -> 'Box': return Box(out_ymin, out_xmin, out_ymin + out_h, out_xmin + out_w) - def make_random_square(self, size: int) -> 'Box': + def make_random_square(self, size: int) -> Self: """Return new randomly positioned square Box that lies inside this Box. Args: @@ -191,7 +197,7 @@ def make_random_square(self, size: int) -> 'Box': return Box.make_square(rand_y, rand_x, size) - def intersection(self, other: 'Box') -> 'Box': + def intersection(self, other: Self) -> Self: """Return the intersection of this Box and the other. Args: @@ -212,7 +218,7 @@ def intersection(self, other: 'Box') -> 'Box': ymax = min(box1.ymax, box2.ymax) return Box(xmin=xmin, ymin=ymin, xmax=xmax, ymax=ymax) - def intersects(self, other: 'Box') -> bool: + def intersects(self, other: Self) -> bool: box1 = self.normalize() box2 = other.normalize() if box1.ymax <= box2.ymin or box1.ymin >= box2.ymax: @@ -221,8 +227,8 @@ def intersects(self, other: 'Box') -> bool: return False return True - @staticmethod - def from_npbox(npbox): + @classmethod + def from_npbox(cls, npbox: np.ndarray) -> Self: """Return new Box based on npbox format. Args: @@ -230,23 +236,23 @@ def from_npbox(npbox): """ return Box(*npbox) - @staticmethod - def from_shapely(shape): + @classmethod + def from_shapely(cls, shape: 'BaseGeometry') -> Self: """Instantiate from the bounds of a shapely geometry.""" xmin, ymin, xmax, ymax = shape.bounds return Box(ymin, xmin, ymax, xmax) @classmethod - def from_rasterio(self, rio_window: RioWindow) -> 'Box': + def from_rasterio(cls, rio_window: RioWindow) -> Self: """Instantiate from a rasterio window.""" yslice, xslice = rio_window.toslices() return Box(yslice.start, xslice.start, yslice.stop, xslice.stop) - def to_xywh(self) -> Tuple[int, int, int, int]: + def to_xywh(self) -> tuple[int, int, int, int]: """Convert to (xmin, ymin, width, height) tuple""" return (self.xmin, self.ymin, self.width, self.height) - def to_xyxy(self) -> Tuple[int, int, int, int]: + def to_xyxy(self) -> tuple[int, int, int, int]: """Convert to (xmin, ymin, xmax, ymax) tuple""" return (self.xmin, self.ymin, self.xmax, self.ymax) @@ -262,19 +268,18 @@ def to_rasterio(self) -> RioWindow: """Convert to a Rasterio Window.""" return RioWindow.from_slices(*self.normalize().to_slices()) - def to_slices(self, - h_step: Optional[int] = None, - w_step: Optional[int] = None) -> Tuple[slice, slice]: + def to_slices(self, h_step: int | None = None, + w_step: int | None = None) -> tuple[slice, slice]: """Convert to slices: ymin:ymax[:h_step], xmin:xmax[:w_step]""" return slice(self.ymin, self.ymax, h_step), slice( self.xmin, self.xmax, w_step) - def translate(self, dy: int, dx: int) -> 'Box': + def translate(self, dy: int, dx: int) -> Self: """Translate window along y and x axes by the given distances.""" ymin, xmin, ymax, xmax = self return Box(ymin + dy, xmin + dx, ymax + dy, xmax + dx) - def to_global_coords(self, bbox: 'Box') -> 'Box': + def to_global_coords(self, bbox: Self) -> Self: """Go from bbox coords to global coords. E.g., Given a box Box(20, 20, 40, 40) and bbox Box(20, 20, 100, 100), @@ -284,7 +289,7 @@ def to_global_coords(self, bbox: 'Box') -> 'Box': """ return self.translate(dy=bbox.ymin, dx=bbox.xmin) - def to_local_coords(self, bbox: 'Box') -> 'Box': + def to_local_coords(self, bbox: Self) -> Self: """Go from to global coords bbox coords. E.g., Given a box Box(40, 40, 60, 60) and bbox Box(20, 20, 100, 100), @@ -294,7 +299,7 @@ def to_local_coords(self, bbox: 'Box') -> 'Box': """ return self.translate(dy=-bbox.ymin, dx=-bbox.xmin) - def reproject(self, transform_fn: Callable) -> 'Box': + def reproject(self, transform_fn: Callable[[tuple], tuple]) -> Self: """Reprojects this box based on a transform function. Args: @@ -308,11 +313,11 @@ def reproject(self, transform_fn: Callable) -> 'Box': return Box(ymin, xmin, ymax, xmax) @staticmethod - def make_square(ymin, xmin, size) -> 'Box': + def make_square(ymin, xmin, size) -> Self: """Return new square Box.""" return Box(ymin, xmin, ymin + size, xmin + size) - def center_crop(self, edge_offset_y: int, edge_offset_x: int) -> 'Box': + def center_crop(self, edge_offset_y: int, edge_offset_x: int) -> Self: """Return Box whose sides are eroded by the given offsets. Box(0, 0, 10, 10).center_crop(2, 4) == Box(2, 4, 8, 6) @@ -320,11 +325,11 @@ def center_crop(self, edge_offset_y: int, edge_offset_x: int) -> 'Box': return Box(self.ymin + edge_offset_y, self.xmin + edge_offset_x, self.ymax - edge_offset_y, self.xmax - edge_offset_x) - def erode(self, erosion_sz) -> 'Box': + def erode(self, erosion_sz) -> Self: """Return new Box whose sides are eroded by erosion_sz.""" return self.center_crop(erosion_sz, erosion_sz) - def buffer(self, buffer_sz: float, max_extent: 'Box') -> 'Box': + def buffer(self, buffer_sz: float, max_extent: Self) -> Self: """Return new Box whose sides are buffered by buffer_sz. The resulting box is clipped so that the values of the corners are @@ -346,7 +351,7 @@ def buffer(self, buffer_sz: float, max_extent: 'Box') -> 'Box': min(max_extent.width, int(self.xmax) + delta_width)) - def pad(self, ymin: int, xmin: int, ymax: int, xmax: int) -> 'Box': + def pad(self, ymin: int, xmin: int, ymax: int, xmax: int) -> Self: """Pad sides by the given amount.""" return Box( ymin=self.ymin - ymin, @@ -354,7 +359,7 @@ def pad(self, ymin: int, xmin: int, ymax: int, xmax: int) -> 'Box': ymax=self.ymax + ymax, xmax=self.xmax + xmax) - def copy(self) -> 'Box': + def copy(self) -> Self: return Box(*self) def get_windows( @@ -363,7 +368,7 @@ def get_windows( stride: PosInt | tuple[PosInt, PosInt], padding: NonNegInt | tuple[NonNegInt, NonNegInt] | None = None, pad_direction: Literal['both', 'start', 'end'] = 'end' - ) -> list['Box']: + ) -> list[Self]: """Return sliding windows for given size, stride, and padding. Each of size, stride, and padding can be either a positive int or @@ -387,7 +392,7 @@ def get_windows( no effect if padding is zero. Defaults to ``'end'``. Returns: - List of Box objects. + List of windows. """ size: tuple[PosInt, PosInt] = ensure_tuple(size) stride: tuple[PosInt, PosInt] = ensure_tuple(stride) @@ -437,7 +442,7 @@ def get_windows( windows.append(Box(ymin, xmin, ymin + h, xmin + w)) return windows - def to_dict(self) -> Dict[str, int]: + def to_dict(self) -> dict[str, int]: """Convert to a dict with keys: ymin, xmin, ymax, xmax.""" return { 'ymin': self.ymin, @@ -447,13 +452,13 @@ def to_dict(self) -> Dict[str, int]: } @classmethod - def from_dict(cls, d: dict) -> 'Box': + def from_dict(cls, d: dict) -> Self: return cls(d['ymin'], d['xmin'], d['ymax'], d['xmax']) @staticmethod - def filter_by_aoi(windows: List['Box'], - aoi_polygons: List[Polygon], - within: bool = True) -> List['Box']: + def filter_by_aoi(windows: list[Self], + aoi_polygons: list[Polygon], + within: bool = True) -> list[Self]: """Filters windows by a list of AOI polygons Args: @@ -473,8 +478,8 @@ def filter_by_aoi(windows: List['Box'], return out @staticmethod - def within_aoi(window: 'Box', - aoi_polygons: Polygon | List[Polygon]) -> bool: + def within_aoi(window: Self, + aoi_polygons: Polygon | list[Polygon]) -> bool: """Check if window is within the union of given AOI polygons.""" aoi_polygons: Polygon | MultiPolygon = unary_union(aoi_polygons) w = window.to_shapely() @@ -482,15 +487,15 @@ def within_aoi(window: 'Box', return out @staticmethod - def intersects_aoi(window: 'Box', - aoi_polygons: Polygon | List[Polygon]) -> bool: + def intersects_aoi(window: Self, + aoi_polygons: Polygon | list[Polygon]) -> bool: """Check if window intersects with the union of given AOI polygons.""" aoi_polygons: Polygon | MultiPolygon = unary_union(aoi_polygons) w = window.to_shapely() out = aoi_polygons.intersects(w) return out - def __contains__(self, query: Union['Box', Sequence]) -> bool: + def __contains__(self, query: Self | tuple[int, int]) -> bool: """Check if box or point is contained within this box. Args: diff --git a/rastervision_core/rastervision/core/cli.py b/rastervision_core/rastervision/core/cli.py index b75b8e2df..545c8c3a0 100644 --- a/rastervision_core/rastervision/core/cli.py +++ b/rastervision_core/rastervision/core/cli.py @@ -1,4 +1,3 @@ -from typing import List, Optional import click from rastervision.pipeline.file_system import get_tmp_dir @@ -68,13 +67,13 @@ def predict(model_bundle: str, image_uri: str, label_uri: str, update_stats: bool = False, - channel_order: Optional[List[str]] = None, - scene_group: Optional[str] = None): + channel_order: list[str] | None = None, + scene_group: str | None = None): """Make predictions on the images at IMAGE_URI using MODEL_BUNDLE and store the prediction output at LABEL_URI. """ if channel_order is not None: - channel_order: List[int] = [int(i) for i in channel_order] + channel_order: list[int] = [int(i) for i in channel_order] with get_tmp_dir() as tmp_dir: predictor = Predictor(model_bundle, tmp_dir, update_stats, @@ -94,7 +93,7 @@ def predict(model_bundle: str, help='Optional URI to serialized Raster Vision PredictOptions config.') def predict_scene(model_bundle_uri: str, scene_config_uri: str, - predict_options_uri: Optional[str] = None): + predict_options_uri: str | None = None): """Use a model-bundle to make predictions on a scene. \b diff --git a/rastervision_core/rastervision/core/data/class_config.py b/rastervision_core/rastervision/core/data/class_config.py index 3594a0de5..8ebc6437e 100644 --- a/rastervision_core/rastervision/core/data/class_config.py +++ b/rastervision_core/rastervision/core/data/class_config.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Self, Tuple, Union +from typing import Self from rastervision.pipeline.config import (Config, register_config, ConfigError, Field, model_validator) @@ -12,17 +12,17 @@ class ClassConfig(Config): """Configure class information for a machine learning task.""" - names: List[str] = Field( + names: list[str] = Field( ..., description='Names of classes. The i-th class in this list will have ' 'class ID = i.') - colors: Optional[List[Union[str, Tuple]]] = Field( + colors: list[str | tuple] | None = Field( None, description= ('Colors used to visualize classes. Can be color strings accepted by ' 'matplotlib or RGB tuples. If None, a random color will be auto-generated ' 'for each class.')) - null_class: Optional[str] = Field( + null_class: str | None = Field( None, description='Optional name of class in `names` to use as the null ' 'class. This is used in semantic segmentation to represent the label ' @@ -121,7 +121,7 @@ def __len__(self) -> int: return len(self.names) @property - def color_triples(self) -> List[Tuple[float, float, float]]: + def color_triples(self) -> list[tuple[float, float, float]]: """Class colors in a normalized form.""" color_triples = [normalize_color(c) for c in self.colors] return color_triples diff --git a/rastervision_core/rastervision/core/data/crs_transformer/crs_transformer.py b/rastervision_core/rastervision/core/data/crs_transformer/crs_transformer.py index 8f1a64d6b..a64163e5b 100644 --- a/rastervision_core/rastervision/core/data/crs_transformer/crs_transformer.py +++ b/rastervision_core/rastervision/core/data/crs_transformer/crs_transformer.py @@ -1,5 +1,5 @@ from abc import ABC, abstractmethod -from typing import Any, Optional, overload, Tuple +from typing import Any, overload import numpy as np from shapely.ops import transform @@ -16,36 +16,35 @@ class CRSTransformer(ABC): """ def __init__(self, - transform: Optional[Any] = None, - image_crs: Optional[str] = None, - map_crs: Optional[str] = None): + transform: Any | None = None, + image_crs: str | None = None, + map_crs: str | None = None): self.transform = transform self.image_crs = image_crs self.map_crs = map_crs @overload - def map_to_pixel(self, - inp: Tuple[float, float], - bbox: Optional[Box] = None) -> Tuple[int, int]: + def map_to_pixel(self, inp: tuple[float, float], + bbox: Box | None = None) -> tuple[int, int]: ... @overload def map_to_pixel(self, - inp: Tuple['np.array', 'np.array'], - bbox: Optional[Box] = None - ) -> Tuple['np.array', 'np.array']: + inp: tuple['np.ndarray', 'np.ndarray'], + bbox: Box | None = None + ) -> tuple['np.ndarray', 'np.ndarray']: ... @overload - def map_to_pixel(self, inp: Box, bbox: Optional[Box] = None) -> Box: + def map_to_pixel(self, inp: Box, bbox: Box | None = None) -> Box: ... @overload def map_to_pixel(self, inp: BaseGeometry, - bbox: Optional[Box] = None) -> BaseGeometry: + bbox: Box | None = None) -> BaseGeometry: ... - def map_to_pixel(self, inp, bbox: Optional[Box] = None): + def map_to_pixel(self, inp, bbox: Box | None = None): """Transform input from map to pixel coords. Args: @@ -90,28 +89,27 @@ def map_to_pixel(self, inp, bbox: Optional[Box] = None): 'Input must be 2-tuple or Box or shapely geometry.') @overload - def pixel_to_map(self, - inp: Tuple[float, float], - bbox: Optional[Box] = None) -> Tuple[float, float]: + def pixel_to_map(self, inp: tuple[float, float], + bbox: Box | None = None) -> tuple[float, float]: ... @overload def pixel_to_map(self, - inp: Tuple['np.array', 'np.array'], - bbox: Optional[Box] = None - ) -> Tuple['np.array', 'np.array']: + inp: tuple['np.ndarray', 'np.ndarray'], + bbox: Box | None = None + ) -> tuple['np.ndarray', 'np.ndarray']: ... @overload - def pixel_to_map(self, inp: Box, bbox: Optional[Box] = None) -> Box: + def pixel_to_map(self, inp: Box, bbox: Box | None = None) -> Box: ... @overload def pixel_to_map(self, inp: BaseGeometry, - bbox: Optional[Box] = None) -> BaseGeometry: + bbox: Box | None = None) -> BaseGeometry: ... - def pixel_to_map(self, inp, bbox: Optional[Box] = None): + def pixel_to_map(self, inp, bbox: Box | None = None): """Transform input from pixel to map coords. Args: @@ -157,7 +155,7 @@ def pixel_to_map(self, inp, bbox: Optional[Box] = None): 'Input must be 2-tuple or Box or shapely geometry.') @abstractmethod - def _map_to_pixel(self, point: Tuple[float, float]) -> Tuple[int, int]: + def _map_to_pixel(self, point: tuple[float, float]) -> tuple[int, int]: """Transform point from map to pixel coordinates. Args: @@ -165,11 +163,11 @@ def _map_to_pixel(self, point: Tuple[float, float]) -> Tuple[int, int]: can be single values or array-like. Returns: - Tuple[int, int]: (x, y) tuple in pixel coordinates. + tuple[int, int]: (x, y) tuple in pixel coordinates. """ @abstractmethod - def _pixel_to_map(self, point: Tuple[float, float]) -> Tuple[float, float]: + def _pixel_to_map(self, point: tuple[int, int]) -> tuple[float, float]: """Transform point from pixel to map coordinates. Args: @@ -177,5 +175,5 @@ def _pixel_to_map(self, point: Tuple[float, float]) -> Tuple[float, float]: single values or array-like. Returns: - Tuple[float, float]: (x, y) tuple in map coordinates (eg. lon/lat). + tuple[float, float]: (x, y) tuple in map coordinates (eg. lon/lat). """ diff --git a/rastervision_core/rastervision/core/data/crs_transformer/rasterio_crs_transformer.py b/rastervision_core/rastervision/core/data/crs_transformer/rasterio_crs_transformer.py index cd526d753..d974eec9e 100644 --- a/rastervision_core/rastervision/core/data/crs_transformer/rasterio_crs_transformer.py +++ b/rastervision_core/rastervision/core/data/crs_transformer/rasterio_crs_transformer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Optional, Union +from typing import Any, Self from pyproj import Transformer import numpy as np @@ -9,9 +9,6 @@ from rastervision.core.data.crs_transformer import (CRSTransformer, IdentityCRSTransformer) -if TYPE_CHECKING: - from typing import Self - class RasterioCRSTransformer(CRSTransformer): """Transformer for a RasterioRasterSource.""" @@ -66,7 +63,8 @@ def __repr__(self) -> str: """ return out - def _map_to_pixel(self, map_point): + def _map_to_pixel(self, map_point: tuple[float, float] | np.ndarray + ) -> tuple[int, int] | np.ndarray: """Transform point from map to pixel-based coordinates. Args: @@ -84,7 +82,8 @@ def _map_to_pixel(self, map_point): pixel_point = (col, row) return pixel_point - def _pixel_to_map(self, pixel_point): + def _pixel_to_map(self, pixel_point: tuple[int, int] | np.ndarray + ) -> tuple[float, float] | np.ndarray: """Transform point from pixel to map-based coordinates. Args: @@ -102,14 +101,15 @@ def _pixel_to_map(self, pixel_point): return map_point @classmethod - def from_dataset( - cls, dataset: Any, map_crs: Optional[str] = 'epsg:4326', **kwargs - ) -> Union[IdentityCRSTransformer, 'RasterioCRSTransformer']: + def from_dataset(cls, + dataset: Any, + map_crs: str | None = 'epsg:4326', + **kwargs) -> 'IdentityCRSTransformer | Self': """Build from rasterio dataset. Args: - dataset (Any): Rasterio dataset. - map_crs (Optional[str]): Target map CRS. Defaults to 'epsg:4326'. + dataset: Rasterio dataset. + map_crs: Target map CRS. Defaults to 'epsg:4326'. **kwargs: Extra args for :meth:`.__init__`. """ transform = dataset.transform @@ -127,13 +127,13 @@ def from_dataset( return cls(transform, image_crs, map_crs, **kwargs) @classmethod - def from_uri(cls, uri: str, map_crs: Optional[str] = 'epsg:4326', - **kwargs) -> 'Self': + def from_uri(cls, uri: str, map_crs: str | None = 'epsg:4326', + **kwargs) -> 'IdentityCRSTransformer | Self': """Build from raster URI. Args: - uri (Any): Raster URI. - map_crs (Optional[str]): Target map CRS. Defaults to 'epsg:4326'. + uri: Raster URI. + map_crs: Target map CRS. Defaults to 'epsg:4326'. **kwargs: Extra args for :meth:`.__init__`. """ with rio.open(uri) as ds: diff --git a/rastervision_core/rastervision/core/data/dataset_config.py b/rastervision_core/rastervision/core/data/dataset_config.py index b2e7348f8..2fd03b09a 100644 --- a/rastervision_core/rastervision/core/data/dataset_config.py +++ b/rastervision_core/rastervision/core/data/dataset_config.py @@ -1,5 +1,3 @@ -from typing import Dict, List, Set - from rastervision.pipeline.config import (Config, register_config, ConfigError, Field) from rastervision.pipeline.utils import split_into_groups @@ -21,13 +19,13 @@ def dataset_config_upgrader(cfg_dict: dict, version: int) -> dict: class DatasetConfig(Config): """Configure train, validation, and test splits for a dataset.""" class_config: ClassConfig - train_scenes: List[SceneConfig] - validation_scenes: List[SceneConfig] - test_scenes: List[SceneConfig] = [] - scene_groups: Dict[str, Set[str]] = Field( + train_scenes: list[SceneConfig] + validation_scenes: list[SceneConfig] + test_scenes: list[SceneConfig] = [] + scene_groups: dict[str, set[str]] = Field( {}, description='Groupings of scenes. Should be a dict of the form: ' - '{: Set(scene_id_1, scene_id_2, ...)}. Three groups are ' + '{: set(scene_id_1, scene_id_2, ...)}. Three groups are ' 'added by default: "train_scenes", "validation_scenes", and ' '"test_scenes"') @@ -88,7 +86,7 @@ def get_split_config(self, split_ind, num_splits): return new_cfg @property - def all_scenes(self) -> List[SceneConfig]: + def all_scenes(self) -> list[SceneConfig]: return self.train_scenes + self.validation_scenes + self.test_scenes def __repr__(self): diff --git a/rastervision_core/rastervision/core/data/label/chip_classification_labels.py b/rastervision_core/rastervision/core/data/label/chip_classification_labels.py index ee7f3dc9e..1d8ddc8d0 100644 --- a/rastervision_core/rastervision/core/data/label/chip_classification_labels.py +++ b/rastervision_core/rastervision/core/data/label/chip_classification_labels.py @@ -1,11 +1,11 @@ -from typing import (TYPE_CHECKING, Any, Dict, Iterable, List, Optional, - Sequence, Tuple) +from typing import (TYPE_CHECKING, Any, Iterable, Self) from dataclasses import dataclass import numpy as np from rastervision.core.box import Box from rastervision.core.data.label import Labels +from rastervision.core.utils.types import Vector if TYPE_CHECKING: from rastervision.core.data import (ClassConfig, CRSTransformer) @@ -15,7 +15,7 @@ @dataclass class ClassificationLabel: class_id: int - scores: Optional[Sequence[float]] = None + scores: Vector | None = None def __iter__(self): return iter((self.class_id, self.scores)) @@ -25,8 +25,8 @@ class ChipClassificationLabels(Labels): """Represents a spatial grid of cells associated with classes.""" def __init__(self, - cell_to_label: Optional[Dict[Box, Tuple[int, Optional[ - Sequence[float]]]]] = None): + cell_to_label: dict[Box, tuple[int, Vector | None]] + | None = None): if cell_to_label is None: cell_to_label = {} @@ -38,12 +38,11 @@ def __init__(self, def __len__(self) -> int: return len(self.cell_to_label) - def __eq__(self, other: 'ChipClassificationLabels') -> bool: + def __eq__(self, other: Self) -> bool: return (isinstance(other, ChipClassificationLabels) and self.cell_to_label == other.cell_to_label) - def __add__(self, other: 'ChipClassificationLabels' - ) -> 'ChipClassificationLabels': + def __add__(self, other: Self) -> Self: result = ChipClassificationLabels() result.extend(self) result.extend(other) @@ -55,8 +54,7 @@ def __contains__(self, cell: Box) -> bool: def __getitem__(self, cell: Box) -> ClassificationLabel: return self.cell_to_label[cell] - def __setitem__(self, window: Box, - value: Tuple[int, Optional[Sequence[float]]]): + def __setitem__(self, window: Box, value: tuple[int, Vector | None]): class_id, scores = value self.set_cell(window, class_id, scores=scores) @@ -68,7 +66,7 @@ def from_predictions(cls, windows: Iterable['Box'], return super().from_predictions(windows, predictions) @classmethod - def make_empty(cls) -> 'ChipClassificationLabels': + def make_empty(cls) -> Self: return ChipClassificationLabels() def filter_by_aoi(self, aoi_polygons: Iterable['Polygon']): @@ -85,32 +83,28 @@ def filter_by_aoi(self, aoi_polygons: Iterable['Polygon']): def set_cell(self, cell: Box, class_id: int, - scores: Optional['np.ndarray'] = None) -> None: + scores: 'np.ndarray | Vector | None' = None) -> None: """Set cell and its class_id. Args: - cell: (Box) - class_id: int - scores: 1d numpy array of probabilities for each class + cell: Cell whose ID to set. + class_id: Class ID. + scores: 1d numpy array of probabilities for each class. """ if scores is not None: - scores = list(map(lambda x: float(x), list(scores))) + scores = list(map(float, scores)) class_id = int(class_id) self.cell_to_label[cell] = ClassificationLabel(class_id, scores) def get_cell_class_id(self, cell: Box) -> int: - """Return class_id for a cell. - - Args: - cell: (Box) - """ + """Return class_id for a cell.""" result = self.cell_to_label.get(cell) if result is not None: return result.class_id else: return None - def get_cell_scores(self, cell: Box) -> Optional[Sequence[float]]: + def get_cell_scores(self, cell: Box) -> Vector | None: """Return scores for a cell. Args: @@ -130,23 +124,23 @@ def get_singleton_labels(self, cell: Box): """ return ChipClassificationLabels({cell: self[cell]}) - def get_cells(self) -> List[Box]: + def get_cells(self) -> list[Box]: """Return list of all cells (list of Box).""" return list(self.cell_to_label.keys()) - def get_class_ids(self) -> List[int]: + def get_class_ids(self) -> list[int]: """Return list of class_ids for all cells.""" return [label.class_id for label in self.cell_to_label.values()] - def get_scores(self) -> List[Optional[Sequence[float]]]: + def get_scores(self) -> list[Vector | None]: """Return list of scores for all cells.""" return [label.scores for label in self.cell_to_label.values()] - def get_values(self) -> List[ClassificationLabel]: + def get_values(self) -> list[ClassificationLabel]: """Return list of class_ids and scores for all cells.""" return list(self.cell_to_label.values()) - def extend(self, labels: 'ChipClassificationLabels') -> None: + def extend(self, labels: Self) -> None: """Adds cells contained in labels. Args: @@ -159,16 +153,16 @@ def save(self, uri: str, class_config: 'ClassConfig', crs_transformer: 'CRSTransformer', - bbox: Optional[Box] = None) -> None: + bbox: Box | None = None) -> None: """Save labels as a GeoJSON file. Args: - uri (str): URI of the output file. - class_config (ClassConfig): ClassConfig to map class IDs to names. - crs_transformer (CRSTransformer): CRSTransformer to convert from - pixel-coords to map-coords before saving. - bbox (Optional[Box]): User-specified crop of the extent. Must be - provided if the corresponding RasterSource has bbox != extent. + uri: URI of the output file. + class_config: ClassConfig to map class IDs to names. + crs_transformer: CRSTransformer to convert from pixel-coords to + map-coords before saving. + bbox: User-specified crop of the extent. Must be provided if the + corresponding RasterSource has bbox != extent. """ from rastervision.core.data import ChipClassificationGeoJSONStore diff --git a/rastervision_core/rastervision/core/data/label/labels.py b/rastervision_core/rastervision/core/data/label/labels.py index 7e5bb8460..5fd5b7c99 100644 --- a/rastervision_core/rastervision/core/data/label/labels.py +++ b/rastervision_core/rastervision/core/data/label/labels.py @@ -1,10 +1,9 @@ """Defines the abstract Labels class.""" -from typing import TYPE_CHECKING, Any, Iterable, List +from typing import TYPE_CHECKING, Any, Iterable, Self from abc import ABC, abstractmethod if TYPE_CHECKING: - from typing import Self from shapely.geometry import Polygon from rastervision.core.box import Box @@ -17,14 +16,14 @@ class Labels(ABC): """ @abstractmethod - def __add__(self, other: 'Self'): + def __add__(self, other: Self): """Add labels to these labels. Returns a concatenation of this and the other labels. """ @abstractmethod - def filter_by_aoi(self, aoi_polygons: List['Polygon']) -> 'Self': + def filter_by_aoi(self, aoi_polygons: list['Polygon']) -> Self: """Return a copy of these labels filtered by given AOI polygons. Args: @@ -42,7 +41,7 @@ def __setitem__(self, key, value): @classmethod @abstractmethod - def make_empty(cls) -> 'Self': + def make_empty(cls) -> Self: """Instantiate an empty instance of this class. Returns: @@ -52,7 +51,7 @@ def make_empty(cls) -> 'Self': @classmethod def from_predictions(cls, windows: Iterable['Box'], - predictions: Iterable[Any]) -> 'Self': + predictions: Iterable[Any]) -> Self: """Instantiate from windows and their corresponding predictions. This makes no assumptions about the type or format of the predictions. diff --git a/rastervision_core/rastervision/core/data/label/object_detection_labels.py b/rastervision_core/rastervision/core/data/label/object_detection_labels.py index 52568697b..f6727ad14 100644 --- a/rastervision_core/rastervision/core/data/label/object_detection_labels.py +++ b/rastervision_core/rastervision/core/data/label/object_detection_labels.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, Iterable, List, Optional +from typing import TYPE_CHECKING, Iterable, Self import numpy as np from shapely.geometry import shape @@ -21,9 +21,9 @@ class ObjectDetectionLabels(Labels): """ def __init__(self, - npboxes: np.array, - class_ids: np.array, - scores: np.array = None): + npboxes: np.ndarray, + class_ids: np.ndarray, + scores: np.ndarray = None): """Construct a set of object detection labels. Args: @@ -43,15 +43,14 @@ def __init__(self, scores = np.ones(class_ids.shape) self.boxlist.add_field('scores', scores) - def __add__(self, - other: 'ObjectDetectionLabels') -> 'ObjectDetectionLabels': + def __add__(self, other: Self) -> Self: return ObjectDetectionLabels.concatenate(self, other) - def __eq__(self, other: 'ObjectDetectionLabels') -> bool: + def __eq__(self, other: Self) -> bool: return (isinstance(other, ObjectDetectionLabels) and self.to_dict() == other.to_dict()) - def __setitem__(self, window: Box, item: Dict[str, np.ndarray]): + def __setitem__(self, window: Box, item: dict[str, np.ndarray]): boxes = item['boxes'] boxes = ObjectDetectionLabels.local_to_global(boxes, window) class_ids = item['class_ids'] @@ -61,10 +60,10 @@ def __setitem__(self, window: Box, item: Dict[str, np.ndarray]): concatenated_labels = self + new_labels self.boxlist = concatenated_labels.boxlist - def __getitem__(self, window: Box) -> 'ObjectDetectionLabels': + def __getitem__(self, window: Box) -> Self: return ObjectDetectionLabels.get_overlapping(self, window) - def assert_equal(self, expected_labels: 'ObjectDetectionLabels'): + def assert_equal(self, expected_labels: Self): np.testing.assert_array_equal(self.get_npboxes(), expected_labels.get_npboxes()) np.testing.assert_array_equal(self.get_class_ids(), @@ -96,7 +95,7 @@ def filter_by_aoi(self, aoi_polygons: Iterable['Polygon']): np.array(new_boxes), np.array(new_class_ids), np.array(new_scores)) @classmethod - def make_empty(cls) -> 'ObjectDetectionLabels': + def make_empty(cls) -> Self: npboxes = np.empty((0, 4)) class_ids = np.empty((0, )) scores = np.empty((0, )) @@ -112,17 +111,17 @@ def from_boxlist(boxlist: NpBoxList): @staticmethod def from_geojson(geojson: dict, - bbox: Optional[Box] = None, + bbox: Box | None = None, ioa_thresh: float = 0.8, - clip: bool = True) -> 'ObjectDetectionLabels': + clip: bool = True) -> Self: """Convert GeoJSON to ObjectDetectionLabels object. If bbox is provided, filter out the boxes that lie "more than a little bit" outside the bbox. Args: - geojson: (dict) normalized GeoJSON (see VectorSource) - bbox: (Box) in pixel coords + geojson: normalized GeoJSON (see VectorSource) + bbox: in pixel coords Returns: ObjectDetectionLabels @@ -145,7 +144,7 @@ def from_geojson(geojson: dict, labels, bbox, ioa_thresh=ioa_thresh, clip=clip) return labels - def get_boxes(self) -> List[Box]: + def get_boxes(self) -> list[Box]: """Return list of Boxes.""" return [Box.from_npbox(npbox) for npbox in self.boxlist.get()] @@ -229,15 +228,15 @@ def normalized_to_local(npboxes: np.ndarray, window: Box): return npboxes * np.array([[height, width, height, width]]) @staticmethod - def get_overlapping(labels: 'ObjectDetectionLabels', + def get_overlapping(labels: Self, window: Box, ioa_thresh: float = 0.5, - clip: bool = False) -> 'ObjectDetectionLabels': + clip: bool = False) -> Self: """Return subset of labels that overlap with window. Args: - labels: ObjectDetectionLabels - window: Box + labels: Labels + window: Window ioa_thresh: The minimum intersection-over-area (IOA) for a box to be considered as overlapping. For each box, IOA is defined as the area of the intersection of the box with the window over @@ -254,24 +253,16 @@ def get_overlapping(labels: 'ObjectDetectionLabels', return ObjectDetectionLabels.from_boxlist(boxlist) @staticmethod - def concatenate( - labels1: 'ObjectDetectionLabels', - labels2: 'ObjectDetectionLabels') -> 'ObjectDetectionLabels': - """Return concatenation of labels. - - Args: - labels1: ObjectDetectionLabels - labels2: ObjectDetectionLabels - """ + def concatenate(labels1: Self, labels2: Self) -> Self: + """Return concatenation of labels.""" new_boxlist = concatenate([labels1.to_boxlist(), labels2.to_boxlist()]) return ObjectDetectionLabels.from_boxlist(new_boxlist) @staticmethod - def prune_duplicates( - labels: 'ObjectDetectionLabels', - score_thresh: float, - merge_thresh: float, - max_output_size: Optional[int] = None) -> 'ObjectDetectionLabels': + def prune_duplicates(labels: Self, + score_thresh: float, + merge_thresh: float, + max_output_size: int | None = None) -> Self: """Remove duplicate boxes via non-maximum suppression. Args: @@ -279,8 +270,8 @@ def prune_duplicates( score_thresh: Prune boxes with score less than this threshold. merge_thresh: Prune boxes with intersection-over-union (IOU) greater than this threshold. - max_output_size (int): Maximum number of retained boxes. - If None, this is set to ``len(abels)``. Defaults to None. + max_output_size: Maximum number of retained boxes. If ``None``, + this is set to ``len(abels)``. Defaults to ``None``. Returns: ObjectDetectionLabels: Pruned labels. @@ -298,15 +289,15 @@ def save(self, uri: str, class_config: 'ClassConfig', crs_transformer: 'CRSTransformer', - bbox: Optional[Box] = None) -> None: + bbox: Box | None = None) -> None: """Save labels as a GeoJSON file. Args: - uri (str): URI of the output file. - class_config (ClassConfig): ClassConfig to map class IDs to names. - crs_transformer (CRSTransformer): CRSTransformer to convert from + uri: URI of the output file. + class_config: ClassConfig to map class IDs to names. + crs_transformer: CRSTransformer to convert from pixel-coords to map-coords before saving. - bbox (Optional[Box]): User-specified crop of the extent. Must be + bbox: User-specified crop of the extent. Must be provided if the corresponding RasterSource has bbox != extent. """ from rastervision.core.data import ObjectDetectionGeoJSONStore diff --git a/rastervision_core/rastervision/core/data/label/semantic_segmentation_labels.py b/rastervision_core/rastervision/core/data/label/semantic_segmentation_labels.py index 192015b72..1a07499c5 100644 --- a/rastervision_core/rastervision/core/data/label/semantic_segmentation_labels.py +++ b/rastervision_core/rastervision/core/data/label/semantic_segmentation_labels.py @@ -1,4 +1,4 @@ -from typing import (TYPE_CHECKING, Any, Iterable, List, Optional, Sequence) +from typing import (TYPE_CHECKING, Any, Iterable, Self, Sequence) from abc import abstractmethod import numpy as np @@ -32,8 +32,7 @@ def __init__(self, extent: Box, num_classes: int, dtype: np.dtype): self.dtype = dtype @abstractmethod - def __add__(self, other: 'SemanticSegmentationLabels' - ) -> 'SemanticSegmentationLabels': + def __add__(self, other: Self) -> Self: """Merge self with other labels.""" def __setitem__(self, window: Box, values: np.ndarray) -> None: @@ -49,7 +48,7 @@ def __getitem__(self, window: Box) -> np.ndarray: """Get labels for the given window.""" @abstractmethod - def add_window(self, window: Box, values: np.ndarray) -> List[Box]: + def add_window(self, window: Box, values: np.ndarray) -> list[Box]: """Set labels for the given window.""" @abstractmethod @@ -69,7 +68,7 @@ def get_score_arr(self, window: Box, def get_class_mask(self, window: Box, class_id: int, - threshold: Optional[float] = None) -> np.ndarray: + threshold: float | None = None) -> np.ndarray: """Get a binary mask representing all pixels of a class.""" scores = self.get_score_arr(window) if threshold is None: @@ -77,7 +76,7 @@ def get_class_mask(self, mask = scores[class_id] >= threshold return mask - def get_windows(self, **kwargs) -> List[Box]: + def get_windows(self, **kwargs) -> list[Box]: """Generate sliding windows over the local extent. The keyword args are passed to :meth:`.Box.get_windows` and can @@ -89,19 +88,19 @@ def get_windows(self, **kwargs) -> List[Box]: Args: **kwargs: Extra args for :meth:`.Box.get_windows`. """ - size: Optional[int] = kwargs.pop('size', None) + size: int | None = kwargs.pop('size', None) if size is None: return [self.extent] return self.extent.get_windows(size, size, **kwargs) - def filter_by_aoi(self, aoi_polygons: List['Polygon'], null_class_id: int, - **kwargs) -> 'SemanticSegmentationLabels': + def filter_by_aoi(self, aoi_polygons: list['Polygon'], null_class_id: int, + **kwargs) -> Self: """Keep only the values that lie inside the AOI. This is an inplace operation. Args: - aoi_polygons (List[Polygon]): AOI polygons to filter by, in pixel + aoi_polygons (list[Polygon]): AOI polygons to filter by, in pixel coordinates. null_class_id (int): Class ID to assign to pixels falling outside the AOI polygons. @@ -121,7 +120,7 @@ def mask_fill(self, window: Box, mask: np.ndarray, for which the mask is ON to the fill_value. """ - def _filter_window_by_aoi(self, window: Box, aoi_polygons: List['Polygon'], + def _filter_window_by_aoi(self, window: Box, aoi_polygons: list['Polygon'], null_class_id: int) -> None: window_geom = window.to_shapely() label_arr = self[window] @@ -156,14 +155,14 @@ def transform_shape(x, y, z=None): @classmethod def make_empty(cls, extent: Box, num_classes: int, - smooth: bool = False) -> 'SemanticSegmentationLabels': + smooth: bool = False) -> Self: """Instantiate an empty instance. Args: extent (Box): The extent of the region to which the labels belong, in global coordinates. num_classes (int): Number of classes. - smooth (bool, optional): If True, creates a + smooth (bool): If True, creates a SemanticSegmentationSmoothLabels object. If False, creates a SemanticSegmentationDiscreteLabels object. Defaults to False. @@ -183,28 +182,27 @@ def make_empty(cls, extent: Box, num_classes: int, extent=extent, num_classes=num_classes) @classmethod - def from_predictions( - cls, - windows: Iterable['Box'], - predictions: Iterable[Any], - extent: Box, - num_classes: int, - smooth: bool = False, - crop_sz: Optional[int] = None) -> 'SemanticSegmentationLabels': + def from_predictions(cls, + windows: Iterable['Box'], + predictions: Iterable[Any], + extent: Box, + num_classes: int, + smooth: bool = False, + crop_sz: int | None = None) -> Self: """Instantiate from windows and their corresponding predictions. Args: - windows (Iterable[Box]): Boxes in pixel coords, specifying chips - in the raster. - predictions (Iterable[Any]): The model predictions for each chip - specified by the windows. - extent (Box): The extent of the region to which the labels belong, - in global coordinates. - num_classes (int): Number of classes. - smooth (bool, optional): If True, creates a - SemanticSegmentationSmoothLabels object. If False, creates a - SemanticSegmentationDiscreteLabels object. Defaults to False. - crop_sz (Optional[int]): Number of rows/columns of pixels from the + windows: Boxes in pixel coords, specifying chips in the raster. + predictions: The model predictions for each chip specified by the + windows. + extent: The extent of the region to which the labels belong, in + global coordinates. + num_classes: Number of classes. + smooth: If ``True``, creates a ``SemanticSegmentationSmoothLabels`` + object. If ``False``, creates a + ``SemanticSegmentationDiscreteLabels`` object. + Defaults to ``False``. + crop_sz: Number of rows/columns of pixels from the edge of prediction windows to discard. This is useful because predictions near edges tend to be lower quality and can result in very visible artifacts near the edges of chips. This should @@ -223,15 +221,14 @@ def from_predictions( def add_predictions(self, windows: Iterable['Box'], predictions: Iterable[Any], - crop_sz: Optional[int] = None) -> None: + crop_sz: int | None = None) -> None: """Populate predictions. Args: - windows (Iterable[Box]): Boxes in pixel coords, specifying chips - in the raster. - predictions (Iterable[Any]): The model predictions for each chip - specified by the windows. - crop_sz (Optional[int]): Number of rows/columns of pixels from the + windows: Boxes in pixel coords, specifying chips in the raster. + predictions: The model predictions for each chip specified by the + windows. + crop_sz: Number of rows/columns of pixels from the edge of prediction windows to discard. This is useful because predictions near edges tend to be lower quality and can result in very visible artifacts near the edges of chips. This should @@ -261,10 +258,10 @@ def __init__(self, extent: Box, num_classes: int, dtype: Any = np.uint8): """Constructor. Args: - extent (Box): The extent of the region to which - the labels belong, in global coordinates. - num_classes (int): Number of classes. - dtype (Any): dtype of the counts array. Defaults to np.uint8. + extent: The extent of the region to which the labels belong, in + global coordinates. + num_classes: Number of classes. + dtype: dtype of the counts array. Defaults to np.uint8. """ super().__init__(extent, num_classes, dtype) @@ -273,8 +270,7 @@ def __init__(self, extent: Box, num_classes: int, dtype: Any = np.uint8): # track which pixels have been hit at all self.hit_mask = np.zeros((self.height, self.width), dtype=bool) - def __add__(self, other: 'SemanticSegmentationDiscreteLabels' - ) -> 'SemanticSegmentationDiscreteLabels': + def __add__(self, other: Self) -> Self: """Merge self with other labels by adding the pixel counts.""" if self.extent != other.extent: raise ValueError('Cannot add labels with unqeual extents.') @@ -282,7 +278,7 @@ def __add__(self, other: 'SemanticSegmentationDiscreteLabels' self.pixel_counts += other.pixel_counts return self - def __eq__(self, other: 'SemanticSegmentationDiscreteLabels') -> bool: + def __eq__(self, other: Self) -> bool: if not isinstance(other, SemanticSegmentationDiscreteLabels): return False if self.extent != other.extent: @@ -351,8 +347,7 @@ def mask_fill(self, window: Box, mask: np.ndarray, self.pixel_counts[class_id, y0:y1, x0:x1][mask] = 1 @classmethod - def make_empty(cls, extent: Box, - num_classes: int) -> 'SemanticSegmentationDiscreteLabels': + def make_empty(cls, extent: Box, num_classes: int) -> Self: """Instantiate an empty instance.""" return cls(extent=extent, num_classes=num_classes) @@ -362,8 +357,7 @@ def from_predictions(cls, predictions: Iterable[Any], extent: Box, num_classes: int, - crop_sz: Optional[int] = None - ) -> 'SemanticSegmentationDiscreteLabels': + crop_sz: int | None = None) -> Self: labels = cls.make_empty(extent, num_classes) labels.add_predictions(windows, predictions, crop_sz=crop_sz) return labels @@ -372,42 +366,41 @@ def save(self, uri: str, crs_transformer: 'CRSTransformer', class_config: 'ClassConfig', - bbox: Optional[Box] = None, - tmp_dir: Optional[str] = None, + bbox: Box | None = None, + tmp_dir: str | None = None, save_as_rgb: bool = False, raster_output: bool = True, rasterio_block_size: int = 512, - vector_outputs: Optional[Sequence['VectorOutputConfig']] = None, - profile_overrides: Optional[dict] = None) -> None: + vector_outputs: 'Sequence[VectorOutputConfig] | None' = None, + profile_overrides: dict | None = None) -> None: """Save labels as a raster and/or vectors. If URI is remote, all files will be first written locally and then uploaded to the URI. Args: - uri (str): URI of directory in which to save all output files. - crs_transformer (CRSTransformer): CRSTransformer to configure CRS - and affine transform of the output GeoTiff. - class_config (ClassConfig): The ClassConfig. - bbox (Optional[Box]): User-specified crop of the extent. Must be - provided if the corresponding RasterSource has bbox != extent. - tmp_dir (Optional[str], optional): Temporary directory to use. If - None, will be auto-generated. Defaults to None. - save_as_rgb (bool, optional): If True, Saves labels as an RGB - image, using the class-color mapping in the class_config. - Defaults to False. - raster_output (bool, optional): If True, saves labels as a raster - of class IDs (one band). Defaults to True. - rasterio_block_size (int, optional): Value to set blockxsize and - blockysize to. Defaults to 512. - vector_outputs (Optional[Sequence[VectorOutputConfig]], optional): - List of VectorOutputConfig's containing vectorization - configuration information. Only classes for which a - VectorOutputConfig is specified will be saved as vectors. - If None, no vector outputs will be produced. Defaults to None. - profile_overrides (Optional[dict], optional): This can be used to - arbitrarily override properties in the profile used to create - the output GeoTiff. Defaults to None. + uri: URI of directory in which to save all output files. + crs_transformer: CRSTransformer to configure CRS and affine + transform of the output GeoTiff. + class_config: The ClassConfig. + bbox: User-specified crop of the extent. Must be provided if the + corresponding RasterSource has ``bbox != extent``. + tmp_dir: Temporary directory to use. If None, will be + auto-generated. Defaults to ``None``. + save_as_rgb: If ``True``, Saves labels as an RGB image, using the + class-color mapping in the class_config. Defaults to ``False``. + raster_output: If ``True``, saves labels as a raster of class IDs + (one band). Defaults to ``True``. + rasterio_block_size: Value to set `blockxsize` and ``blockysize`` + to. Defaults to ``512``. + vector_outputs: List of VectorOutputConfig's containing + vectorization configuration information. Only classes for which + a ``VectorOutputConfig`` is specified will be saved as vectors. + If ``None``, no vector outputs will be produced. + Defaults to ``None``. + profile_overrides: This can be used to arbitrarily override + properties in the profile used to create the output GeoTiff. + Defaults to ``None``. """ from rastervision.core.data import SemanticSegmentationLabelStore @@ -442,11 +435,11 @@ def __init__(self, """Constructor. Args: - extent (Box): The extent of the region to which - the labels belong, in global coordinates. - num_classes (int): Number of classes. - dtype (Any): dtype of the scores array. Defaults to np.float16. - dtype_hits (Any): dtype of the hits array. Defaults to np.uint8. + extent: The extent of the region to which the labels belong, + in global coordinates. + num_classes: Number of classes. + dtype: ``dtype`` of the scores array. Defaults to ``np.float16``. + dtype_hits: ``dtype`` of the hits array. Defaults to ``np.uint8``. """ super().__init__(extent, num_classes, dtype) @@ -454,8 +447,7 @@ def __init__(self, (self.num_classes, self.height, self.width), dtype=self.dtype) self.pixel_hits = np.zeros((self.height, self.width), dtype=dtype_hits) - def __add__(self, other: 'SemanticSegmentationSmoothLabels' - ) -> 'SemanticSegmentationSmoothLabels': + def __add__(self, other: Self) -> Self: """Merge self with other by adding pixel scores and hits.""" if self.extent != other.extent: raise ValueError('Cannot add labels with unqeual extents.') @@ -464,7 +456,7 @@ def __add__(self, other: 'SemanticSegmentationSmoothLabels' self.pixel_hits += other.pixel_hits return self - def __eq__(self, other: 'SemanticSegmentationSmoothLabels') -> bool: + def __eq__(self, other: Self) -> bool: if not isinstance(other, SemanticSegmentationSmoothLabels): return False if self.extent != other.extent: @@ -531,8 +523,7 @@ def mask_fill(self, window: Box, mask: np.ndarray, self.pixel_hits[y0:y1, x0:x1][mask] = 1 @classmethod - def make_empty(cls, extent: Box, - num_classes: int) -> 'SemanticSegmentationSmoothLabels': + def make_empty(cls, extent: Box, num_classes: int) -> Self: """Instantiate an empty instance.""" return cls(extent=extent, num_classes=num_classes) @@ -542,8 +533,7 @@ def from_predictions(cls, predictions: Iterable[Any], extent: Box, num_classes: int, - crop_sz: Optional[int] = None - ) -> 'SemanticSegmentationSmoothLabels': + crop_sz: int | None = None) -> Self: labels = cls.make_empty(extent, num_classes) labels.add_predictions(windows, predictions, crop_sz=crop_sz) return labels @@ -552,50 +542,50 @@ def save(self, uri: str, crs_transformer: 'CRSTransformer', class_config: 'ClassConfig', - bbox: Optional[Box] = None, - tmp_dir: Optional[str] = None, + bbox: Box | None = None, + tmp_dir: str | None = None, save_as_rgb: bool = False, discrete_output: bool = True, smooth_output: bool = True, smooth_as_uint8: bool = False, rasterio_block_size: int = 512, - vector_outputs: Optional[Sequence['VectorOutputConfig']] = None, - profile_overrides: Optional[dict] = None) -> None: + vector_outputs: 'Sequence[VectorOutputConfig] | None' = None, + profile_overrides: dict | None = None) -> None: """Save labels as rasters and/or vectors. If URI is remote, all files will be first written locally and then uploaded to the URI. Args: - uri (str): URI of directory in which to save all output files. - crs_transformer (CRSTransformer): CRSTransformer to configure CRS - and affine transform of the output GeoTiff(s). - class_config (ClassConfig): The ClassConfig. - bbox (Optional[Box]): User-specified crop of the extent. Must be - provided if the corresponding RasterSource has bbox != extent. - tmp_dir (Optional[str], optional): Temporary directory to use. If - None, will be auto-generated. Defaults to None. - save_as_rgb (bool, optional): If True, saves labels as an RGB - image, using the class-color mapping in the class_config. - Defaults to False. - discrete_output (bool, optional): If True, saves labels as a raster - of class IDs (one band). Defaults to True. - smooth_output (bool, optional): If True, saves labels as a raster - of class scores (one band for each class). Defaults to True. - smooth_as_uint8 (bool, optional): If True, stores smooth class - scores as np.uint8 (0-255) values rather than as np.float32 + uri: URI of directory in which to save all output files. + crs_transformer: CRSTransformer to configure CRS and affine + transform of the output GeoTiff(s). + class_config: The ClassConfig. + bbox: User-specified crop of the extent. Must be provided if the + corresponding RasterSource has bbox != extent. + tmp_dir: Temporary directory to use. If ``None``, will be + auto-generated. Defaults to ``None``. + save_as_rgb: If True, saves labels as an RGB image, using the + class-color mapping in the ``class_config``. + Defaults to ``False``. + discrete_output: If ``True``, saves labels as a raster of class IDs + (one band). Defaults to ``True``. + smooth_output: If ``True``, saves labels as a raster of class + scores (one band for each class). Defaults to ``True``. + smooth_as_uint8: If ``True``, stores smooth class scores as + ``np.uint8`` (0-255) values rather than as ``np.float32`` discrete labels, to help save memory/disk space. - Defaults to False. - rasterio_block_size (int, optional): Value to set blockxsize and - blockysize to. Defaults to 512. - vector_outputs (Optional[Sequence[VectorOutputConfig]], optional): - List of VectorOutputConfig's containing vectorization - configuration information. Only classes for which a - VectorOutputConfig is specified will be saved as vectors. - If None, no vector outputs will be produced. Defaults to None. - profile_overrides (Optional[dict], optional): This can be used to - arbitrarily override properties in the profile used to create - the output GeoTiff(s). Defaults to None. + Defaults to ``False``. + rasterio_block_size: Value to set ``blockxsize`` and ``blockysize`` + to. Defaults to ``512``. + vector_outputs: List of VectorOutputConfig's containing + vectorization configuration information. Only classes for which + a ``VectorOutputConfig`` is specified will be saved as vectors. + If ``None``, no vector outputs will be produced. + Defaults to ``None``. + profile_overrides: This can be used to arbitrarily override + properties in the profile used to create the output GeoTiff(s). + Defaults to ``None``. """ from rastervision.core.data import SemanticSegmentationLabelStore diff --git a/rastervision_core/rastervision/core/data/label/tfod_utils/np_box_list.py b/rastervision_core/rastervision/core/data/label/tfod_utils/np_box_list.py index fc69d873c..620ac49fa 100644 --- a/rastervision_core/rastervision/core/data/label/tfod_utils/np_box_list.py +++ b/rastervision_core/rastervision/core/data/label/tfod_utils/np_box_list.py @@ -14,8 +14,6 @@ # ============================================================================== """Numpy BoxList classes and functions.""" -from typing import List, Tuple - import numpy as np @@ -53,7 +51,7 @@ def num_boxes(self) -> int: """Return number of boxes held in collections.""" return self.data['boxes'].shape[0] - def get_extra_fields(self) -> List[str]: + def get_extra_fields(self) -> list[str]: """Return all non-box fields.""" return [k for k in self.data.keys() if k != 'boxes'] @@ -102,11 +100,11 @@ def get_field(self, name: str) -> np.ndarray: raise ValueError(f'field {name} does not exist') def get_coordinates( - self) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + self) -> tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: """Get corner coordinates of boxes. Returns: - Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: a 4-tuple + tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray]: a 4-tuple of 1-d numpy arrays [y_min, x_min, y_max, x_max]. """ boxes = self.get_field('boxes') diff --git a/rastervision_core/rastervision/core/data/label/tfod_utils/np_box_list_ops.py b/rastervision_core/rastervision/core/data/label/tfod_utils/np_box_list_ops.py index 9e89b60b4..4790ca41b 100644 --- a/rastervision_core/rastervision/core/data/label/tfod_utils/np_box_list_ops.py +++ b/rastervision_core/rastervision/core/data/label/tfod_utils/np_box_list_ops.py @@ -13,7 +13,6 @@ # limitations under the License. # ============================================================================== """Bounding Box List operations for Numpy BoxLists.""" -from typing import List, Optional, Tuple from tqdm.auto import trange import numpy as np @@ -93,7 +92,7 @@ def ioa(boxlist1: NpBoxList, boxlist2: NpBoxList) -> np.ndarray: def gather(boxlist: NpBoxList, indices: np.ndarray, - fields: Optional[List[str]] = None) -> NpBoxList: + fields: list[str] | None = None) -> NpBoxList: """Gather boxes from BoxList according to indices and return new BoxList. By default, gather returns boxes corresponding to the input index list, as @@ -104,7 +103,7 @@ def gather(boxlist: NpBoxList, Args: boxlist (BoxList): BoxList holding N boxes. indices (np.ndarray): A 1-d numpy array of type int. - fields (Optional[List[str]]): List of fields to also gather from. If + fields (list[str] | None): List of fields to also gather from. If None, all fields are gathered from. Pass an empty fields list to only gather the box coordinates. Defaults to None. @@ -137,7 +136,7 @@ def sort_by_field(boxlist: NpBoxList, Args: boxlist (BoxList): A BoxList holding N boxes. field (str): A BoxList field for sorting and reordering the BoxList. - order (SortOrder, optional): 'descend' or 'ascend'. Default is descend. + order (SortOrder): 'descend' or 'ascend'. Default is descend. Returns: BoxList: A sorted BoxList with the field in the specified order. @@ -408,7 +407,7 @@ def prune_non_overlapping_boxes(boxlist1: NpBoxList, def prune_outside_window(boxlist: NpBoxList, - window: np.ndarray) -> Tuple[NpBoxList, np.ndarray]: + window: np.ndarray) -> tuple[NpBoxList, np.ndarray]: """Prunes bounding boxes that fall outside a given window. This function prunes bounding boxes that even partially fall outside the @@ -422,7 +421,7 @@ def prune_outside_window(boxlist: NpBoxList, [ymin, xmin, ymax, xmax] of the window. Returns: - Tuple[BoxList, np.ndarray]: Pruned Boxlist of length <= M_in and + tuple[BoxList, np.ndarray]: Pruned Boxlist of length <= M_in and an array of shape [M_out] indexing the valid bounding boxes in the input tensor. """ @@ -444,8 +443,8 @@ def prune_outside_window(boxlist: NpBoxList, return pruned_boxlist, valid_indices -def concatenate(boxlists: List[NpBoxList], - fields: Optional[List[str]] = None) -> NpBoxList: +def concatenate(boxlists: list[NpBoxList], + fields: list[str] | None = None) -> NpBoxList: """Concatenate list of BoxLists. This op concatenates a list of input BoxLists into a larger BoxList. It also @@ -453,8 +452,8 @@ def concatenate(boxlists: List[NpBoxList], are equal except for the first dimension. Args: - boxlists (List[BoxList]): List of BoxList objects. - fields (Optional[List[str]]): Optional list of fields to also + boxlists (list[BoxList]): List of BoxList objects. + fields (list[str] | None): Optional list of fields to also concatenate. If None, all fields from the first BoxList in the list are included in the concatenation. Defaults to None. diff --git a/rastervision_core/rastervision/core/data/label/utils.py b/rastervision_core/rastervision/core/data/label/utils.py index 5e0a10116..2f0dbe228 100644 --- a/rastervision_core/rastervision/core/data/label/utils.py +++ b/rastervision_core/rastervision/core/data/label/utils.py @@ -1,4 +1,4 @@ -from typing import (TYPE_CHECKING, Iterable, Iterator, List, Tuple) +from typing import (TYPE_CHECKING, Iterable, Iterator) if TYPE_CHECKING: import numpy as np from rastervision.core.box import Box @@ -6,7 +6,7 @@ def discard_prediction_edges( windows: Iterable['Box'], predictions: Iterable['np.ndarray'], - crop_sz: int) -> Tuple[List['Box'], Iterator['np.ndarray']]: + crop_sz: int) -> tuple[list['Box'], Iterator['np.ndarray']]: """Discard the edges of predicted chips. Args: @@ -15,7 +15,7 @@ def discard_prediction_edges( crop_sz (int): Number of pixel rows/cols to discard. Returns: - Tuple[Iterator[Box], Iterator[np.ndarray]]: Cropped windows and chips. + tuple[Iterator[Box], Iterator[np.ndarray]]: Cropped windows and chips. """ windows_cropped = [w.center_crop(crop_sz, crop_sz) for w in windows] array_slices = [ diff --git a/rastervision_core/rastervision/core/data/label_source/chip_classification_label_source.py b/rastervision_core/rastervision/core/data/label_source/chip_classification_label_source.py index 7ab73a132..8037f5914 100644 --- a/rastervision_core/rastervision/core/data/label_source/chip_classification_label_source.py +++ b/rastervision_core/rastervision/core/data/label_source/chip_classification_label_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Iterable, List, Optional +from typing import TYPE_CHECKING, Any, Iterable import geopandas as gpd @@ -11,7 +11,7 @@ CRSTransformer, VectorSource) -def infer_cells(cells: List[Box], labels_df: gpd.GeoDataFrame, +def infer_cells(cells: list[Box], labels_df: gpd.GeoDataFrame, ioa_thresh: float, use_intersection_over_cell: bool, pick_min_class_id: bool, background_class_id: int) -> ChipClassificationLabels: @@ -28,20 +28,17 @@ def infer_cells(cells: List[Box], labels_df: gpd.GeoDataFrame, considered null or background. Args: - ioa_thresh: (float) the minimum IOA of a polygon and cell for that + ioa_thresh: the minimum IOA of a polygon and cell for that polygon to be a candidate for setting the class_id - use_intersection_over_cell: (bool) If true, then use the area of the + use_intersection_over_cell: If ``True``, then use the area of the cell as the denominator in the IOA. Otherwise, use the area of the polygon. - background_class_id: (None or int) If not None, class_id to use as the + background_class_id: If not ``None``, class_id to use as the background class; ie. the one that is used when a window contains no boxes. - pick_min_class_id: If true, the class_id for a cell is the minimum + pick_min_class_id: If ``True``, the class_id for a cell is the minimum class_id of the boxes in that cell. Otherwise, pick the class_id of the box covering the greatest area. - - Returns: - ChipClassificationLabels """ cells_df = gpd.GeoDataFrame( data={'cell_id': range(len(cells))}, @@ -93,21 +90,18 @@ def infer_cells(cells: List[Box], labels_df: gpd.GeoDataFrame, def read_labels(labels_df: gpd.GeoDataFrame, - bbox: Optional[Box] = None) -> ChipClassificationLabels: - """Convert GeoDataFrame to ChipClassificationLabels. + bbox: Box | None = None) -> ChipClassificationLabels: + """Convert ``GeoDataFrame`` to ``ChipClassificationLabels``. - If the GeoDataFrame already contains a grid of cells, then - ChipClassificationLabels can be constructed in a straightforward manner + If the ``GeoDataFrame`` already contains a grid of cells, then + ``ChipClassificationLabels`` can be constructed in a straightforward manner without having to infer the class of cells. - If bbox is given, only labels that intersect with it are returned. + If ``bbox`` is given, only labels that intersect with it are returned. Args: geojson: dict in normalized GeoJSON format (see VectorSource) bbox: Box in pixel coords - - Returns: - ChipClassificationLabels """ boxes = [Box.from_shapely(g).to_int() for g in labels_df.geometry] if bbox is not None: @@ -139,18 +133,17 @@ class ChipClassificationLabelSource(LabelSource): def __init__(self, label_source_config: 'ChipClassificationLabelSourceConfig', vector_source: 'VectorSource', - bbox: Optional[Box] = None, + bbox: Box | None = None, lazy: bool = False): - """Constructs a LabelSource for chip classification. + """Constructor. Args: - label_source_config (ChipClassificationLabelSourceConfig): Config - for class inference. - vector_source (VectorSource): Source of vector labels. - bbox (Optional[Box], optional): User-specified crop of the extent. - If None, the full extent available in the source file is used. - lazy (bool): If True, labels are not populated during - initialization. Defaults to False. + label_source_config: Config for class inference. + vector_source: Source of vector labels. + bbox: User-specified crop of the extent. If ``None``, the full + extent available in the source file is used. + lazy: If ``True``, labels are not populated during initialization. + Defaults to ``False``. """ self.cfg = label_source_config self.vector_source = vector_source @@ -165,8 +158,8 @@ def __init__(self, if not self.lazy: self.populate_labels() - def populate_labels(self, cells: Optional[Iterable[Box]] = None) -> None: - """Populate self.labels by either reading or inferring. + def populate_labels(self, cells: Iterable[Box] | None = None) -> None: + """Populate ``self.labels`` by either reading or inferring. If cfg.infer_cells is True or specific cells are given, the labels are inferred. Otherwise, they are read from the geojson. @@ -176,14 +169,14 @@ def populate_labels(self, cells: Optional[Iterable[Box]] = None) -> None: else: self.labels = read_labels(self.labels_df, bbox=self.bbox) - def infer_cells(self, cells: Optional[Iterable[Box]] = None + def infer_cells(self, cells: Iterable[Box] | None = None ) -> ChipClassificationLabels: - """Infer labels for a list of cells. Only cells whose labels are not - already known are inferred. + """Infer labels for a list of cells. + + Only cells whose labels are not already known are inferred. Args: - cells (Optional[Iterable[Box]], optional): Cells whose labels are - to be inferred. Defaults to None. + cells: Cells whose labels are to be inferred. Defaults to ``None``. Returns: ChipClassificationLabels: labels @@ -214,7 +207,7 @@ def infer_cells(self, cells: Optional[Iterable[Box]] = None return labels def get_labels(self, - window: Optional[Box] = None) -> ChipClassificationLabels: + window: Box | None = None) -> ChipClassificationLabels: if window is None: return self.labels window = window.to_global_coords(self.bbox) diff --git a/rastervision_core/rastervision/core/data/label_source/chip_classification_label_source_config.py b/rastervision_core/rastervision/core/data/label_source/chip_classification_label_source_config.py index a1ceaed1c..70b73c5eb 100644 --- a/rastervision_core/rastervision/core/data/label_source/chip_classification_label_source_config.py +++ b/rastervision_core/rastervision/core/data/label_source/chip_classification_label_source_config.py @@ -1,4 +1,4 @@ -from typing import Optional, Self +from typing import Self from rastervision.core.data.vector_source import (VectorSourceConfig) from rastervision.core.data.label_source import (LabelSourceConfig, @@ -25,7 +25,7 @@ class ChipClassificationLabelSourceConfig(LabelSourceConfig): This can be provided explicitly as a grid of cells, or a grid of cells can be inferred from arbitrary polygons. """ - vector_source: Optional[VectorSourceConfig] = None + vector_source: VectorSourceConfig | None = None ioa_thresh: float = Field( 0.5, description= @@ -42,7 +42,7 @@ class ChipClassificationLabelSourceConfig(LabelSourceConfig): ('If True, the class_id for a cell is the minimum class_id of the boxes in that ' 'cell. Otherwise, pick the class_id of the box covering the greatest area.' )) - background_class_id: Optional[int] = Field( + background_class_id: int | None = Field( None, description= ('If not None, class_id to use as the background class; ie. the one that is used ' @@ -51,7 +51,7 @@ class ChipClassificationLabelSourceConfig(LabelSourceConfig): infer_cells: bool = Field( False, description='If True, infers a grid of cells based on the cell_sz.') - cell_sz: Optional[int] = Field( + cell_sz: int | None = Field( None, description= ('Size of a cell to use in pixels. If None, and this Config is part ' diff --git a/rastervision_core/rastervision/core/data/label_source/label_source.py b/rastervision_core/rastervision/core/data/label_source/label_source.py index cd09f0863..bf6386a31 100644 --- a/rastervision_core/rastervision/core/data/label_source/label_source.py +++ b/rastervision_core/rastervision/core/data/label_source/label_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any from abc import ABC, abstractmethod from rastervision.core.box import Box @@ -18,7 +18,7 @@ class LabelSource(ABC): """ @abstractmethod - def get_labels(self, window: Optional['Box'] = None) -> 'Labels': + def get_labels(self, window: 'Box | None' = None) -> 'Labels': """Return labels overlapping with window. Args: diff --git a/rastervision_core/rastervision/core/data/label_source/label_source_config.py b/rastervision_core/rastervision/core/data/label_source/label_source_config.py index de265e9f0..967dd0385 100644 --- a/rastervision_core/rastervision/core/data/label_source/label_source_config.py +++ b/rastervision_core/rastervision/core/data/label_source/label_source_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.config import Config, register_config @@ -16,11 +16,11 @@ class LabelSourceConfig(Config): def build(self, class_config: 'ClassConfig', crs_transformer: 'CRSTransformer', - bbox: Optional['Box'] = None, - tmp_dir: Optional[str] = None) -> 'LabelSource': + bbox: 'Box | None' = None, + tmp_dir: str | None = None) -> 'LabelSource': raise NotImplementedError() def update(self, - pipeline: Optional['RVPipelineConfig'] = None, - scene: Optional['SceneConfig'] = None) -> None: + pipeline: 'RVPipelineConfig | None' = None, + scene: 'SceneConfig | None' = None) -> None: pass diff --git a/rastervision_core/rastervision/core/data/label_source/object_detection_label_source.py b/rastervision_core/rastervision/core/data/label_source/object_detection_label_source.py index 13bb7cb4c..badd5ec34 100644 --- a/rastervision_core/rastervision/core/data/label_source/object_detection_label_source.py +++ b/rastervision_core/rastervision/core/data/label_source/object_detection_label_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Optional, Tuple +from typing import TYPE_CHECKING, Any import numpy as np @@ -17,19 +17,19 @@ class ObjectDetectionLabelSource(LabelSource): def __init__(self, vector_source: VectorSource, - bbox: Optional[Box] = None, - ioa_thresh: Optional[float] = None, + bbox: Box | None = None, + ioa_thresh: float | None = None, clip: bool = False): """Constructor. Args: - vector_source (VectorSource): A VectorSource. - bbox (Optional[Box], optional): User-specified crop of the extent. - If None, the full extent available in the source file is used. - ioa_thresh (Optional[float], optional): IOA threshold to apply when - retrieving labels for a window. Defaults to None. - clip (bool, optional): Clip bounding boxes to window limits when - retrieving labels for a window. Defaults to False. + vector_source: A ``VectorSource``. + bbox: User-specified crop of the extent. If ``None``, the full + extent available in the source file is used. + ioa_thresh: IOA threshold to apply when retrieving labels for a + window. Defaults to ``None``. + clip: Clip bounding boxes to window limits when retrieving labels + for a window. Defaults to ``False``. """ self.vector_source = vector_source geojson = self.vector_source.get_geojson() @@ -42,18 +42,18 @@ def __init__(self, self.clip = clip def get_labels(self, - window: Box = None, + window: Box | None = None, ioa_thresh: float = 1e-6, clip: bool = False) -> ObjectDetectionLabels: """Get labels (in global coords) for a window. Args: - window (Box): Window coords. + window: Window coords. Returns: - ObjectDetectionLabels: Labels with sufficient overlap with the - window. The returned labels are in global coods (i.e. coords within - the full extent of the source). + Labels with sufficient overlap with the window. The returned labels + are in global coods (i.e. coords within the full extent of the + source). """ if window is None: return self.labels @@ -61,7 +61,7 @@ def get_labels(self, return ObjectDetectionLabels.get_overlapping( self.labels, window, ioa_thresh=ioa_thresh, clip=clip) - def __getitem__(self, key: Any) -> Tuple[np.ndarray, np.ndarray, str]: + def __getitem__(self, key: Any) -> tuple[np.ndarray, np.ndarray, str]: """Get labels (in window coords) for a window. Returns a 3-tuple: (npboxes, class_ids, box_format). @@ -76,7 +76,7 @@ class labels for each of the boxes. window (Box): Window coords. Returns: - Tuple[np.ndarray, np.ndarray, str]: 3-tuple of + tuple[np.ndarray, np.ndarray, str]: 3-tuple of (npboxes, class_ids, box_format). The returned npboxes are in window coords (i.e. coords within the window). """ diff --git a/rastervision_core/rastervision/core/data/label_source/semantic_segmentation_label_source.py b/rastervision_core/rastervision/core/data/label_source/semantic_segmentation_label_source.py index 96717dce1..3d4e4989e 100644 --- a/rastervision_core/rastervision/core/data/label_source/semantic_segmentation_label_source.py +++ b/rastervision_core/rastervision/core/data/label_source/semantic_segmentation_label_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import numpy as np @@ -19,17 +19,17 @@ class SemanticSegmentationLabelSource(LabelSource): def __init__(self, raster_source: RasterSource, class_config: ClassConfig, - bbox: Optional[Box] = None): + bbox: Box | None = None): """Constructor. Args: - raster_source (RasterSource): A raster source that returns a single - channel raster with class_ids as values. - null_class_id (int): the null class id used as fill values for when + raster_source: A raster source that returns a single channel raster + with class_ids as values. + null_class_id: the null class id used as fill values for when windows go over the edge of the label array. This can be - retrieved using class_config.null_class_id. - bbox (Optional[Box], optional): User-specified crop of the extent. - If None, the full extent available in the source file is used. + retrieved using ``class_config.null_class_id``. + bbox: User-specified crop of the extent. If ``None``, the full + extent available in the source file is used. """ self.raster_source = raster_source self.class_config = class_config @@ -37,15 +37,15 @@ def __init__(self, self.set_bbox(bbox) def get_labels(self, - window: Optional[Box] = None) -> SemanticSegmentationLabels: + window: Box | None = None) -> SemanticSegmentationLabels: """Get labels for a window. Args: - window (Optional[Box], optional): Window to get labels for. If - None, returns labels covering the full extent of the scene. + window: Window to get labels for. If None, returns labels covering + the full extent of the scene. Returns: - SemanticSegmentationLabels: The labels. + The labels. """ if window is None: window = self.extent @@ -59,7 +59,7 @@ def get_labels(self, return labels - def get_label_arr(self, window: Optional[Box] = None) -> np.ndarray: + def get_label_arr(self, window: Box | None = None) -> np.ndarray: """Get labels for a window. The returned array will be the same size as the input window. If window @@ -67,12 +67,11 @@ def get_label_arr(self, window: Optional[Box] = None) -> np.ndarray: of the null class as defined by the class_config. Args: - window (Optional[Box], optional): Window (in pixel coords) to get - labels for. If None, returns a label array covering the full - extent of the scene. + window: Window (in pixel coords) to get labels for. If ``None``, + returns a label array covering the full extent of the scene. Returns: - np.ndarray: Label array. + Label array. """ if window is None: window = self.extent diff --git a/rastervision_core/rastervision/core/data/label_source/semantic_segmentation_label_source_config.py b/rastervision_core/rastervision/core/data/label_source/semantic_segmentation_label_source_config.py index 0d2cac802..c6943425e 100644 --- a/rastervision_core/rastervision/core/data/label_source/semantic_segmentation_label_source_config.py +++ b/rastervision_core/rastervision/core/data/label_source/semantic_segmentation_label_source_config.py @@ -1,5 +1,3 @@ -from typing import Union - from rastervision.core.data.raster_source import (RasterSourceConfig, RasterizedSourceConfig) from rastervision.core.data.label_source import ( @@ -21,7 +19,7 @@ def ss_label_source_config_upgrader(cfg_dict: dict, class SemanticSegmentationLabelSourceConfig(LabelSourceConfig): """Configure a :class:`.SemanticSegmentationLabelSource`.""" - raster_source: Union[RasterSourceConfig, RasterizedSourceConfig] = Field( + raster_source: RasterSourceConfig | RasterizedSourceConfig = Field( ..., description='The labels in the form of rasters.') def build(self, class_config, crs_transformer, bbox=None, diff --git a/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store.py b/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store.py index f2b319f89..52092bd13 100644 --- a/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store.py +++ b/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.file_system import json_to_file from rastervision.core.data.label import ChipClassificationLabels @@ -20,7 +20,7 @@ def __init__(self, uri: str, class_config: 'ClassConfig', crs_transformer: 'CRSTransformer', - bbox: Optional['Box'] = None): + bbox: 'Box | None' = None): """Constructor. Args: @@ -28,7 +28,7 @@ def __init__(self, class_config: ClassConfig crs_transformer: CRSTransformer to convert from map coords in label in GeoJSON file to pixel coords. - bbox (Optional[Box], optional): User-specified crop of the extent. + bbox (Box | None): User-specified crop of the extent. If provided, only labels falling inside it are returned by :meth:`.ChipClassificationGeoJSONStore.get_labels`. Must be provided if the corresponding RasterSource has bbox != extent. diff --git a/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store_config.py b/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store_config.py index 35244de22..dddc2a88e 100644 --- a/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store_config.py +++ b/rastervision_core/rastervision/core/data/label_store/chip_classification_geojson_store_config.py @@ -1,4 +1,3 @@ -from typing import Optional from os.path import join from rastervision.core.data.label_store import (LabelStoreConfig, @@ -10,7 +9,7 @@ class ChipClassificationGeoJSONStoreConfig(LabelStoreConfig): """Configure a :class:`.ChipClassificationGeoJSONStore`.""" - uri: Optional[str] = Field( + uri: str | None = Field( None, description= ('URI of GeoJSON file with predictions. If None, and this Config is part of ' diff --git a/rastervision_core/rastervision/core/data/label_store/label_store.py b/rastervision_core/rastervision/core/data/label_store/label_store.py index 49265151d..89440e12b 100644 --- a/rastervision_core/rastervision/core/data/label_store/label_store.py +++ b/rastervision_core/rastervision/core/data/label_store/label_store.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from abc import ABC, abstractmethod if TYPE_CHECKING: @@ -24,7 +24,7 @@ def get_labels(self): @property @abstractmethod - def bbox(self) -> Optional['Box']: + def bbox(self) -> 'Box | None': """Bounding box applied to the source.""" @property diff --git a/rastervision_core/rastervision/core/data/label_store/label_store_config.py b/rastervision_core/rastervision/core/data/label_store/label_store_config.py index ece7b97d3..472a5d524 100644 --- a/rastervision_core/rastervision/core/data/label_store/label_store_config.py +++ b/rastervision_core/rastervision/core/data/label_store/label_store_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.config import Config, register_config @@ -16,11 +16,11 @@ class LabelStoreConfig(Config): def build(self, class_config: 'ClassConfig', crs_transformer: 'CRSTransformer', - bbox: Optional['Box'] = None, - tmp_dir: Optional[str] = None) -> 'LabelStore': + bbox: 'Box | None' = None, + tmp_dir: str | None = None) -> 'LabelStore': raise NotImplementedError() def update(self, - pipeline: Optional['RVPipelineConfig'] = None, - scene: Optional['SceneConfig'] = None) -> None: + pipeline: 'RVPipelineConfig | None' = None, + scene: 'SceneConfig | None' = None) -> None: pass diff --git a/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store.py b/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store.py index 8c2dae032..7cd49cc5f 100644 --- a/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store.py +++ b/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import logging from rastervision.core.data.label import ObjectDetectionLabels @@ -21,7 +21,7 @@ def __init__(self, uri: str, class_config: 'ClassConfig', crs_transformer: 'CRSTransformer', - bbox: Optional['Box'] = None): + bbox: 'Box | None' = None): """Constructor. Args: @@ -30,7 +30,7 @@ def __init__(self, (or label) field crs_transformer: CRSTransformer to convert from map coords in label in GeoJSON file to pixel coords. - bbox (Optional[Box], optional): User-specified crop of the extent. + bbox: User-specified crop of the extent. If provided, only labels falling inside it are returned by :meth:`.ObjectDetectionGeoJSONStore.get_labels`. Must be provided if the corresponding RasterSource has bbox != extent. diff --git a/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store_config.py b/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store_config.py index 0d5d77e1a..386d0d0c6 100644 --- a/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store_config.py +++ b/rastervision_core/rastervision/core/data/label_store/object_detection_geojson_store_config.py @@ -1,4 +1,3 @@ -from typing import Optional from os.path import join from rastervision.core.data.label_store import (LabelStoreConfig, @@ -10,7 +9,7 @@ class ObjectDetectionGeoJSONStoreConfig(LabelStoreConfig): """Configure an :class:`.ObjectDetectionGeoJSONStore`.""" - uri: Optional[str] = Field( + uri: str | None = Field( None, description= ('URI of GeoJSON file with predictions. If None, and this Config is part of ' @@ -24,5 +23,4 @@ def build(self, class_config, crs_transformer, bbox=None, tmp_dir=None): def update(self, pipeline=None, scene=None): if pipeline is not None and scene is not None: if self.uri is None: - self.uri = join(pipeline.predict_uri, - '{}.json'.format(scene.id)) + self.uri = join(pipeline.predict_uri, f'{scene.id}.json') diff --git a/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store.py b/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store.py index 474bdebb1..e43d90680 100644 --- a/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store.py +++ b/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Sequence, Tuple +from typing import TYPE_CHECKING, Sequence from os.path import join import logging @@ -34,53 +34,54 @@ class SemanticSegmentationLabelStore(LabelStore): and can optionally vectorize predictions and store them as GeoJSON files. """ - def __init__( - self, - uri: str, - crs_transformer: CRSTransformer, - class_config: ClassConfig, - bbox: Optional[Box] = None, - tmp_dir: Optional[str] = None, - vector_outputs: Optional[Sequence['VectorOutputConfig']] = None, - save_as_rgb: bool = False, - discrete_output: bool = True, - smooth_output: bool = False, - smooth_as_uint8: bool = False, - rasterio_block_size: int = 512): + def __init__(self, + uri: str, + crs_transformer: CRSTransformer, + class_config: ClassConfig, + bbox: Box | None = None, + tmp_dir: str | None = None, + vector_outputs: 'Sequence[VectorOutputConfig] | None' = None, + save_as_rgb: bool = False, + discrete_output: bool = True, + smooth_output: bool = False, + smooth_as_uint8: bool = False, + rasterio_block_size: int = 512): """Constructor. Args: - uri (str): Path to directory where the predictions are/will be - stored. Smooth scores will be saved as "uri/scores.tif", - discrete labels will be stored as "uri/labels.tif", and vector - outputs will be saved in "uri/vector_outputs/". - crs_transformer (CRSTransformer): CRS transformer for correctly - mapping from pixel coords to map coords. - class_config (ClassConfig): Class config. - bbox (Optional[Box], optional): User-specified crop of the extent. + uri: Path to directory where the predictions are/will be + stored. Smooth scores will be saved as ``"uri/scores.tif"``, + discrete labels will be stored as ``"uri/labels.tif"``, and + vector outputs will be saved in ``"uri/vector_outputs/"``. + crs_transformer: CRS transformer for correctly mapping from pixel + coords to map coords. + class_config: Class config. + bbox: User-specified crop of the extent. If provided, only labels falling inside it are returned by :meth:`.SemanticSegmentationLabelStore.get_labels`. Must be - provided if the corresponding RasterSource has bbox != extent. - tmp_dir (Optional[str], optional): Temporary directory to use. If - None, will be auto-generated. Defaults to None. - vector_outputs (Optional[Sequence[VectorOutputConfig]], optional): + provided if the corresponding ``RasterSource`` has + ``bbox != extent``. + tmp_dir: Temporary directory to use. If ``None``, will be + auto-generated. Defaults to ``None``. + vector_outputs: List of VectorOutputConfig's containing vectorization configuration information. Only classes for which a - VectorOutputConfig is specified will be saved as vectors. - If None, no vector outputs will be produced. Defaults to None. - save_as_rgb (bool, optional): If True, saves labels as an RGB - image, using the class-color mapping in the class_config. - Defaults to False. - discrete_output (bool, optional): If True, saves labels as a raster - of class IDs (one band). Defaults to False. - smooth_output (bool, optional): If True, saves labels as a raster - of class scores (one band for each class). Defaults to False. - smooth_as_uint8 (bool, optional): If True, stores smooth class - scores as np.uint8 (0-255) values rather than as np.float32 - discrete labels, to help save memory/disk space. - Defaults to False. - rasterio_block_size (int, optional): Value to set blockxsize and - blockysize to. Defaults to 512. + ``VectorOutputConfig`` is specified will be saved as vectors. + If ``None``, no vector outputs will be produced. + Defaults to ``None``. + save_as_rgb: If ``True``, saves labels as an RGB + image, using the class-color mapping in the ``class_config``. + Defaults to ``False``. + discrete_output: If ``True``, saves labels as a raster + of class IDs (one band). Defaults to ``False``. + smooth_output: If ``True``, saves labels as a raster + of class scores (one band for each class). Defaults to ``False``. + smooth_as_uint8: If ``True``, stores smooth class + scores as ``np.uint8`` (0-255) values rather than as + ``np.float32`` discrete labels, to help save memory/disk space. + Defaults to ``False``. + rasterio_block_size: Value to set ``blockxsize`` and ``blockysize`` + to. Defaults to ``512``. """ self.root_uri = uri @@ -191,7 +192,7 @@ def get_scores(self) -> 'SemanticSegmentationSmoothLabels': def save(self, labels: SemanticSegmentationLabels, - profile: Optional[dict] = None) -> None: + profile: dict | None = None) -> None: """Save labels to disk. More info on rasterio IO: @@ -335,8 +336,8 @@ def write_vector_output(self, vo: 'VectorOutputConfig', mask: np.ndarray, def _clip_to_extent(self, extent: Box, window: Box, - arr: Optional[np.ndarray] = None - ) -> Tuple[Box, Optional[np.ndarray]]: + arr: np.ndarray | None = None + ) -> tuple[Box, np.ndarray | None]: clipped_window = window.intersection(extent) if arr is not None: h, w = clipped_window.size diff --git a/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store_config.py b/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store_config.py index 115b94b28..03d770495 100644 --- a/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store_config.py +++ b/rastervision_core/rastervision/core/data/label_store/semantic_segmentation_label_store_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Iterator, List, Optional +from typing import TYPE_CHECKING, Iterator from os.path import join from rastervision.pipeline.config import register_config, Config, Field @@ -41,7 +41,7 @@ class VectorOutputConfig(Config): 'intensive (especially for large images). Larger values will remove ' 'more noise and make vectorization faster but might also remove ' 'legitimate detections.') - threshold: Optional[float] = Field( + threshold: float | None = Field( None, description='Probability threshold for creating the binary mask for ' 'the pixels of this class. Pixels will be considered to belong to ' @@ -57,7 +57,7 @@ def vectorize(self, mask: 'np.ndarray') -> Iterator['BaseGeometry']: raise NotImplementedError() def get_uri(self, root: str, - class_config: Optional['ClassConfig'] = None) -> str: + class_config: 'ClassConfig | None' = None) -> str: if class_config is not None: class_name = class_config.get_name(self.class_id) uri = join(root, f'class-{self.class_id}-{class_name}.json') @@ -125,13 +125,13 @@ class SemanticSegmentationLabelStoreConfig(LabelStoreConfig): Stores class raster as GeoTIFF, and can optionally vectorizes predictions and stores them in GeoJSON files. """ - uri: Optional[str] = Field( + uri: str | None = Field( None, description=( 'URI of file with predictions. If None, and this Config is part of ' 'a SceneConfig inside an RVPipelineConfig, this fiend will be ' 'auto-generated.')) - vector_output: List[VectorOutputConfig] = [] + vector_output: list[VectorOutputConfig] = [] rgb: bool = Field( False, description= @@ -156,7 +156,7 @@ def build(self, class_config: 'ClassConfig', crs_transformer: 'CRSTransformer', bbox: 'Box', - tmp_dir: Optional[str] = None) -> SemanticSegmentationLabelStore: + tmp_dir: str | None = None) -> SemanticSegmentationLabelStore: class_config.ensure_null_class() label_store = SemanticSegmentationLabelStore( @@ -174,8 +174,8 @@ def build(self, return label_store def update(self, - pipeline: Optional['RVPipelineConfig'] = None, - scene: Optional['SceneConfig'] = None): + pipeline: 'RVPipelineConfig | None' = None, + scene: 'SceneConfig | None' = None): if pipeline is not None and scene is not None: if self.uri is None: self.uri = join(pipeline.predict_uri, f'{scene.id}') diff --git a/rastervision_core/rastervision/core/data/label_store/utils.py b/rastervision_core/rastervision/core/data/label_store/utils.py index 74911f057..3fa8707f1 100644 --- a/rastervision_core/rastervision/core/data/label_store/utils.py +++ b/rastervision_core/rastervision/core/data/label_store/utils.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Sequence, Union +from typing import TYPE_CHECKING, Sequence from tqdm.auto import tqdm @@ -11,27 +11,25 @@ PROGRESSBAR_DELAY_SEC = 5 -def boxes_to_geojson( - boxes: Sequence['Box'], - class_ids: Sequence[int], - crs_transformer: 'CRSTransformer', - class_config: 'ClassConfig', - scores: Optional[Sequence[Union[float, Sequence[float]]]] = None, - bbox: Optional['Box'] = None) -> dict: +def boxes_to_geojson(boxes: Sequence['Box'], + class_ids: Sequence[int], + crs_transformer: 'CRSTransformer', + class_config: 'ClassConfig', + scores: Sequence[float | Sequence[float]] | None = None, + bbox: 'Box | None' = None) -> dict: """Convert boxes and associated data into a GeoJSON dict. Args: - boxes (Sequence[Box]): List of Box in pixel row/col format. - class_ids (Sequence[int]): List of int (one for each box) - crs_transformer (CRSTransformer): CRSTransformer used to convert pixel - coords to map coords in the GeoJSON. - class_config (ClassConfig): ClassConfig - scores (Optional[Sequence[Union[float, Sequence[float]]]], optional): - Optional list of score or scores. If floats (one for each box), + boxes: List of Box in pixel row/col format. + class_ids: List of int (one for each box) + crs_transformer: CRSTransformer used to convert pixel coords to map + coords in the GeoJSON. + class_config: ClassConfig + scores: Optional list of score or scores. If floats (one for each box), property name will be "score". If lists of floats, property name - will be "scores". Defaults to None. - bbox (Optional[Box]): User-specified crop of the extent. Must be - provided if the corresponding RasterSource has bbox != extent. + will be "scores". Defaults to ``None``. + bbox: User-specified crop of the extent. Must be provided if the + corresponding :class:`.RasterSource` has ``bbox != extent``. Returns: dict: Serialized GeoJSON. diff --git a/rastervision_core/rastervision/core/data/raster_source/multi_raster_source.py b/rastervision_core/rastervision/core/data/raster_source/multi_raster_source.py index d86627d10..4aa0de03a 100644 --- a/rastervision_core/rastervision/core/data/raster_source/multi_raster_source.py +++ b/rastervision_core/rastervision/core/data/raster_source/multi_raster_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Sequence, Self, Tuple +from typing import TYPE_CHECKING, Sequence, Self from pydantic import NonNegativeInt as NonNegInt import numpy as np @@ -171,7 +171,7 @@ def primary_source(self) -> RasterSource: return self.raster_sources[self.primary_source_idx] @property - def shape(self) -> Tuple[int, ...]: + def shape(self) -> tuple[int, ...]: """Shape of the raster as a (..., H, W, C) tuple.""" *shape, _ = self.primary_source.shape return (*shape, self.num_channels) @@ -186,7 +186,7 @@ def crs_transformer(self) -> 'CRSTransformer': def _get_sub_chips(self, window: Box, - out_shape: Optional[Tuple[int, int]] = None + out_shape: tuple[int, int] | None = None ) -> list[np.ndarray]: """Return chips from sub raster sources as a list. @@ -204,20 +204,17 @@ def _get_sub_chips(self, reference chip from the primary sub raster source Args: - window (Box): The window for which to get the chip, in pixel - coordinates. - out_shape (Optional[Tuple[int, int]]): (height, width) to resize - the chip to. + window: The window for which to get the chip, in pixel coordinates. + out_shape: (height, width) to resize the chip to. Returns: - List[np.ndarray]: List of chips from each sub raster source. + List of chips from each sub raster source. """ - def get_chip( - rs: RasterSource, - window: Box, - map: bool = False, - out_shape: Optional[Tuple[int, int]] = None) -> np.ndarray: + def get_chip(rs: RasterSource, + window: Box, + map: bool = False, + out_shape: tuple[int, int] | None = None) -> np.ndarray: if map: func = rs.get_chip_by_map_window else: @@ -244,27 +241,25 @@ def get_chip( return sub_chips - def _get_chip(self, - window: Box, - out_shape: Optional[Tuple[int, int]] = None) -> np.ndarray: + def _get_chip(self, window: Box, + out_shape: tuple[int, int] | None = None) -> np.ndarray: """Get chip w/o applying channel_order and transformers. Args: window (Box): The window for which to get the chip, in pixel coordinates. - out_shape (Optional[Tuple[int, int]]): (height, width) to resize + out_shape (tuple[int, int] | None): (height, width) to resize the chip to. Returns: - [height, width, channels] numpy array + Array of shape (height, width, channels). """ sub_chips = self._get_sub_chips(window, out_shape=out_shape) chip = np.concatenate(sub_chips, axis=-1) return chip - def get_chip(self, - window: Box, - out_shape: Optional[Tuple[int, int]] = None) -> np.ndarray: + def get_chip(self, window: Box, + out_shape: tuple[int, int] | None = None) -> np.ndarray: """Return the transformed chip in the window. Get processed chips from sub raster sources (with their respective @@ -272,13 +267,11 @@ def get_chip(self, channel dimension, apply channel_order, followed by transformations. Args: - window (Box): The window for which to get the chip, in pixel - coordinates. - out_shape (Optional[Tuple[int, int]]): (height, width) to resize - the chip to. + window: The window for which to get the chip, in pixel coordinates. + out_shape: (height, width) to resize the chip to. Returns: - np.ndarray with shape [height, width, channels] + Array of shape (height, width, channels). """ sub_chips = self._get_sub_chips(window, out_shape=out_shape) chip = np.concatenate(sub_chips, axis=-1) diff --git a/rastervision_core/rastervision/core/data/raster_source/multi_raster_source_config.py b/rastervision_core/rastervision/core/data/raster_source/multi_raster_source_config.py index 19f31c40e..3f942ab9a 100644 --- a/rastervision_core/rastervision/core/data/raster_source/multi_raster_source_config.py +++ b/rastervision_core/rastervision/core/data/raster_source/multi_raster_source_config.py @@ -1,4 +1,4 @@ -from typing import List, Optional, Self +from typing import Self from typing_extensions import Annotated from pydantic import NonNegativeInt as NonNegInt @@ -28,7 +28,7 @@ class MultiRasterSourceConfig(RasterSourceConfig): Or :class:`.TemporalMultiRasterSource`, if ``temporal=True``. """ - raster_sources: Annotated[List[ + raster_sources: Annotated[list[ RasterSourceConfig], Field(min_length=1)] = Field( ..., description='List of RasterSourceConfig to combine.') primary_source_idx: NonNegInt = Field( @@ -61,8 +61,7 @@ def validate_temporal(self) -> Self: 'Setting channel_order is not allowed if temporal=True.') return self - def build(self, - tmp_dir: Optional[str] = None, + def build(self, tmp_dir: str | None = None, use_transformers: bool = True) -> MultiRasterSource: if use_transformers: raster_transformers = [t.build() for t in self.transformers] diff --git a/rastervision_core/rastervision/core/data/raster_source/raster_source.py b/rastervision_core/rastervision/core/data/raster_source/raster_source.py index da851d4e6..17c298848 100644 --- a/rastervision_core/rastervision/core/data/raster_source/raster_source.py +++ b/rastervision_core/rastervision/core/data/raster_source/raster_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, List, Optional, Tuple +from typing import TYPE_CHECKING, Any from abc import ABC, abstractmethod import numpy as np @@ -12,7 +12,7 @@ class ChannelOrderError(Exception): - def __init__(self, channel_order: List[int], num_channels_raw: int): + def __init__(self, channel_order: list[int], num_channels_raw: int): self.channel_order = channel_order self.num_channels_raw = num_channels_raw msg = (f'The channel_order ({channel_order}) contains an ' @@ -28,10 +28,10 @@ class RasterSource(ABC): """ def __init__(self, - channel_order: Optional[List[int]], + channel_order: list[int] | None, num_channels_raw: int, bbox: Box, - raster_transformers: List['RasterTransformer'] = []): + raster_transformers: list['RasterTransformer'] = []): """Constructor. Args: @@ -60,7 +60,7 @@ def num_channels(self) -> int: return len(self.channel_order) @property - def shape(self) -> Tuple[int, int, int]: + def shape(self) -> tuple[int, int, int]: """Shape of the raster as a (height, width, num_channels) tuple.""" H, W = self.bbox.size return H, W, self.num_channels @@ -98,12 +98,12 @@ def set_bbox(self, bbox: 'Box') -> None: @abstractmethod def _get_chip(self, window: 'Box', - out_shape: Optional[Tuple[int, int]] = None) -> 'np.ndarray': + out_shape: tuple[int, int] | None = None) -> 'np.ndarray': """Return raw chip without applying channel_order or transforms. Args: window (Box): The window for which to get the chip. - out_shape (Optional[Tuple[int, int]]): (height, width) to resize + out_shape (tuple[int, int] | None): (height, width) to resize the chip to. Returns: @@ -123,9 +123,8 @@ def __getitem__(self, key: Any) -> 'np.ndarray': return chip - def get_chip(self, - window: 'Box', - out_shape: Optional[Tuple[int, int]] = None) -> 'np.ndarray': + def get_chip(self, window: 'Box', + out_shape: tuple[int, int] | None = None) -> 'np.ndarray': """Return the transformed chip in the window. Get a raw chip, extract subset of channels using channel_order, and then apply @@ -133,7 +132,7 @@ def get_chip(self, Args: window (Box): The window for which to get the chip. - out_shape (Optional[Tuple[int, int]]): (height, width) to resize + out_shape (tuple[int, int] | None): (height, width) to resize the chip to. Returns: @@ -165,8 +164,7 @@ def _get_chip_by_map_window(self, window_map_coords: 'Box', *args, def get_raw_chip(self, window: 'Box', - out_shape: Optional[Tuple[int, int]] = None - ) -> 'np.ndarray': + out_shape: tuple[int, int] | None = None) -> 'np.ndarray': """Return raw chip without applying channel_order or transforms. Args: @@ -179,7 +177,7 @@ def get_raw_chip(self, def resize(self, chip: 'np.ndarray', - out_shape: Optional[Tuple[int, int]] = None) -> 'np.ndarray': + out_shape: tuple[int, int] | None = None) -> 'np.ndarray': out_shape = chip.shape[:-3] + out_shape out = resize(chip, out_shape, preserve_range=True, anti_aliasing=True) out = out.round(6).astype(chip.dtype) diff --git a/rastervision_core/rastervision/core/data/raster_source/raster_source_config.py b/rastervision_core/rastervision/core/data/raster_source/raster_source_config.py index c23a3df17..ab014bc7f 100644 --- a/rastervision_core/rastervision/core/data/raster_source/raster_source_config.py +++ b/rastervision_core/rastervision/core/data/raster_source/raster_source_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING from rastervision.pipeline.config import (Config, register_config, Field, ConfigError) @@ -33,24 +33,23 @@ def rs_config_upgrader(cfg_dict: dict, class RasterSourceConfig(Config): """Configure a :class:`.RasterSource`.""" - channel_order: Optional[List[int]] = Field( + channel_order: list[int] | None = Field( None, description= 'The sequence of channel indices to use when reading imagery.') - transformers: List[RasterTransformerConfig] = [] - bbox: Optional[Tuple[int, int, int, int]] = Field( + transformers: list[RasterTransformerConfig] = [] + bbox: tuple[int, int, int, int] | None = Field( None, description='User-specified bbox in pixel coords in the form ' '(ymin, xmin, ymax, xmax). Useful for cropping the raster source so ' 'that only part of the raster is read from.') - def build(self, - tmp_dir: Optional[str] = None, + def build(self, tmp_dir: str | None = None, use_transformers: bool = True) -> 'RasterSource': raise NotImplementedError() def update(self, - pipeline: Optional['RVPipelineConfig'] = None, - scene: Optional['SceneConfig'] = None) -> None: + pipeline: 'RVPipelineConfig | None' = None, + scene: 'SceneConfig | None' = None) -> None: for t in self.transformers: t.update(pipeline, scene) diff --git a/rastervision_core/rastervision/core/data/raster_source/rasterio_source.py b/rastervision_core/rastervision/core/data/raster_source/rasterio_source.py index 2061838b7..6c8d3fd29 100644 --- a/rastervision_core/rastervision/core/data/raster_source/rasterio_source.py +++ b/rastervision_core/rastervision/core/data/raster_source/rasterio_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Any, Sequence import logging import numpy as np @@ -39,30 +39,30 @@ class RasterioSource(RasterSource): """ def __init__(self, - uris: Union[str, List[str]], - raster_transformers: List['RasterTransformer'] = [], + uris: str | list[str], + raster_transformers: list['RasterTransformer'] = [], allow_streaming: bool = False, - channel_order: Optional[Sequence[int]] = None, - bbox: Optional[Box] = None, - tmp_dir: Optional[str] = None): + channel_order: Sequence[int] | None = None, + bbox: Box | None = None, + tmp_dir: str | None = None): """Constructor. Args: - uris (Union[str, List[str]]): One or more URIs of images. If more - than one, the images will be mosaiced together using GDAL. - raster_transformers (List['RasterTransformer']): RasterTransformers - to use to transform chips after they are read. - allow_streaming (bool): If True, read data without downloading the - entire file first. Defaults to False. - channel_order (Optional[Sequence[int]]): List of indices of - channels to extract from raw imagery. Can be a subset of the - available channels. If None, all channels available in the - image will be read. Defaults to None. - bbox (Optional[Box], optional): User-specified crop of the extent. - If None, the full extent available in the source file is used. - tmp_dir (Optional[str]): Directory to use for storing the VRT - (needed if multiple uris or allow_streaming=True). If None, - will be auto-generated. Defaults to None. + uris: One or more URIs of images. If more than one, the images will + be mosaiced together using GDAL. + raster_transformers: RasterTransformers to use to transform chips + after they are read. + allow_streaming: If ``True``, read data without downloading the + entire file first. Defaults to ``False``. + channel_order: List of indices of channels to extract from raw + imagery. Can be a subset of the available channels. If + ``None``, all channels available in the image will be read. + Defaults to ``None``. + bbox: User-specified crop of the extent. If ``None``, the full + extent available in the source file is used. + tmp_dir: Directory to use for storing the VRT (needed if multiple + ``uris`` or ``allow_streaming=True``). If ``None``, + will be auto-generated. Defaults to ``None``. """ self.uris = listify_uris(uris) self.allow_streaming = allow_streaming @@ -139,8 +139,7 @@ def _set_info_from_chip(self): self._dtype = test_chip.dtype self._num_channels = test_chip.shape[-1] - def download_data(self, - vrt_dir: Optional[str] = None, + def download_data(self, vrt_dir: str | None = None, stream: bool = False) -> str: """Download any data needed for this raster source. @@ -158,8 +157,8 @@ def download_data(self, def _get_chip(self, window: Box, - bands: Optional[Sequence[int]] = None, - out_shape: Optional[Tuple[int, ...]] = None) -> np.ndarray: + bands: Sequence[int] | None = None, + out_shape: tuple[int, ...] | None = None) -> np.ndarray: window = window.to_global_coords(self.bbox) chip = read_window( self.image_dataset, @@ -172,20 +171,18 @@ def _get_chip(self, def get_chip(self, window: Box, - bands: Optional[Union[Sequence[int], slice]] = None, - out_shape: Optional[Tuple[int, ...]] = None) -> np.ndarray: + bands: Sequence[int] | slice | None = None, + out_shape: tuple[int, ...] | None = None) -> np.ndarray: """Read a chip specified by a window from the file. Args: - window (Box): Bounding box of chip in pixel coordinates. - bands (Optional[Union[Sequence[int], slice]], optional): Subset of - bands to read. Note that this will be applied on top of the - channel_order (if specified). So if this is an RGB image and - channel_order=[2, 1, 0], then using bands=[0] will return the - B-channel. Defaults to None. - out_shape (Optional[Tuple[int, ...]], optional): (height, width) of - the output chip. If None, no resizing is done. - Defaults to None. + window: Bounding box of chip in pixel coordinates. + bands: Subset of bands to read. Note that this will be applied on + top of the ``channel_order`` (if specified). E.g. if this is an + RGB image and ``channel_order=[2, 1, 0]``, then ``bands=[0]`` + will return the B-channel. Defaults to ``None``. + out_shape: (height, width) of the output chip. If ``None``, no + resizing is done. Defaults to ``None``. Returns: np.ndarray: A chip of shape (height, width, channels). diff --git a/rastervision_core/rastervision/core/data/raster_source/rasterio_source_config.py b/rastervision_core/rastervision/core/data/raster_source/rasterio_source_config.py index 438c35f5a..b7d13fd16 100644 --- a/rastervision_core/rastervision/core/data/raster_source/rasterio_source_config.py +++ b/rastervision_core/rastervision/core/data/raster_source/rasterio_source_config.py @@ -1,5 +1,3 @@ -from typing import List, Union - from rastervision.core.box import Box from rastervision.core.data.raster_source import RasterSourceConfig, RasterioSource from rastervision.pipeline.config import ConfigError, Field, register_config @@ -26,7 +24,7 @@ def rasterio_source_config_upgrader(cfg_dict: dict, class RasterioSourceConfig(RasterSourceConfig): """Configure a :class:`.RasterioSource`.""" - uris: Union[str, List[str]] = Field( + uris: str | list[str] = Field( ..., description='One or more image URIs that comprise the imagery for a ' 'scene. The format of each file can be any that can be read by ' diff --git a/rastervision_core/rastervision/core/data/raster_source/rasterized_source.py b/rastervision_core/rastervision/core/data/raster_source/rasterized_source.py index 363d83501..3ddab9941 100644 --- a/rastervision_core/rastervision/core/data/raster_source/rasterized_source.py +++ b/rastervision_core/rastervision/core/data/raster_source/rasterized_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional, Tuple +from typing import TYPE_CHECKING import logging from rasterio.features import rasterize @@ -63,18 +63,18 @@ class RasterizedSource(RasterSource): def __init__(self, vector_source: 'VectorSource', background_class_id: int, - bbox: Optional['Box'] = None, + bbox: 'Box | None' = None, all_touched: bool = False, - raster_transformers: List['RasterTransformer'] = []): + raster_transformers: list['RasterTransformer'] = []): """Constructor. Args: vector_source (VectorSource): The VectorSource to rasterize. background_class_id (int): The class_id to use for any background pixels, ie. pixels not covered by a polygon. - bbox (Optional[Box], optional): User-specified crop of the extent. + bbox (Box | None): User-specified crop of the extent. If None, the full extent available in the source file is used. - all_touched (bool, optional): If True, all pixels touched by + all_touched (bool): If True, all pixels touched by geometries will be burned in. If false, only pixels whose center is within the polygon or that are selected by Bresenham's line algorithm will be burned in. @@ -107,7 +107,7 @@ def crs_transformer(self): def _get_chip(self, window: 'Box', - out_shape: Optional[Tuple[int, int]] = None) -> np.ndarray: + out_shape: tuple[int, int] | None = None) -> np.ndarray: """Return the chip located in the window. Polygons falling within the window are rasterized using their diff --git a/rastervision_core/rastervision/core/data/raster_source/rasterized_source_config.py b/rastervision_core/rastervision/core/data/raster_source/rasterized_source_config.py index efc30bd35..e574298fb 100644 --- a/rastervision_core/rastervision/core/data/raster_source/rasterized_source_config.py +++ b/rastervision_core/rastervision/core/data/raster_source/rasterized_source_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from rastervision.core.data.raster_source import (RasterizedSource) from rastervision.core.data.vector_source import (VectorSourceConfig) @@ -65,7 +65,7 @@ def update(self, pipeline=None, scene=None): def build(self, class_config: 'ClassConfig', crs_transformer: 'CRSTransformer', - bbox: Optional['Box'] = None) -> RasterizedSource: + bbox: 'Box | None' = None) -> RasterizedSource: vector_source = self.vector_source.build(class_config, crs_transformer) return RasterizedSource( vector_source=vector_source, diff --git a/rastervision_core/rastervision/core/data/raster_source/stac_config.py b/rastervision_core/rastervision/core/data/raster_source/stac_config.py index ee7df6f09..0249105b8 100644 --- a/rastervision_core/rastervision/core/data/raster_source/stac_config.py +++ b/rastervision_core/rastervision/core/data/raster_source/stac_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.config import (Config, Field, register_config) from rastervision.pipeline.file_system.utils import file_to_json @@ -12,7 +12,7 @@ class STACItemConfig(Config): """Specify a raster via a STAC Item.""" uri: str = Field(..., description='URI to a JSON-serialized STAC Item.') - assets: Optional[List[str]] = Field( + assets: list[str] | None = Field( None, description= 'Subset of assets to use. This should be a list of asset keys') @@ -32,7 +32,7 @@ class STACItemCollectionConfig(Config): uri: str = Field( ..., description='URI to a JSON-serialized STAC ItemCollection.') - assets: Optional[List[str]] = Field( + assets: list[str] | None = Field( None, description= 'Subset of assets to use. This should be a list of asset keys') @@ -47,7 +47,7 @@ def build(self) -> 'ItemCollection': return items -def subset_assets(item: 'Item', assets: List[str]) -> 'Item': +def subset_assets(item: 'Item', assets: list[str]) -> 'Item': """Return a copy of the Item with assets subsetted.""" item = item.clone() src_assets = item.assets diff --git a/rastervision_core/rastervision/core/data/raster_source/temporal_multi_raster_source.py b/rastervision_core/rastervision/core/data/raster_source/temporal_multi_raster_source.py index 79672a4ba..000655ecc 100644 --- a/rastervision_core/rastervision/core/data/raster_source/temporal_multi_raster_source.py +++ b/rastervision_core/rastervision/core/data/raster_source/temporal_multi_raster_source.py @@ -1,4 +1,4 @@ -from typing import Any, Optional, Sequence, Tuple +from typing import Any, Sequence from pydantic import NonNegativeInt as NonNegInt import numpy as np @@ -17,7 +17,7 @@ def __init__(self, primary_source_idx: NonNegInt = 0, force_same_dtype: bool = False, raster_transformers: Sequence = [], - bbox: Optional[Box] = None): + bbox: Box | None = None): """Constructor. Args: @@ -30,7 +30,7 @@ def __init__(self, conversion is done, just a quick cast. Use with caution. raster_transformers (Sequence): Sequence of transformers. Defaults to []. - bbox (Optional[Box]): User-specified crop of the extent. + bbox (Box | None): User-specified crop of the extent. If given, the primary raster source's bbox is set to this. If None, the full extent available in the source file of the primary raster source is used. @@ -77,15 +77,14 @@ def from_stac(cls, *args, **kwargs): 'Create raster sources by calling MultiRasterSource.from_stac() ' 'on each Item and then pass them to TemporalMultiRasterSource.') - def _get_chip(self, - window: Box, - out_shape: Optional[Tuple[int, int]] = None) -> np.ndarray: + def _get_chip(self, window: Box, + out_shape: tuple[int, int] | None = None) -> np.ndarray: """Get chip w/o applying channel_order and transformers. Args: window (Box): The window for which to get the chip, in pixel coordinates. - out_shape (Optional[Tuple[int, int]]): (height, width) to resize + out_shape (tuple[int, int] | None): (height, width) to resize the chip to. Returns: @@ -95,9 +94,8 @@ def _get_chip(self, chip = np.stack(sub_chips) return chip - def get_chip(self, - window: Box, - out_shape: Optional[Tuple[int, int]] = None) -> np.ndarray: + def get_chip(self, window: Box, + out_shape: tuple[int, int] | None = None) -> np.ndarray: """Return the transformed chip in the window. Get processed chips from sub raster sources (with their respective @@ -107,7 +105,7 @@ def get_chip(self, Args: window (Box): The window for which to get the chip, in pixel coordinates. - out_shape (Optional[Tuple[int, int]]): (height, width) to resize + out_shape (tuple[int, int] | None): (height, width) to resize the chip to. Returns: @@ -136,5 +134,5 @@ def __getitem__(self, key: Any) -> 'np.ndarray': return chip @property - def shape(self) -> Tuple[int, int, int, int]: + def shape(self) -> tuple[int, int, int, int]: return (len(self.raster_sources), *self.primary_source.shape) diff --git a/rastervision_core/rastervision/core/data/raster_source/xarray_source.py b/rastervision_core/rastervision/core/data/raster_source/xarray_source.py index 2e81fe8c3..adbdf30f3 100644 --- a/rastervision_core/rastervision/core/data/raster_source/xarray_source.py +++ b/rastervision_core/rastervision/core/data/raster_source/xarray_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Any, Sequence import logging import numpy as np @@ -25,26 +25,26 @@ class XarraySource(RasterSource): def __init__(self, data_array: DataArray, crs_transformer: 'CRSTransformer', - raster_transformers: List['RasterTransformer'] = [], - channel_order: Optional[Sequence[int]] = None, - bbox: Optional[Box] = None, + raster_transformers: list['RasterTransformer'] = [], + channel_order: Sequence[int] | None = None, + bbox: Box | None = None, temporal: bool = False): """Constructor. Args: - uris (Union[str, List[str]]): One or more URIs of images. If more - than one, the images will be mosaiced together using GDAL. - crs_transformer (CRSTransformer): A CRSTransformer defining the - mapping between pixel and map coords. - raster_transformers (List['RasterTransformer']): RasterTransformers - to use to transform chips after they are read. - channel_order (Optional[Sequence[int]]): List of indices of - channels to extract from raw imagery. Can be a subset of the - available channels. If None, all channels available in the - image will be read. Defaults to None. - bbox (Optional[Box], optional): User-specified crop of the extent. - If None, the full extent available in the source file is used. - temporal (bool): If True, data_array is expected to have a "time" + uris: One or more URIs of images. If more than one, the images will + be mosaiced together using GDAL. + crs_transformer: A CRSTransformer defining the mapping between + pixel and map coords. + raster_transformers: RasterTransformers to use to transform chips + after they are read. + channel_order: List of indices of channels to extract from raw + imagery. Can be a subset of the available channels. If + ``None``, all channels available in the image will be read. + Defaults to ``None``. + bbox: User-specified crop of the extent. If ``None``, the full + extent available in the source file is used. + temporal: If ``True``, data_array is expected to have a "time" dimension and the chips returned will be of shape (T, H, W, C). """ self.temporal = temporal @@ -167,7 +167,7 @@ def from_stac( return raster_source @property - def shape(self) -> Tuple[int, int, int]: + def shape(self) -> tuple[int, int, int]: """Shape of the raster as a (height, width, num_channels) tuple.""" H, W = self.bbox.size if self.temporal: @@ -208,9 +208,9 @@ def _set_info_from_chip(self): def _get_chip(self, window: Box, - bands: Union[int, Sequence[int], slice] = slice(None), - time: Union[int, Sequence[int], slice] = slice(None), - out_shape: Optional[Tuple[int, ...]] = None) -> np.ndarray: + bands: int | Sequence[int] | slice = slice(None), + time: int | Sequence[int] | slice = slice(None), + out_shape: tuple[int, ...] | None = None) -> np.ndarray: window = window.to_global_coords(self.bbox) window_within_bbox = window.intersection(self.bbox) @@ -239,21 +239,19 @@ def _get_chip(self, def get_chip(self, window: Box, - bands: Optional[Union[int, Sequence[int], slice]] = None, - time: Union[int, Sequence[int], slice] = slice(None), - out_shape: Optional[Tuple[int, ...]] = None) -> np.ndarray: + bands: int | Sequence[int] | slice | None = None, + time: int | Sequence[int] | slice = slice(None), + out_shape: tuple[int, ...] | None = None) -> np.ndarray: """Read a chip specified by a window from the file. Args: - window (Box): Bounding box of chip in pixel coordinates. - bands (Optional[Union[Sequence[int], slice]], optional): Subset of - bands to read. Note that this will be applied on top of the - channel_order (if specified). So if this is an RGB image and - channel_order=[2, 1, 0], then using bands=[0] will return the - B-channel. Defaults to None. - out_shape (Optional[Tuple[int, ...]], optional): (height, width) of - the output chip. If None, no resizing is done. - Defaults to None. + window: Bounding box of chip in pixel coordinates. + bands: Subset of bands to read. Note that this will be applied on + top of the ``channel_order`` (if specified). So if this is an + RGB image and ``channel_order=[2, 1, 0]``, then using + ``bands=[0]`` will return the B-channel. Defaults to ``None``. + out_shape: (height, width) of the output chip. If ``None``, no + resizing is done. Defaults to ``None``. Returns: np.ndarray: A chip of shape (height, width, channels). diff --git a/rastervision_core/rastervision/core/data/raster_source/xarray_source_config.py b/rastervision_core/rastervision/core/data/raster_source/xarray_source_config.py index 8331e77ba..fa97cb951 100644 --- a/rastervision_core/rastervision/core/data/raster_source/xarray_source_config.py +++ b/rastervision_core/rastervision/core/data/raster_source/xarray_source_config.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, Optional, Tuple, Union +from typing import Any import logging from rastervision.pipeline.config import Field, register_config @@ -15,14 +15,14 @@ class XarraySourceConfig(RasterSourceConfig): """Configure an :class:`.XarraySource`.""" - stac: Union[STACItemConfig, STACItemCollectionConfig] = Field( + stac: STACItemConfig | STACItemCollectionConfig = Field( ..., description='STAC Item or ItemCollection to build the DataArray from.') allow_streaming: bool = Field( True, description='If False, load the entire DataArray into memory. ' 'Defaults to True.') - bbox_map_coords: Optional[Tuple[float, float, float, float]] = Field( + bbox_map_coords: tuple[float, float, float, float] | None = Field( None, description='Optional user-specified bbox in EPSG:4326 coords of the ' 'form (ymin, xmin, ymax, xmax). Useful for cropping the raster source ' @@ -30,11 +30,10 @@ class XarraySourceConfig(RasterSourceConfig): 'bbox is also specified. Defaults to None.') temporal: bool = Field( False, description='Whether the data is a time-series.') - stackstac_args: Dict[str, Any] = Field( + stackstac_args: dict[str, Any] = Field( {}, description='Optional arguments to pass to stackstac.stack().') - def build(self, - tmp_dir: Optional[str] = None, + def build(self, tmp_dir: str | None = None, use_transformers: bool = True) -> XarraySource: item_or_item_collection = self.stac.build() raster_transformers = ([rt.build() for rt in self.transformers] diff --git a/rastervision_core/rastervision/core/data/raster_transformer/cast_transformer.py b/rastervision_core/rastervision/core/data/raster_transformer/cast_transformer.py index 309bbcce8..2ff1bb2ab 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/cast_transformer.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/cast_transformer.py @@ -1,5 +1,3 @@ -from typing import Optional - from rastervision.core.data.raster_transformer.raster_transformer \ import RasterTransformer from rastervision.pipeline.utils import repr_with_args @@ -22,7 +20,7 @@ def __repr__(self): return repr_with_args(self, to_dtype=str(self.to_dtype)) def transform(self, chip: np.ndarray, - channel_order: Optional[list] = None) -> np.ndarray: + channel_order: list | None = None) -> np.ndarray: """Cast chip to self.to_dtype. Args: diff --git a/rastervision_core/rastervision/core/data/raster_transformer/min_max_transformer.py b/rastervision_core/rastervision/core/data/raster_transformer/min_max_transformer.py index 3eb46482d..aa2797db4 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/min_max_transformer.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/min_max_transformer.py @@ -1,5 +1,3 @@ -from typing import List, Optional - import numpy as np from rastervision.core.data.raster_transformer import RasterTransformer @@ -10,7 +8,7 @@ class MinMaxTransformer(RasterTransformer): def transform(self, chip: np.ndarray, - channel_order: Optional[List[int]] = None) -> np.ndarray: + channel_order: list[int] | None = None) -> np.ndarray: c = chip.shape[-1] pixels = chip.reshape(-1, c) channel_mins = pixels.min(axis=0) diff --git a/rastervision_core/rastervision/core/data/raster_transformer/nan_transformer_config.py b/rastervision_core/rastervision/core/data/raster_transformer/nan_transformer_config.py index e4b75714d..59470c67c 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/nan_transformer_config.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/nan_transformer_config.py @@ -1,5 +1,3 @@ -from typing import Optional - from rastervision.pipeline.config import register_config, Field from rastervision.core.data.raster_transformer.raster_transformer_config import ( # noqa RasterTransformerConfig) @@ -11,7 +9,7 @@ class NanTransformerConfig(RasterTransformerConfig): """Configure a :class:`.NanTransformer`.""" - to_value: Optional[float] = Field( + to_value: float | None = Field( 0.0, description=('Turn all NaN values into this value.')) def build(self): diff --git a/rastervision_core/rastervision/core/data/raster_transformer/reclass_transformer.py b/rastervision_core/rastervision/core/data/raster_transformer/reclass_transformer.py index 97c2d8c58..74fe64df7 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/reclass_transformer.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/reclass_transformer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, List, Optional +from typing import TYPE_CHECKING from rastervision.core.data.raster_transformer import RasterTransformer if TYPE_CHECKING: @@ -8,7 +8,7 @@ class ReclassTransformer(RasterTransformer): """Maps class IDs in a label raster to other values.""" - def __init__(self, mapping: Dict[int, int]): + def __init__(self, mapping: dict[int, int]): """Construct a new ReclassTransformer. Args: @@ -18,7 +18,7 @@ def __init__(self, mapping: Dict[int, int]): def transform(self, chip: 'np.ndarray', - channel_order: Optional[List[int]] = None): + channel_order: list[int] | None = None): """Transform a chip. Reclassify a label raster using the given mapping. diff --git a/rastervision_core/rastervision/core/data/raster_transformer/reclass_transformer_config.py b/rastervision_core/rastervision/core/data/raster_transformer/reclass_transformer_config.py index 3ed94785c..893d0f522 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/reclass_transformer_config.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/reclass_transformer_config.py @@ -1,5 +1,3 @@ -from typing import Dict - from rastervision.pipeline.config import register_config, Field from rastervision.core.data.raster_transformer import (RasterTransformerConfig, ReclassTransformer) @@ -9,7 +7,7 @@ class ReclassTransformerConfig(RasterTransformerConfig): """Configure a :class:`.ReclassTransformer`.""" - mapping: Dict[int, int] = Field( + mapping: dict[int, int] = Field( ..., description=('The reclassification mapping.')) def build(self) -> ReclassTransformer: diff --git a/rastervision_core/rastervision/core/data/raster_transformer/rgb_class_transformer.py b/rastervision_core/rastervision/core/data/raster_transformer/rgb_class_transformer.py index ca61db9e1..e16e0acb3 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/rgb_class_transformer.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/rgb_class_transformer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING import numpy as np @@ -39,12 +39,12 @@ def __init__(self, class_config: 'ClassConfig'): def transform(self, chip: np.ndarray, - channel_order: Optional[List[int]] = None) -> np.ndarray: + channel_order: list[int] | None = None) -> np.ndarray: """Transform RGB array to array of class IDs or vice versa. Args: chip (np.ndarray): Numpy array of shape (H, W, 3). - channel_order (Optional[List[int]], optional): List of indices of + channel_order (list[int] | None): List of indices of channels that were extracted from the raw imagery. Defaults to None. diff --git a/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py b/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py index e8e81a4b5..f345a269c 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional, Sequence +from typing import TYPE_CHECKING, Self, Sequence import numpy as np @@ -44,7 +44,7 @@ def __init__(self, def transform(self, chip: np.ndarray, - channel_order: Optional[Sequence[int]] = None) -> np.ndarray: + channel_order: Sequence[int] | None = None) -> np.ndarray: """Transform a chip. Transforms non-uint8 to uint8 values using raster_stats. @@ -95,19 +95,19 @@ def transform(self, @classmethod def from_raster_sources(cls, - raster_sources: List['RasterSource'], - sample_prob: Optional[float] = 0.1, + raster_sources: list['RasterSource'], + sample_prob: float | None = 0.1, max_stds: float = 3., - chip_sz: int = 300) -> 'StatsTransformer': + chip_sz: int = 300) -> Self: """Build with stats from the given raster sources. Args: - raster_sources (List[RasterSource]): List of raster sources to + raster_sources (list[RasterSource]): List of raster sources to compute stats from. - sample_prob (float, optional): Fraction of each raster to sample + sample_prob (float): Fraction of each raster to sample for computing stats. For details see docs for RasterStats.compute(). Defaults to 0.1. - max_stds (float, optional): Number of standard deviations to clip + max_stds (float): Number of standard deviations to clip the distribution to on both sides. Defaults to 3. Returns: @@ -123,7 +123,7 @@ def from_raster_sources(cls, return stats_transformer @classmethod - def from_stats_json(cls, uri: str, **kwargs) -> 'StatsTransformer': + def from_stats_json(cls, uri: str, **kwargs) -> Self: """Build with stats from a JSON file. The file is expected to be in the same format as written by @@ -141,8 +141,7 @@ def from_stats_json(cls, uri: str, **kwargs) -> 'StatsTransformer': return stats_transformer @classmethod - def from_raster_stats(cls, stats: RasterStats, - **kwargs) -> 'StatsTransformer': + def from_raster_stats(cls, stats: RasterStats, **kwargs) -> Self: """Build with stats from a :class:`.RasterStats` instance. The file is expected to be in the same format as written by diff --git a/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer_config.py b/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer_config.py index d4e9a983c..923f6ace5 100644 --- a/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer_config.py +++ b/rastervision_core/rastervision/core/data/raster_transformer/stats_transformer_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from os.path import join from rastervision.pipeline.config import register_config, Field @@ -26,7 +26,7 @@ def stats_transformer_config_upgrader(cfg_dict: dict, version: int) -> dict: class StatsTransformerConfig(RasterTransformerConfig): """Configure a :class:`.StatsTransformer`.""" - stats_uri: Optional[str] = Field( + stats_uri: str | None = Field( None, description='The URI of the output of the StatsAnalyzer. ' 'If None, and this Config is inside an RVPipeline, ' @@ -37,8 +37,8 @@ class StatsTransformerConfig(RasterTransformerConfig): 'to "train_scenes".') def update(self, - pipeline: Optional['RVPipelineConfig'] = None, - scene: Optional['SceneConfig'] = None) -> None: + pipeline: 'RVPipelineConfig | None' = None, + scene: 'SceneConfig | None' = None) -> None: if pipeline is not None and self.stats_uri is None: self.stats_uri = join(pipeline.analyze_uri, 'stats', self.scene_group, 'stats.json') diff --git a/rastervision_core/rastervision/core/data/scene.py b/rastervision_core/rastervision/core/data/scene.py index cbbfec9c2..9db2a4a0d 100644 --- a/rastervision_core/rastervision/core/data/scene.py +++ b/rastervision_core/rastervision/core/data/scene.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, List, Optional, Tuple +from typing import TYPE_CHECKING, Any from rastervision.core.data.utils import match_bboxes, geoms_to_bbox_coords @@ -14,9 +14,9 @@ class Scene: def __init__(self, id: str, raster_source: 'RasterSource', - label_source: Optional['LabelSource'] = None, - label_store: Optional['LabelStore'] = None, - aoi_polygons: Optional[List['BaseGeometry']] = None): + label_source: 'LabelSource | None' = None, + label_store: 'LabelStore | None' = None, + aoi_polygons: list['BaseGeometry'] | None = None): """Constructor. During initialization, ``Scene`` attempts to set the extents of the @@ -68,7 +68,7 @@ def bbox(self) -> 'Box': """Bounding box applied to the source data.""" return self.raster_source.bbox - def __getitem__(self, key: Any) -> Tuple[Any, Any]: + def __getitem__(self, key: Any) -> tuple[Any, Any]: x = self.raster_source[key] if self.label_source is not None: y = self.label_source[key] diff --git a/rastervision_core/rastervision/core/data/scene_config.py b/rastervision_core/rastervision/core/data/scene_config.py index f82c63bb1..e438bfaba 100644 --- a/rastervision_core/rastervision/core/data/scene_config.py +++ b/rastervision_core/rastervision/core/data/scene_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.config import (Config, ConfigError, register_config, Field) @@ -35,9 +35,9 @@ class SceneConfig(Config): id: str raster_source: RasterSourceConfig - label_source: Optional[LabelSourceConfig] = None - label_store: Optional[LabelStoreConfig] = None - aoi_uris: Optional[List[str]] = Field( + label_source: LabelSourceConfig | None = None + label_store: LabelStoreConfig | None = None + aoi_uris: list[str] | None = Field( None, description='List of URIs of GeoJSON files that define the AOIs for ' 'the scene. Each polygon defines an AOI which is a piece of the scene ' @@ -46,7 +46,7 @@ class SceneConfig(Config): def build(self, class_config: 'ClassConfig', - tmp_dir: Optional[str] = None, + tmp_dir: str | None = None, use_transformers: bool = True) -> Scene: raster_source = self.raster_source.build( tmp_dir, use_transformers=use_transformers) @@ -80,7 +80,7 @@ def build(self, label_store=label_store, aoi_polygons=aoi_polygons) - def update(self, pipeline: Optional['RVPipelineConfig'] = None) -> None: + def update(self, pipeline: 'RVPipelineConfig | None' = None) -> None: super().update() self.raster_source.update(pipeline=pipeline, scene=self) diff --git a/rastervision_core/rastervision/core/data/utils/aoi_sampler.py b/rastervision_core/rastervision/core/data/utils/aoi_sampler.py index dfce42625..9cf049602 100644 --- a/rastervision_core/rastervision/core/data/utils/aoi_sampler.py +++ b/rastervision_core/rastervision/core/data/utils/aoi_sampler.py @@ -1,18 +1,17 @@ -from typing import Sequence, Tuple, Union +from typing import Sequence import numpy as np from shapely.geometry import Polygon, MultiPolygon, LinearRing from shapely.ops import unary_union -class AoiSampler(): - """Given a set of polygons representing the AOI, allows efficiently - sampling points inside the AOI uniformly at random. +class AoiSampler: + """Allows efficiently sampling points inside an AOI uniformly at random. - To achieve this, each polygon is first partitioned into triangles - (triangulation). Then, to sample a single point, we first sample a triangle - at random with probability proportional to its area and then sample a point - within that triangle uniformly at random. + To achieve this, each polygon in the AOI is first partitioned into + triangles (triangulation). Then, to sample a single point, we first sample + a triangle at random with probability proportional to its area and then + sample a point within that triangle uniformly at random. """ def __init__(self, polygons: Sequence[Polygon]) -> None: @@ -43,7 +42,7 @@ def sample(self, n: int = 1) -> np.ndarray: - Return the final position. Args: - n (int, optional): Number of points to sample. Defaults to 1. + n (int): Number of points to sample. Defaults to 1. Returns: np.ndarray: (n, 2) 2D coordinates of the sampled points. @@ -63,8 +62,10 @@ def sample(self, n: int = 1) -> np.ndarray: return loc def triangulate_polygon(self, polygon: Polygon) -> dict: - """Extract vertices and edges from the polygon (and its holes, if any) - and pass them to the Triangle library for triangulation. + """Triangulate polygon. + + Extracts vertices and edges from the polygon (and its holes, if any) + and passes them to the Triangle library for triangulation. """ from triangle import triangulate @@ -111,17 +112,16 @@ def triangulate_polygon(self, polygon: Polygon) -> dict: } return out - def polygon_to_graph(self, polygon: Union[Polygon, LinearRing] - ) -> Tuple[np.ndarray, np.ndarray]: + def polygon_to_graph(self, polygon: Polygon | LinearRing + ) -> tuple[np.ndarray, np.ndarray]: """Given a polygon, return its graph representation. Args: - polygon (Union[Polygon, LinearRing]): A polygon or - polygon-exterior. + polygon: A polygon or polygon-exterior. Returns: - Tuple[np.ndarray, np.ndarray]: An (N, 2) array of vertices and - an (N, 2) array of indices to vertices representing edges. + An (N, 2) array of vertices and an (N, 2) array of indices to + vertices representing edges. """ exterior = getattr(polygon, 'exterior', polygon) vertices = np.array(exterior.coords) @@ -138,17 +138,17 @@ def polygon_to_graph(self, polygon: Union[Polygon, LinearRing] return vertices, edges def triangle_side_lengths(self, vertices: np.ndarray, simplices: np.ndarray - ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: """Calculate lengths of all 3 sides of each triangle specified by the simplices array. Args: - vertices (np.ndarray): (N, 2) array of vertex coords in 2D. - simplices (np.ndarray): (N, 3) array of indexes to entries in the + vertices: (N, 2) array of vertex coords in 2D. + simplices: (N, 3) array of indexes to entries in the vertices array. Each row represents one triangle. Returns: - Tuple[np.ndarray, np.ndarray, np.ndarray]: ||AB||, ||BC||, ||AC|| + tuple[np.ndarray, np.ndarray, np.ndarray]: ||AB||, ||BC||, ||AC|| """ A = vertices[simplices[:, 0]] B = vertices[simplices[:, 1]] @@ -161,12 +161,12 @@ def triangle_side_lengths(self, vertices: np.ndarray, simplices: np.ndarray def triangle_origin_and_basis( self, vertices: np.ndarray, simplices: np.ndarray - ) -> Tuple[np.ndarray, Tuple[np.ndarray, np.ndarray]]: + ) -> tuple[np.ndarray, tuple[np.ndarray, np.ndarray]]: """For each triangle ABC, return point A, vector AB, and vector AC. Args: - vertices (np.ndarray): (N, 2) array of vertex coords in 2D. - simplices (np.ndarray): (N, 3) array of indexes to entries in the + vertices: (N, 2) array of vertex coords in 2D. + simplices: (N, 3) array of indexes to entries in the vertices array. Each row represents one triangle. Returns: @@ -186,8 +186,8 @@ def triangle_area(self, vertices: np.ndarray, using Heron's formula. Args: - vertices (np.ndarray): (N, 2) array of vertex coords in 2D. - simplices (np.ndarray): (N, 3) array of indexes to entries in the + vertices: (N, 2) array of vertex coords in 2D. + simplices: (N, 3) array of indexes to entries in the vertices array. Each row represents one triangle. Returns: diff --git a/rastervision_core/rastervision/core/data/utils/factory.py b/rastervision_core/rastervision/core/data/utils/factory.py index 2aeb8cb6b..3fe3ec12f 100644 --- a/rastervision_core/rastervision/core/data/utils/factory.py +++ b/rastervision_core/rastervision/core/data/utils/factory.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING from uuid import uuid4 from rastervision.core.data.utils import listify_uris, get_polygons_from_uris @@ -7,54 +7,53 @@ from rastervision.core.data import ClassConfig, Scene -def make_ss_scene(image_uri: Union[str, List[str]], - label_raster_uri: Optional[Union[str, List[str]]] = None, - class_config: Optional['ClassConfig'] = None, - label_vector_uri: Optional[str] = None, - aoi_uri: Union[str, List[str]] = [], - label_vector_default_class_id: Optional[int] = None, +def make_ss_scene(image_uri: str | list[str], + label_raster_uri: str | list[str] | None = None, + class_config: 'ClassConfig | None' = None, + label_vector_uri: str | None = None, + aoi_uri: str | list[str] = [], + label_vector_default_class_id: int | None = None, image_raster_source_kw: dict = {}, label_raster_source_kw: dict = {}, label_vector_source_kw: dict = {}, - scene_id: Optional[str] = None) -> 'Scene': + scene_id: str | None = None) -> 'Scene': """Create a semantic segmentation scene from image and label URIs. This is a convenience method. For more fine-grained control, it is recommended to use the default constructor. Args: - image_uri (Union[str, List[str]]): URI or list of URIs of GeoTIFFs to - use as the source of image data. - label_raster_uri (Optional[Union[str, List[str]]], optional): URI or - list of URIs of GeoTIFFs to use as the source of segmentation label - data. If the labels are in the form of GeoJSONs, use - label_vector_uri instead. Defaults to None. - label_vector_uri (Optional[str], optional): URI of GeoJSON file to use - as the source of segmentation label data. If the labels are in the - form of GeoTIFFs, use label_raster_uri instead. Defaults to None. - class_config (Optional[ClassConfig]): The ClassConfig. Must be - non-None if creating a scene without a LabelSource. - Defaults to None. - aoi_uri (Union[str, List[str]], optional): URI or list of URIs of - GeoJSONs that specify the area-of-interest. If provided, the - dataset will only access data from this area. Defaults to []. - label_vector_default_class_id (Optional[int], optional): If using - label_vector_uri and all polygons in that file belong to the same - class and they do not contain a `class_id` property, then use this - argument to map all of the polygons to the appropriate class ID. - See docs for ClassInferenceTransformer for more details. - Defaults to None. - image_raster_source_kw (dict, optional): Additional arguments to pass - to the RasterioSource used for image data. See docs for - RasterioSource for more details. Defaults to {}. - label_raster_source_kw (dict, optional): Additional arguments to pass - to the RasterioSource used for label data, if label_raster_uri is - used. See docs for RasterioSource for more details. Defaults to {}. - label_vector_source_kw (dict, optional): Additional arguments to pass - to the GeoJSONVectorSource used for label data, if label_vector_uri - is used. See docs for GeoJSONVectorSource for more details. - Defaults to {}. - scene_id (Optional[str]): Optional scene ID. If None, will be randomly + image_uri: URI or list of URIs of GeoTIFFs to use as the source of + image data. + label_raster_uri: URI or list of URIs of GeoTIFFs to use as the source + of segmentation label data. If the labels are in the form of + GeoJSONs, use ``label_vector_uri`` instead. Defaults to ``None``. + label_vector_uri: URI of GeoJSON file to use as the source of + segmentation label data. If the labels are in the form of GeoTIFFs, + use ``label_raster_uri`` instead. Defaults to ``None``. + class_config: The ``ClassConfig``. Can be ``None`` if not using any + labels. + aoi_uri: URI or list of URIs of GeoJSONs that specify the + area-of-interest. If provided, the dataset will only access data + from this area. Defaults to ``[]``. + label_vector_default_class_id: If using ``label_vector_uri`` and all + polygons in that file belong to the same class and they do not + contain a ``class_id`` property, then use this argument to map all + of the polygons to the appropriate class ID. See docs for + :class:`.ClassInferenceTransformer` for more details. + Defaults to ``None``. + image_raster_source_kw: Additional arguments to pass to the + :class:`.RasterioSource` used for image data. See docs for + :class:`.RasterioSource` for more details. Defaults to ``{}``. + label_raster_source_kw: Additional arguments to pass + to the :class:`.RasterioSource` used for label data, if + ``label_raster_uri`` is used. See docs for :class:`.RasterioSource` + for more details. Defaults to ``{}``. + label_vector_source_kw: Additional arguments to pass to the + :class:`.GeoJSONVectorSource` used for label data, if + ``label_vector_uri`` is used. See docs for + :class:`.GeoJSONVectorSource` for more details. Defaults to ``{}``. + scene_id: Optional scene ID. If None, will be randomly generated. Defaults to None. Raises: @@ -123,51 +122,50 @@ class and they do not contain a `class_id` property, then use this return scene -def make_cc_scene(image_uri: Union[str, List[str]], - label_vector_uri: Optional[str] = None, - class_config: Optional['ClassConfig'] = None, - aoi_uri: Union[str, List[str]] = [], - label_vector_default_class_id: Optional[int] = None, +def make_cc_scene(image_uri: str | list[str], + label_vector_uri: str | None = None, + class_config: 'ClassConfig | None' = None, + aoi_uri: str | list[str] = [], + label_vector_default_class_id: int | None = None, image_raster_source_kw: dict = {}, label_vector_source_kw: dict = {}, label_source_kw: dict = {}, - scene_id: Optional[str] = None) -> 'Scene': + scene_id: str | None = None) -> 'Scene': """Create a chip classification scene from image and label URIs. This is a convenience method. For more fine-grained control, it is recommended to use the default constructor. Args: - image_uri (Union[str, List[str]]): URI or list of URIs of GeoTIFFs to - use as the source of image data. - label_vector_uri (Optional[str], optional): URI of GeoJSON file to use - as the source of segmentation label data. Defaults to None. - class_config (Optional[ClassConfig]): The ClassConfig. Must be - non-None if creating a scene without a LabelSource. - Defaults to None. - aoi_uri (Union[str, List[str]], optional): URI or list of URIs of - GeoJSONs that specify the area-of-interest. If provided, the - dataset will only access data from this area. Defaults to []. - label_vector_default_class_id (Optional[int], optional): If using - label_vector_uri and all polygons in that file belong to the same - class and they do not contain a `class_id` property, then use this - argument to map all of the polygons to the appropriate class ID. - See docs for ClassInferenceTransformer for more details. - Defaults to None. - image_raster_source_kw (dict, optional): Additional arguments to pass - to the RasterioSource used for image data. See docs for - RasterioSource for more details. Defaults to {}. - label_vector_source_kw (dict, optional): Additional arguments to pass - to the GeoJSONVectorSourceConfig used for label data, if - label_vector_uri is set. See docs for GeoJSONVectorSourceConfig - for more details. Defaults to {}. - label_source_kw (dict, optional): Additional arguments to pass - to the ChipClassificationLabelSourceConfig used for label data, if - label_vector_uri is set. See docs for - ChipClassificationLabelSourceConfig for more details. - Defaults to {}. - scene_id (Optional[str]): Optional scene ID. If None, will be randomly - generated. Defaults to None. + image_uri: URI or list of URIs of GeoTIFFs to use as the source of + image data. + label_vector_uri: URI of GeoJSON file to use as the source of + segmentation label data. Defaults to ``None``. + class_config: The ClassConfig. Must be non-``None`` if creating a scene + without a ``LabelSource``. Defaults to ``None``. + aoi_uri: URI or list of URIs of GeoJSONs that specify the + area-of-interest. If provided, the dataset will only access data + from this area. Defaults to ``[]``. + label_vector_default_class_id: If using ``label_vector_uri`` and all + polygons in that file belong to the same class and they do not + contain a `class_id` property, then use this argument to map all of + the polygons to the appropriate class ID. See docs for + :class:`.ClassInferenceTransformer` for more details. + Defaults to ``None``. + image_raster_source_kw: Additional arguments to pass to the + :class:`.RasterioSource` used for image data. See docs for + :class:`.RasterioSource` for more details. Defaults to ``{}``. + label_vector_source_kw: Additional arguments to pass to the + :class:`.GeoJSONVectorSource` used for label data, if + ``label_vector_uri`` is used. See docs for + :class:`.GeoJSONVectorSource` for more details. Defaults to ``{}``. + label_source_kw: Additional arguments to pass to the + :class:`.ChipClassificationLabelSourceConfig` used for label data, + if ``label_vector_uri`` is set. See docs for + :class:`.ChipClassificationLabelSourceConfig` for more details. + Defaults to ``{}``. + scene_id: Optional scene ID. If ``None``, will be randomly + generated. Defaults to ``None``. Returns: Scene: A chip classification scene. @@ -212,50 +210,50 @@ class and they do not contain a `class_id` property, then use this return scene -def make_od_scene(image_uri: Union[str, List[str]], - label_vector_uri: Optional[str] = None, - class_config: Optional['ClassConfig'] = None, - aoi_uri: Union[str, List[str]] = [], - label_vector_default_class_id: Optional[int] = None, +def make_od_scene(image_uri: str | list[str], + label_vector_uri: str | None = None, + class_config: 'ClassConfig | None' = None, + aoi_uri: str | list[str] = [], + label_vector_default_class_id: int | None = None, image_raster_source_kw: dict = {}, label_vector_source_kw: dict = {}, label_source_kw: dict = {}, - scene_id: Optional[str] = None) -> 'Scene': + scene_id: str | None = None) -> 'Scene': """Create an object detection scene from image and label URIs. This is a convenience method. For more fine-grained control, it is recommended to use the default constructor. Args: - image_uri (Union[str, List[str]]): URI or list of URIs of GeoTIFFs to - use as the source of image data. - label_vector_uri (Optional[str], optional): URI of GeoJSON file to use - as the source of segmentation label data. Defaults to None. - class_config (Optional[ClassConfig]): The ClassConfig. Must be - non-None if creating a scene without a LabelSource. - Defaults to None. - aoi_uri (Union[str, List[str]], optional): URI or list of URIs of + image_uri: URI or list of URIs of GeoTIFFs to use as the source of + image data. + label_vector_uri: URI of GeoJSON file to use as the source of label. + Defaults to ``None``. + class_config: The ClassConfig. Must be non-None if creating a scene + without a ``LabelSource``. Defaults to ``None``. + aoi_uri: URI or list of URIs of GeoJSONs that specify the area-of-interest. If provided, the - dataset will only access data from this area. Defaults to []. - label_vector_default_class_id (Optional[int], optional): If using + dataset will only access data from this area. Defaults to ``[]``. + label_vector_default_class_id: If using label_vector_uri and all polygons in that file belong to the same class and they do not contain a `class_id` property, then use this argument to map all of the polygons to the appropriate class ID. See docs for ClassInferenceTransformer for more details. - Defaults to None. - image_raster_source_kw (dict, optional): Additional arguments to pass - to the RasterioSource used for image data. See docs for - RasterioSource for more details. Defaults to {}. - label_vector_source_kw (dict, optional): Additional arguments to pass - to the GeoJSONVectorSourceConfig used for label data, if - label_vector_uri is set. See docs for GeoJSONVectorSourceConfig - for more details. Defaults to {}. - label_source_kw (dict, optional): Additional arguments to pass - to the ObjectDetectionLabelSourceConfig used for label data, if + Defaults to ``None``. + image_raster_source_kw: Additional arguments to pass + to the :class:`.RasterioSource` used for image data. See docs for + :class:`.RasterioSource` for more details. Defaults to ``{}``. + label_vector_source_kw: Additional arguments to pass + to the :class:`.GeoJSONVectorSourceConfig` used for label data, if + label_vector_uri is set. See docs for + :class:`.GeoJSONVectorSourceConfig` for more details. + Defaults to ``{}``. + label_source_kw: Additional arguments to pass + to the :class:`.ObjectDetectionLabelSourceConfig` used for label data, if label_vector_uri is set. See docs for - ObjectDetectionLabelSourceConfig for more details. - Defaults to {}. - scene_id (Optional[str]): Optional scene ID. If None, will be randomly + :class:`.ObjectDetectionLabelSourceConfig` for more details. + Defaults to ``{}``. + scene_id: Optional scene ID. If None, will be randomly generated. Defaults to None. Returns: diff --git a/rastervision_core/rastervision/core/data/utils/geojson.py b/rastervision_core/rastervision/core/data/utils/geojson.py index d30cb2b85..74c2ae207 100644 --- a/rastervision_core/rastervision/core/data/utils/geojson.py +++ b/rastervision_core/rastervision/core/data/utils/geojson.py @@ -1,5 +1,5 @@ -from typing import (TYPE_CHECKING, Callable, Dict, Iterable, Iterator, List, - Optional, Union) +from typing import (TYPE_CHECKING, Iterable, Iterator) +from collections.abc import Callable from copy import deepcopy from shapely.geometry import shape, mapping @@ -17,8 +17,7 @@ PROGRESSBAR_DELAY_SEC = 5 -def geometry_to_feature(mapping: dict, - properties: Optional[dict] = None) -> dict: +def geometry_to_feature(mapping: dict, properties: dict | None = None) -> dict: """Convert a serialized geometry to a serialized GeoJSON feature.""" already_a_feature = mapping.get('type') == 'Feature' if already_a_feature: @@ -35,15 +34,15 @@ def geometries_to_geojson(geometries: Iterable[dict]) -> dict: return features_to_geojson(features) -def features_to_geojson(features: List[dict]) -> dict: +def features_to_geojson(features: list[dict]) -> dict: """Convert GeoJSON-like mapping of Features to a FeatureCollection.""" return {'type': 'FeatureCollection', 'features': features} -def map_features(func: Callable, +def map_features(func: Callable[[dict], dict], geojson: dict, include_geom_types: Iterable[str] = [], - progressbar_kw: Optional[dict] = None) -> dict: + progressbar_kw: dict | None = None) -> dict: """Map GeoJSON features to new features. Returns a new GeoJSON dict.""" features_in = geojson['features'] @@ -68,13 +67,21 @@ def map_features(func: Callable, return features_to_geojson(features_out) -def map_geoms(func: Callable, +def map_geoms(func: Callable[['BaseGeometry', dict], dict], geojson: dict, include_geom_types: Iterable[str] = [], - progressbar_kw: Optional[dict] = None) -> dict: - """Map GeoJSON features to new features by applying func to geometries. + progressbar_kw: dict | None = None) -> dict: + """Map GeoJSON features to new features by applying ``func`` to geometries. - Returns a new GeoJSON dict. + For each feature, the geometry is deserialized to a shapely geom, ``func`` + is applied, and its output is serialized to a new feature. + + ``func`` must be a function that takes a shapely geom and a keyword arg + named ``feature`` (to which the feature dict will be passed) and returns + a shapely geom. + + Returns: + A new GeoJSON dict. """ def feat_func(feature_in: dict) -> dict: @@ -106,7 +113,7 @@ def geojson_to_geoms(geojson: dict) -> Iterator['BaseGeometry']: def geoms_to_geojson(geoms: Iterable['BaseGeometry'], - properties: Optional[Iterable[dict]] = None) -> dict: + properties: Iterable[dict] | None = None) -> dict: """Serialize shapely geometries to GeoJSON.""" with tqdm( geoms, @@ -122,16 +129,16 @@ def geoms_to_geojson(geoms: Iterable['BaseGeometry'], def geom_to_feature(geom: 'BaseGeometry', - properties: Optional[dict] = None) -> dict: + properties: dict | None = None) -> dict: """Serialize a single shapely geometry to a GeoJSON Feature.""" geometry = mapping(geom) feature = geometry_to_feature(geometry, properties=properties) return feature -def filter_features(func: Callable, +def filter_features(func: Callable[[dict], bool], geojson: dict, - progressbar_kw: Optional[dict] = None) -> dict: + progressbar_kw: dict | None = None) -> dict: """Filter GeoJSON features. Returns a new GeoJSON dict.""" features_in = geojson['features'] @@ -160,7 +167,7 @@ def is_empty_feature(f: dict) -> bool: Returns: bool: Whether the feature contains any geometry. """ - g: Optional[dict] = f.get('geometry') + g: dict | None = f.get('geometry') if not g: return True no_geometries = not g.get('geometries') @@ -194,7 +201,7 @@ def split_multi_geometries(geojson: dict) -> dict: dict: FeatureCollection without multi-part geometries. """ - def split_geom(geom: 'BaseGeometry') -> List['BaseGeometry']: + def split_geom(geom: 'BaseGeometry') -> list['BaseGeometry']: # Split GeometryCollection into list of geoms. if geom.geom_type == 'GeometryCollection': geoms = list(geom.geoms) @@ -286,26 +293,25 @@ def simplify_polygons(geojson: dict) -> dict: def buffer_geoms(geojson: dict, geom_type: str, - class_bufs: Dict[int, Optional[float]] = {}, - default_buf: Optional[float] = 1) -> dict: + class_bufs: dict[int, float | None] = {}, + default_buf: float | None = 1) -> dict: """Buffer geometries. Geometries in features without a class_id property will be ignored. Args: - geojson (dict): A GeoJSON-like mapping of a FeatureCollection. - geom_type (str): Shapely geometry type to apply the buffering to. Other + geojson: A GeoJSON-like mapping of a FeatureCollection. + geom_type: Shapely geometry type to apply the buffering to. Other types of geometries will not be affected. - class_bufs (Dict[int, Optional[float]]): Optional - mapping from class ID to buffer distance (in pixel units) for - geom_type geometries. + class_bufs: Optional mapping from class ID to buffer distance + (in pixel units) for geom_type geometries. Returns: dict: FeatureCollection with buffered geometries. """ def buffer_geom(geom: 'BaseGeometry', - feature: Optional[dict] = None) -> 'BaseGeometry': + feature: dict | None = None) -> 'BaseGeometry': has_class_id = (('properties' in feature) and ('class_id' in feature['properties'])) if has_class_id: @@ -338,10 +344,10 @@ def all_geoms_valid(geojson: dict): return all(g.is_valid for g in geoms) -def get_polygons_from_uris(uris: Union[str, List[str]], +def get_polygons_from_uris(uris: str | list[str], crs_transformer: 'CRSTransformer', - bbox: Optional['Box'] = None, - map_coords: bool = False) -> List['BaseGeometry']: + bbox: 'Box | None' = None, + map_coords: bool = False) -> list['BaseGeometry']: """Load and return polygons (in pixel coords) from one or more URIs.""" # use local imports to avoid circular import problems diff --git a/rastervision_core/rastervision/core/data/utils/misc.py b/rastervision_core/rastervision/core/data/utils/misc.py index 95ccd46f9..a26328d32 100644 --- a/rastervision_core/rastervision/core/data/utils/misc.py +++ b/rastervision_core/rastervision/core/data/utils/misc.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, List, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Any, Sequence import logging import numpy as np @@ -14,7 +14,7 @@ def color_to_triple( - color: Optional[Union[str, Sequence]] = None) -> Tuple[int, int, int]: + color: str | Sequence | None = None) -> tuple[int, int, int]: """Given a PIL ImageColor string, return a triple of integers representing the red, green, and blue values. @@ -57,7 +57,7 @@ def color_to_integer(color: str) -> int: def normalize_color( - color: Union[str, tuple, list]) -> Tuple[float, float, float]: + color: str | tuple | list | None) -> tuple[float, float, float]: """Convert color representation to a float 3-tuple with values in [0-1].""" if isinstance(color, str): color = color_to_triple(color) @@ -87,19 +87,19 @@ def all_equal(it: list): return it.count(it[0]) == len(it) -def listify_uris(uris: Union[str, List[str]]) -> List[str]: +def listify_uris(uris: str | list[str]) -> list[str]: """Convert to URI to list if needed.""" if isinstance(uris, (list, tuple)): pass elif isinstance(uris, str): uris = [uris] else: - raise TypeError(f'Expected str or List[str], but got {type(uris)}.') + raise TypeError(f'Expected str or list[str], but got {type(uris)}.') return uris def match_bboxes(raster_source: 'RasterSource', - label_source: Union['LabelSource', 'LabelStore']) -> None: + label_source: 'LabelSource | LabelStore') -> None: """Set ``label_souce`` bbox equal to ``raster_source`` bbox. Logs a warning if ``raster_source`` and ``label_source`` extents don't @@ -107,7 +107,7 @@ def match_bboxes(raster_source: 'RasterSource', Args: raster_source (RasterSource): Source of imagery for a scene. - label_source (Union[LabelSource, LabelStore]): Source of labels for a + label_source (LabelSource | LabelStore): Source of labels for a scene. Can be a ``LabelStore``. """ crs_tf_img = raster_source.crs_transformer @@ -126,21 +126,21 @@ def match_bboxes(raster_source: 'RasterSource', label_source.set_bbox(bbox_label_pixel) -def parse_array_slices_2d(key: Union[tuple, slice], - extent: Box) -> Tuple[Box, List[Optional[Any]]]: +def parse_array_slices_2d(key: tuple | slice, + extent: Box) -> tuple[Box, list[Any | None]]: """Parse 2D array-indexing inputs into a Box and slices.""" return parse_array_slices_Nd(key, extent, dims=2, h_dim=0, w_dim=1) -def parse_array_slices_Nd(key: Union[tuple, slice], +def parse_array_slices_Nd(key: tuple | slice, extent: Box, dims: int = 3, h_dim: int = -3, - w_dim: int = -2) -> Tuple[Box, List[Optional[Any]]]: + w_dim: int = -2) -> tuple[Box, list[Any | None]]: """Parse multi-dim array-indexing inputs into a Box and slices. Args: - key (Union[tuple, slice]): Input to __getitem__. + key (tuple | slice): Input to __getitem__. extent (Box): Extent of the raster/label source being indexed. dims (int): Total available indexable dims. Defaults to 3. h_dim (int): Index of height dim. Defaults to -3. @@ -156,7 +156,7 @@ def parse_array_slices_Nd(key: Union[tuple, slice], NotImplementedError: If input contains negative values. Returns: - Tuple[Box, list]: A Box representing the h and w slices and a list + tuple[Box, list]: A Box representing the h and w slices and a list containing slices/index-values for all the dims. """ if isinstance(key, slice): diff --git a/rastervision_core/rastervision/core/data/utils/raster.py b/rastervision_core/rastervision/core/data/utils/raster.py index e55071b13..827809c78 100644 --- a/rastervision_core/rastervision/core/data/utils/raster.py +++ b/rastervision_core/rastervision/core/data/utils/raster.py @@ -13,7 +13,7 @@ def fill_overflow(bbox: Box, bbox (Box): Bounding box. window (Box): Window corresponding to the ``chip``. chip (np.ndarray): (H, W, C) array. - fill_value (int, optional): Value to set overflowing pixels to. + fill_value (int): Value to set overflowing pixels to. Defaults to 0. Returns: @@ -43,7 +43,7 @@ def pad_to_window_size(chip: np.ndarray, window: Box, bbox: Box, chip (np.ndarray): (H, W[, C]) array. bbox (Box): Bounding box. window (Box): Window corresponding to the ``chip``. - fill_value (int, optional): Value to pad with. Defaults to 0. + fill_value (int): Value to pad with. Defaults to 0. Returns: np.ndarray: Chip of size equal to window size with edges padded with diff --git a/rastervision_core/rastervision/core/data/utils/rasterio.py b/rastervision_core/rastervision/core/data/utils/rasterio.py index 466f74154..00a3ac4df 100644 --- a/rastervision_core/rastervision/core/data/utils/rasterio.py +++ b/rastervision_core/rastervision/core/data/utils/rasterio.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional, Sequence, Tuple, Union +from typing import TYPE_CHECKING, Sequence import os import subprocess import logging @@ -22,13 +22,13 @@ def write_window(dataset: 'DatasetReader', arr: np.ndarray, - window: Optional[Box] = None) -> None: + window: Box | None = None) -> None: """Write a (H, W[, C]) array out to a rasterio dataset. Args: dataset (DatasetReader): Rasterio dataset, opened for writing. arr (np.ndarray): Array to write. - window (Optional[Box]): Window (in pixel coords) to write to. + window (Box | None): Window (in pixel coords) to write to. Defaults to None. """ if window is not None: @@ -75,7 +75,7 @@ def write_bbox(path: str, arr: np.ndarray, bbox: Box, crs_wkt: str, **kwargs): def write_geotiff_like_geojson(path: str, arr: np.ndarray, geojson_path: str, - crs: Optional[str] = None, + crs: str | None = None, **kwargs) -> None: """Write array to GeoTIFF, georeferenced to same bbox as the given GeoJSON. @@ -130,12 +130,12 @@ def crop_geotiff(src_uri: str, window: Box, dst_uri: str): upload_or_copy(crop_path, dst_uri) -def build_vrt(vrt_path: str, image_uris: List[str]) -> None: +def build_vrt(vrt_path: str, image_uris: list[str]) -> None: """Build a VRT for a set of TIFF files. Args: vrt_path (str): Local path for the VRT to be created. - image_uris (List[str]): Image URIs. + image_uris (list[str]): Image URIs. """ log.info('Building VRT...') cmd = ['gdalbuildvrt', vrt_path] @@ -143,15 +143,15 @@ def build_vrt(vrt_path: str, image_uris: List[str]) -> None: subprocess.run(cmd) -def download_and_build_vrt(image_uris: List[str], +def download_and_build_vrt(image_uris: list[str], vrt_dir: str, stream: bool = False) -> str: """Download images (if needed) and build a VRT for a set of TIFF files. Args: - image_uris (List[str]): Image URIs. + image_uris (list[str]): Image URIs. vrt_dir (str): Dir where the VRT will be created. - stream (bool, optional): If true, do not download images. + stream (bool): If true, do not download images. Defaults to False. Returns: @@ -164,26 +164,23 @@ def download_and_build_vrt(image_uris: List[str], return vrt_path -def read_window( - dataset: 'DatasetReader', - bands: Optional[Union[int, Sequence[int]]] = None, - window: Optional[Tuple[Tuple[int, int], Tuple[int, int]]] = None, - is_masked: bool = False, - out_shape: Optional[Tuple[int, ...]] = None) -> np.ndarray: +def read_window(dataset: 'DatasetReader', + bands: int | Sequence[int] | None = None, + window: tuple[tuple[int, int], tuple[int, int]] | None = None, + is_masked: bool = False, + out_shape: tuple[int, ...] | None = None) -> np.ndarray: """Load a window of an image using Rasterio. Args: dataset: a Rasterio dataset. - bands (Optional[Union[int, Sequence[int]]]): Band index or indices to - read. Must be 1-indexed. - window (Optional[Tuple[Tuple[int, int], Tuple[int, int]]]): - ((row_start, row_stop), (col_start, col_stop)) or - ((y_min, y_max), (x_min, x_max)). If None, reads the entire raster. - Defaults to None. - is_masked (bool): If True, read a masked array from rasterio. - Defaults to False. - out_shape (Optional[Tuple[int, int]]): (height, width) of the output - chip. If None, no resizing is done. Defaults to None. + bands: Band index or indices to read. Must be 1-indexed. + window: ``((row_start, row_stop), (col_start, col_stop))`` or + ``((y_min, y_max), (x_min, x_max))``. If ``None``, reads the entire + raster. Defaults to ``None``. + is_masked: If ``True``, read a masked array from rasterio. + Defaults to ``False``. + out_shape: (height, width) of the output chip. If ``None``, no + resizing is done. Defaults to ``None``. Returns: np.ndarray: array of shape (height, width, channels). @@ -217,7 +214,7 @@ def read_window( return im -def get_channel_order_from_dataset(dataset: 'DatasetReader') -> List[int]: +def get_channel_order_from_dataset(dataset: 'DatasetReader') -> list[int]: """Get channel order from rasterio image dataset. Accounts for dataset's ``colorinterp`` if defined. @@ -226,7 +223,7 @@ def get_channel_order_from_dataset(dataset: 'DatasetReader') -> List[int]: dataset (DatasetReader): Rasterio image dataset. Returns: - List[int]: List of channel indices. + list[int]: List of channel indices. """ colorinterp = dataset.colorinterp if colorinterp: diff --git a/rastervision_core/rastervision/core/data/utils/vectorization.py b/rastervision_core/rastervision/core/data/utils/vectorization.py index 377232014..7f9a13b56 100644 --- a/rastervision_core/rastervision/core/data/utils/vectorization.py +++ b/rastervision_core/rastervision/core/data/utils/vectorization.py @@ -2,7 +2,7 @@ # Ported over from https://github.com/azavea/mask-to-polygons. ############################################################################### -from typing import TYPE_CHECKING, Iterator, Optional, Tuple +from typing import TYPE_CHECKING, Iterator from itertools import chain import numpy as np @@ -13,17 +13,17 @@ if TYPE_CHECKING: from shapely.geometry.base import BaseGeometry -RotatedRectange = Tuple[Tuple[float, float], Tuple[float, float], float] +RotatedRectange = tuple[tuple[float, float], tuple[float, float], float] -def mask_to_polygons(mask: np.ndarray, transform: Optional[rio.Affine] = None +def mask_to_polygons(mask: np.ndarray, transform: rio.Affine | None = None ) -> Iterator['BaseGeometry']: """Polygonize a raster mask. Wrapper around rasterio.features.shapes. Args: - mask (np.ndarray): The mask containing buildings to polygonize. - transform (Optional[rio.Affine]): Affine transform to use during - polygonization. Defaults to None (i.e. identity transform). + mask: The mask containing buildings to polygonize. + transform: Affine transform to use during polygonization. + Defaults to ``None`` (i.e. identity transform). Returns: Iterator[BaseGeometry]: Generator of shapely polygons. @@ -37,7 +37,7 @@ def mask_to_polygons(mask: np.ndarray, transform: Optional[rio.Affine] = None def mask_to_building_polygons( mask: np.ndarray, - transform: Optional[rio.Affine] = None, + transform: rio.Affine | None = None, min_area: float = 100, width_factor: float = 0.5, thickness: float = 0.001) -> Iterator['BaseGeometry']: @@ -60,7 +60,7 @@ def mask_to_building_polygons( Args: mask (np.ndarray): The mask containing buildings to polygonize. - transform (Optional[rio.Affine]): Affine transform to use during + transform (rio.Affine|None): Affine transform to use during polygonization. Defaults to None (i.e. identity transform). min_area (float): Minimum area (in pixels^2) of anything that can be considered to be a building or cluster of buildings. The goal is to @@ -103,7 +103,7 @@ def mask_to_building_polygons( return chain.from_iterable(iterators) -def get_rectangle(buildings: np.ndarray) -> Optional[RotatedRectange]: +def get_rectangle(buildings: np.ndarray) -> RotatedRectange | None: contours, _ = cv2.findContours(buildings, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) if len(contours) > 0: @@ -115,7 +115,7 @@ def get_rectangle(buildings: np.ndarray) -> Optional[RotatedRectange]: def get_kernel(rectangle: RotatedRectange, width_factor: float = 0.5, - thickness: float = 0.001) -> Optional[np.ndarray]: + thickness: float = 0.001) -> np.ndarray | None: ((cx, cy), (xwidth, ywidth), angle) = rectangle width = int(width_factor * min(xwidth, ywidth)) diff --git a/rastervision_core/rastervision/core/data/vector_source/geojson_vector_source.py b/rastervision_core/rastervision/core/data/vector_source/geojson_vector_source.py index 27f53ef86..3ba012f92 100644 --- a/rastervision_core/rastervision/core/data/vector_source/geojson_vector_source.py +++ b/rastervision_core/rastervision/core/data/vector_source/geojson_vector_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional, Union +from typing import TYPE_CHECKING import logging import geopandas as gpd @@ -18,21 +18,21 @@ class GeoJSONVectorSource(VectorSource): """A :class:`.VectorSource` for reading GeoJSON files.""" def __init__(self, - uris: Union[str, List[str]], + uris: str | list[str], crs_transformer: 'CRSTransformer', - vector_transformers: List['VectorTransformer'] = [], - bbox: Optional[Box] = None): + vector_transformers: list['VectorTransformer'] = [], + bbox: Box | None = None): """Constructor. Args: - uris (Union[str, List[str]]): URI(s) of the GeoJSON file(s). - crs_transformer: A ``CRSTransformer`` to convert - between map and pixel coords. Normally this is obtained from a + uris: URI(s) of the GeoJSON file(s). + crs_transformer: A ``CRSTransformer`` to convert between map and + pixel coords. Normally this is obtained from a :class:`.RasterSource`. vector_transformers: ``VectorTransformers`` for transforming geometries. Defaults to ``[]``. - bbox (Optional[Box]): User-specified crop of the extent. If None, - the full extent available in the source file is used. + bbox: User-specified crop of the extent. If ``None``, the full + extent available in the source file is used. """ self.uris = listify_uris(uris) super().__init__( diff --git a/rastervision_core/rastervision/core/data/vector_source/geojson_vector_source_config.py b/rastervision_core/rastervision/core/data/vector_source/geojson_vector_source_config.py index 99fd3bdd3..5e9f86ad5 100644 --- a/rastervision_core/rastervision/core/data/vector_source/geojson_vector_source_config.py +++ b/rastervision_core/rastervision/core/data/vector_source/geojson_vector_source_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Union +from typing import TYPE_CHECKING from rastervision.core.data.vector_source import (VectorSourceConfig, GeoJSONVectorSource) from rastervision.pipeline.config import register_config, Field @@ -22,7 +22,7 @@ def geojson_vector_source_config_upgrader(cfg_dict: dict, class GeoJSONVectorSourceConfig(VectorSourceConfig): """Configure a :class:`.GeoJSONVectorSource`.""" - uris: Union[str, List[str]] = Field( + uris: str | list[str] = Field( ..., description='URI(s) of GeoJSON file(s).') def build(self, diff --git a/rastervision_core/rastervision/core/data/vector_source/vector_source.py b/rastervision_core/rastervision/core/data/vector_source/vector_source.py index 02bdbe2d8..3ec2226ef 100644 --- a/rastervision_core/rastervision/core/data/vector_source/vector_source.py +++ b/rastervision_core/rastervision/core/data/vector_source/vector_source.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING from abc import ABC, abstractmethod import logging @@ -22,18 +22,18 @@ class VectorSource(ABC): def __init__(self, crs_transformer: 'CRSTransformer', - vector_transformers: List['VectorTransformer'] = [], - bbox: Optional[Box] = None): + vector_transformers: list['VectorTransformer'] = [], + bbox: Box | None = None): """Constructor. Args: crs_transformer (CRSTransformer): A ``CRSTransformer`` to convert between map and pixel coords. Normally this is obtained from a :class:`.RasterSource`. - vector_transformers (List[VectorTransformer]): + vector_transformers (list[VectorTransformer]): ``VectorTransformers`` for transforming geometries. Defaults to ``[]``. - bbox (Optional[Box]): User-specified crop of the extent. If None, + bbox (Box | None): User-specified crop of the extent. If None, the full extent available in the source file is used. """ self.crs_transformer = crs_transformer @@ -44,7 +44,7 @@ def __init__(self, self._bbox = bbox def get_geojson(self, - window: Optional[Box] = None, + window: Box | None = None, to_map_coords: bool = False) -> dict: """Return transformed GeoJSON. @@ -59,7 +59,7 @@ def get_geojson(self, VectorTransformers in vector_transformers are also applied. Args: - window (Optional[Box]): If specified, return only the features that + window (Box | None): If specified, return only the features that intersect with this window; otherwise, return all features. Defaults to None. to_map_coords (bool): If true, will return GeoJSON in map @@ -87,16 +87,15 @@ def get_geojson(self, return filter_geojson_to_window(geojson, window) return geojson - def get_geoms(self, - window: Optional[Box] = None, - to_map_coords: bool = False) -> List['BaseGeometry']: + def get_geoms(self, window: Box | None = None, + to_map_coords: bool = False) -> list['BaseGeometry']: """Returns all geometries in the transformed GeoJSON as Shapely geoms. Args: to_map_coords: If true, will return geoms in map coordinates. Returns: - List['BaseGeometry']: List of Shapely geoms. + list['BaseGeometry']: List of Shapely geoms. """ geojson = self.get_geojson(window=window, to_map_coords=to_map_coords) return list(geojson_to_geoms(geojson)) @@ -106,12 +105,12 @@ def _get_geojson(self) -> dict: """Return raw GeoJSON.""" def get_dataframe(self, - window: Optional[Box] = None, + window: Box | None = None, to_map_coords: bool = False) -> gpd.GeoDataFrame: """Return geometries as a :class:`~geopandas.GeoDataFrame`. Arguments: - window (Optional[Box]): If specified, return only the features that + window (Box | None): If specified, return only the features that intersect with this window; otherwise, return all features. Defaults to None. to_map_coords (bool): If true, will return GeoJSON in map @@ -162,7 +161,7 @@ def sanitize_geojson(geojson: dict, geojson (dict): A GeoJSON-like mapping of a FeatureCollection. crs_transformer (CRSTransformer): A CRS transformer for coordinate transformation. - to_map_coords (bool, optional): If True, transform geometries back to + to_map_coords (bool): If True, transform geometries back to map coordinates before returning. Defaults to False. Returns: diff --git a/rastervision_core/rastervision/core/data/vector_source/vector_source_config.py b/rastervision_core/rastervision/core/data/vector_source/vector_source_config.py index 29076d55b..1e8e68596 100644 --- a/rastervision_core/rastervision/core/data/vector_source/vector_source_config.py +++ b/rastervision_core/rastervision/core/data/vector_source/vector_source_config.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.config import Config, register_config, Field from rastervision.core.data.vector_transformer import VectorTransformerConfig @@ -47,7 +47,7 @@ def vector_source_config_upgrader(cfg_dict: dict, class VectorSourceConfig(Config): """Configure a :class:`.VectorSource`.""" - transformers: List[VectorTransformerConfig] = Field( + transformers: list[VectorTransformerConfig] = Field( [], description='List of VectorTransformers.') @abstractmethod @@ -56,6 +56,6 @@ def build(self, class_config: 'ClassConfig', pass def update(self, - pipeline: Optional['RVPipelineConfig'] = None, - scene: Optional['SceneConfig'] = None) -> None: + pipeline: 'RVPipelineConfig | None' = None, + scene: 'SceneConfig | None' = None) -> None: pass diff --git a/rastervision_core/rastervision/core/data/vector_transformer/buffer_transformer.py b/rastervision_core/rastervision/core/data/vector_transformer/buffer_transformer.py index 55030c9b0..dc6b16771 100644 --- a/rastervision_core/rastervision/core/data/vector_transformer/buffer_transformer.py +++ b/rastervision_core/rastervision/core/data/vector_transformer/buffer_transformer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING from rastervision.core.data.utils.geojson import buffer_geoms from rastervision.core.data.vector_transformer import VectorTransformer @@ -12,24 +12,22 @@ class BufferTransformer(VectorTransformer): def __init__(self, geom_type: str, - class_bufs: Optional[Dict[int, Optional[float]]] = None, - default_buf: Optional[float] = None): + class_bufs: dict[int, float | None] | None = None, + default_buf: float | None = None): """Constructor. Args: - geom_type (str): The geometry type to apply this transform to. + geom_type: The geometry type to apply this transform to. E.g. "LineString", "Point", "Polygon". - class_bufs (Dict[int, Optional[float]], optional): Mapping from - class IDs to buffer amounts (in pixels). If a class ID is not - found in the mapping, the value specified by the default_buf - field will be used. If the buffer value for a class is None, - then no buffering will be applied to the geoms of that - class. - Defaults to {}. - default_buf (Optional[float], optional): Default buffer to apply to - classes not in class_bufs. If None, no buffering will be - applied to the geoms of those missing classes. Defaults to - None. + class_bufs: Mapping from class IDs to buffer amounts (in pixels). + If a class ID is not found in the mapping, the value specified + by the ``default_buf`` field will be used. If the buffer value + for a class is ``None``, then no buffering will be applied to + the geoms of that class. Defaults to ``{}``. + default_buf: Default buffer to apply to + classes not in ``class_bufs``. If ``None``, no buffering will + be applied to the geoms of those missing classes. Defaults to + ``None``. """ self.geom_type = geom_type self.class_bufs = class_bufs if class_bufs is not None else {} @@ -37,7 +35,7 @@ class IDs to buffer amounts (in pixels). If a class ID is not def transform(self, geojson: dict, - crs_transformer: Optional['CRSTransformer'] = None) -> dict: + crs_transformer: 'CRSTransformer | None' = None) -> dict: return buffer_geoms( geojson, self.geom_type, diff --git a/rastervision_core/rastervision/core/data/vector_transformer/buffer_transformer_config.py b/rastervision_core/rastervision/core/data/vector_transformer/buffer_transformer_config.py index b3ba9815d..ffd387001 100644 --- a/rastervision_core/rastervision/core/data/vector_transformer/buffer_transformer_config.py +++ b/rastervision_core/rastervision/core/data/vector_transformer/buffer_transformer_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.config import register_config, Field from rastervision.core.data.vector_transformer import (VectorTransformerConfig, @@ -19,7 +19,7 @@ class BufferTransformerConfig(VectorTransformerConfig): ..., description='The geometry type to apply this transform to. ' 'E.g. "LineString", "Point", "Polygon".') - class_bufs: Dict[int, Optional[float]] = Field( + class_bufs: dict[int, float | None] = Field( {}, description='Mapping from class IDs to buffer amounts (in pixels). ' 'If a class ID is not found in the mapping, the value specified by ' @@ -28,13 +28,13 @@ class BufferTransformerConfig(VectorTransformerConfig): 'class and the geom won\'t get converted to a Polygon. Not converting ' 'to Polygon is incompatible with the currently available ' 'LabelSources, but may be useful in the future.') - default_buf: Optional[float] = Field( + default_buf: float | None = Field( 1, description='Default buffer to apply to classes not in class_bufs. ' 'If None, no buffering will be applied to the geoms of those classes.') - def build(self, class_config: Optional['ClassConfig'] = None - ) -> BufferTransformer: + def build(self, + class_config: 'ClassConfig | None' = None) -> BufferTransformer: return BufferTransformer( self.geom_type, class_bufs=self.class_bufs, diff --git a/rastervision_core/rastervision/core/data/vector_transformer/class_inference_transformer.py b/rastervision_core/rastervision/core/data/vector_transformer/class_inference_transformer.py index 6d3887a58..2df54ff86 100644 --- a/rastervision_core/rastervision/core/data/vector_transformer/class_inference_transformer.py +++ b/rastervision_core/rastervision/core/data/vector_transformer/class_inference_transformer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, Optional +from typing import TYPE_CHECKING from copy import deepcopy import logging @@ -27,10 +27,10 @@ class ClassInferenceTransformer(VectorTransformer): """ def __init__(self, - default_class_id: Optional[int], - class_config: Optional['ClassConfig'] = None, - class_id_to_filter: Optional[Dict[int, list]] = None, - class_name_mapping: Optional[dict[str, str]] = None): + default_class_id: int | None, + class_config: 'ClassConfig | None' = None, + class_id_to_filter: dict[int, list] | None = None, + class_name_mapping: dict[str, str] | None = None): """Constructor. Args: @@ -74,11 +74,10 @@ def __init__(self, @staticmethod def infer_feature_class_id( feature: dict, - default_class_id: Optional[int], - class_config: Optional['ClassConfig'] = None, - class_id_to_filter: Optional[Dict[int, list]] = None, - class_name_mapping: Optional[dict[str, str]] = None - ) -> Optional[int]: + default_class_id: int | None, + class_config: 'ClassConfig | None' = None, + class_id_to_filter: dict[int, list] | None = None, + class_name_mapping: dict[str, str] | None = None) -> int | None: """Infer the class ID for a GeoJSON feature. Rules: @@ -115,7 +114,7 @@ def infer_feature_class_id( ``dict(car="vehicle", truck="vehicle")``. Defaults to ``None``. Returns: - Optional[int]: Inferred class ID. + int | None: Inferred class ID. """ if class_name_mapping is not None and class_config is None: raise ValueError( @@ -146,7 +145,7 @@ def infer_feature_class_id( def transform(self, geojson: dict, - crs_transformer: Optional['CRSTransformer'] = None) -> dict: + crs_transformer: 'CRSTransformer | None' = None) -> dict: """Add class_id to feature properties and drop features with no class. For each feature in geojson, the class_id is inferred and is set into diff --git a/rastervision_core/rastervision/core/data/vector_transformer/shift_transformer.py b/rastervision_core/rastervision/core/data/vector_transformer/shift_transformer.py index 956902baf..01e2019bb 100644 --- a/rastervision_core/rastervision/core/data/vector_transformer/shift_transformer.py +++ b/rastervision_core/rastervision/core/data/vector_transformer/shift_transformer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import numpy as np from shapely.ops import transform @@ -32,7 +32,7 @@ def __init__(self, def transform(self, geojson: dict, - crs_transformer: Optional['CRSTransformer'] = None) -> dict: + crs_transformer: 'CRSTransformer | None' = None) -> dict: # https://gis.stackexchange.com/questions/2951/algorithm-for-offsetting-a-latitude-longitude-by-some-amount-of-meters # noqa def shift(x, y, z=None): diff --git a/rastervision_core/rastervision/core/data/vector_transformer/shift_transformer_config.py b/rastervision_core/rastervision/core/data/vector_transformer/shift_transformer_config.py index d78c7fe35..61195ce4b 100644 --- a/rastervision_core/rastervision/core/data/vector_transformer/shift_transformer_config.py +++ b/rastervision_core/rastervision/core/data/vector_transformer/shift_transformer_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.config import register_config, Field from rastervision.core.data.vector_transformer import (VectorTransformerConfig, @@ -24,8 +24,8 @@ class ShiftTransformerConfig(VectorTransformerConfig): True, descriptions='Whether to round shifted pixel values to integers.') - def build(self, class_config: Optional['ClassConfig'] = None - ) -> ShiftTransformer: + def build(self, + class_config: 'ClassConfig | None' = None) -> ShiftTransformer: return ShiftTransformer( x_shift=self.x_shift, y_shift=self.y_shift, diff --git a/rastervision_core/rastervision/core/data/vector_transformer/vector_transformer.py b/rastervision_core/rastervision/core/data/vector_transformer/vector_transformer.py index eb32f15a8..112ddc2ce 100644 --- a/rastervision_core/rastervision/core/data/vector_transformer/vector_transformer.py +++ b/rastervision_core/rastervision/core/data/vector_transformer/vector_transformer.py @@ -1,5 +1,5 @@ from abc import (ABC, abstractmethod) -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING if TYPE_CHECKING: from rastervision.core.data import CRSTransformer @@ -10,13 +10,13 @@ class VectorTransformer(ABC): def __call__(self, geojson: dict, - crs_transformer: Optional['CRSTransformer'] = None, + crs_transformer: 'CRSTransformer | None' = None, **kwargs) -> dict: """Shortcut for :meth:`.transform`. Args: geojson (dict): A GeoJSON-like mapping of a FeatureCollection. - crs_transformer (Optional[CRSTransformer]): CRSTransformer. + crs_transformer (CRSTransformer | None): CRSTransformer. Defaults to None. **kwargs: Extra args for :meth:`.transform`. @@ -29,7 +29,7 @@ def __call__(self, @abstractmethod def transform(self, geojson: dict, - crs_transformer: Optional['CRSTransformer'] = None) -> dict: + crs_transformer: 'CRSTransformer | None' = None) -> dict: """Transform a GeoJSON mapping of vector data. Args: diff --git a/rastervision_core/rastervision/core/data/vector_transformer/vector_transformer_config.py b/rastervision_core/rastervision/core/data/vector_transformer/vector_transformer_config.py index 686add279..ea63202a7 100644 --- a/rastervision_core/rastervision/core/data/vector_transformer/vector_transformer_config.py +++ b/rastervision_core/rastervision/core/data/vector_transformer/vector_transformer_config.py @@ -1,5 +1,5 @@ from abc import abstractmethod -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from rastervision.pipeline.config import Config, register_config @@ -14,8 +14,8 @@ class VectorTransformerConfig(Config): """Configure a :class:`.VectorTransformer`.""" def update(self, - pipeline: Optional['RVPipelineConfig'] = None, - scene: Optional['SceneConfig'] = None) -> None: + pipeline: 'RVPipelineConfig | None' = None, + scene: 'SceneConfig | None' = None) -> None: pass @abstractmethod diff --git a/rastervision_core/rastervision/core/data_sample.py b/rastervision_core/rastervision/core/data_sample.py index 0e5005f68..1a31fbabe 100644 --- a/rastervision_core/rastervision/core/data_sample.py +++ b/rastervision_core/rastervision/core/data_sample.py @@ -1,4 +1,4 @@ -from typing import Any, Literal, Optional +from typing import Any, Literal from dataclasses import dataclass from numpy import ndarray @@ -10,7 +10,7 @@ class DataSample: """A chip and labels along with metadata.""" chip: ndarray - label: Optional[Any] = None - split: Optional[Literal['train', 'valid', 'test']] = None - scene_id: Optional[str] = None - window: Optional[Box] = None + label: Any | None = None + split: Literal['train', 'valid', 'test'] | None = None + scene_id: str | None = None + window: Box | None = None diff --git a/rastervision_core/rastervision/core/evaluation/chip_classification_evaluator_config.py b/rastervision_core/rastervision/core/evaluation/chip_classification_evaluator_config.py index 98d926aec..7eac694c7 100644 --- a/rastervision_core/rastervision/core/evaluation/chip_classification_evaluator_config.py +++ b/rastervision_core/rastervision/core/evaluation/chip_classification_evaluator_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Iterable from rastervision.pipeline.config import register_config from rastervision.core.evaluation.classification_evaluator_config import ( ClassificationEvaluatorConfig) @@ -15,7 +15,7 @@ class ChipClassificationEvaluatorConfig(ClassificationEvaluatorConfig): def build(self, class_config: 'ClassConfig', - scene_group: Optional[Tuple[str, Iterable[str]]] = None + scene_group: tuple[str, Iterable[str]] | None = None ) -> ChipClassificationEvaluator: if scene_group is None: output_uri = self.get_output_uri() diff --git a/rastervision_core/rastervision/core/evaluation/class_evaluation_item.py b/rastervision_core/rastervision/core/evaluation/class_evaluation_item.py index 670dcc571..7f981070c 100644 --- a/rastervision_core/rastervision/core/evaluation/class_evaluation_item.py +++ b/rastervision_core/rastervision/core/evaluation/class_evaluation_item.py @@ -1,7 +1,5 @@ """Defines ``ClassEvaluationItem``.""" - -from typing import Optional - +from typing import Self import numpy as np from rastervision.core.evaluation import EvaluationItem @@ -33,7 +31,7 @@ def __init__(self, tp: int, fp: int, fn: int, - tn: Optional[int] = None, + tn: int | None = None, **kwargs): """Constructor. @@ -43,7 +41,7 @@ def __init__(self, tp (int): True positive count. fp (int): False positive count. fn (int): False negative count. - tn (Optional[int], optional): True negative count. + tn (int | None): True negative count. Defaults to None. **kwargs: Additional data can be provided as keyword arguments. These will be included in the dict returned by ``to_json()``. @@ -57,8 +55,7 @@ def __init__(self, @classmethod def from_multiclass_conf_mat(cls, conf_mat: np.ndarray, class_id: int, - class_name: str, - **kwargs) -> 'ClassEvaluationItem': + class_name: str, **kwargs) -> Self: """Construct from a multi-class confusion matrix and a target class ID. Args: @@ -110,11 +107,11 @@ def true_pos(self) -> int: return self.conf_mat[1, 1] @property - def true_neg(self) -> Optional[int]: + def true_neg(self) -> int | None: """True negative count. Returns: - Optional[int]: Count as int if available. Otherwise, None. + int | None: Count as int if available. Otherwise, None. """ tn = self.conf_mat[0, 0] if tn < 0: @@ -144,7 +141,7 @@ def sensitivity(self) -> float: return self.recall @property - def specificity(self) -> Optional[float]: + def specificity(self) -> float | None: """``TN / (TN + FP)``""" if self.true_neg is None: return None diff --git a/rastervision_core/rastervision/core/evaluation/classification_evaluation.py b/rastervision_core/rastervision/core/evaluation/classification_evaluation.py index 93035ef88..00d5a7009 100644 --- a/rastervision_core/rastervision/core/evaluation/classification_evaluation.py +++ b/rastervision_core/rastervision/core/evaluation/classification_evaluation.py @@ -1,6 +1,6 @@ """Defines abstract base evaluation class for all tasks.""" -from typing import TYPE_CHECKING, Any, Dict, Optional, Union +from typing import TYPE_CHECKING, Any from abc import (ABC, abstractmethod) import copy import json @@ -20,20 +20,17 @@ class ClassificationEvaluation(ABC): Evaluations can be keyed, for instance, if evaluations happen per class. Attributes: - class_to_eval_item (Dict[int, ClassEvaluationItem]): Mapping from class - IDs to ``ClassEvaluationItem``s. - scene_to_eval (Dict[str, ClassificationEvaluation]): Mapping from scene - IDs to ``ClassificationEvaluation``s. - avg_item (Optional[Dict[str, Any]]): Averaged evaluation over all - classes. - conf_mat (Optional[np.ndarray]): Confusion matrix. + class_to_eval_item: Mapping from class IDs to ``ClassEvaluationItem``s. + scene_to_eval: Mapping from scene IDs to ``ClassificationEvaluation``s. + avg_item: Averaged evaluation over all classes. + conf_mat: Confusion matrix. """ def __init__(self): - self.class_to_eval_item: Dict[int, 'ClassEvaluationItem'] - self.scene_to_eval: Dict[str, 'ClassificationEvaluation'] - self.avg_item: Optional[Dict[str, Any]] - self.conf_mat: Optional[np.ndarray] + self.class_to_eval_item: dict[int, 'ClassEvaluationItem'] + self.scene_to_eval: dict[str, 'ClassificationEvaluation'] + self.avg_item: dict[str, Any] | None + self.conf_mat: np.ndarray | None self.reset() def reset(self): @@ -43,11 +40,11 @@ def reset(self): self.avg_item = None self.conf_mat = None - def to_json(self) -> Union[dict, list]: + def to_json(self) -> dict | list: """Serialize to a dict or list. Returns: - Union[dict, list]: Class-wise and (if available) scene-wise + dict | list: Class-wise and (if available) scene-wise evaluations. """ out = [item.to_json() for item in self.class_to_eval_item.values()] @@ -77,7 +74,7 @@ def save(self, output_uri: str) -> None: def merge(self, other: 'ClassificationEvaluation', - scene_id: Optional[str] = None) -> None: + scene_id: str | None = None) -> None: """Merge Evaluation for another Scene into this one. This is useful for computing the average metrics of a set of scenes. @@ -85,7 +82,7 @@ def merge(self, Args: other (ClassificationEvaluation): Evaluation to merge into this one - scene_id (Optional[str], optional): ID of scene. If specified, + scene_id (str | None): ID of scene. If specified, (a copy of) ``other`` will be saved and be available in ``to_json()``'s output. Defaults to None. """ diff --git a/rastervision_core/rastervision/core/evaluation/classification_evaluator.py b/rastervision_core/rastervision/core/evaluation/classification_evaluator.py index 263438c0b..1735688c7 100644 --- a/rastervision_core/rastervision/core/evaluation/classification_evaluator.py +++ b/rastervision_core/rastervision/core/evaluation/classification_evaluator.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Iterable, Optional +from typing import TYPE_CHECKING, Iterable from abc import (abstractmethod) import logging @@ -17,7 +17,7 @@ class ClassificationEvaluator(Evaluator): def __init__(self, class_config: 'ClassConfig', - output_uri: Optional[str] = None): + output_uri: str | None = None): self.class_config = class_config self.output_uri = output_uri @@ -26,7 +26,7 @@ def create_evaluation(self) -> 'ClassificationEvaluation': pass def process(self, scenes: Iterable['Scene'], - tmp_dir: Optional[str] = None) -> None: + tmp_dir: str | None = None) -> None: if self.output_uri is not None: evaluation_global = self.create_evaluation() for scene in scenes: diff --git a/rastervision_core/rastervision/core/evaluation/evaluator_config.py b/rastervision_core/rastervision/core/evaluation/evaluator_config.py index 511a71c7b..8c93693b7 100644 --- a/rastervision_core/rastervision/core/evaluation/evaluator_config.py +++ b/rastervision_core/rastervision/core/evaluation/evaluator_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Iterable from os.path import join from rastervision.pipeline.config import register_config, Config, Field @@ -13,7 +13,7 @@ class EvaluatorConfig(Config): """Configure an :class:`.Evaluator`.""" - output_uri: Optional[str] = Field( + output_uri: str | None = Field( None, description='URI of directory where evaluator output will be saved. ' 'Evaluations for each scene-group will be save in a JSON file at ' @@ -22,15 +22,15 @@ class EvaluatorConfig(Config): def build(self, class_config: 'ClassConfig', - scene_group: Optional[Tuple[str, Iterable[str]]] = None + scene_group: tuple[str, Iterable[str]] | None = None ) -> 'Evaluator': pass - def get_output_uri(self, scene_group_name: Optional[str] = None) -> str: + def get_output_uri(self, scene_group_name: str | None = None) -> str: if scene_group_name is None: return join(self.output_uri, 'eval.json') return join(self.output_uri, scene_group_name, 'eval.json') - def update(self, pipeline: Optional['RVPipelineConfig'] = None) -> None: + def update(self, pipeline: 'RVPipelineConfig | None' = None) -> None: if pipeline is not None and self.output_uri is None: self.output_uri = pipeline.eval_uri diff --git a/rastervision_core/rastervision/core/evaluation/object_detection_evaluation.py b/rastervision_core/rastervision/core/evaluation/object_detection_evaluation.py index 519be44c1..7baf4a909 100644 --- a/rastervision_core/rastervision/core/evaluation/object_detection_evaluation.py +++ b/rastervision_core/rastervision/core/evaluation/object_detection_evaluation.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, Tuple +from typing import TYPE_CHECKING import numpy as np import geopandas as gpd @@ -15,7 +15,7 @@ def compute_metrics( gt_labels: 'ObjectDetectionLabels', pred_labels: 'ObjectDetectionLabels', num_classes: int, - iou_thresh: float = 0.5) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + iou_thresh: float = 0.5) -> tuple[np.ndarray, np.ndarray, np.ndarray]: """Compute per-class true positives, false positives, and false negatives. Does the following: @@ -97,7 +97,7 @@ def compute_eval_items( gt_labels: 'ObjectDetectionLabels', pred_labels: 'ObjectDetectionLabels', class_config: 'ClassConfig', - iou_thresh: float = 0.5) -> Dict[int, ClassEvaluationItem]: + iou_thresh: float = 0.5) -> dict[int, ClassEvaluationItem]: num_classes = len(class_config) tps, fps, fns = compute_metrics(gt_labels, pred_labels, num_classes, iou_thresh) diff --git a/rastervision_core/rastervision/core/evaluation/object_detection_evaluator_config.py b/rastervision_core/rastervision/core/evaluation/object_detection_evaluator_config.py index 11a7dd013..db0066891 100644 --- a/rastervision_core/rastervision/core/evaluation/object_detection_evaluator_config.py +++ b/rastervision_core/rastervision/core/evaluation/object_detection_evaluator_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Iterable from rastervision.pipeline.config import register_config from rastervision.core.evaluation.classification_evaluator_config import ( @@ -16,7 +16,7 @@ class ObjectDetectionEvaluatorConfig(ClassificationEvaluatorConfig): def build(self, class_config: 'ClassConfig', - scene_group: Optional[Tuple[str, Iterable[str]]] = None + scene_group: tuple[str, Iterable[str]] | None = None ) -> ObjectDetectionEvaluator: if scene_group is None: output_uri = self.get_output_uri() diff --git a/rastervision_core/rastervision/core/evaluation/semantic_segmentation_evaluator_config.py b/rastervision_core/rastervision/core/evaluation/semantic_segmentation_evaluator_config.py index 87943d4c5..c1b6e64f0 100644 --- a/rastervision_core/rastervision/core/evaluation/semantic_segmentation_evaluator_config.py +++ b/rastervision_core/rastervision/core/evaluation/semantic_segmentation_evaluator_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Iterable, Optional, Tuple +from typing import TYPE_CHECKING, Iterable from rastervision.pipeline.config import register_config from rastervision.core.evaluation.classification_evaluator_config import ( @@ -25,7 +25,7 @@ class SemanticSegmentationEvaluatorConfig(ClassificationEvaluatorConfig): def build(self, class_config: 'ClassConfig', - scene_group: Optional[Tuple[str, Iterable[str]]] = None + scene_group: tuple[str, Iterable[str]] | None = None ) -> SemanticSegmentationEvaluator: if scene_group is None: output_uri = self.get_output_uri() diff --git a/rastervision_core/rastervision/core/predictor.py b/rastervision_core/rastervision/core/predictor.py index 7781ce632..8077d51b4 100644 --- a/rastervision_core/rastervision/core/predictor.py +++ b/rastervision_core/rastervision/core/predictor.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING from os.path import join import logging @@ -27,8 +27,8 @@ def __init__(self, model_bundle_uri: str, tmp_dir: str, update_stats: bool = False, - channel_order: Optional[List[int]] = None, - scene_group: Optional[str] = None): + channel_order: list[int] | None = None, + scene_group: str | None = None): """Creates a new Predictor. Args: @@ -95,7 +95,7 @@ def __init__(self, self.pipeline = None - def predict(self, image_uris: List[str], label_uri: str) -> None: + def predict(self, image_uris: list[str], label_uri: str) -> None: """Generate predictions for the given image. Args: @@ -142,7 +142,7 @@ class ScenePredictor: def __init__(self, model_bundle_uri: str, predict_options: 'str | dict | PredictOptions | None' = None, - tmp_dir: Optional[str] = None): + tmp_dir: str | None = None): """Creates a new Predictor. Args: diff --git a/rastervision_core/rastervision/core/raster_stats.py b/rastervision_core/rastervision/core/raster_stats.py index 0a9c0ace0..764280101 100644 --- a/rastervision_core/rastervision/core/raster_stats.py +++ b/rastervision_core/rastervision/core/raster_stats.py @@ -1,5 +1,4 @@ -from typing import (TYPE_CHECKING, Iterable, Iterator, Optional, Sequence, - Tuple, Union) +from typing import (TYPE_CHECKING, Iterable, Iterator, Self, Sequence) import numpy as np from tqdm.auto import tqdm @@ -17,24 +16,23 @@ class RasterStats: """Band-wise means and standard deviations.""" def __init__(self, - means: Optional[np.ndarray] = None, - stds: Optional[np.ndarray] = None, - counts: Optional[np.ndarray] = None): + means: np.ndarray | None = None, + stds: np.ndarray | None = None, + counts: np.ndarray | None = None): """Constructor. Args: - means (Optional[np.ndarray]): Band means. Defaults to None. - stds (Optional[np.ndarray]): Band standard deviations. - Defaults to None. - counts (Optional[np.ndarray]): Band pixel counts (used to compute - the specified means and stds). Defaults to None. + means: Band means. Defaults to ``None``. + stds: Band standard deviations. Defaults to ``None``. + counts: Band pixel counts (used to compute the specified means and + stds). Defaults to ``None``. """ self.means = means self.stds = stds self.counts = counts @classmethod - def load(cls, stats_uri: str) -> 'RasterStats': + def load(cls, stats_uri: str) -> Self: """Load stats from file.""" stats_json = file_to_json(stats_uri) assert 'means' in stats_json and 'stds' in stats_json @@ -46,10 +44,10 @@ def load(cls, stats_uri: str) -> 'RasterStats': def compute(self, raster_sources: Sequence['RasterSource'], - sample_prob: Optional[float] = None, + sample_prob: float | None = None, chip_sz: int = 300, - stride: Optional[int] = None, - nodata_value: Optional[float] = 0) -> None: + stride: int | None = None, + nodata_value: float | None = 0) -> None: """Compute the mean and stds over all the raster_sources. This ignores NODATA values if nodata_value is not None. @@ -64,11 +62,11 @@ def compute(self, stats. Args: - raster_sources Sequence['RasterSource']: List of RasterSources. - sample_prob (Optional[float]): Pixel sampling probability. See - notes above. Defaults to None. - nodata_value (Optional[float]): NODATA value. If set, these pixels - will be ignored when computing stats. + raster_sources: List of RasterSources. + sample_prob: Pixel sampling probability. See notes above. + Defaults to ``None``. + nodata_value: NODATA value. If set, these pixels will be ignored + when computing stats. """ if sample_prob is None: if stride is None: @@ -99,10 +97,10 @@ def compute(self, def compute_from_chips( self, chips: Iterable[np.ndarray], - running_mean: Optional[np.ndarray] = None, - running_var: Optional[np.ndarray] = None, - running_count: Optional[np.ndarray] = None) -> Union[Tuple[ - None, None, None], Tuple[np.ndarray, np.ndarray, np.ndarray]]: + running_mean: np.ndarray | None = None, + running_var: np.ndarray | None = None, + running_count: np.ndarray | None = None + ) -> tuple[None, None, None] | tuple[np.ndarray, np.ndarray, np.ndarray]: """Compute running mean and var from chips in stream.""" with tqdm(chips, desc='Analyzing chips') as bar: for chip in bar: @@ -117,10 +115,10 @@ def compute_from_chips( def compute_from_pixels(self, pixels: np.ndarray, - running_mean: Optional[np.ndarray] = None, - running_var: Optional[np.ndarray] = None, - running_count: Optional[np.ndarray] = None - ) -> Tuple[np.ndarray, np.ndarray, np.ndarray]: + running_mean: np.ndarray | None = None, + running_var: np.ndarray | None = None, + running_count: np.ndarray | None = None + ) -> tuple[np.ndarray, np.ndarray, np.ndarray]: """Update running mean and var from pixel values.""" running_stats = [running_mean, running_var, running_count] has_running_stats = any(s is not None for s in running_stats) @@ -156,7 +154,7 @@ def save(self, stats_uri: str) -> None: json_to_file(stats_dict, stats_uri) @property - def vars(self) -> Optional[np.ndarray]: + def vars(self) -> np.ndarray | None: """Channel variances, if self.stds is set.""" if self.stds is None: return None @@ -214,7 +212,7 @@ def sliding_chip_stream( raster_sources: Iterable['RasterSource'], chip_sz: int, stride: int, - nodata_value: Optional[float] = 0) -> Iterator[np.ndarray]: + nodata_value: float | None = 0) -> Iterator[np.ndarray]: """Get stream of chips using a sliding window.""" for raster_source in raster_sources: windows = raster_source.extent.get_windows(chip_sz, stride) @@ -225,11 +223,10 @@ def sliding_chip_stream( yield chip -def random_chip_stream( - raster_sources: Iterable['RasterSource'], - chip_sz: int, - sample_prob: float, - nodata_value: Optional[float] = 0) -> Iterator[np.ndarray]: +def random_chip_stream(raster_sources: Iterable['RasterSource'], + chip_sz: int, + sample_prob: float, + nodata_value: float | None = 0) -> Iterator[np.ndarray]: """Get random stream of chips.""" for raster_source in raster_sources: extent = raster_source.extent @@ -251,7 +248,7 @@ def random_chip_stream( def get_chip(raster_source: 'RasterSource', window: 'Box', - nodata_value: Optional[float] = 0) -> Optional[np.ndarray]: + nodata_value: float | None = 0) -> np.ndarray | None: """Return chip or None if all values are NODATA.""" chip = raster_source.get_raw_chip(window).astype(float) diff --git a/rastervision_core/rastervision/core/rv_pipeline/chip_options.py b/rastervision_core/rastervision/core/rv_pipeline/chip_options.py index 037c91c8d..0bdca95f9 100644 --- a/rastervision_core/rastervision/core/rv_pipeline/chip_options.py +++ b/rastervision_core/rastervision/core/rv_pipeline/chip_options.py @@ -1,4 +1,4 @@ -from typing import (Any, Dict, Literal, Optional, Self, Tuple, Union) +from typing import (Any, Literal, Self) from enum import Enum from pydantic import NonNegativeInt as NonNegInt, PositiveInt as PosInt @@ -27,17 +27,17 @@ class WindowSamplingConfig(Config): method: WindowSamplingMethod = Field( WindowSamplingMethod.sliding, description='') - size: Union[PosInt, Tuple[PosInt, PosInt]] = Field( + size: PosInt | tuple[PosInt, PosInt] = Field( ..., description='If method = sliding, this is the size of sliding window. ' 'If method = random, this is the size that all the windows are ' 'resized to before they are returned. If method = random and neither ' 'size_lims nor h_lims and w_lims have been specified, then size_lims ' 'is set to (size, size + 1).') - stride: Optional[Union[PosInt, Tuple[PosInt, PosInt]]] = Field( + stride: PosInt | tuple[PosInt, PosInt] | None = Field( None, description='Stride of sliding window. Only used if method = sliding.') - padding: Optional[Union[NonNegInt, Tuple[NonNegInt, NonNegInt]]] = Field( + padding: NonNegInt | tuple[NonNegInt, NonNegInt] | None = Field( None, description='How many pixels are windows allowed to overflow ' 'the edges of the raster source.') @@ -46,7 +46,7 @@ class WindowSamplingConfig(Config): description='If "end", only pad ymax and xmax (bottom and right). ' 'If "start", only pad ymin and xmin (top and left). If "both", ' 'pad all sides. Has no effect if padding is zero. Defaults to "end".') - size_lims: Optional[Tuple[PosInt, PosInt]] = Field( + size_lims: tuple[PosInt, PosInt] | None = Field( None, description='[min, max) interval from which window sizes will be ' 'uniformly randomly sampled. The upper limit is exclusive. To fix the ' @@ -55,11 +55,11 @@ class WindowSamplingConfig(Config): 'h_lims and w_lims, but not both. If neither size_lims nor h_lims ' 'and w_lims have been specified, then this will be set to ' '(size, size + 1).') - h_lims: Optional[Tuple[PosInt, PosInt]] = Field( + h_lims: tuple[PosInt, PosInt] | None = Field( None, description='[min, max] interval from which window heights will be ' 'uniformly randomly sampled. Only used if method = random.') - w_lims: Optional[Tuple[PosInt, PosInt]] = Field( + w_lims: tuple[PosInt, PosInt] | None = Field( None, description='[min, max] interval from which window widths will be ' 'uniformly randomly sampled. Only used if method = random.') @@ -110,9 +110,8 @@ def validate_options(self) -> Self: @register_config('chip_options') class ChipOptions(Config): """Configure the sampling and filtering of chips.""" - sampling: Union[WindowSamplingConfig, Dict[ - str, WindowSamplingConfig]] = Field( - ..., description='Window sampling config.') + sampling: WindowSamplingConfig | dict[str, WindowSamplingConfig] = Field( + ..., description='Window sampling config.') nodata_threshold: Proportion = Field( 1., description='Discard chips where the proportion of NODATA values is ' @@ -121,11 +120,11 @@ class ChipOptions(Config): 'caution. If 1.0, only chips that are fully NODATA will be discarded. ' 'Defaults to 1.0.') - def get_chip_sz(self, scene_id: Optional[str] = None) -> int: + def get_chip_sz(self, scene_id: str | None = None) -> int: if isinstance(self.sampling, dict): if scene_id is None: raise KeyError( - 'sampling is a Dict[scene_id, WindowSamplingConfig], so ' + 'sampling is a dict[scene_id, WindowSamplingConfig], so ' 'there is no single chip size. Specify a valid scene_id ' 'to get the chip size for a particular scene.') return self.sampling[scene_id].size diff --git a/rastervision_core/rastervision/core/rv_pipeline/object_detection_config.py b/rastervision_core/rastervision/core/rv_pipeline/object_detection_config.py index 16cbc4741..59a00dc53 100644 --- a/rastervision_core/rastervision/core/rv_pipeline/object_detection_config.py +++ b/rastervision_core/rastervision/core/rv_pipeline/object_detection_config.py @@ -1,4 +1,4 @@ -from typing import Optional, Self +from typing import Self from rastervision.pipeline.config import (Field, register_config, model_validator) @@ -21,7 +21,7 @@ class ObjectDetectionWindowSamplingConfig(WindowSamplingConfig): False, description='Clip bounding boxes to window limits when retrieving ' 'labels for a window.') - neg_ratio: Optional[float] = Field( + neg_ratio: float | None = Field( None, description='The ratio of negative chips (those containing no ' 'bounding boxes) to positive chips. This can be useful if the ' @@ -43,7 +43,7 @@ class ObjectDetectionChipOptions(ChipOptions): @register_config('object_detection_predict_options') class ObjectDetectionPredictOptions(PredictOptions): - stride: Optional[int] = Field( + stride: int | None = Field( None, description='Stride of the sliding window for generating chips. ' 'Defaults to half of ``chip_sz``.') @@ -71,8 +71,8 @@ def validate_stride(self) -> Self: class ObjectDetectionConfig(RVPipelineConfig): """Configure an :class:`.ObjectDetection` pipeline.""" - chip_options: Optional[ObjectDetectionChipOptions] = None - predict_options: Optional[ObjectDetectionPredictOptions] = None + chip_options: ObjectDetectionChipOptions | None = None + predict_options: ObjectDetectionPredictOptions | None = None def build(self, tmp_dir): from rastervision.core.rv_pipeline.object_detection import ObjectDetection diff --git a/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline.py b/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline.py index c94b30281..d5afb8b72 100644 --- a/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline.py +++ b/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline.py @@ -1,8 +1,8 @@ -import logging +from typing import TYPE_CHECKING from os.path import join +import logging import tempfile import shutil -from typing import TYPE_CHECKING, Optional, List from functools import lru_cache import click @@ -45,7 +45,7 @@ class RVPipeline(Pipeline): def __init__(self, config: 'RVPipelineConfig', tmp_dir: str): super().__init__(config, tmp_dir) - self.backend: Optional['Backend'] = None + self.backend: 'Backend | None' = None self.config: 'RVPipelineConfig' @property @@ -93,7 +93,7 @@ def build_scene(scene_id: str) -> Scene: f'scene group "{group_name}"...') analyzer.process(group_scenes, self.tmp_dir) - def get_train_windows(self, scene: Scene) -> List[Box]: + def get_train_windows(self, scene: Scene) -> list[Box]: """Return the training windows for a Scene. Each training window represents the spatial extent of a training chip to @@ -134,7 +134,7 @@ def post_process_sample(self, sample: DataSample) -> DataSample: """ return sample - def post_process_batch(self, windows: List[Box], chips: np.ndarray, + def post_process_batch(self, windows: list[Box], chips: np.ndarray, labels: Labels) -> Labels: """Post-process a batch of predictions.""" return labels @@ -233,6 +233,6 @@ def bundle(self): zipdir(bundle_dir, model_bundle_path) upload_or_copy(model_bundle_path, model_bundle_uri) - def build_backend(self, uri: Optional[str] = None) -> None: + def build_backend(self, uri: str | None = None) -> None: self.backend = self.config.backend.build(self.config, self.tmp_dir) self.backend.load_model(uri) diff --git a/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline_config.py b/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline_config.py index 468a7942e..3b1d32e2e 100644 --- a/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline_config.py +++ b/rastervision_core/rastervision/core/rv_pipeline/rv_pipeline_config.py @@ -1,4 +1,4 @@ -from typing import (TYPE_CHECKING, List, Optional, Self) +from typing import TYPE_CHECKING, Self from os.path import join from rastervision.pipeline.pipeline_config import PipelineConfig @@ -20,7 +20,7 @@ class PredictOptions(Config): chip_sz: int = Field( 300, description='Size of predictions chips in pixels.') - stride: Optional[int] = Field( + stride: int | None = Field( None, description='Stride of the sliding window for generating chips.' 'Defaults to ``chip_sz``.') @@ -65,47 +65,47 @@ class RVPipelineConfig(PipelineConfig): 'Dataset containing train, validation, and optional test scenes.') backend: BackendConfig = Field( ..., description='Backend to use for interfacing with ML library.') - evaluators: List[EvaluatorConfig] = Field( + evaluators: list[EvaluatorConfig] = Field( [], description=( 'Evaluators to run during analyzer command. If list is empty ' 'the default evaluator is added.')) - analyzers: List[AnalyzerConfig] = Field( + analyzers: list[AnalyzerConfig] = Field( [], description= ('Analyzers to run during analyzer command. A StatsAnalyzer will be added ' 'automatically if any scenes have a RasterTransformer.')) - analyze_uri: Optional[str] = Field( + analyze_uri: str | None = Field( None, description= 'URI for output of analyze. If None, will be auto-generated.') - chip_uri: Optional[str] = Field( + chip_uri: str | None = Field( None, description='URI for output of chip. If None, will be auto-generated.') - train_uri: Optional[str] = Field( + train_uri: str | None = Field( None, description='URI for output of train. If None, will be auto-generated.' ) - predict_uri: Optional[str] = Field( + predict_uri: str | None = Field( None, description= 'URI for output of predict. If None, will be auto-generated.') - eval_uri: Optional[str] = Field( + eval_uri: str | None = Field( None, description='URI for output of eval. If None, will be auto-generated.') - bundle_uri: Optional[str] = Field( + bundle_uri: str | None = Field( None, description='URI for output of bundle. If None, will be auto-generated.' ) - source_bundle_uri: Optional[str] = Field( + source_bundle_uri: str | None = Field( None, description='If provided, the model will be loaded from this bundle ' 'for the train stage. Useful for fine-tuning.') - chip_options: Optional[ChipOptions] = Field( + chip_options: ChipOptions | None = Field( None, description='Config for chip stage.') - predict_options: Optional[PredictOptions] = Field( + predict_options: PredictOptions | None = Field( None, description='Config for predict stage.') def update(self): diff --git a/rastervision_core/rastervision/core/rv_pipeline/semantic_segmentation_config.py b/rastervision_core/rastervision/core/rv_pipeline/semantic_segmentation_config.py index a5e8c490b..1046c83e1 100644 --- a/rastervision_core/rastervision/core/rv_pipeline/semantic_segmentation_config.py +++ b/rastervision_core/rastervision/core/rv_pipeline/semantic_segmentation_config.py @@ -1,4 +1,4 @@ -from typing import (List, Literal, Optional, Self, Union) +from typing import Literal, Self import logging from pydantic import NonNegativeInt as NonNegInt @@ -32,7 +32,7 @@ def ss_chip_options_upgrader(cfg_dict: dict, version: int) -> dict: 'semantic_segmentation_chip_options', upgrader=ss_chip_options_upgrader) class SemanticSegmentationChipOptions(ChipOptions): """Chipping options for semantic segmentation.""" - target_class_ids: Optional[List[int]] = Field( + target_class_ids: list[int] | None = Field( None, description= ('List of class ids considered as targets (ie. those to prioritize when ' @@ -77,12 +77,12 @@ def enough_target_pixels(self, label_arr: np.ndarray) -> bool: @register_config('semantic_segmentation_predict_options') class SemanticSegmentationPredictOptions(PredictOptions): - stride: Optional[int] = Field( + stride: int | None = Field( None, description='Stride of the sliding window for generating chips. ' 'Allows aggregating multiple predictions for each pixel if less than ' 'the chip size. Defaults to ``chip_sz``.') - crop_sz: Optional[Union[NonNegInt, Literal['auto']]] = Field( + crop_sz: NonNegInt | Literal['auto'] | None = Field( None, description= 'Number of rows/columns of pixels from the edge of prediction ' diff --git a/rastervision_core/rastervision/core/rv_pipeline/utils.py b/rastervision_core/rastervision/core/rv_pipeline/utils.py index 4ec59465e..400615404 100644 --- a/rastervision_core/rastervision/core/rv_pipeline/utils.py +++ b/rastervision_core/rastervision/core/rv_pipeline/utils.py @@ -10,7 +10,7 @@ def nodata_below_threshold(chip: np.ndarray, chip (np.ndarray): Raster as (..., H, W[, C]) numpy array. threshold (float): Threshold to check the fraction of NODATA pixels against. - nodata_val (int, optional): Value that represents NODATA pixels. + nodata_val (int): Value that represents NODATA pixels. Defaults to 0. Returns: diff --git a/rastervision_core/rastervision/core/utils/stac.py b/rastervision_core/rastervision/core/utils/stac.py index 2719d864e..355fe1571 100644 --- a/rastervision_core/rastervision/core/utils/stac.py +++ b/rastervision_core/rastervision/core/utils/stac.py @@ -1,4 +1,3 @@ -from typing import List, Optional from urllib.parse import urlparse import logging from itertools import islice @@ -50,7 +49,7 @@ def is_label_item(item: Item) -> bool: return False -def get_linked_image_item(label_item: Item) -> Optional[Item]: +def get_linked_image_item(label_item: Item) -> Item | None: """Find link in the item that has "rel" == "source" and return its "target" item. If no such link, return None. If multiple such links, raise an exception.""" @@ -63,7 +62,7 @@ def get_linked_image_item(label_item: Item) -> Optional[Item]: return image_item -def parse_stac(stac_uri: str, item_limit: Optional[int] = None) -> List[dict]: +def parse_stac(stac_uri: str, item_limit: int | None = None) -> list[dict]: """Parse a STAC catalog JSON file to extract label URIs, images URIs, and AOIs. @@ -74,7 +73,7 @@ def parse_stac(stac_uri: str, item_limit: Optional[int] = None) -> List[dict]: stac_uri (str): Path to the STAC catalog JSON file. Returns: - List[dict]: A list of dicts with keys: "label_uri", "image_uris", + list[dict]: A list of dicts with keys: "label_uri", "image_uris", "label_bbox", "image_bbox", "bboxes_intersect", and "aoi_geometry". Each dict corresponds to one label item and its associated image assets in the STAC catalog. @@ -100,7 +99,7 @@ def parse_stac(stac_uri: str, item_limit: Optional[int] = None) -> List[dict]: for label_item, image_item in zip(label_items, image_items): label_uri: str = list(label_item.assets.values())[0].href label_bbox = box(*label_item.bbox) - aoi_geometry: Optional[dict] = label_item.geometry + aoi_geometry: dict | None = label_item.geometry if image_item is not None: image_assets = [ @@ -126,8 +125,8 @@ def parse_stac(stac_uri: str, item_limit: Optional[int] = None) -> List[dict]: return out -def read_stac(uri: str, extract_dir: Optional[str] = None, - **kwargs) -> List[dict]: +def read_stac(uri: str, extract_dir: str | None = None, + **kwargs) -> list[dict]: """Parse the contents of a STAC catalog. The file is downloaded if needed. If it is a zip file, it is unzipped and @@ -136,7 +135,7 @@ def read_stac(uri: str, extract_dir: Optional[str] = None, Args: uri (str): Either a URI to a STAC catalog JSON file or a URI to a zip file containing a STAC catalog JSON file. - extract_dir (Optional[str]): Dir to extract to, if URI is a zip file. + extract_dir (str | None): Dir to extract to, if URI is a zip file. If None, a temporary dir will be used. Defaults to None. **kwargs: Extra args for :func:`.parse_stac`. @@ -145,7 +144,7 @@ def read_stac(uri: str, extract_dir: Optional[str] = None, Exception: If multiple catalog.json's are found inside the zip file. Returns: - List[dict]: A list of dicts with keys: "label_uri", "image_uris", + list[dict]: A list of dicts with keys: "label_uri", "image_uris", "label_bbox", "image_bbox", "bboxes_intersect", and "aoi_geometry". Each dict corresponds to one label item and its associated image assets in the STAC catalog. diff --git a/rastervision_core/rastervision/core/utils/types.py b/rastervision_core/rastervision/core/utils/types.py index 058347860..6b75d33a8 100644 --- a/rastervision_core/rastervision/core/utils/types.py +++ b/rastervision_core/rastervision/core/utils/types.py @@ -1,3 +1,4 @@ +from typing import Sequence from typing_extensions import Annotated from pydantic.types import StringConstraints @@ -6,3 +7,4 @@ NonEmptyStr = Annotated[str, StringConstraints(strip_whitespace=True, min_length=1)] Proportion = Annotated[float, Field(ge=0, le=1)] +Vector = Sequence[float] diff --git a/rastervision_gdal_vsi/rastervision/gdal_vsi/vsi_file_system.py b/rastervision_gdal_vsi/rastervision/gdal_vsi/vsi_file_system.py index 7d590dc78..e9994a6d4 100644 --- a/rastervision_gdal_vsi/rastervision/gdal_vsi/vsi_file_system.py +++ b/rastervision_gdal_vsi/rastervision/gdal_vsi/vsi_file_system.py @@ -1,4 +1,3 @@ -from typing import List, Optional import os from os.path import join from pathlib import Path @@ -195,18 +194,18 @@ def local_path(vsipath: str, download_dir: str) -> str: return join(download_dir, filename) @staticmethod - def last_modified(vsipath: str) -> Optional[datetime]: + def last_modified(vsipath: str) -> datetime | None: stats = gdal.VSIStatL(vsipath) return datetime.fromtimestamp(stats.mtime) if stats else None @staticmethod - def list_paths(vsipath: str, ext: Optional[str] = None) -> List[str]: + def list_paths(vsipath: str, ext: str | None = None) -> list[str]: items = VsiFileSystem.list_children(vsipath, ext=ext) paths = [join(vsipath, item) for item in items] return paths @staticmethod - def list_children(vsipath: str, ext: Optional[str] = None) -> List[str]: + def list_children(vsipath: str, ext: str | None = None) -> list[str]: """List filenames of children rooted at URI. Optionally only includes filenames with a certain file extension. diff --git a/rastervision_pipeline/rastervision/pipeline/config.py b/rastervision_pipeline/rastervision/pipeline/config.py index 486adcaa8..c3628386e 100644 --- a/rastervision_pipeline/rastervision/pipeline/config.py +++ b/rastervision_pipeline/rastervision/pipeline/config.py @@ -1,5 +1,5 @@ -from typing import (TYPE_CHECKING, Callable, Dict, List, Literal, Optional, - Self, Type, Union) +from typing import TYPE_CHECKING, Literal, Self +from collections.abc import Callable import inspect import logging @@ -84,12 +84,12 @@ def recursive_validate_config(self): for c in child_configs: c.recursive_validate_config() - def validate_list(self, field: str, valid_options: List[str]): + def validate_list(self, field: str, valid_options: list[str]): """Validate a list field. Args: field (str): name of field to validate - valid_options (List[str]): values that field is allowed to take + valid_options (list[str]): values that field is allowed to take Raises: ConfigError: if field is invalid @@ -197,8 +197,8 @@ def save_pipeline_config(cfg: 'PipelineConfig', output_uri: str) -> None: str_to_file(cfg_json, output_uri) -def build_config(x: Union[dict, List[Union[dict, Config]], Config] - ) -> Union[Config, List[Config]]: +def build_config( + x: dict | list[dict | Config] | Config) -> Config | list[Config]: """Build a Config from various types of input. This is useful for deserializing from JSON. It implements polymorphic @@ -209,7 +209,7 @@ def build_config(x: Union[dict, List[Union[dict, Config]], Config] x: some representation of Config(s) Returns: - Config: the corresponding Config(s) + The corresponding Config(s). """ if isinstance(x, dict): new_x = {k: build_config(v) for k, v in x.items()} @@ -224,8 +224,8 @@ def build_config(x: Union[dict, List[Union[dict, Config]], Config] return x -def _upgrade_config(x: Union[dict, List[dict]], plugin_versions: Dict[str, int] - ) -> Union[dict, List[dict]]: +def _upgrade_config(x: dict | list[dict], + plugin_versions: dict[str, int]) -> dict | list[dict]: """Upgrade serialized Config(s) to the latest version. Used to implement backward compatibility of Configs using upgraders stored @@ -265,7 +265,7 @@ def _upgrade_config(x: Union[dict, List[dict]], plugin_versions: Dict[str, int] return x -def upgrade_plugin_versions(plugin_versions: Dict[str, int]) -> Dict[str, int]: +def upgrade_plugin_versions(plugin_versions: dict[str, int]) -> dict[str, int]: """Update the names of the plugins using the plugin aliases in the registry. This allows changing the names of plugins over time and maintaining backward @@ -289,8 +289,7 @@ def upgrade_plugin_versions(plugin_versions: Dict[str, int]) -> Dict[str, int]: return new_plugin_versions -def upgrade_config( - config_dict: Union[dict, List[dict]]) -> Union[dict, List[dict]]: +def upgrade_config(config_dict: dict | list[dict]) -> dict | list[dict]: """Upgrade serialized Config(s) to the latest version. Used to implement backward compatibility of Configs using upgraders stored @@ -312,7 +311,7 @@ def upgrade_config( return out -def get_plugin(config_cls: Type) -> str: +def get_plugin(config_cls: type) -> str: """Infer the module path of the plugin where a Config class is defined. This only works correctly if the plugin is in a module under rastervision. @@ -322,8 +321,9 @@ def get_plugin(config_cls: Type) -> str: def register_config(type_hint: str, - plugin: Optional[str] = None, - upgrader: Optional[Callable] = None) -> Callable: + plugin: str | None = None, + upgrader: Callable[[dict, int], dict] | None = None + ) -> Callable[[], Config]: """Class decorator used to register Config classes with registry. All Configs must be registered! Registering a Config does the following: @@ -335,24 +335,22 @@ def register_config(type_hint: str, type_hint. Args: - type_hint (str): a type hint used to deserialize Configs. Must be - unique across all registered Configs. - plugin (Optional[str], optional): the module path of the plugin where - the Config is defined. If None, will be inferred. - Defauilts to None. - upgrader (Optional[Callable], optional): a function of the form - upgrade(config_dict, version) which returns the corresponding - config dict of version = version + 1. This can be useful for - maintaining backward compatibility by allowing old configs using an - outdated schema to be upgraded to the current schema. - Defaults to None. + type_hint: a type hint used to deserialize Configs. Must be unique + across all registered Configs. + plugin: the module path of the plugin where the ``Config`` is defined. + If ``None``, will be inferred. Defauilts to ``None``. + upgrader: a function of the form ``upgrade(config_dict, version)`` + which returns the corresponding config dict of + ``version = version + 1``. This can be useful for maintaining + backward compatibility by allowing old configs using an outdated + schema to be upgraded to the current schema. Defaults to ``None``. Returns: - Callable: A function that returns a new class that is identical to the - input Config with an additional ``type_hint`` field. + A function that returns a new class that is identical to the input + ``Config`` with an additional ``type_hint`` field. """ - def _register_config(cls: Type): + def _register_config(cls: type): new_cls = create_model( cls.__name__, __base__=cls, diff --git a/rastervision_pipeline/rastervision/pipeline/file_system/file_system.py b/rastervision_pipeline/rastervision/pipeline/file_system/file_system.py index 9ffb3f20b..c0cdae391 100644 --- a/rastervision_pipeline/rastervision/pipeline/file_system/file_system.py +++ b/rastervision_pipeline/rastervision/pipeline/file_system/file_system.py @@ -1,6 +1,5 @@ from abc import (ABC, abstractmethod) from datetime import datetime -from typing import Optional, List from rastervision.pipeline import registry_ as registry @@ -135,7 +134,7 @@ def local_path(uri: str, download_dir: str) -> str: @staticmethod @abstractmethod - def last_modified(uri: str) -> Optional[datetime]: + def last_modified(uri: str) -> datetime | None: """Get the last modified date of a file. Args: @@ -148,7 +147,7 @@ def last_modified(uri: str) -> Optional[datetime]: @staticmethod @abstractmethod - def list_paths(uri: str, ext: Optional[str] = None) -> List[str]: + def list_paths(uri: str, ext: str | None = None) -> list[str]: """List paths rooted at URI. Optionally only includes paths with a certain file extension. diff --git a/rastervision_pipeline/rastervision/pipeline/file_system/utils.py b/rastervision_pipeline/rastervision/pipeline/file_system/utils.py index 5048d3053..27025c05f 100644 --- a/rastervision_pipeline/rastervision/pipeline/file_system/utils.py +++ b/rastervision_pipeline/rastervision/pipeline/file_system/utils.py @@ -1,3 +1,4 @@ +from typing import TYPE_CHECKING import os from os.path import join, normpath, relpath import shutil @@ -6,7 +7,6 @@ import logging import json import zipfile -from typing import TYPE_CHECKING, Optional, List from tqdm.auto import tqdm @@ -21,9 +21,8 @@ log = logging.getLogger(__name__) -def get_local_path(uri: str, - download_dir: str, - fs: Optional[FileSystem] = None) -> str: +def get_local_path(uri: str, download_dir: str, + fs: FileSystem | None = None) -> str: """Return the path where a local copy of URI should be stored. If URI is local, return it. If it's remote, we generate a path for it @@ -52,7 +51,7 @@ def get_local_path(uri: str, def sync_to_dir(src_dir: str, dst_dir_uri: str, delete: bool = False, - fs: Optional[FileSystem] = None): + fs: FileSystem | None = None): """Synchronize a local source directory to destination directory. Transfers files from source to destination directories so that the @@ -75,7 +74,7 @@ def sync_to_dir(src_dir: str, def sync_from_dir(src_dir_uri: str, dst_dir: str, delete: bool = False, - fs: Optional[FileSystem] = None): + fs: FileSystem | None = None): """Synchronize a source directory to local destination directory. Transfers files from source to destination directories so that the @@ -98,7 +97,7 @@ def sync_from_dir(src_dir_uri: str, def start_sync(src_dir: str, dst_dir_uri: str, sync_interval: int = 600, - fs: Optional[FileSystem] = None): # pragma: no cover + fs: FileSystem | None = None): # pragma: no cover """Repeatedly sync a local source directory to a destination on a schedule. Calls sync_to_dir on a schedule. @@ -133,8 +132,8 @@ def __exit__(self, type, value, traceback): def download_if_needed(uri: str, - download_dir: Optional[str] = None, - fs: Optional[FileSystem] = None, + download_dir: str | None = None, + fs: FileSystem | None = None, use_cache: bool = True) -> str: """Download a file into a directory if it's remote. @@ -142,12 +141,12 @@ def download_if_needed(uri: str, Args: uri (str): URI of file to download. - download_dir (Optional[str], optional): Local directory to download + download_dir (str | None): Local directory to download file into. If None, the file will be downloaded to cache dir as defined by RVConfig. Defaults to None. - fs (Optional[FileSystem], optional): If provided, use fs instead of + fs (FileSystem | None): If provided, use fs instead of the automatically chosen FileSystem for uri. Defaults to None. - use_cache (bool, optional): If False and the file is remote, download + use_cache (bool): If False and the file is remote, download it regardless of whether it exists in cache. Defaults to True. Returns: @@ -180,7 +179,7 @@ def download_if_needed(uri: str, def download_or_copy(uri: str, target_dir: str, delete_tmp: bool = False, - fs: Optional[FileSystem] = None) -> str: + fs: FileSystem | None = None) -> str: """Downloads or copies a file to a directory. Downloads or copies URI into target_dir. @@ -219,10 +218,8 @@ def file_exists(uri, fs=None, include_dir=True) -> bool: return fs.file_exists(uri, include_dir) -def list_paths(uri: str, - ext: str = '', - fs: Optional[FileSystem] = None, - **kwargs) -> List[str]: +def list_paths(uri: str, ext: str = '', fs: FileSystem | None = None, + **kwargs) -> list[str]: """List paths rooted at URI. Optionally only includes paths with a certain file extension. @@ -243,9 +240,8 @@ def list_paths(uri: str, return fs.list_paths(uri, ext=ext, **kwargs) -def upload_or_copy(src_path: str, - dst_uri: str, - fs: Optional[FileSystem] = None) -> None: +def upload_or_copy(src_path: str, dst_uri: str, + fs: FileSystem | None = None) -> None: """Upload or copy a file. If dst_uri is local, the file is copied. Otherwise, it is uploaded. @@ -273,7 +269,7 @@ def upload_or_copy(src_path: str, fs.copy_to(src_path, dst_uri) -def file_to_str(uri: str, fs: Optional[FileSystem] = None) -> str: +def file_to_str(uri: str, fs: FileSystem | None = None) -> str: """Load contents of text file into a string. Args: @@ -291,7 +287,7 @@ def file_to_str(uri: str, fs: Optional[FileSystem] = None) -> str: return fs.read_str(uri) -def str_to_file(content_str: str, uri: str, fs: Optional[FileSystem] = None): +def str_to_file(content_str: str, uri: str, fs: FileSystem | None = None): """Writes string to text file. Args: @@ -359,8 +355,8 @@ def is_archive(uri: str) -> bool: def extract(uri: str, - target_dir: Optional[str] = None, - download_dir: Optional[str] = None) -> str: + target_dir: str | None = None, + download_dir: str | None = None) -> str: """Extract a compressed file.""" if target_dir is None: target_dir = rv_config.get_cache_dir() diff --git a/rastervision_pipeline/rastervision/pipeline/pipeline.py b/rastervision_pipeline/rastervision/pipeline/pipeline.py index afeeeabfb..8e87adf5a 100644 --- a/rastervision_pipeline/rastervision/pipeline/pipeline.py +++ b/rastervision_pipeline/rastervision/pipeline/pipeline.py @@ -1,5 +1,5 @@ +from typing import TYPE_CHECKING import logging -from typing import List, TYPE_CHECKING log = logging.getLogger(__name__) @@ -25,9 +25,9 @@ def my_command(self, split_ind: int = 0, num_splits: int = 1) gpu_commands: names of commands that should be executed on GPUs if available """ - commands: List[str] = ['test_cpu', 'test_gpu'] - split_commands: List[str] = ['test_cpu'] - gpu_commands: List[str] = ['test_gpu'] + commands: list[str] = ['test_cpu', 'test_gpu'] + split_commands: list[str] = ['test_cpu'] + gpu_commands: list[str] = ['test_gpu'] def __init__(self, config: 'PipelineConfig', tmp_dir: str): """Constructor diff --git a/rastervision_pipeline/rastervision/pipeline/registry.py b/rastervision_pipeline/rastervision/pipeline/registry.py index dcc65a002..817035c34 100644 --- a/rastervision_pipeline/rastervision/pipeline/registry.py +++ b/rastervision_pipeline/rastervision/pipeline/registry.py @@ -1,4 +1,5 @@ -from typing import Iterable, List, Type, TYPE_CHECKING, Optional, Callable +from typing import TYPE_CHECKING, Iterable +from collections.abc import Callable import inspect from click import Command @@ -33,16 +34,16 @@ def add_plugin_command(self, cmd: Command): """Add a click command contributed by a plugin.""" self.plugin_commands.append(cmd) - def get_plugin_commands(self) -> List[Command]: + def get_plugin_commands(self) -> list[Command]: """Get the click commands contributed by plugins.""" return self.plugin_commands - def set_plugin_aliases(self, plugin: str, aliases: List[str]): + def set_plugin_aliases(self, plugin: str, aliases: list[str]): self.alias_to_plugin[plugin] = plugin for alias in aliases: self.alias_to_plugin[alias] = plugin - def get_plugin_from_alias(self, alias: str) -> Optional[str]: + def get_plugin_from_alias(self, alias: str) -> str | None: if alias in self.plugin_versions: return alias return self.alias_to_plugin.get(alias) @@ -67,7 +68,7 @@ def register_renamed_type_hints(self, type_hint_old: str, """ self.renamed_type_hints[type_hint_old] = type_hint_new - def get_type_hint_lineage(self, type_hint: str) -> List[str]: + def get_type_hint_lineage(self, type_hint: str) -> list[str]: """Get the lineage for a type hint. Returns: @@ -88,11 +89,12 @@ def get_plugin(self, type_hint: str) -> str: """Get module path of plugin when Config class with type_hint is defined.""" return self.type_hint_to_plugin[type_hint] - def get_upgrader(self, type_hint: str) -> Optional[Callable]: + def get_upgrader(self, + type_hint: str) -> Callable[[dict, int], dict] | None: """Get function that upgrades config dicts for type_hint.""" return self.type_hint_to_upgrader.get(type_hint) - def add_runner(self, runner_name: str, runner: Type['Runner']): + def add_runner(self, runner_name: str, runner: type['Runner']): """Add a Runner. Args: @@ -105,7 +107,7 @@ def add_runner(self, runner_name: str, runner: Type['Runner']): self.runners[runner_name] = runner - def get_runner(self, runner_name: str) -> Type['Runner']: # noqa + def get_runner(self, runner_name: str) -> type['Runner']: # noqa """Return a Runner class based on its name.""" runner = self.runners.get(runner_name) if runner: @@ -122,7 +124,7 @@ def add_file_system(self, file_system: 'FileSystem'): self.file_systems.append(file_system) def get_file_system(self, uri: str, - mode: str = 'r') -> Type['FileSystem']: # noqa + mode: str = 'r') -> type['FileSystem']: # noqa """Get a FileSystem used to handle the file type of a URI. Args: @@ -144,7 +146,7 @@ def get_file_system(self, uri: str, def add_config(self, type_hint: str, - config: Type['Config'], + config: type['Config'], plugin: str, upgrader=None): """Add a Config. @@ -165,7 +167,7 @@ def add_config(self, self.update_config_info() - def get_config(self, type_hint: str) -> Type['Config']: + def get_config(self, type_hint: str) -> type['Config']: """Get a Config class associated with a type_hint.""" config = self.configs.get(type_hint) if config: @@ -178,7 +180,7 @@ def get_config(self, type_hint: str) -> Type['Config']: 'file for the plugin.') def add_rv_config_schema(self, config_section: str, - config_fields: List[str]): + config_fields: list[str]): """Add section of schema used by RVConfig. Args: @@ -240,8 +242,7 @@ def iter_namespace(ns_pkg): ] return discovered_plugins - def load_plugins(self, - plugin_names: Optional[Iterable[str]] = None) -> None: + def load_plugins(self, plugin_names: Iterable[str] | None = None) -> None: """Load plugins and register their resources. Import each Python module within the rastervision namespace package diff --git a/rastervision_pipeline/rastervision/pipeline/runner/inprocess_runner.py b/rastervision_pipeline/rastervision/pipeline/runner/inprocess_runner.py index 9a5606821..1a8fa7d5e 100644 --- a/rastervision_pipeline/rastervision/pipeline/runner/inprocess_runner.py +++ b/rastervision_pipeline/rastervision/pipeline/runner/inprocess_runner.py @@ -1,4 +1,3 @@ -from typing import List from rastervision.pipeline.cli import _run_command from rastervision.pipeline.runner.runner import Runner @@ -25,6 +24,6 @@ def run(self, else: _run_command(cfg_json_uri, command, 0, 1) - def run_command(self, cmd: List[str]): + def run_command(self, cmd: list[str]): raise NotImplementedError( 'Use LocalRunner.run_command to run a command locally.') diff --git a/rastervision_pipeline/rastervision/pipeline/runner/local_runner.py b/rastervision_pipeline/rastervision/pipeline/runner/local_runner.py index fa810032e..abcef6e6b 100644 --- a/rastervision_pipeline/rastervision/pipeline/runner/local_runner.py +++ b/rastervision_pipeline/rastervision/pipeline/runner/local_runner.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional +from typing import TYPE_CHECKING import sys from os.path import dirname, join from subprocess import Popen @@ -15,7 +15,7 @@ def make_run_cmd_invocation(cfg_json_uri: str, command: str, - opts: Optional[dict] = None) -> str: + opts: dict | None = None) -> str: opts_str = '' if opts is not None: opts_str = ' ' + ' '.join(f'{k} {v}' for k, v in opts.items()) @@ -24,10 +24,10 @@ def make_run_cmd_invocation(cfg_json_uri: str, def make_makefile_entry_for_cmd(curr_command_ind: int, - prev_command_inds: List[int], + prev_command_inds: list[int], cfg_json_uri: str, command: str, - opts: Optional[dict] = None) -> str: + opts: dict | None = None) -> str: out = f'{curr_command_ind}: ' out += ' '.join([str(ci) for ci in prev_command_inds]) out += '\n' @@ -47,7 +47,7 @@ class LocalRunner(Runner): def run(self, cfg_json_uri: str, pipeline: 'Pipeline', - commands: List[str], + commands: list[str], num_splits: int = 1, pipeline_run_name: str = 'raster-vision'): makefile = self.build_makefile_string(cfg_json_uri, pipeline, commands, @@ -57,7 +57,7 @@ def run(self, makefile_path_local = download_if_needed(makefile_path) return self.run_command(['make', '-j', '-f', makefile_path_local]) - def run_command(self, cmd: List[str]): + def run_command(self, cmd: list[str]): process = Popen(cmd) terminate_at_exit(process) exitcode = process.wait() @@ -69,7 +69,7 @@ def run_command(self, cmd: List[str]): def build_makefile_string(self, cfg_json_uri: str, pipeline: 'Pipeline', - commands: List[str], + commands: list[str], num_splits: int = 1) -> str: num_commands = 0 for command in commands: diff --git a/rastervision_pipeline/rastervision/pipeline/runner/runner.py b/rastervision_pipeline/rastervision/pipeline/runner/runner.py index a3909f1c0..a622291ca 100644 --- a/rastervision_pipeline/rastervision/pipeline/runner/runner.py +++ b/rastervision_pipeline/rastervision/pipeline/runner/runner.py @@ -1,5 +1,4 @@ from abc import abstractmethod -from typing import Optional, List from rastervision.pipeline.pipeline import Pipeline @@ -15,7 +14,7 @@ class Runner(): def run(self, cfg_json_uri: str, pipeline: Pipeline, - commands: List[str], + commands: list[str], num_splits: int = 1, pipeline_run_name: str = 'raster-vision'): """Run commands in a Pipeline using a serialized PipelineConfig. @@ -28,14 +27,14 @@ def run(self, """ @abstractmethod - def run_command(self, cmd: List[str]): + def run_command(self, cmd: list[str]): """Run a single command. Args: cmd: The command to run. """ - def get_split_ind(self) -> Optional[int]: + def get_split_ind(self) -> int | None: """Get the split_ind for the process. For split commands, the split_ind determines which split of work to perform diff --git a/rastervision_pipeline/rastervision/pipeline/rv_config.py b/rastervision_pipeline/rastervision/pipeline/rv_config.py index 389dad9b2..df19c8ebd 100644 --- a/rastervision_pipeline/rastervision/pipeline/rv_config.py +++ b/rastervision_pipeline/rastervision/pipeline/rv_config.py @@ -1,4 +1,4 @@ -from typing import Any, Dict, List, Optional +from typing import Any import os from tempfile import TemporaryDirectory from pathlib import Path @@ -86,7 +86,7 @@ def get_tmp_dir_root(self) -> str: """Return the root of all temp dirs.""" return self.tmp_dir_root - def set_tmp_dir_root(self, tmp_dir_root: Optional[str] = None): + def set_tmp_dir_root(self, tmp_dir_root: str | None = None): """Set root of all temporary directories. To set the value, the following rules are used in decreasing priority: @@ -140,7 +140,7 @@ def get_cache_dir(self) -> TemporaryDirectory: def set_everett_config(self, profile: str = None, rv_home: str = None, - config_overrides: Dict[str, str] = None): + config_overrides: dict[str, str] = None): """Set Everett config. This sets up any other configuration using the Everett library. @@ -217,8 +217,8 @@ def get_namespace_config(self, namespace: str) -> ConfigManager: def get_namespace_option(self, namespace: str, key: str, - default: Optional[Any] = None, - as_bool: bool = False) -> Optional[Any]: + default: Any | None = None, + as_bool: bool = False) -> Any | None: """Get the value of an option from a namespace.""" namespace_options = self.config.with_namespace(namespace) try: @@ -232,7 +232,7 @@ def get_namespace_option(self, return default def get_config_dict( - self, rv_config_schema: Dict[str, List[str]]) -> Dict[str, str]: + self, rv_config_schema: dict[str, list[str]]) -> dict[str, str]: """Get all Everett configuration. This method is used to serialize an Everett configuration so it can be used on @@ -257,7 +257,7 @@ def get_config_dict( return config_dict - def _discover_config_file_locations(self, profile) -> List[str]: + def _discover_config_file_locations(self, profile) -> list[str]: """Discover the location of RV config files. Args: diff --git a/rastervision_pipeline/rastervision/pipeline/utils.py b/rastervision_pipeline/rastervision/pipeline/utils.py index 69eff9965..a22227235 100644 --- a/rastervision_pipeline/rastervision/pipeline/utils.py +++ b/rastervision_pipeline/rastervision/pipeline/utils.py @@ -1,4 +1,5 @@ -from typing import Any, Callable, Optional +from typing import Any +from collections.abc import Callable import os import atexit import logging @@ -49,8 +50,8 @@ def repr_with_args(obj: Any, **kwargs) -> str: def get_env_var(key: str, - default: Optional[Any] = None, - out_type: Optional[type | Callable] = None) -> Optional[Any]: + default: Any | None = None, + out_type: type | Callable | None = None) -> Any | None: val = os.environ.get(key, default) if val is not None and out_type is not None: if out_type == bool: diff --git a/rastervision_pipeline/rastervision/pipeline_example_plugin1/sample_pipeline.py b/rastervision_pipeline/rastervision/pipeline_example_plugin1/sample_pipeline.py index 9d06513b2..41f9399c2 100644 --- a/rastervision_pipeline/rastervision/pipeline_example_plugin1/sample_pipeline.py +++ b/rastervision_pipeline/rastervision/pipeline_example_plugin1/sample_pipeline.py @@ -1,4 +1,3 @@ -from typing import List, Optional from os.path import join from rastervision.pipeline.pipeline import Pipeline @@ -14,8 +13,8 @@ class SamplePipelineConfig(PipelineConfig): # Config classes are configuration schemas. Each field is an attributes # with a type and optional default value. - names: List[str] = ['alice', 'bob'] - message_uris: Optional[List[str]] = None + names: list[str] = ['alice', 'bob'] + message_uris: list[str] | None = None def build(self, tmp_dir): # The build method is used to instantiate the corresponding object @@ -34,7 +33,7 @@ def update(self): class SamplePipeline(Pipeline): # The order in which commands run. Each command correspond to a method. - commands: List[str] = ['save_messages', 'print_messages'] + commands: list[str] = ['save_messages', 'print_messages'] # Split commands can be split up and run in parallel. split_commands = ['save_messages'] diff --git a/rastervision_pipeline/rastervision/pipeline_example_plugin1/sample_pipeline2.py b/rastervision_pipeline/rastervision/pipeline_example_plugin1/sample_pipeline2.py index 214ae77c3..ac14f760b 100644 --- a/rastervision_pipeline/rastervision/pipeline_example_plugin1/sample_pipeline2.py +++ b/rastervision_pipeline/rastervision/pipeline_example_plugin1/sample_pipeline2.py @@ -1,4 +1,3 @@ -from typing import List, Optional from os.path import join from rastervision.pipeline.pipeline import Pipeline @@ -22,13 +21,13 @@ def __init__(self, config): def make_message(self, name): # Use the greeting field to make the message. - return '{} {}!'.format(self.config.greeting, name) + return f'{self.config.greeting} {name}!' @register_config('pipeline_example_plugin1.sample_pipeline2') class SamplePipeline2Config(PipelineConfig): - names: List[str] = ['alice', 'bob'] - message_uris: Optional[List[str]] = None + names: list[str] = ['alice', 'bob'] + message_uris: list[str] | None = None # Fields can have other Configs as types. message_maker: MessageMakerConfig = MessageMakerConfig() @@ -38,13 +37,12 @@ def build(self, tmp_dir): def update(self): if self.message_uris is None: self.message_uris = [ - join(self.root_uri, '{}.txt'.format(name)) - for name in self.names + join(self.root_uri, f'{name}.txt') for name in self.names ] class SamplePipeline2(Pipeline): - commands: List[str] = ['save_messages', 'print_messages'] + commands: list[str] = ['save_messages', 'print_messages'] split_commands = ['save_messages'] gpu_commands = [] diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/chip_classification/spacenet_rio.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/chip_classification/spacenet_rio.py index f9c2097d8..93086d1c4 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/chip_classification/spacenet_rio.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/chip_classification/spacenet_rio.py @@ -37,19 +37,19 @@ def get_config(runner, processed_uri (str): Directory for storing processed data. E.g. crops for testing. root_uri (str): Directory where all the output will be written. - external_model (bool, optional): If True, use an external model defined + external_model (bool): If True, use an external model defined by the ExternalModuleConfig. Defaults to False. - external_loss (bool, optional): If True, use an external loss defined + external_loss (bool): If True, use an external loss defined by the ExternalModuleConfig. Defaults to False. - augment (bool, optional): If True, use custom data augmentation + augment (bool): If True, use custom data augmentation transforms. Some basic data augmentation is done even if this is False. To completely disable, specify augmentors=[] is the dat config. Defaults to False. - nochip (bool, optional): If True, read directly from the TIFF during + nochip (bool): If True, read directly from the TIFF during training instead of from pre-generated chips. The analyze and chip commands should not be run, if this is set to True. Defaults to True. - test (bool, optional): If True, does the following simplifications: + test (bool): If True, does the following simplifications: (1) Uses only the first 1 scene (2) Uses only a 600x600 crop of the scenes (3) Trains for only 4 epochs. diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/object_detection/cowc_potsdam.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/object_detection/cowc_potsdam.py index c9b932f43..c04911203 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/object_detection/cowc_potsdam.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/object_detection/cowc_potsdam.py @@ -40,16 +40,16 @@ def get_config(runner, processed_uri (str): Directory for storing processed data. E.g. crops for testing. root_uri (str): Directory where all the output will be written. - nochip (bool, optional): If True, read directly from the TIFF during + nochip (bool): If True, read directly from the TIFF during training instead of from pre-generated chips. The analyze and chip commands should not be run, if this is set to True. Defaults to False. - multiband (bool, optional): If True, all 4 channels (R, G, B, & IR) + multiband (bool): If True, all 4 channels (R, G, B, & IR) available in the raster source will be used. If False, only IR, R, G (in that order) will be used. Defaults to False. - external_model (bool, optional): If True, use an external model defined + external_model (bool): If True, use an external model defined by the ExternalModuleConfig. Defaults to True. - test (bool, optional): If True, does the following simplifications: + test (bool): If True, does the following simplifications: (1) Uses only the first 2 scenes (2) Uses only a 2000x2000 crop of the scenes (3) Trains for only 2 epochs. diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/object_detection/xview.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/object_detection/xview.py index 41ace648e..4e0e59061 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/object_detection/xview.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/object_detection/xview.py @@ -32,11 +32,11 @@ def get_config(runner, processed_uri (str): Directory for storing processed data. E.g. crops for testing. root_uri (str): Directory where all the output will be written. - nochip (bool, optional): If True, read directly from the TIFF during + nochip (bool): If True, read directly from the TIFF during training instead of from pre-generated chips. The analyze and chip commands should not be run, if this is set to True. Defaults to False. - test (bool, optional): If True, does the following simplifications: + test (bool): If True, does the following simplifications: (1) Uses only the first 2 scenes. (2) Uses only a 2000x2000 crop of the scenes. (3) Trains for only 2 epochs. diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/isprs_potsdam.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/isprs_potsdam.py index 8c2917f8c..1232e5a7e 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/isprs_potsdam.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/isprs_potsdam.py @@ -1,4 +1,3 @@ -from typing import Optional from os.path import join, basename import albumentations as A @@ -40,7 +39,7 @@ def get_config(runner, raw_uri: str, root_uri: str, - processed_uri: Optional[str] = None, + processed_uri: str | None = None, multiband: bool = False, external_model: bool = True, augment: bool = False, @@ -57,22 +56,22 @@ def get_config(runner, root_uri (str): Directory where all the output will be written. processed_uri (str): Directory for storing processed data. E.g. crops for testing. Defaults to None. - multiband (bool, optional): If True, all 4 channels (R, G, B, & IR) + multiband (bool): If True, all 4 channels (R, G, B, & IR) available in the raster source will be used. If False, only IR, R, G (in that order) will be used. Defaults to False. - external_model (bool, optional): If True, use an external model defined + external_model (bool): If True, use an external model defined by the ExternalModuleConfig. Defaults to True. - augment (bool, optional): If True, use custom data augmentation + augment (bool): If True, use custom data augmentation transforms. Some basic data augmentation is done even if this is False. To completely disable, specify augmentors=[] is the dat config. Defaults to False. - nochip (bool, optional): If True, read directly from the TIFF during + nochip (bool): If True, read directly from the TIFF during training instead of from pre-generated chips. The analyze and chip commands should not be run, if this is set to True. Defaults to True. num_epochs (int): Number of epochs to train for. batch_sz (int): Batch size. - test (bool, optional): If True, does the following simplifications: + test (bool): If True, does the following simplifications: (1) Uses only the first 2 scenes (2) Uses only a 600x600 crop of the scenes (3) Trains for only 2 epochs and uses a batch size of 2. diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/isprs_potsdam_multi_source.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/isprs_potsdam_multi_source.py index db0054d65..b8092d047 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/isprs_potsdam_multi_source.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/isprs_potsdam_multi_source.py @@ -1,5 +1,4 @@ from functools import partial -from typing import Tuple, Union from rastervision.core.rv_pipeline import ( SceneConfig, DatasetConfig, SemanticSegmentationChipOptions, @@ -89,11 +88,11 @@ def get_config(runner, processed_uri (str): Directory for storing processed data. E.g. crops for testing. root_uri (str): Directory where all the output will be written. - nochip (bool, optional): If True, read directly from the TIFF during + nochip (bool): If True, read directly from the TIFF during training instead of from pre-generated chips. The analyze and chip commands should not be run, if this is set to True. Defaults to True. - test (bool, optional): If True, does the following simplifications: + test (bool): If True, does the following simplifications: (1) Uses only the first 2 scenes (2) Uses only a 600x600 crop of the scenes (3) Trains for only 2 epochs and uses a batch size of 2. @@ -237,8 +236,8 @@ def make_scene(raw_uri: UriPath, def make_multi_raster_source( - rgbir_raster_uri: Union[UriPath, str], - elevation_raster_uri: Union[UriPath, str]) -> MultiRasterSourceConfig: + rgbir_raster_uri: UriPath | str, + elevation_raster_uri: UriPath | str) -> MultiRasterSourceConfig: """ Create multi raster source by combining rgbir and elevation sources. """ rgbir_raster_uri = str(rgbir_raster_uri) elevation_raster_uri = str(elevation_raster_uri) @@ -257,7 +256,7 @@ def make_multi_raster_source( def make_crop(processed_uri: UriPath, raster_uri: UriPath, - label_uri: UriPath = None) -> Tuple[UriPath, UriPath]: + label_uri: UriPath = None) -> tuple[UriPath, UriPath]: crop_uri = processed_uri / TEST_CROP_DIR / raster_uri.name if label_uri is not None: label_crop_uri = processed_uri / TEST_CROP_DIR / label_uri.name @@ -275,8 +274,8 @@ def make_crop(processed_uri: UriPath, return crop_uri, label_crop_uri -def make_label_source(class_config: ClassConfig, label_uri: Union[UriPath, str] - ) -> Tuple[SemanticSegmentationLabelSourceConfig, +def make_label_source(class_config: ClassConfig, label_uri: UriPath | str + ) -> tuple[SemanticSegmentationLabelSourceConfig, SemanticSegmentationLabelStoreConfig]: label_uri = str(label_uri) # Using with_rgb_class_map because label TIFFs have classes encoded as diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/spacenet_vegas.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/spacenet_vegas.py index e7fde5b13..ff5d1889c 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/spacenet_vegas.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/semantic_segmentation/spacenet_vegas.py @@ -1,4 +1,3 @@ -from typing import Optional import re import random import os @@ -101,7 +100,7 @@ def get_class_id_to_filter(self): def build_scene(spacenet_cfg: SpacenetConfig, id: str, - channel_order: Optional[list] = None) -> SceneConfig: + channel_order: list | None = None) -> SceneConfig: image_uri = spacenet_cfg.get_raster_source_uri(id) label_uri = spacenet_cfg.get_geojson_uri(id) @@ -148,11 +147,11 @@ def get_config(runner, raw_uri (str): Directory where the raw data resides root_uri (str): Directory where all the output will be written. target (str): "buildings" | "roads". Defaults to "buildings". - nochip (bool, optional): If True, read directly from the TIFF during + nochip (bool): If True, read directly from the TIFF during training instead of from pre-generated chips. The analyze and chip commands should not be run, if this is set to True. Defaults to True. - test (bool, optional): If True, does the following simplifications: + test (bool): If True, does the following simplifications: (1) Uses only a small subset of training and validation scenes. (2) Trains for only 2 epochs. Defaults to False. diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/utils.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/utils.py index acda49abd..5866d6600 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/utils.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/examples/utils.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import os import csv from io import StringIO @@ -23,13 +23,13 @@ def get_scene_info(csv_uri: str) -> list: # pragma: no cover def save_image_crop( image_uri: str, image_crop_uri: str, - label_uri: Optional[str] = None, - label_crop_uri: Optional[str] = None, + label_uri: str | None = None, + label_crop_uri: str | None = None, size: int = 600, min_features: int = 10, vector_labels: bool = True, default_class_id: int = 0, - class_config: Optional['ClassConfig'] = None): # pragma: no cover + class_config: 'ClassConfig | None' = None): # pragma: no cover """Save a crop of an image to use for testing. If label_uri is set, the crop needs to cover >= min_features. diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/pytorch_learner_backend.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/pytorch_learner_backend.py index 92198975b..8c5520fef 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/pytorch_learner_backend.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/pytorch_learner_backend.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING from os.path import join, splitext import tempfile @@ -127,13 +127,13 @@ def train(self, source_bundle_uri=None): learner = self.learner_cfg.build(self.tmp_dir, training=True) learner.main() - def load_model(self, uri: Optional[str] = None): + def load_model(self, uri: str | None = None): self.learner = self._build_learner_from_bundle( bundle_uri=uri, training=False) def _build_learner_from_bundle(self, - bundle_uri: Optional[str] = None, - cfg: Optional['LearnerConfig'] = None, + bundle_uri: str | None = None, + cfg: 'LearnerConfig | None' = None, training: bool = False): if bundle_uri is None: bundle_uri = self.learner_cfg.get_model_bundle_uri() @@ -166,7 +166,7 @@ def chip_pytorch_dataset( dataset: 'Dataset', sample_writer: 'PyTorchLearnerSampleWriter', chip_options: 'ChipOptions', - split: Optional[str] = None, + split: str | None = None, dataloader_kw: dict = {}, ) -> None: from torch.utils.data import DataLoader @@ -204,7 +204,7 @@ def chip_pytorch_dataset( def predict_scene(self, scene: 'Scene', chip_sz: int, - stride: Optional[int] = None): + stride: int | None = None): raise NotImplementedError() def _make_chip_data_config(self, dataset: 'DatasetConfig', diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/pytorch_learner_backend_config.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/pytorch_learner_backend_config.py index 8d0a5e1e1..31fb5c912 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/pytorch_learner_backend_config.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/pytorch_learner_backend_config.py @@ -1,4 +1,3 @@ -from typing import Optional, List import logging from rastervision.pipeline.config import (register_config, Field) @@ -44,7 +43,7 @@ class PyTorchLearnerBackendConfig(BackendConfig): def get_bundle_filenames(self): return ['model-bundle.zip'] - def update(self, pipeline: Optional[RVPipelineConfig] = None): + def update(self, pipeline: RVPipelineConfig | None = None): super().update(pipeline=pipeline) if isinstance(self.data, ImageDataConfig): @@ -57,13 +56,13 @@ def update(self, pipeline: Optional[RVPipelineConfig] = None): if not self.data.img_channels: self.data.img_channels = self.get_img_channels(pipeline) - def get_learner_config(self, pipeline: Optional[RVPipelineConfig]): + def get_learner_config(self, pipeline: RVPipelineConfig | None): raise NotImplementedError() - def build(self, pipeline: Optional[RVPipelineConfig], tmp_dir: str): + def build(self, pipeline: RVPipelineConfig | None, tmp_dir: str): raise NotImplementedError() - def filter_commands(self, commands: List[str]) -> List[str]: + def filter_commands(self, commands: list[str]) -> list[str]: nochip = isinstance(self.data, GeoDataConfig) if nochip and 'chip' in commands: commands = [c for c in commands if c != 'chip'] diff --git a/rastervision_pytorch_backend/rastervision/pytorch_backend/utils.py b/rastervision_pytorch_backend/rastervision/pytorch_backend/utils.py index 5b59bd9ec..ac1e90285 100644 --- a/rastervision_pytorch_backend/rastervision/pytorch_backend/utils.py +++ b/rastervision_pytorch_backend/rastervision/pytorch_backend/utils.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Tuple +from typing import TYPE_CHECKING import numpy as np @@ -9,24 +9,24 @@ from rastervision.core.box import Box -def chip_collate_fn_ss(batch: List[Tuple[Tuple[np.ndarray, np.ndarray], 'Box']] - ) -> Tuple[Tuple[np.ndarray, np.ndarray], List['Box']]: +def chip_collate_fn_ss(batch: list[tuple[tuple[np.ndarray, np.ndarray], 'Box']] + ) -> tuple[tuple[np.ndarray, np.ndarray], list['Box']]: xs = np.stack([x for (x, _), _ in batch]) ys = [None if np.isnan(y).all() else y for (_, y), _ in batch] ws = [w for (_, _), w in batch] return (xs, ys), ws -def chip_collate_fn_cc(batch: List[Tuple[Tuple[np.ndarray, np.ndarray], 'Box']] - ) -> Tuple[Tuple[np.ndarray, np.ndarray], List['Box']]: +def chip_collate_fn_cc(batch: list[tuple[tuple[np.ndarray, np.ndarray], 'Box']] + ) -> tuple[tuple[np.ndarray, np.ndarray], list['Box']]: xs = np.stack([x for (x, _), _ in batch]) ys = [None if np.isnan(y).all() else y for (_, y), _ in batch] ws = [w for (_, _), w in batch] return (xs, ys), ws -def chip_collate_fn_od(batch: List[Tuple[Tuple['Tensor', 'BoxList'], 'Box']] - ) -> Tuple[Tuple[np.ndarray, 'BoxList'], List['Box']]: +def chip_collate_fn_od(batch: list[tuple[tuple['Tensor', 'BoxList'], 'Box']] + ) -> tuple[tuple[np.ndarray, 'BoxList'], list['Box']]: xs = np.stack([x.numpy() for (x, _), _ in batch]) # (..., c, h, w) --> (..., h, w, c) xs = xs.swapaxes(-3, -2).swapaxes(-2, -1) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/classification_learner_config.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/classification_learner_config.py index 2caf6ed34..6153cec0d 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/classification_learner_config.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/classification_learner_config.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Callable, Iterable, Optional, Union +from typing import TYPE_CHECKING, Iterable +from collections.abc import Callable from enum import Enum import logging @@ -58,18 +59,18 @@ class ClassificationGeoDataConfig(ClassificationDataConfig, GeoDataConfig): def build_scenes(self, scene_configs: Iterable['SceneConfig'], - tmp_dir: Optional[str] = None): + tmp_dir: str | None = None): for s in scene_configs: if s.label_source is not None: s.label_source.lazy = True return super().build_scenes(scene_configs, tmp_dir=tmp_dir) - def scene_to_dataset(self, - scene: Scene, - transform: Optional[A.BasicTransform] = None, - for_chipping: bool = False - ) -> Union[ClassificationSlidingWindowGeoDataset, - ClassificationRandomWindowGeoDataset]: + def scene_to_dataset( + self, + scene: Scene, + transform: A.BasicTransform | None = None, + for_chipping: bool = False + ) -> ClassificationSlidingWindowGeoDataset | ClassificationRandomWindowGeoDataset: if isinstance(self.sampling, dict): opts = self.sampling[scene.id] else: @@ -155,7 +156,7 @@ def build_default_model(self, num_classes: int, class ClassificationLearnerConfig(LearnerConfig): """Configure a :class:`.ClassificationLearner`.""" - model: Optional[ClassificationModelConfig] + model: ClassificationModelConfig | None def build(self, tmp_dir=None, diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/classification_dataset.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/classification_dataset.py index 5cd9b5f78..c5cc6cbc6 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/classification_dataset.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/classification_dataset.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Iterable, List, Optional, Union +from typing import TYPE_CHECKING, Iterable import logging from rastervision.pytorch_learner.dataset import ( @@ -20,14 +20,13 @@ class ClassificationImageDataset(ImageDataset): directories are located in the same parent directory. """ - def __init__(self, data_dir: str, class_names: Optional[Iterable[str]], - *args, **kwargs): + def __init__(self, data_dir: str, class_names: Iterable[str] | None, *args, + **kwargs): """Constructor. Args: - data_dir (str): Root directory containing class dirs. - class_names (Optional[Iterable[str]]): Class names. Should match - class dir names. + data_dir: Root directory containing class dirs. + class_names: Class names. Should match class dir names. *args: See :meth:`.ImageDataset.__init__`. **kwargs: See :meth:`.ImageDataset.__init__`. """ @@ -37,11 +36,11 @@ class dir names. def make_cc_geodataset(cls, - image_uri: Union[str, List[str]], - label_vector_uri: Optional[str] = None, - class_config: Optional['ClassConfig'] = None, - aoi_uri: Union[str, List[str]] = [], - label_vector_default_class_id: Optional[int] = None, + image_uri: str | list[str], + label_vector_uri: str | None = None, + class_config: 'ClassConfig | None' = None, + aoi_uri: str | list[str] = [], + label_vector_default_class_id: int | None = None, image_raster_source_kw: dict = {}, label_vector_source_kw: dict = {}, label_source_kw: dict = {}, @@ -52,34 +51,35 @@ def make_cc_geodataset(cls, recommended to use the default constructor. Args: - class_config (ClassConfig): The ClassConfig. - image_uri (Union[str, List[str]]): URI or list of URIs of GeoTIFFs to - use as the source of image data. - label_vector_uri (Optional[str], optional): URI of GeoJSON file to use - as the source of segmentation label data. Defaults to None. - class_config (Optional['ClassConfig']): The ClassConfig. Can be None if - not using any labels. - aoi_uri (Union[str, List[str]], optional): URI or list of URIs of - GeoJSONs that specify the area-of-interest. If provided, the - dataset will only access data from this area. Defaults to []. - label_vector_default_class_id (Optional[int], optional): If using - label_vector_uri and all polygons in that file belong to the same - class and they do not contain a `class_id` property, then use this - argument to map all of the polygons to the appropriate class ID. - See docs for ClassInferenceTransformer for more details. - Defaults to None. - image_raster_source_kw (dict, optional): Additional arguments to pass + class_config: The ``ClassConfig``. + image_uri: URI or list of URIs of GeoTIFFs to use as the source of + image data. + label_vector_uri: URI of GeoJSON file to use as the source of + segmentation label data. Defaults to ``None``. + class_config: The ``ClassConfig``. Can be ``None`` if not using any + labels. + aoi_uri: URI or list of URIs of GeoJSONs that specify the + area-of-interest. If provided, the dataset will only access data + from this area. Defaults to ``[]``. + label_vector_default_class_id: If using ``label_vector_uri`` and all + polygons in that file belong to the same class and they do not + contain a `class_id` property, then use this argument to map all of + the polygons to the appropriate class ID. See docs for + :class:`.ClassInferenceTransformer` for more details. + Defaults to ``None``. + image_raster_source_kw: Additional arguments to pass to the RasterioSource used for image data. See docs for - RasterioSource for more details. Defaults to {}. - label_vector_source_kw (dict, optional): Additional arguments to pass - to the GeoJSONVectorSourceConfig used for label data, if - label_vector_uri is set. See docs for GeoJSONVectorSourceConfig - for more details. Defaults to {}. - label_source_kw (dict, optional): Additional arguments to pass - to the ChipClassificationLabelSourceConfig used for label data, if - label_vector_uri is set. See docs for - ChipClassificationLabelSourceConfig for more details. - Defaults to {}. + RasterioSource for more details. Defaults to ``{}``. + label_vector_source_kw: Additional arguments to pass to the + :class:`.GeoJSONVectorSourceConfig` used for label data, if + ``label_vector_uri`` is set. See docs for + :class:`.GeoJSONVectorSourceConfig` for more details. + Defaults to ``{}``. + label_source_kw: Additional arguments to pass + to the :class:`.ChipClassificationLabelSourceConfig` used for + label data, if ``label_vector_uri`` is set. See docs for + :class:`.ChipClassificationLabelSourceConfig` for more details. + Defaults to ``{}``. **kwargs: All other keyword args are passed to the default constructor for this class. diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/dataset.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/dataset.py index 0548ed93c..1a1d6c5ea 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/dataset.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/dataset.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Literal, Optional, Tuple, Union +from typing import TYPE_CHECKING, Any, Literal, Self import logging import numpy as np @@ -26,30 +26,30 @@ class AlbumentationsDataset(Dataset): def __init__(self, orig_dataset: Any, - transform: Optional[A.BasicTransform] = None, + transform: A.BasicTransform | None = None, transform_type: TransformType = TransformType.noop, normalize=True, to_pytorch=True): """Constructor. Args: - orig_dataset (Any): An object with a __getitem__ and __len__. - transform (A.BasicTransform, optional): Albumentations - transform to apply to the windows. Defaults to None. - Each transform in Albumentations takes images of type uint8, and - sometimes other data types. The data type requirements can be - seen at https://albumentations.ai/docs/api_reference/augmentations/transforms/ # noqa + orig_dataset: An object with a __getitem__ and __len__. + transform: Albumentations transform to apply to the windows. + Defaults to ``None``. Each transform in Albumentations takes + images of type uint8, and sometimes other data types. The data + type requirements can be seen at + https://albumentations.ai/docs/api_reference/augmentations/transforms/ # noqa If there is a mismatch between the data type of imagery and the transform requirements, a RasterTransformer should be set on the RasterSource that converts to uint8, such as - MinMaxTransformer or StatsTransformer. - transform_type (TransformType): The type of transform so that its - inputs and outputs can be handled correctly. Defaults to - TransformType.noop. - normalize (bool, optional): If True, x is normalized to [0, 1] - based on its data type. Defaults to True. - to_pytorch (bool, optional): If True, x and y are converted to - pytorch tensors. Defaults to True. + :class:`.MinMaxTransformer` or :class:`.StatsTransformer`. + transform_type: The type of transform so that its inputs and + outputs can be handled correctly. + Defaults to ``TransformType.noop``. + normalize: If ``True``, the sampled chips are normalized to [0, 1] + based on their data type. Defaults to ``True``. + to_pytorch: If ``True``, the sampled chips and labels are converted + to pytorch tensors. Defaults to ``True``. """ self.orig_dataset = orig_dataset self.normalize = normalize @@ -63,7 +63,7 @@ def __init__(self, self.normalize = False self.to_pytorch = False - def __getitem__(self, key) -> Tuple[torch.Tensor, torch.Tensor]: + def __getitem__(self, key) -> tuple[torch.Tensor, torch.Tensor]: val = self.orig_dataset[key] try: @@ -107,26 +107,25 @@ class GeoDataset(AlbumentationsDataset): (i.e. a raster source and a label source). """ - def __init__( - self, - scene: Scene, - out_size: Optional[Union[PosInt, Tuple[PosInt, PosInt]]] = None, - within_aoi: bool = True, - transform: Optional[A.BasicTransform] = None, - transform_type: Optional[TransformType] = None, - normalize: bool = True, - to_pytorch: bool = True, - return_window: bool = False): + def __init__(self, + scene: Scene, + out_size: PosInt | tuple[PosInt, PosInt] | None = None, + within_aoi: bool = True, + transform: A.BasicTransform | None = None, + transform_type: TransformType | None = None, + normalize: bool = True, + to_pytorch: bool = True, + return_window: bool = False): """Constructor. Args: - scene (Scene): A Scene object. + scene: A Scene instance. out_size: Resize chips to this size before returning. within_aoi: If True and if the scene has an AOI, only sample windows that lie fully within the AOI. If False, windows only partially intersecting the AOI will also be allowed. Defaults to True. - transform (Optional[A.BasicTransform], optional): Albumentations + transform (A.BasicTransform | None): Albumentations transform to apply to the windows. Defaults to None. Each transform in Albumentations takes images of type uint8, and sometimes other data types. The data type requirements can be @@ -135,14 +134,13 @@ def __init__( transform requirements, a RasterTransformer should be set on the RasterSource that converts to uint8, such as MinMaxTransformer or StatsTransformer. - transform_type (Optional[TransformType], optional): Type of - transform. Defaults to None. - normalize (bool, optional): If True, x is normalized to [0, 1] - based on its data type. Defaults to True. - to_pytorch (bool, optional): If True, x and y are converted to - pytorch tensors. Defaults to True. - return_window (bool, optional): Make __getitem__ return the window - coordinates used to generate the image. Defaults to False. + transform_type: Type of transform. Defaults to ``None``. + normalize: If True, x is normalized to [0, 1] based on its data + type. Defaults to ``True``. + normalize: If ``True``, the sampled chips are normalized to [0, 1] + based on their data type. Defaults to ``True``. + to_pytorch: If ``True``, the sampled chips and labels are converted + to pytorch tensors. Defaults to ``True``. """ self.scene = scene self.within_aoi = within_aoi @@ -175,7 +173,7 @@ def __len__(self): raise NotImplementedError() @classmethod - def from_uris(cls, *args, **kwargs) -> 'GeoDataset': + def from_uris(cls, *args, **kwargs) -> Self: raise NotImplementedError() @@ -186,56 +184,53 @@ class SlidingWindowGeoDataset(GeoDataset): def __init__( self, scene: Scene, - size: Union[PosInt, Tuple[PosInt, PosInt]], - stride: Union[PosInt, Tuple[PosInt, PosInt]], - out_size: Optional[Union[PosInt, Tuple[PosInt, PosInt]]] = None, - padding: Optional[Union[NonNegInt, Tuple[NonNegInt, - NonNegInt]]] = None, + size: PosInt | tuple[PosInt, PosInt], + stride: PosInt | tuple[PosInt, PosInt], + out_size: PosInt | tuple[PosInt, PosInt] | None = None, + padding: NonNegInt | tuple[NonNegInt, NonNegInt] | None = None, pad_direction: Literal['both', 'start', 'end'] = 'end', within_aoi: bool = True, - transform: Optional[A.BasicTransform] = None, - transform_type: Optional[TransformType] = None, + transform: A.BasicTransform | None = None, + transform_type: TransformType | None = None, normalize: bool = True, to_pytorch: bool = True, return_window: bool = False): """Constructor. Args: - scene (Scene): A Scene object. - size (Union[PosInt, Tuple[PosInt, PosInt]]): Window size. - stride (Union[PosInt, Tuple[PosInt, PosInt]]): Step size between - windows. - out_size: Resize chips to this size before returning. Defaults to + scene A Scene object. + size: Window size. + stride: Step size between windows. + out_size Resize chips to this size before returning. Defaults to ``None``. - padding (Optional[Union[NonNegInt, Tuple[NonNegInt, NonNegInt]]]): - How many pixels the windows are allowed to overflow the sides - of the raster source. If None, padding is set to size // 2. - Defaults to None. - pad_direction (Literal['both', 'start', 'end']): If 'end', only pad - ymax and xmax (bottom and right). If 'start', only pad ymin and - xmin (top and left). If 'both', pad all sides. Has no effect if - padding is zero. Defaults to 'end'. - within_aoi: If True and if the scene has an AOI, only sample + padding: How many pixels the windows are allowed to overflow the + sides of the raster source. If ``None``, will be automatically + calculated such that the windows cover the entire extent. + Defaults to ``None``. + pad_direction: If ``'end'``, only pad ymax and xmax (bottom and + right). If ``'start'``, only pad ymin and xmin (top and left). + If ``'both'``, pad all sides. If ``'both'`` pad all sides. Has + no effect if padding is zero. Defaults to ``'end'``. + within_aoi If ``True`` and if the scene has an AOI, only sample windows that lie fully within the AOI. If False, windows only partially intersecting the AOI will also be allowed. - Defaults to True. - transform (Optional[A.BasicTransform], optional): Albumentations - transform to apply to the windows. Defaults to None. - Each transform in Albumentations takes images of type uint8, and - sometimes other data types. The data type requirements can be - seen at https://albumentations.ai/docs/api_reference/augmentations/transforms/ # noqa + Defaults to ``True``. + transform: Albumentations transform to apply to the windows. + Defaults to ``None``. Each transform in Albumentations takes + images of type uint8, and sometimes other data types. The data + type requirements can be seen at + https://albumentations.ai/docs/api_reference/augmentations/transforms/ # noqa If there is a mismatch between the data type of imagery and the transform requirements, a RasterTransformer should be set on the RasterSource that converts to uint8, such as - MinMaxTransformer or StatsTransformer. - transform_type (Optional[TransformType], optional): Type of - transform. Defaults to None. - normalize (bool, optional): If True, x is normalized to [0, 1] - based on its data type. Defaults to True. - to_pytorch (bool, optional): If True, x and y are converted to - pytorch tensors. Defaults to True. - return_window (bool, optional): Make __getitem__ return the window - coordinates used to generate the image. Defaults to False. + :class:`.MinMaxTransformer` or :class:`.StatsTransformer`. + transform_type: Type of transform. Defaults to ``None``. + normalize: If ``True``, the sampled chips are normalized to [0, 1] + based on their data type. Defaults to ``True``. + to_pytorch: If ``True``, the sampled chips and labels are converted + to pytorch tensors. Defaults to ``True``. + return_window: Make ``__getitem__`` return the window coordinates + used to generate the image. Defaults to ``False``. """ super().__init__( scene=scene, @@ -283,23 +278,23 @@ class RandomWindowGeoDataset(GeoDataset): """Read the scene by sampling random window sizes and locations. """ - def __init__(self, - scene: Scene, - out_size: Optional[Union[PosInt, Tuple[PosInt, PosInt]]], - size_lims: Optional[Tuple[PosInt, PosInt]] = None, - h_lims: Optional[Tuple[PosInt, PosInt]] = None, - w_lims: Optional[Tuple[PosInt, PosInt]] = None, - padding: Optional[Union[NonNegInt, Tuple[NonNegInt, - NonNegInt]]] = None, - max_windows: Optional[NonNegInt] = None, - max_sample_attempts: PosInt = 100, - efficient_aoi_sampling: bool = True, - within_aoi: bool = True, - transform: Optional[A.BasicTransform] = None, - transform_type: Optional[TransformType] = None, - normalize: bool = True, - to_pytorch: bool = True, - return_window: bool = False): + def __init__( + self, + scene: Scene, + out_size: PosInt | tuple[PosInt, PosInt] | None, + size_lims: tuple[PosInt, PosInt] | None = None, + h_lims: tuple[PosInt, PosInt] | None = None, + w_lims: tuple[PosInt, PosInt] | None = None, + padding: NonNegInt | tuple[NonNegInt, NonNegInt] | None = None, + max_windows: NonNegInt | None = None, + max_sample_attempts: PosInt = 100, + efficient_aoi_sampling: bool = True, + within_aoi: bool = True, + transform: A.BasicTransform | None = None, + transform_type: TransformType | None = None, + normalize: bool = True, + to_pytorch: bool = True, + return_window: bool = False): """Constructor. Will sample square windows if size_lims is specified. Otherwise, will @@ -307,27 +302,22 @@ def __init__(self, h_lims and w_lims. Args: - scene (Scene): A Scene object. - out_size (Optional[Union[PosInt, Tuple[PosInt, PosInt]]]]): Resize - windows to this size before returning. This is to aid in - collating the windows into a batch. If None, windows are - returned without being normalized or converted to pytorch, and - will be of different sizes in successive reads. - size_lims (Optional[Tuple[PosInt, PosInt]]): Interval from which to - sample window size. - h_lims (Optional[Tuple[PosInt, PosInt]]): Interval from which to - sample window height. - w_lims (Optional[Tuple[PosInt, PosInt]]): Interval from which to - sample window width. - padding (Optional[Union[NonNegInt, Tuple[NonNegInt, NonNegInt]]]): - How many pixels the windows are allowed to overflow the sides - of the raster source. If None, padding = size. - Defaults to None. - max_windows (Optional[NonNegInt]): Max allowed reads. Will raise - StopIteration on further read attempts. If None, will be set to - np.inf. Defaults to None. - transform (Optional[A.BasicTransform], optional): Albumentations - transform to apply to the windows. Defaults to None. + scene: A Scene object. + out_size: Resize windows to this size before returning. This is to + aid in collating the windows into a batch. If ``None``, windows + are returned without being normalized or converted to pytorch, + and will be of different sizes in successive reads. + size_lims: Interval from which to sample window size. + h_lims: Interval from which to sample window height. + w_lims: Interval from which to sample window width. + padding: How many pixels the windows are allowed to overflow the + sides of the raster source. If ``None``, ``padding = size``. + Defaults to ``None``. + max_windows: Max allowed reads. Will raise ``StopIteration`` on + further read attempts. If None, will be set to ``np.inf``. + Defaults to ``None``. + transform: Albumentations + transform to apply to the windows. Defaults to ``None``. Each transform in Albumentations takes images of type uint8, and sometimes other data types. The data type requirements can be seen at https://albumentations.ai/docs/api_reference/augmentations/transforms/ @@ -335,32 +325,30 @@ def __init__(self, transform requirements, a RasterTransformer should be set on the RasterSource that converts to uint8, such as MinMaxTransformer or StatsTransformer. - transform_type (Optional[TransformType], optional): Type of - transform. Defaults to None. - max_sample_attempts (NonNegInt, optional): Max attempts when trying - to find a window within the AOI of the scene. Only used if the - scene has aoi_polygons specified. StopIteratioin is raised if - this is exceeded. Defaults to 100. - efficient_aoi_sampling (bool, optional): If the scene has AOIs, + transform_type: Type of transform. Defaults to ``None``. + max_sample_attempts: Max attempts when trying to find a window + within the AOI of the scene. Only used if the scene has + ``aoi_polygons`` specified. ``StopIteratioin`` is raised if + this is exceeded. Defaults to ``100``. + efficient_aoi_sampling: If the scene has AOIs, sampling windows at random anywhere in the extent and then checking if they fall within any of the AOIs can be very inefficient. This flag enables the use of an alternate algorithm that only samples window locations inside the AOIs. - Defaults to True. - within_aoi: If True and if the scene has an AOI, only sample + Defaults to ``True``. + within_aoi If ``True`` and if the scene has an AOI, only sample windows that lie fully within the AOI. If False, windows only partially intersecting the AOI will also be allowed. - Defaults to True. - transform (Optional[A.BasicTransform], optional): Albumentations - transform to apply to the windows. Defaults to None. - transform_type (Optional[TransformType], optional): Type of - transform. Defaults to None. - normalize (bool, optional): If True, x is normalized to [0, 1] - based on its data type. Defaults to True. - to_pytorch (bool, optional): If True, x and y are converted to - pytorch tensors. Defaults to True. - return_window (bool, optional): Make __getitem__ return the window - coordinates used to generate the image. Defaults to False. + Defaults to ``True``. + transform: Albumentations transform to apply to the windows. + Defaults to ``None``. + transform_type: Type of transform. Defaults to ``None``. + normalize: If ``True``, the sampled chips are normalized to [0, 1] + based on their data type. Defaults to ``True``. + to_pytorch: If ``True``, the sampled chips and labels are converted + to pytorch tensors. Defaults to ``True``. + return_window: Make ``__getitem__`` return the window coordinates + used to generate the image. Defaults to ``False``. """ # noqa has_size_lims = size_lims is not None has_h_lims = h_lims is not None @@ -371,7 +359,7 @@ def __init__(self, raise ValueError('h_lims and w_lims must both be specified') if out_size is None: - log.warning(f'out_size is None, chips will not be normalized or ' + log.warning('out_size is None, chips will not be normalized or ' 'converted to PyTorch Tensors.') normalize, to_pytorch = False, False @@ -415,7 +403,7 @@ def __init__(self, self.has_aoi_polygons = len(aoi_polygons) > 0 if self.has_aoi_polygons: extent_polygon = self.extent.to_shapely() - aoi: 'Polygon' | 'MultiPolygon' = unary_union(aoi_polygons) + aoi: 'Polygon | MultiPolygon' = unary_union(aoi_polygons) # only sample from polygons that intersect w/ the extent self.aoi = aoi.intersection(extent_polygon) if efficient_aoi_sampling: @@ -437,7 +425,7 @@ def max_size(self): return self.size_lims[1], self.size_lims[1] return self.h_lims[1], self.w_lims[1] - def sample_window_size(self) -> Tuple[int, int]: + def sample_window_size(self) -> tuple[int, int]: """Randomly sample the window size.""" if self.size_lims is not None: sz_min, sz_max = self.size_lims @@ -451,7 +439,7 @@ def sample_window_size(self) -> Tuple[int, int]: w = torch.randint(low=wmin, high=wmax, size=(1, )).item() return h, w - def sample_window_loc(self, h: int, w: int) -> Tuple[int, int]: + def sample_window_loc(self, h: int, w: int) -> tuple[int, int]: """Randomly sample coordinates of the top left corner of the window.""" if not self.aoi_sampler: ymin, xmin, ymax, xmax = self.extent diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/object_detection_dataset.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/object_detection_dataset.py index fbbb24370..67b3b5e53 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/object_detection_dataset.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/object_detection_dataset.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Tuple, List, Dict, Union +from typing import TYPE_CHECKING from os.path import join from collections import defaultdict import logging @@ -27,15 +27,15 @@ def __init__(self, img_dir: str, annotation_uri: str): """Constructor. Args: - img_dir (str): Directory containing the images. Image filenames + img_dir: Directory containing the images. Image filenames must match the image IDs in the annotations file. - annotation_uri (str): URI to a JSON file containing annotations in + annotation_uri: URI to a JSON file containing annotations in the COCO format. """ self.annotation_uri = annotation_uri ann_json = file_to_json(annotation_uri) - self.img_ids: List[str] = [img['id'] for img in ann_json['images']] + self.img_ids: list[str] = [img['id'] for img in ann_json['images']] self.img_paths = { img['id']: join(img_dir, img['file_name']) for img in ann_json['images'] @@ -47,10 +47,10 @@ def __init__(self, img_dir: str, annotation_uri: str): img_ann['category_id'].append(ann['category_id']) def __getitem__(self, ind: int - ) -> Tuple[np.ndarray, Tuple[np.ndarray, np.ndarray, str]]: + ) -> tuple[np.ndarray, tuple[np.ndarray, np.ndarray, str]]: img_id = self.img_ids[ind] path = self.img_paths[img_id] - ann: Dict[str, list] = self.img_anns[img_id] + ann: dict[str, list] = self.img_anns[img_id] x = load_image(path) bboxes = np.array(ann['bboxes']) @@ -75,9 +75,9 @@ def __init__(self, img_dir: str, annotation_uri: str, *args, **kwargs): """Constructor. Args: - img_dir (str): Directory containing the images. Image filenames + img_dir: Directory containing the images. Image filenames must match the image IDs in the annotations file. - annotation_uri (str): URI to a JSON file containing annotations in + annotation_uri: URI to a JSON file containing annotations in the COCO format. *args: See :meth:`.ImageDataset.__init__`. **kwargs: See :meth:`.ImageDataset.__init__`. @@ -88,11 +88,11 @@ def __init__(self, img_dir: str, annotation_uri: str, *args, **kwargs): def make_od_geodataset(cls, - image_uri: Union[str, List[str]], - label_vector_uri: Optional[str] = None, - class_config: Optional['ClassConfig'] = None, - aoi_uri: Union[str, List[str]] = [], - label_vector_default_class_id: Optional[int] = None, + image_uri: str | list[str], + label_vector_uri: str | None = None, + class_config: 'ClassConfig | None' = None, + aoi_uri: str | list[str] = [], + label_vector_default_class_id: int | None = None, image_raster_source_kw: dict = {}, label_vector_source_kw: dict = {}, label_source_kw: dict = {}, @@ -103,33 +103,34 @@ def make_od_geodataset(cls, recommended to use the default constructor. Args: - image_uri (Union[str, List[str]]): URI or list of URIs of GeoTIFFs to - use as the source of image data. - label_vector_uri (Optional[str], optional): URI of GeoJSON file to use - as the source of segmentation label data. Defaults to None. - class_config (Optional['ClassConfig']): The ClassConfig. Can be None if - not using any labels. - aoi_uri (Union[str, List[str]], optional): URI or list of URIs of + image_uri: URI or list of URIs of GeoTIFFs to use as the source of + image data. + label_vector_uri: URI of GeoJSON file to use as the source of label. + Defaults to ``None``. + class_config: The ClassConfig. Must be non-None if creating a scene + without a ``LabelSource``. Defaults to ``None``. + aoi_uri: URI or list of URIs of GeoJSONs that specify the area-of-interest. If provided, the - dataset will only access data from this area. Defaults to []. - label_vector_default_class_id (Optional[int], optional): If using + dataset will only access data from this area. Defaults to ``[]``. + label_vector_default_class_id: If using label_vector_uri and all polygons in that file belong to the same class and they do not contain a `class_id` property, then use this argument to map all of the polygons to the appropriate class ID. See docs for ClassInferenceTransformer for more details. - Defaults to None. - image_raster_source_kw (dict, optional): Additional arguments to pass - to the RasterioSource used for image data. See docs for - RasterioSource for more details. Defaults to {}. - label_vector_source_kw (dict, optional): Additional arguments to pass - to the GeoJSONVectorSourceConfig used for label data, if - label_vector_uri is set. See docs for GeoJSONVectorSourceConfig - for more details. Defaults to {}. - label_source_kw (dict, optional): Additional arguments to pass - to the ObjectDetectionLabelSourceConfig used for label data, if + Defaults to ``None``. + image_raster_source_kw: Additional arguments to pass + to the :class:`.RasterioSource` used for image data. See docs for + :class:`.RasterioSource` for more details. Defaults to ``{}``. + label_vector_source_kw: Additional arguments to pass + to the :class:`.GeoJSONVectorSourceConfig` used for label data, if label_vector_uri is set. See docs for - ObjectDetectionLabelSourceConfig for more details. - Defaults to {}. + :class:`.GeoJSONVectorSourceConfig` for more details. + Defaults to ``{}``. + label_source_kw: Additional arguments to pass + to the :class:`.ObjectDetectionLabelSourceConfig` used for label data, if + label_vector_uri is set. See docs for + :class:`.ObjectDetectionLabelSourceConfig` for more details. + Defaults to ``{}``. **kwargs: All other keyword args are passed to the default constructor for this class. @@ -167,38 +168,36 @@ def __init__(self, *args, **kwargs): *args: See :meth:`.RandomWindowGeoDataset.__init__`. Keyword Args: - bbox_params (Optional[A.BboxParams], optional): Optional - bbox_params to use when resizing windows. Defaults to None. - ioa_thresh (float, optional): Minimum IoA of a bounding box with a - given window for it to be included in the labels for that - window. Defaults to 0.9. - clip (bool, optional): Clip bounding boxes to window limits when - retrieving labels for a window. Defaults to False. - neg_ratio (Optional[float], optional): Ratio of sampling - probabilities of negative windows (windows w/o bboxes) vs - positive windows (windows w/ at least 1 bbox). E.g. neg_ratio=2 - means 2/3 probability of sampling a negative window. - If None, the default sampling behavior of - RandomWindowGeoDataset is used, without taking bboxes into - account. Defaults to None. - neg_ioa_thresh (float, optional): A window will be considered - negative if its max IoA with any bounding box is less than this - threshold. Defaults to 0.2. + bbox_params: Optional ``bbox_params`` to use when resizing windows. + Defaults to ``None``. + ioa_thresh: Minimum IoA of a bounding box with a given window for + it to be included in the labels for that window. + Defaults to ``0.9``. + clip: Clip bounding boxes to window limits when retrieving labels + for a window. Defaults to ``False``. + neg_ratio: Ratio of sampling probabilities of negative windows + (windows w/o bboxes) vs positive windows (windows w/ at least 1 + bbox). E.g. ``neg_ratio=2`` means 2/3 probability of sampling a + negative window. If ``None``, the default sampling behavior of + ``RandomWindowGeoDataset`` is used, without taking bboxes into + account. Defaults to ``None``. + neg_ioa_thresh: A window will be considered negative if its max IoA + with any bounding box is less than this threshold. + Defaults to ``0.2``. **kwargs: See :meth:`.RandomWindowGeoDataset.__init__`. """ from rastervision.pytorch_learner import DEFAULT_BBOX_PARAMS - self.bbox_params: Optional[A.BboxParams] = kwargs.pop( + self.bbox_params: A.BboxParams | None = kwargs.pop( 'bbox_params', DEFAULT_BBOX_PARAMS) ioa_thresh: float = kwargs.pop('ioa_thresh', 0.9) clip: bool = kwargs.pop('clip', False) - neg_ratio: Optional[float] = kwargs.pop('neg_ratio', None) + neg_ratio: float | None = kwargs.pop('neg_ratio', None) neg_ioa_thresh: float = kwargs.pop('neg_ioa_thresh', 0.2) super().__init__( *args, **kwargs, transform_type=TransformType.object_detection) - label_source: Optional[ - 'ObjectDetectionLabelSource'] = self.scene.label_source + label_source: 'ObjectDetectionLabelSource | None' = self.scene.label_source if label_source is not None: label_source.ioa_thresh = ioa_thresh label_source.clip = clip diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/regression_dataset.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/regression_dataset.py index 6364c9b3a..8e2c48749 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/regression_dataset.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/regression_dataset.py @@ -1,4 +1,4 @@ -from typing import Iterable, Tuple +from typing import Iterable from os.path import join import logging import csv @@ -29,7 +29,7 @@ def __init__(self, data_dir: str, class_names: Iterable[str]): self.targets = [[float(row[i]) for i in class_inds] for row in rows] self.img_paths = [join(img_dir, row[0]) for row in rows] - def __getitem__(self, ind) -> Tuple[np.ndarray, np.ndarray]: + def __getitem__(self, ind) -> tuple[np.ndarray, np.ndarray]: img_path = self.img_paths[ind] targets = self.targets[ind] x = load_image(img_path) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/semantic_segmentation_dataset.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/semantic_segmentation_dataset.py index d41aa991d..192037838 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/semantic_segmentation_dataset.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/semantic_segmentation_dataset.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, List, Optional, Tuple, Union +from typing import TYPE_CHECKING from pathlib import Path import logging @@ -48,7 +48,7 @@ def validate_paths(self) -> None: f'Name mismatch between image file {img_path.stem} ' f'and label file {label_path.stem}.') - def __getitem__(self, ind: int) -> Tuple[np.ndarray, np.ndarray]: + def __getitem__(self, ind: int) -> tuple[np.ndarray, np.ndarray]: img_path = self.img_paths[ind] label_path = self.label_paths[ind] @@ -85,54 +85,53 @@ def __init__(self, img_dir: str, label_dir: str, *args, **kwargs): transform_type=TransformType.semantic_segmentation) -def make_ss_geodataset( - cls, - image_uri: Union[str, List[str]], - label_raster_uri: Optional[Union[str, List[str]]] = None, - label_vector_uri: Optional[str] = None, - class_config: Optional['ClassConfig'] = None, - aoi_uri: Union[str, List[str]] = [], - label_vector_default_class_id: Optional[int] = None, - image_raster_source_kw: dict = {}, - label_raster_source_kw: dict = {}, - label_vector_source_kw: dict = {}, - **kwargs): +def make_ss_geodataset(cls, + image_uri: str | list[str], + label_raster_uri: str | list[str] | None = None, + label_vector_uri: str | None = None, + class_config: 'ClassConfig | None' = None, + aoi_uri: str | list[str] = [], + label_vector_default_class_id: int | None = None, + image_raster_source_kw: dict = {}, + label_raster_source_kw: dict = {}, + label_vector_source_kw: dict = {}, + **kwargs): """Create an instance of this class from image and label URIs. This is a convenience method. For more fine-grained control, it is recommended to use the default constructor. Args: - image_uri (Union[str, List[str]]): URI or list of URIs of GeoTIFFs to - use as the source of image data. - label_raster_uri (Optional[Union[str, List[str]]], optional): URI or - list of URIs of GeoTIFFs to use as the source of segmentation label - data. If the labels are in the form of GeoJSONs, use - label_vector_uri instead. Defaults to None. - label_vector_uri (Optional[str], optional): URI of GeoJSON file to use - as the source of segmentation label data. If the labels are in the - form of GeoTIFFs, use label_raster_uri instead. Defaults to None. - class_config (Optional['ClassConfig']): The ClassConfig. Can be None if - not using any labels. - aoi_uri (Union[str, List[str]], optional): URI or list of URIs of - GeoJSONs that specify the area-of-interest. If provided, the - dataset will only access data from this area. Defaults to []. - label_vector_default_class_id (Optional[int], optional): If using - label_vector_uri and all polygons in that file belong to the same - class and they do not contain a `class_id` property, then use this - argument to map all of the polygons to the appropriate class ID. - See docs for ClassInferenceTransformer for more details. - Defaults to None. - image_raster_source_kw (dict, optional): Additional arguments to pass - to the RasterioSource used for image data. See docs for - RasterioSource for more details. Defaults to {}. - label_raster_source_kw (dict, optional): Additional arguments to pass - to the RasterioSource used for label data, if label_raster_uri is - used. See docs for RasterioSource for more details. Defaults to {}. - label_vector_source_kw (dict, optional): Additional arguments to pass - to the GeoJSONVectorSource used for label data, if label_vector_uri - is used. See docs for GeoJSONVectorSource for more details. - Defaults to {}. + image_uri: URI or list of URIs of GeoTIFFs to use as the source of + image data. + label_raster_uri: URI or list of URIs of GeoTIFFs to use as the source + of segmentation label data. If the labels are in the form of + GeoJSONs, use ``label_vector_uri`` instead. Defaults to ``None``. + label_vector_uri: URI of GeoJSON file to use as the source of + segmentation label data. If the labels are in the form of GeoTIFFs, + use ``label_raster_uri`` instead. Defaults to ``None``. + class_config: The ``ClassConfig``. Can be ``None`` if not using any + labels. + aoi_uri: URI or list of URIs of GeoJSONs that specify the + area-of-interest. If provided, the dataset will only access data + from this area. Defaults to ``[]``. + label_vector_default_class_id: If using ``label_vector_uri`` and all + polygons in that file belong to the same class and they do not + contain a ``class_id`` property, then use this argument to map all + of the polygons to the appropriate class ID. See docs for + :class:`.ClassInferenceTransformer` for more details. + Defaults to ``None``. + image_raster_source_kw: Additional arguments to pass to the + :class:`.RasterioSource` used for image data. See docs for + :class:`.RasterioSource` for more details. Defaults to ``{}``. + label_raster_source_kw: Additional arguments to pass + to the :class:`.RasterioSource` used for label data, if + ``label_raster_uri`` is used. See docs for :class:`.RasterioSource` + for more details. Defaults to ``{}``. + label_vector_source_kw: Additional arguments to pass to the + :class:`.GeoJSONVectorSource` used for label data, if + ``label_vector_uri`` is used. See docs for + :class:`.GeoJSONVectorSource` for more details. Defaults to ``{}``. **kwargs: All other keyword args are passed to the default constructor for this class. diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/transform.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/transform.py index a6c216ea1..c6f110d0e 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/transform.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/transform.py @@ -1,14 +1,17 @@ -from typing import Optional, Tuple, Any, Callable, Dict -from pydantic import PositiveInt as PosInt +from typing import Any +from collections.abc import Callable from enum import Enum +from pydantic import PositiveInt as PosInt import numpy as np import albumentations as A - import torch from rastervision.pytorch_learner.object_detection_utils import BoxList +TransformFunc = Callable[[tuple[np.ndarray, Any], A.BasicTransform], tuple[ + np.ndarray, Any]] + class TransformType(Enum): noop = 'noop' @@ -25,11 +28,11 @@ def apply_transform(transform: A.BasicTransform, **kwargs) -> dict: This is useful for when the images represent a time-series. Args: - transform (A.BasicTransform): An albumentations transform. + transform: An albumentations transform. **kwargs: Extra args for ``transform``. Returns: - dict: Output of ``transform``. If ndim == 4, the transformed image in + Output of ``transform``. If ``ndim == 4``, the transformed image in the dict is also 4-dimensional. """ img = kwargs['image'] @@ -55,9 +58,9 @@ def apply_transform(transform: A.BasicTransform, **kwargs) -> dict: return out -def classification_transformer(inp: Tuple[np.ndarray, Optional[int]], - transform=Optional[A.BasicTransform] - ) -> Tuple[np.ndarray, Optional[np.ndarray]]: +def classification_transformer(inp: tuple[np.ndarray, int | None], + transform=A.BasicTransform | None + ) -> tuple[np.ndarray, np.ndarray | None]: """Apply transform to image only.""" x, y = inp x = np.array(x) @@ -69,9 +72,9 @@ def classification_transformer(inp: Tuple[np.ndarray, Optional[int]], return x, y -def regression_transformer(inp: Tuple[np.ndarray, Optional[Any]], - transform=Optional[A.BasicTransform] - ) -> Tuple[np.ndarray, Optional[np.ndarray]]: +def regression_transformer(inp: tuple[np.ndarray, Any | None], + transform=A.BasicTransform | None + ) -> tuple[np.ndarray, np.ndarray | None]: """Apply transform to image only.""" x, y = inp x = np.array(x) @@ -84,7 +87,7 @@ def regression_transformer(inp: Tuple[np.ndarray, Optional[Any]], def yxyx_to_albu(yxyx: np.ndarray, - img_size: Tuple[PosInt, PosInt]) -> np.ndarray: + img_size: tuple[PosInt, PosInt]) -> np.ndarray: """Unnormalized [ymin, xmin, ymax, xmax] to Albumentations format i.e. normalized [ymin, xmin, ymax, xmax]. """ @@ -103,7 +106,7 @@ def yxyx_to_albu(yxyx: np.ndarray, def xywh_to_albu(xywh: np.ndarray, - img_size: Tuple[PosInt, PosInt]) -> np.ndarray: + img_size: tuple[PosInt, PosInt]) -> np.ndarray: """Unnormalized [xmin, ymin, w, h] to Albumentations format i.e. normalized [ymin, xmin, ymax, xmax]. """ @@ -123,7 +126,7 @@ def xywh_to_albu(xywh: np.ndarray, def albu_to_yxyx(xyxy: np.ndarray, - img_size: Tuple[PosInt, PosInt]) -> np.ndarray: + img_size: tuple[PosInt, PosInt]) -> np.ndarray: """Albumentations format (i.e. normalized [ymin, xmin, ymax, xmax]) to unnormalized [ymin, xmin, ymax, xmax]. """ @@ -141,9 +144,9 @@ def albu_to_yxyx(xyxy: np.ndarray, def object_detection_transformer( - inp: Tuple[np.ndarray, Optional[Tuple[np.ndarray, np.ndarray, str]]], - transform: Optional[A.BasicTransform] = None -) -> Tuple[torch.Tensor, Optional[BoxList]]: + inp: tuple[np.ndarray, tuple[np.ndarray, np.ndarray, str] | None], + transform: A.BasicTransform | None = None +) -> tuple[torch.Tensor, BoxList | None]: """Apply transform to image, bounding boxes, and labels. Also perform normalization and conversion to pytorch tensors. @@ -151,17 +154,15 @@ def object_detection_transformer( 'albumentations' (i.e. normalized [ymin, xmin, ymax, xmax]). Args: - inp (Tuple[np.ndarray, Optional[Tuple[np.ndarray, np.ndarray, str]]]): - Tuple of the form: (image, (boxes, class_ids, box_format)). - box_format must be 'yxyx' or 'xywh'. - transform (Optional[A.BasicTransform], optional): A transform. - Defaults to None. + inp: Tuple of the form: ``(image, (boxes, class_ids, box_format))``. + box_format must be ``'yxyx'`` or ``'xywh'``. + transform: A transform. Defaults to ``None``. Raises: - NotImplementedError: If box_format is not 'yxyx' or 'xywh'. + NotImplementedError: If box_format is not ``'yxyx'`` or ``'xywh'``. Returns: - Tuple[torch.Tensor, BoxList]: Transformed image and boxes. + Transformed image and boxes. """ x, y = inp img_size = x.shape[:2] @@ -214,9 +215,9 @@ def object_detection_transformer( def semantic_segmentation_transformer( - inp: Tuple[np.ndarray, Optional[np.ndarray]], - transform=Optional[A.BasicTransform] -) -> Tuple[np.ndarray, Optional[np.ndarray]]: + inp: tuple[np.ndarray, np.ndarray | None], + transform=A.BasicTransform | None +) -> tuple[np.ndarray, np.ndarray | None]: """Apply transform to image and mask.""" x, y = inp x = np.array(x) @@ -232,7 +233,7 @@ def semantic_segmentation_transformer( return x, y -TF_TYPE_TO_TF_FUNC: Dict[TransformType, Callable] = { +TF_TYPE_TO_TF_FUNC: dict[TransformType, TransformFunc] = { TransformType.noop: lambda x, tf: x, TransformType.classification: classification_transformer, TransformType.regression: regression_transformer, diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/utils/utils.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/utils/utils.py index d29cada31..521b2fd5b 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/utils/utils.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/utils/utils.py @@ -1,4 +1,4 @@ -from typing import Dict, Iterable, List, Optional, Tuple +from typing import Iterable from os import PathLike from os.path import join, splitext from pathlib import Path @@ -25,7 +25,7 @@ class GeoDatasetError(DatasetError): def discover_images(dir: PathLike, - extensions: Iterable[str] = IMG_EXTENSIONS) -> List[Path]: + extensions: Iterable[str] = IMG_EXTENSIONS) -> list[Path]: """Find all images with the given ``extensions`` in ``dir``.""" dir = Path(dir) img_paths = chain.from_iterable( @@ -52,9 +52,8 @@ def load_image(path: PathLike) -> np.ndarray: return img -def make_image_folder_dataset(data_dir: str, - classes: Optional[Iterable[str]] = None - ) -> DatasetFolder: +def make_image_folder_dataset( + data_dir: str, classes: Iterable[str] | None = None) -> DatasetFolder: """Initializes and returns an ImageFolder. If classes is specified, ImageFolder's default class-to-index mapping @@ -76,7 +75,7 @@ def make_image_folder_dataset(data_dir: str, class ImageFolder(DatasetFolder): def find_classes(self, - directory: str) -> Tuple[List[str], Dict[str, int]]: + directory: str) -> tuple[list[str], dict[str, int]]: """Override to force mapping from class name to class index.""" return classes_present, class_to_id diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/classification_visualizer.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/classification_visualizer.py index 1700aebbd..ebd2f9127 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/classification_visualizer.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/classification_visualizer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Sequence +from typing import TYPE_CHECKING, Sequence from textwrap import wrap import torch @@ -17,8 +17,8 @@ class ClassificationVisualizer(Visualizer): def plot_xyz(self, axs: Sequence['Axes'], x: torch.Tensor, - y: Optional[int] = None, - z: Optional[int] = None, + y: int | None = None, + z: int | None = None, plot_title: bool = True) -> None: channel_groups = self.get_channel_display_groups(x.shape[1]) @@ -61,7 +61,7 @@ def plot_pred(self, ax: 'Axes', class_names: Sequence[str], z: torch.Tensor, - y: Optional[torch.Tensor] = None): + y: torch.Tensor | None = None): """Plot predictions. Plots predicted class probabilities as a horizontal bar plot. If ground diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/object_detection_visualizer.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/object_detection_visualizer.py index eea6cb63b..ec48fea7e 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/object_detection_visualizer.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/object_detection_visualizer.py @@ -1,4 +1,4 @@ -from typing import (Sequence, Optional) +from typing import Sequence import torch @@ -18,8 +18,8 @@ def get_collate_fn(self): def plot_xyz(self, axs: Sequence, x: torch.Tensor, - y: Optional[BoxList] = None, - z: Optional[BoxList] = None, + y: BoxList | None = None, + z: BoxList | None = None, plot_title: bool = True) -> None: channel_groups = self.get_channel_display_groups(x.shape[1]) imgs = channel_groups_to_imgs(x, channel_groups) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/regression_visualizer.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/regression_visualizer.py index f2ec7282b..b92e64f65 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/regression_visualizer.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/regression_visualizer.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Sequence +from typing import TYPE_CHECKING, Sequence from textwrap import wrap import torch @@ -20,7 +20,7 @@ def plot_xyz(self, axs: Sequence, x: torch.Tensor, y: int, - z: Optional[int] = None, + z: int | None = None, plot_title: bool = True) -> None: channel_groups = self.get_channel_display_groups(x.shape[1]) @@ -59,7 +59,7 @@ def plot_pred(self, ax: 'Axes', class_names: Sequence[str], z: torch.Tensor, - y: Optional[torch.Tensor] = None): + y: torch.Tensor | None = None): """Plot targets and predictions as a grouped horizontal bar plot.""" # display targets and predictions as a grouped horizontal bar plot bar_thickness = 0.35 if y is not None else 0.70 diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/semantic_segmentation_visualizer.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/semantic_segmentation_visualizer.py index 3274d46b9..1ad64f5dc 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/semantic_segmentation_visualizer.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/semantic_segmentation_visualizer.py @@ -1,11 +1,11 @@ -from typing import TYPE_CHECKING, Optional, Sequence, Union +from typing import TYPE_CHECKING, Sequence import torch import numpy as np import matplotlib.colors as mcolors import matplotlib.patches as mpatches -from rastervision.pytorch_learner.dataset.visualizer import Visualizer # NOQA +from rastervision.pytorch_learner.dataset.visualizer import Visualizer from rastervision.pytorch_learner.utils import ( color_to_triple, plot_channel_groups, channel_groups_to_imgs) @@ -20,8 +20,8 @@ class SemanticSegmentationVisualizer(Visualizer): def plot_xyz(self, axs: Sequence, x: torch.Tensor, - y: Optional[Union[torch.Tensor, np.ndarray]] = None, - z: Optional[torch.Tensor] = None, + y: torch.Tensor | np.ndarray | None = None, + z: torch.Tensor | None = None, plot_title: bool = True) -> None: channel_groups = self.get_channel_display_groups(x.shape[1]) @@ -68,7 +68,7 @@ def plot_xyz(self, loc='center left', bbox_to_anchor=(1., 0.5)) - def plot_gt(self, ax: 'Axes', y: Union[torch.Tensor, np.ndarray], + def plot_gt(self, ax: 'Axes', y: torch.Tensor | np.ndarray, num_classes: int, cmap: 'Colormap', **kwargs): ax.imshow( y, @@ -80,7 +80,7 @@ def plot_gt(self, ax: 'Axes', y: Union[torch.Tensor, np.ndarray], ax.set_xticks([]) ax.set_yticks([]) - def plot_pred(self, ax: 'Axes', z: Union[torch.Tensor, np.ndarray], + def plot_pred(self, ax: 'Axes', z: torch.Tensor | np.ndarray, num_classes: int, cmap: 'Colormap', **kwargs): if z.ndim == 3: z = z.argmax(dim=0) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/visualizer.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/visualizer.py index 423a6d68a..7b19ff3f9 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/visualizer.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/dataset/visualizer/visualizer.py @@ -1,6 +1,6 @@ -from typing import (TYPE_CHECKING, Sequence, Optional, List, Dict, Union, - Tuple, Any) +from typing import TYPE_CHECKING, Any, Sequence from abc import ABC, abstractmethod +from collections.abc import Callable import numpy as np import torch @@ -32,11 +32,11 @@ class Visualizer(ABC): scale: float = 3. def __init__(self, - class_names: List[str], - class_colors: Optional[List[Union[str, RGBTuple]]] = None, - transform: Optional[Dict] = None, - channel_display_groups: Optional[Union[Dict[ - str, ChannelInds], Sequence[ChannelInds]]] = None): + class_names: list[str], + class_colors: list[str | RGBTuple] | None = None, + transform: dict | None = None, + channel_display_groups: dict[str, ChannelInds] + | Sequence[ChannelInds] | None = None): """Constructor. Args: @@ -79,8 +79,8 @@ def class_colors(self): def plot_xyz(self, axs, x: Tensor, - y: Optional[Sequence] = None, - z: Optional[Sequence] = None, + y: Sequence | None = None, + z: Sequence | None = None, plot_title: bool = True): """Plot image, ground truth labels, and predicted labels. @@ -93,10 +93,10 @@ def plot_xyz(self, def plot_batch(self, x: Tensor, - y: Optional[Sequence] = None, - output_path: Optional[str] = None, - z: Optional[Sequence] = None, - batch_limit: Optional[int] = None, + y: Sequence | None = None, + output_path: str | None = None, + z: Sequence | None = None, + batch_limit: int | None = None, show: bool = False): """Plot a whole batch in a grid using plot_xyz. @@ -161,10 +161,10 @@ def _plot_batch( self, fig: 'Figure', axs: Sequence, - plot_xyz_args: List[dict], + plot_xyz_args: list[dict], x: Tensor, - y: Optional[Sequence] = None, - z: Optional[Sequence] = None, + y: Sequence | None = None, + z: Sequence | None = None, ): # (N, c, h, w) --> (N, h, w, c) x = x.permute(0, 2, 3, 1) @@ -184,7 +184,7 @@ def _plot_batch( def get_channel_display_groups( self, nb_img_channels: int - ) -> Union[Dict[str, ChannelInds], Sequence[ChannelInds]]: + ) -> dict[str, ChannelInds] | Sequence[ChannelInds]: # The default channel_display_groups object depends on the number of # channels in the image. This number is not known when the Visualizer # is constructed which is why it needs to be created later. @@ -192,7 +192,7 @@ def get_channel_display_groups( return self._channel_display_groups return get_default_channel_display_groups(nb_img_channels) - def get_collate_fn(self) -> Optional[callable]: + def get_collate_fn(self) -> Callable | None: """Returns a custom collate_fn to use in DataLoader. None is returned if default collate_fn should be used. @@ -202,7 +202,7 @@ def get_collate_fn(self) -> Optional[callable]: return None def get_batch(self, dataset: 'Dataset', batch_sz: int = 4, - **kwargs) -> Tuple[Tensor, Any]: + **kwargs) -> tuple[Tensor, Any]: """Generate a batch from a dataset. This is a convenience method for generating a batch of data to plot. @@ -213,7 +213,7 @@ def get_batch(self, dataset: 'Dataset', batch_sz: int = 4, **kwargs: Extra args for :class:`~torch.utils.data.DataLoader`. Returns: - Tuple[Tensor, Any]: (x, y) tuple where x is images and y is labels. + tuple[Tensor, Any]: (x, y) tuple where x is images and y is labels. """ collate_fn = self.get_collate_fn() dl = DataLoader(dataset, batch_sz, collate_fn=collate_fn, **kwargs) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/learner.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/learner.py index cb1a4b205..f9762f83b 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/learner.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/learner.py @@ -1,6 +1,6 @@ -from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterator, List, - Literal, Optional, Tuple, Union, Type) +from typing import (TYPE_CHECKING, Any, Iterator, Literal, Self) from abc import ABC, abstractmethod +from collections.abc import Callable from os.path import join, isfile, basename, isdir import warnings from time import perf_counter @@ -40,7 +40,6 @@ from rastervision.pytorch_learner.dataset.visualizer import Visualizer if TYPE_CHECKING: - from typing import Self from torch.optim import Optimizer from torch.optim.lr_scheduler import _LRScheduler from torch.utils.data import Dataset, Sampler @@ -56,7 +55,7 @@ log = logging.getLogger(__name__) -MetricDict = Dict[str, float] +MetricDict = dict[str, float] class Learner(ABC): @@ -90,65 +89,61 @@ class Learner(ABC): def __init__(self, cfg: 'LearnerConfig', - output_dir: Optional[str] = None, - train_ds: Optional['Dataset'] = None, - valid_ds: Optional['Dataset'] = None, - test_ds: Optional['Dataset'] = None, - model: Optional[nn.Module] = None, - loss: Optional[Callable] = None, - optimizer: Optional['Optimizer'] = None, - epoch_scheduler: Optional['_LRScheduler'] = None, - step_scheduler: Optional['_LRScheduler'] = None, - tmp_dir: Optional[str] = None, - model_weights_path: Optional[str] = None, - model_def_path: Optional[str] = None, - loss_def_path: Optional[str] = None, + output_dir: str | None = None, + train_ds: 'Dataset | None' = None, + valid_ds: 'Dataset | None' = None, + test_ds: 'Dataset | None' = None, + model: nn.Module | None = None, + loss: Callable[..., Tensor] | None = None, + optimizer: 'Optimizer | None' = None, + epoch_scheduler: '_LRScheduler | None' = None, + step_scheduler: '_LRScheduler | None' = None, + tmp_dir: str | None = None, + model_weights_path: str | None = None, + model_def_path: str | None = None, + loss_def_path: str | None = None, training: bool = True): """Constructor. Args: - cfg (LearnerConfig): LearnerConfig. - train_ds (Optional[Dataset], optional): The dataset to use for - training. If None, will be generated from cfg.data. - Defaults to None. - valid_ds (Optional[Dataset], optional): The dataset to use for - validation. If None, will be generated from cfg.data. - Defaults to None. - test_ds (Optional[Dataset], optional): The dataset to use for - testing. If None, will be generated from cfg.data. - Defaults to None. - model (Optional[nn.Module], optional): The model. If None, - will be generated from cfg.model. Defaults to None. - loss (Optional[Callable], optional): The loss function. - If None, will be generated from cfg.solver. - Defaults to None. - optimizer (Optional[Optimizer], optional): The optimizer. - If None, will be generated from cfg.solver. - Defaults to None. - epoch_scheduler (Optional[_LRScheduler], optional): The scheduler - that updates after each epoch. If None, will be generated from - cfg.solver. Defaults to None. - step_scheduler (Optional[_LRScheduler], optional): The scheduler - that updates after each optimizer-step. If None, will be - generated from cfg.solver. Defaults to None. - tmp_dir (Optional[str], optional): A temporary directory to use for - downloads etc. If None, will be auto-generated. - Defaults to None. - model_weights_path (Optional[str], optional): URI of model weights - to initialize the model with. Defaults to None. - model_def_path (Optional[str], optional): A local path to a - directory with a hubconf.py. If provided, the model definition - is imported from here. This is used when loading an external - model from a model-bundle. Defaults to None. - loss_def_path (Optional[str], optional): A local path to a - directory with a hubconf.py. If provided, the loss function - definition is imported from here. This is used when loading an - external loss function from a model-bundle. Defaults to None. - training (bool, optional): If False, the training apparatus (loss, + cfg: LearnerConfig. + train_ds: The dataset to use for training. If ``None``, will be + generated from ``cfg.data``. Defaults to ``None``. + valid_ds: The dataset to use for + validation. If ``None``, will be generated from ``cfg.data``. + Defaults to ``None``. + test_ds: The dataset to use for + testing. If ``None``, will be generated from ``cfg.data``. + Defaults to ``None``. + model: The model. If ``None``, + will be generated from cfg.model. Defaults to ``None``. + loss: The loss function. If ``None``, will be generated from + ``cfg.solver``. Defaults to ``None``. + optimizer: The optimizer. If ``None``, will be generated from + ``cfg.solver``. Defaults to ``None``. + epoch_scheduler: The scheduler that updates after each epoch. + If ``None``, will be generated from ``cfg.solver``. + Defaults to ``None``. + step_scheduler: The scheduler that updates after each + optimizer-step. If ``None``, will be generated from + ``cfg.solver``. Defaults to ``None``. + tmp_dir: A temporary directory to use for downloads etc. If + ``None``, will be auto-generated. Defaults to ``None``. + model_weights_path: URI of model weights to initialize the model + with. Defaults to ``None``. + model_def_path: A local path to a directory with a ``hubconf.py`` + file. If provided, the model definition is imported from here. + This is used when loading an external model from a + model-bundle. Defaults to ``None``. + loss_def_path: A local path to a directory with a ``hubconf.py`` + file. If provided, the loss function definition is imported + from here. This is used when loading an external loss function + from a model-bundle. Defaults to ``None``. + training: If ``False``, the training apparatus (loss, optimizer, scheduler, logging, etc.) will not be set up and the - model will be put into eval mode. If True, the training + model will be put into eval mode. If ``True``, the training apparatus will be set up and the model will be put into - training mode. Defaults to True. + training mode. Defaults to ``True``. """ self.cfg = cfg self.training = training @@ -264,13 +259,13 @@ def __init__(self, cfg.data.plot_options.channel_display_groups) @classmethod - def from_model_bundle(cls: Type, + def from_model_bundle(cls: type, model_bundle_uri: str, - tmp_dir: Optional[str] = None, - cfg: Optional['LearnerConfig'] = None, + tmp_dir: str | None = None, + cfg: 'LearnerConfig | None' = None, training: bool = False, - use_onnx_model: Optional[bool] = None, - **kwargs) -> 'Self': + use_onnx_model: bool | None = None, + **kwargs) -> Self: """Create a Learner from a model bundle. .. note:: @@ -279,19 +274,19 @@ def from_model_bundle(cls: Type, ``bundle/model-bundle.zip``. Args: - model_bundle_uri (str): URI of the model bundle. - tmp_dir (Optional[str], optional): Optional temporary directory. + model_bundle_uri: URI of the model bundle. + tmp_dir: Optional temporary directory. Will be used for unzipping bundle and also passed to the default constructor. If None, will be auto-generated. Defaults to None. - cfg (Optional[LearnerConfig], optional): If None, will be read from + cfg: If None, will be read from the bundle. Defaults to None. - training (bool, optional): If False, the training apparatus (loss, + training: If False, the training apparatus (loss, optimizer, scheduler, logging, etc.) will not be set up and the model will be put into eval mode. If True, the training apparatus will be set up and the model will be put into training mode. Defaults to True. - use_onnx_model (Optional[bool]): If True and training=False and a + use_onnx_model: If True and training=False and a model.onnx file is available in the bundle, use that for inference rather than the PyTorch weights. Defaults to the boolean environment variable RASTERVISION_USE_ONNX if set, @@ -426,7 +421,7 @@ def _main(self): ########################### # Training and validation ########################### - def train(self, epochs: Optional[int] = None): + def train(self, epochs: int | None = None): """Run training loop, resuming training if appropriate""" start_epoch, end_epoch = self.get_start_and_end_epochs(epochs) @@ -513,8 +508,8 @@ def _run_train_distributed(self, rank: int, world_size: int, def train_epoch( self, optimizer: 'Optimizer', - dataloader: Optional[DataLoader] = None, - step_scheduler: Optional['_LRScheduler'] = None) -> MetricDict: + dataloader: DataLoader | None = None, + step_scheduler: '_LRScheduler | None' = None) -> MetricDict: """Train for a single epoch.""" self.model.train() if dataloader is None: @@ -565,8 +560,8 @@ def on_train_start(self): self.log_data_stats() self.plot_dataloaders(self.cfg.data.preview_batch_limit) - def train_end(self, outputs: List[Dict[str, Union[float, Tensor]]] - ) -> MetricDict: + def train_end(self, + outputs: list[dict[str, float | Tensor]]) -> MetricDict: """Aggregate the output of train_step at the end of the epoch. Args: @@ -661,8 +656,8 @@ def validate_step(self, batch: Any, batch_ind: int) -> MetricDict: dict with metric names mapped to metric values """ - def validate_end(self, outputs: List[Dict[str, Union[float, Tensor]]] - ) -> MetricDict: + def validate_end(self, + outputs: list[dict[str, float | Tensor]]) -> MetricDict: """Aggregate the output of validate_step at the end of the epoch. Args: @@ -736,27 +731,27 @@ def predict_dataset(self, dataloader_kw: dict = {}, progress_bar: bool = True, progress_bar_kw: dict = {} - ) -> Union[Iterator[Any], Iterator[Tuple[Any, ...]]]: + ) -> Iterator[Any] | Iterator[tuple[Any, ...]]: """Returns an iterator over predictions on the given dataset. Args: dataset (Dataset): The dataset to make predictions on. - return_format (Literal['xyz', 'yz', 'z'], optional): Format of the + return_format (Literal['xyz', 'yz', 'z']): Format of the return elements of the returned iterator. Must be one of: 'xyz', 'yz', and 'z'. If 'xyz', elements are 3-tuples of x, y, and z. If 'yz', elements are 2-tuples of y and z. If 'z', elements are (non-tuple) values of z. Where x = input image, y = ground truth, and z = prediction. Defaults to 'z'. - raw_out (bool, optional): If true, return raw predicted scores. + raw_out (bool): If true, return raw predicted scores. Defaults to True. - numpy_out (bool, optional): If True, convert predictions to numpy + numpy_out (bool): If True, convert predictions to numpy arrays before returning. Defaults to False. predict_kw (dict): Dict with keywords passed to Learner.predict(). Useful if a Learner subclass implements a custom predict() method. dataloader_kw (dict): Dict with keywords passed to the DataLoader constructor. - progress_bar (bool, optional): If True, display a progress bar. + progress_bar (bool): If True, display a progress bar. Since this function returns an iterator, the progress bar won't be visible until the iterator is consumed. Defaults to True. progress_bar_kw (dict): Dict with keywords passed to tqdm. @@ -810,28 +805,27 @@ def predict_dataset(self, return preds - def predict_dataloader( - self, - dl: DataLoader, - batched_output: bool = True, - return_format: Literal['xyz', 'yz', 'z'] = 'z', - raw_out: bool = True, - predict_kw: dict = {} - ) -> Union[Iterator[Any], Iterator[Tuple[Any, ...]]]: + def predict_dataloader(self, + dl: DataLoader, + batched_output: bool = True, + return_format: Literal['xyz', 'yz', 'z'] = 'z', + raw_out: bool = True, + predict_kw: dict = {} + ) -> Iterator[Any] | Iterator[tuple[Any, ...]]: """Returns an iterator over predictions on the given dataloader. Args: dl (DataLoader): The dataloader to make predictions on. - batched_output (bool, optional): If True, return batches of + batched_output (bool): If True, return batches of x, y, z as defined by the dataloader. If False, unroll the batches into individual items. Defaults to True. - return_format (Literal['xyz', 'yz', 'z'], optional): Format of the + return_format (Literal['xyz', 'yz', 'z']): Format of the return elements of the returned iterator. Must be one of: 'xyz', 'yz', and 'z'. If 'xyz', elements are 3-tuples of x, y, and z. If 'yz', elements are 2-tuples of y and z. If 'z', elements are (non-tuple) values of z. Where x = input image, y = ground truth, and z = prediction. Defaults to 'z'. - raw_out (bool, optional): If true, return raw predicted scores. + raw_out (bool): If true, return raw predicted scores. Defaults to True. predict_kw (dict): Dict with keywords passed to Learner.predict(). Useful if a Learner subclass implements a custom predict() @@ -841,10 +835,9 @@ def predict_dataloader( ValueError: If return_format is not one of the allowed values. Returns: - Union[Iterator[Any], Iterator[Tuple[Any, ...]]]: If return_format - is 'z', the returned value is an iterator of whatever type the - predictions are. Otherwise, the returned value is an iterator - of tuples. + If ``return_format`` is ``'z'``, the returned value is an iterator + of whatever type the predictions are. Otherwise, the returned value + is an iterator of tuples. """ if return_format not in {'xyz', 'yz', 'z'}: @@ -868,15 +861,15 @@ def _predict_dataloader( dl: DataLoader, raw_out: bool = True, batched_output: bool = True, - predict_kw: dict = {}) -> Iterator[Tuple[Tensor, Any, Any]]: + predict_kw: dict = {}) -> Iterator[tuple[Tensor, Any, Any]]: """Returns an iterator over predictions on the given dataloader. Args: dl (DataLoader): The dataloader to make predictions on. - batched_output (bool, optional): If True, return batches of + batched_output (bool): If True, return batches of x, y, z as defined by the dataloader. If False, unroll the batches into individual items. Defaults to True. - raw_out (bool, optional): If true, return raw predicted scores. + raw_out (bool): If true, return raw predicted scores. Defaults to True. predict_kw (dict): Dict with keywords passed to Learner.predict(). Useful if a Learner subclass implements a custom predict() @@ -886,7 +879,7 @@ def _predict_dataloader( ValueError: If return_format is not one of the allowed values. Yields: - Iterator[Tuple[Tensor, Any, Any]]: 3-tuples of x, y, and z, which + Iterator[tuple[Tensor, Any, Any]]: 3-tuples of x, y, and z, which might or might not be batched depending on the batched_output argument. """ @@ -1013,7 +1006,7 @@ def setup_ddp_params(self): log.info(f'DDP rank: {self.ddp_rank}') log.info(f'DDP local rank: {self.ddp_local_rank}') - def setup_training(self, loss_def_path: Optional[str] = None) -> None: + def setup_training(self, loss_def_path: str | None = None) -> None: """Set up model, data, loss, optimizers and various paths. The exact behavior differs based on whether this method is called in @@ -1073,8 +1066,8 @@ def setup_training(self, loss_def_path: Optional[str] = None) -> None: if self.ddp_start_method == 'fork': self.setup_data() - def get_start_and_end_epochs( - self, epochs: Optional[int] = None) -> Tuple[int, int]: + def get_start_and_end_epochs(self, + epochs: int | None = None) -> tuple[int, int]: """Get start and end epochs given epochs.""" start_epoch = self.get_start_epoch() if epochs is None: @@ -1101,15 +1094,15 @@ def get_start_epoch(self) -> int: return start_epoch def setup_model(self, - model_weights_path: Optional[str] = None, - model_def_path: Optional[str] = None) -> None: + model_weights_path: str | None = None, + model_def_path: str | None = None) -> None: """Setup self.model. Args: - model_weights_path (Optional[str], optional): Path to model + model_weights_path (str | None): Path to model weights. Will be available when loading from a bundle. Defaults to None. - model_def_path (Optional[str], optional): Path to model definition. + model_def_path (str | None): Path to model definition. Will be available when loading from a bundle. Defaults to None. """ if self.onnx_mode: @@ -1124,7 +1117,7 @@ def setup_model(self, self.model = DDP(self.model, device_ids=[self.ddp_local_rank]) self.load_init_weights(model_weights_path=model_weights_path) - def build_model(self, model_def_path: Optional[str] = None) -> nn.Module: + def build_model(self, model_def_path: str | None = None) -> nn.Module: """Build a PyTorch model.""" cfg = self.cfg @@ -1141,7 +1134,7 @@ def build_model(self, model_def_path: Optional[str] = None) -> nn.Module: ddp_rank=self.ddp_local_rank) return model - def setup_data(self, distributed: Optional[bool] = None): + def setup_data(self, distributed: bool | None = None): """Set datasets and dataLoaders for train, validation, and test sets. """ if distributed is None: @@ -1175,7 +1168,7 @@ def setup_data(self, distributed: Optional[bool] = None): self.train_dl, self.valid_dl, self.test_dl = self.build_dataloaders( distributed=distributed) - def build_datasets(self) -> Tuple['Dataset', 'Dataset', 'Dataset']: + def build_datasets(self) -> tuple['Dataset', 'Dataset', 'Dataset']: """Build Datasets for train, validation, and test splits.""" log.info(f'Building datasets ...') train_ds, val_ds, test_ds = self.cfg.data.build(tmp_dir=self.tmp_dir) @@ -1188,8 +1181,8 @@ def build_dataset(self, ds = self.cfg.data.build_dataset(split=split, tmp_dir=self.tmp_dir) return ds - def build_dataloaders(self, distributed: Optional[bool] = None - ) -> Tuple[DataLoader, DataLoader, DataLoader]: + def build_dataloaders(self, distributed: bool | None = None + ) -> tuple[DataLoader, DataLoader, DataLoader]: """Build DataLoaders for train, validation, and test splits.""" if distributed is None: distributed = self.distributed @@ -1205,7 +1198,7 @@ def build_dataloaders(self, distributed: Optional[bool] = None def build_dataloader(self, split: Literal['train', 'valid', 'test'], - distributed: Optional[bool] = None, + distributed: bool | None = None, **kwargs) -> DataLoader: """Build DataLoader for split.""" if distributed is None: @@ -1253,7 +1246,7 @@ def build_dataloader(self, dl = DataLoader(ds, **args) return dl - def get_collate_fn(self) -> Optional[callable]: + def get_collate_fn(self) -> Callable | None: """Returns a custom collate_fn to use in DataLoader. None is returned if default collate_fn should be used. @@ -1265,7 +1258,7 @@ def get_collate_fn(self) -> Optional[callable]: def build_sampler(self, ds: 'Dataset', split: Literal['train', 'valid', 'test'], - distributed: bool = False) -> Optional['Sampler']: + distributed: bool = False) -> 'Sampler | None': """Build an optional sampler for the split's dataloader.""" split = split.lower() sampler = None @@ -1285,11 +1278,11 @@ def build_sampler(self, rank=self.ddp_rank) return sampler - def setup_loss(self, loss_def_path: Optional[str] = None) -> None: + def setup_loss(self, loss_def_path: str | None = None) -> None: """Setup self.loss. Args: - loss_def_path (str, optional): Loss definition path. Will be + loss_def_path (str): Loss definition path. Will be available when loading from a bundle. Defaults to None. """ if self.loss is None: @@ -1298,7 +1291,8 @@ def setup_loss(self, loss_def_path: Optional[str] = None) -> None: if self.loss is not None and isinstance(self.loss, nn.Module): self.loss.to(self.device) - def build_loss(self, loss_def_path: Optional[str] = None) -> Callable: + def build_loss(self, + loss_def_path: str | None = None) -> Callable[..., Tensor]: """Build a loss Callable.""" cfg = self.cfg loss = cfg.solver.build_loss( @@ -1327,12 +1321,12 @@ def build_epoch_scheduler(self, start_epoch: int = 0) -> '_LRScheduler': # Visualization ################ @abstractmethod - def get_visualizer_class(self) -> Type[Visualizer]: + def get_visualizer_class(self) -> type[Visualizer]: """Returns a Visualizer class object for plotting data samples.""" def plot_predictions(self, split: Literal['train', 'valid', 'test'], - batch_limit: Optional[int] = None, + batch_limit: int | None = None, show: bool = False): """Plot predictions for a split. @@ -1356,7 +1350,7 @@ def plot_predictions(self, def plot_dataloader(self, dl: DataLoader, output_path: str, - batch_limit: Optional[int] = None, + batch_limit: int | None = None, show: bool = False): """Plot images and ground truth labels for a DataLoader.""" x, y = next(iter(dl)) @@ -1364,7 +1358,7 @@ def plot_dataloader(self, x, y, output_path, batch_limit=batch_limit, show=show) def plot_dataloaders(self, - batch_limit: Optional[int] = None, + batch_limit: int | None = None, show: bool = False): """Plot images and ground truth labels for all DataLoaders.""" if self.train_dl: @@ -1448,28 +1442,28 @@ def _bundle_model(self, model_bundle_dir: str, def export_to_onnx(self, path: str, - model: Optional['nn.Module'] = None, - sample_input: Optional[Tensor] = None, + model: nn.Module | None = None, + sample_input: Tensor | None = None, validate_export: bool = True, **kwargs) -> None: """Export model to ONNX format via :func:`torch.onnx.export`. Args: - path (str): File path to save to. - model (Optional[nn.Module]): The model to export. If None, - self.model will be used. Defaults to None. - sample_input (Optional[Tensor]): Sample input to the model. If - None, a single batch from any available DataLoader in this - Learner will be used. Defaults to None. - validate_export (bool): If True, use - :func:`onnx.checker.check_model` to validate exported model. - An exception is raised if the check fails. Defaults to True. - **kwargs (dict): Keyword args to pass to :func:`torch.onnx.export`. + path: File path to save to. + model: The model to export. If ``None``, + ``self.model`` will be used. Defaults to ``None``. + sample_input: Sample input to the model. If ``None``, a single + batch from any available ``DataLoader`` in this ``Learner`` + will be used. Defaults to ``None``. + validate_export: If ``True``, use :func:`onnx.checker.check_model` + to validate exported model. An exception is raised if the check + fails. Defaults to ``True``. + **kwargs: Keyword args to pass to :func:`torch.onnx.export`. These override the default values used in the function definition. Raises: - ValueError: If sample_input is None and the Learner has no valid + ValueError: If sample_input is ``None`` and the Learner has no valid DataLoaders. """ if model is None: @@ -1557,7 +1551,7 @@ def _bundle_transforms(self, model_bundle_dir: str) -> None: ######### # Misc. ######### - def ddp(self, rank: Optional[int] = None, world_size: Optional[int] = None + def ddp(self, rank: int | None = None, world_size: int | None = None ) -> DDPContextManager: # pragma: no cover """Return a :class:`DDPContextManager`. @@ -1611,7 +1605,7 @@ def to_batch(self, x: Tensor) -> Tensor: x = x[None, ...] return x - def to_device(self, x: Any, device: Union[str, torch.device]) -> Any: + def to_device(self, x: Any, device: str | torch.device) -> Any: """Load Tensors onto a device. Args: @@ -1627,7 +1621,7 @@ def to_device(self, x: Any, device: Union[str, torch.device]) -> Any: return x.to(device) def get_dataset(self, split: Literal['train', 'valid', 'test'] - ) -> Optional[DataLoader]: + ) -> DataLoader | None: """Get the Dataset for a split. Args: @@ -1673,8 +1667,7 @@ def normalize_input(self, x: np.ndarray) -> np.ndarray: x = x.astype(float) / max_val return x - def load_init_weights(self, - model_weights_path: Optional[str] = None) -> None: + def load_init_weights(self, model_weights_path: str | None = None) -> None: """Load the weights to initialize model.""" cfg = self.cfg uri = None diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/learner_config.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/learner_config.py index 50d32bfb0..c9326ad1e 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/learner_config.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/learner_config.py @@ -1,5 +1,5 @@ -from typing import (TYPE_CHECKING, Any, Callable, Dict, Iterable, List, - Literal, Optional, Self, Sequence, Tuple, Union) +from typing import (TYPE_CHECKING, Any, Iterable, Literal, Self, Sequence) +from collections.abc import Callable import os from os.path import join, isdir from enum import Enum @@ -45,7 +45,7 @@ ] # types -RGBTuple = Tuple[int, int, int] +RGBTuple = tuple[int, int, int] ChannelInds = Sequence[NonNegInt] @@ -131,22 +131,23 @@ def int_to_str(x): @register_config('external-module') class ExternalModuleConfig(Config): """Config describing an object to be loaded via Torch Hub.""" - uri: Optional[NonEmptyStr] = Field( + uri: NonEmptyStr | None = Field( None, description=('Local uri of a zip file, or local uri of a directory,' 'or remote uri of zip file.')) - github_repo: Optional[Annotated[ - str, StringConstraints( - strip_whitespace=True, pattern=r'.+/.+')]] = Field( - None, description='/[:tag]') - name: Optional[NonEmptyStr] = Field( + github_repo: ( + Annotated[str, + StringConstraints(strip_whitespace=True, pattern=r'.+/.+')] + | None) = Field( + None, description='/[:tag]') + name: NonEmptyStr | None = Field( None, description= 'Name of the folder in which to extract/copy the definition files.') entrypoint: NonEmptyStr = Field( ..., - description=('Name of a callable present in hubconf.py. ' - 'See docs for torch.hub for details.')) + description=('Name of a Callable present in ``hubconf.py``. ' + 'See docs for ``torch.hub`` for details.')) entrypoint_args: list = Field( [], description='Args to pass to the entrypoint. Must be serializable.') @@ -168,16 +169,16 @@ def check_either_uri_or_repo(self) -> Self: def build(self, save_dir: str, - hubconf_dir: Optional[str] = None, - ddp_rank: Optional[int] = None) -> Any: + hubconf_dir: str | None = None, + ddp_rank: int | None = None) -> Any: """Load an external module via torch.hub. Note: Loading a PyTorch module is the typical use case, but there are no type restrictions on the object loaded through torch.hub. Args: - save_dir (str, optional): The module def will be saved here. - hubconf_dir (str, optional): Path to existing definition. + save_dir (str): The module def will be saved here. + hubconf_dir (str): Path to existing definition. If provided, the definition will not be fetched from the external source but instead from this dir. Defaults to None. @@ -236,7 +237,7 @@ class ModelConfig(Config): description=( 'If True, use ImageNet weights. If False, use random initialization.' )) - init_weights: Optional[str] = Field( + init_weights: str | None = Field( None, description=('URI of PyTorch model weights used to initialize model. ' 'If set, this supersedes the pretrained option.')) @@ -246,7 +247,7 @@ class ModelConfig(Config): 'If True, the keys in the state dict referenced by init_weights ' 'must match exactly. Setting this to False can be useful if you ' 'just want to load the backbone of a model.')) - external_def: Optional[ExternalModuleConfig] = Field( + external_def: ExternalModuleConfig | None = Field( None, description='If specified, the model will be built from the ' 'definition from this external source, using Torch Hub.') @@ -262,19 +263,19 @@ def get_backbone_str(self): def build(self, num_classes: int, in_channels: int, - save_dir: Optional[str] = None, - hubconf_dir: Optional[str] = None, - ddp_rank: Optional[int] = None, + save_dir: str | None = None, + hubconf_dir: str | None = None, + ddp_rank: int | None = None, **kwargs) -> nn.Module: """Build and return a model based on the config. Args: num_classes (int): Number of classes. - in_channels (int, optional): Number of channels in the images that + in_channels (int): Number of channels in the images that will be fed into the model. Defaults to 3. - save_dir (Optional[str], optional): Used for building external_def + save_dir (str|None): Used for building external_def if specified. Defaults to None. - hubconf_dir (Optional[str], optional): Used for building + hubconf_dir (str|None): Used for building external_def if specified. Defaults to None. **kwargs: Extra args for :meth:`.build_default_model`. @@ -292,7 +293,7 @@ def build_default_model(self, num_classes: int, in_channels: int, Args: num_classes (int): Number of classes. - in_channels (int, optional): Number of channels in the images that + in_channels (int): Number of channels in the images that will be fed into the model. Defaults to 3. Returns: @@ -302,13 +303,13 @@ def build_default_model(self, num_classes: int, in_channels: int, def build_external_model(self, save_dir: str, - hubconf_dir: Optional[str] = None, - ddp_rank: Optional[int] = None) -> nn.Module: + hubconf_dir: str | None = None, + ddp_rank: int | None = None) -> nn.Module: """Build and return an external model. Args: save_dir (str): The module def will be saved here. - hubconf_dir (Optional[str], optional): Path to existing definition. + hubconf_dir (str|None): Path to existing definition. Defaults to None. Returns: @@ -350,18 +351,18 @@ class SolverConfig(Config): description= ('If True, use triangular LR scheduler with a single cycle across all ' 'epochs with start and end LR being lr/10 and the peak being lr.')) - multi_stage: List = Field( + multi_stage: list[int] = Field( [], description=('List of epoch indices at which to divide LR by 10.')) - class_loss_weights: Optional[Sequence[float]] = Field( + class_loss_weights: Sequence[float] | None = Field( None, description=('Class weights for weighted loss.')) - ignore_class_index: Optional[int] = Field( + ignore_class_index: int | None = Field( None, description='If specified, this index is ignored when computing the ' 'loss. See pytorch documentation for nn.CrossEntropyLoss for more ' 'details. This can also be negative, in which case it is treated as a ' 'negative slice index i.e. -1 = last index, -2 = second-last index, ' 'and so on.') - external_loss_def: Optional[ExternalModuleConfig] = Field( + external_loss_def: ExternalModuleConfig | None = Field( None, description='If specified, the loss will be built from the definition ' 'from this external source, using Torch Hub.') @@ -381,18 +382,19 @@ def check_no_loss_opts_if_external(self) -> Self: 'with external_loss_def.') return self - def build_loss(self, - num_classes: int, - save_dir: Optional[str] = None, - hubconf_dir: Optional[str] = None) -> Callable: + def build_loss( + self, + num_classes: int, + save_dir: str | None = None, + hubconf_dir: str | None = None) -> Callable[..., torch.Tensor]: """Build and return a loss function based on the config. Args: - num_classes (int): Number of classes. - save_dir (Optional[str], optional): Used for building - external_loss_def if specified. Defaults to None. - hubconf_dir (Optional[str], optional): Used for building - external_loss_def if specified. Defaults to None. + num_classes: Number of classes. + save_dir: Used for building ``external_loss_def`` if specified. + Defaults to ``None``. + hubconf_dir: Used for building ``external_loss_def`` if specified. + Defaults to ``None``. Returns: Loss function. @@ -435,7 +437,7 @@ def build_step_scheduler(self, optimizer: optim.Optimizer, train_ds_sz: int, last_epoch: int = -1, - **kwargs) -> Optional[_LRScheduler]: + **kwargs) -> _LRScheduler | None: """Returns an LR scheduler that changes the LR each step. This is used to implement the "one cycle" schedule popularized by @@ -476,7 +478,7 @@ def build_step_scheduler(self, def build_epoch_scheduler(self, optimizer: optim.Optimizer, last_epoch: int = -1, - **kwargs) -> Optional[_LRScheduler]: + **kwargs) -> _LRScheduler | None: """Returns an LR scheduler that changes the LR each epoch. This is used to divide the learning rate by 10 at certain epochs. @@ -507,7 +509,7 @@ def build_epoch_scheduler(self, def get_default_channel_display_groups( - nb_img_channels: int) -> Dict[str, ChannelInds]: + nb_img_channels: int) -> dict[str, ChannelInds]: """Returns the default channel_display_groups object. See PlotOptions.channel_display_groups. @@ -520,8 +522,8 @@ def get_default_channel_display_groups( return {'Input': list(range(num_display_channels))} -def validate_channel_display_groups(groups: Optional[Union[Dict[ - str, ChannelInds], Sequence[ChannelInds]]]): +def validate_channel_display_groups( + groups: dict[str, ChannelInds] | Sequence[ChannelInds] | None): """Validate channel display groups object. See PlotOptions.channel_display_groups. @@ -530,7 +532,7 @@ def validate_channel_display_groups(groups: Optional[Union[Dict[ return None elif len(groups) == 0: raise ConfigError( - f'channel_display_groups cannot be empty. Set to None instead.') + 'channel_display_groups cannot be empty. Set to None instead.') elif not isinstance(groups, dict): # if in list/tuple form, convert to dict s.t. # [(0, 1, 2), (4, 3, 5)] --> { @@ -552,7 +554,7 @@ def validate_channel_display_groups(groups: Optional[Union[Dict[ @register_config('plot_options') class PlotOptions(Config): """Config related to plotting.""" - transform: Optional[dict] = Field( + transform: dict | None = Field( A.to_dict(MinMaxNormalize()), description='An Albumentations transform serialized as a dict that ' 'will be applied to each image before it is plotted. Mainly useful ' @@ -562,25 +564,26 @@ class PlotOptions(Config): 'the plotting function. This default is useful for cases where the values after ' 'normalization are close to zero which makes the plot difficult to see.' ) - channel_display_groups: Optional[Union[Dict[str, ChannelInds], Sequence[ - ChannelInds]]] = Field( - None, - description= - ('Groups of image channels to display together as a subplot ' - 'when plotting the data and predictions. ' - 'Can be a list or tuple of groups (e.g. [(0, 1, 2), (3,)]) or a ' - 'dict containing title-to-group mappings ' - '(e.g. {"RGB": [0, 1, 2], "IR": [3]}), ' - 'where each group is a list or tuple of channel indices and ' - 'title is a string that will be used as the title of the subplot ' - 'for that group.')) + channel_display_groups: ( + dict[str, ChannelInds] | Sequence[ChannelInds] | None + ) = Field( + None, + description=( + 'Groups of image channels to display together as a subplot ' + 'when plotting the data and predictions. ' + 'Can be a list or tuple of groups (e.g. [(0, 1, 2), (3,)]) or a ' + 'dict containing title-to-group mappings ' + '(e.g. {"RGB": [0, 1, 2], "IR": [3]}), ' + 'where each group is a list or tuple of channel indices and ' + 'title is a string that will be used as the title of the subplot ' + 'for that group.')) # validators _tf = field_validator('transform')(validate_albumentation_transform) def update(self, **kwargs) -> None: super().update() - img_channels: Optional[int] = kwargs.get('img_channels') + img_channels: int | None = kwargs.get('img_channels') if self.channel_display_groups is None and img_channels is not None: self.channel_display_groups = get_default_channel_display_groups( img_channels) @@ -588,9 +591,8 @@ def update(self, **kwargs) -> None: @field_validator('channel_display_groups') @classmethod def validate_channel_display_groups( - cls, v: Optional[Union[Dict[str, Sequence[NonNegInt]], Sequence[ - Sequence[NonNegInt]]]] - ) -> Optional[Dict[str, List[NonNegInt]]]: + cls, v: dict[str, ChannelInds] | Sequence[ChannelInds] | None + ) -> dict[str, ChannelInds] | None: return validate_channel_display_groups(v) @@ -611,44 +613,44 @@ def data_config_upgrader(cfg_dict: dict, version: int) -> dict: class DataConfig(Config): """Config related to dataset for training and testing.""" class_config: ClassConfig | None = Field(None, description='Class config.') - img_channels: Optional[PosInt] = Field( + img_channels: PosInt | None = Field( None, description='The number of channels of the training images.') img_sz: PosInt = Field( 256, description= ('Length of a side of each image in pixels. This is the size to transform ' 'it to during training, not the size in the raw dataset.')) - train_sz: Optional[int] = Field( + train_sz: int | None = Field( None, description= ('If set, the number of training images to use. If fewer images exist, ' 'then an exception will be raised.')) - train_sz_rel: Optional[float] = Field( + train_sz_rel: float | None = Field( None, description='If set, the proportion of training images to use.') num_workers: int = Field( 4, description='Number of workers to use when DataLoader makes batches.') - augmentors: List[str] = Field( + augmentors: list[str] = Field( default_augmentors, description='Names of albumentations augmentors to use for training ' f'batches. Choices include: {augmentors}. Alternatively, a custom ' 'transform can be provided via the aug_transform option.') - base_transform: Optional[dict] = Field( + base_transform: dict | None = Field( None, description='An Albumentations transform serialized as a dict that ' 'will be applied to all datasets: training, validation, and test. ' 'This transformation is in addition to the resizing due to img_sz. ' 'This is useful for, for example, applying the same normalization to ' 'all datasets.') - aug_transform: Optional[dict] = Field( + aug_transform: dict | None = Field( None, description='An Albumentations transform serialized as a dict that ' 'will be applied as data augmentation to the training dataset. This ' 'transform is applied before base_transform. If provided, the ' 'augmentors option is ignored.') - plot_options: Optional[PlotOptions] = Field( + plot_options: PlotOptions | None = Field( PlotOptions(), description='Options to control plotting.') - preview_batch_limit: Optional[int] = Field( + preview_batch_limit: int | None = Field( None, description= ('Optional limit on the number of items in the preview plots produced ' @@ -690,7 +692,7 @@ def validate_plot_options(self) -> Self: self.plot_options.update(img_channels=self.img_channels) return self - def get_custom_albumentations_transforms(self) -> List[dict]: + def get_custom_albumentations_transforms(self) -> list[dict]: """Returns all custom transforms found in this config. This should return all serialized albumentations transforms with @@ -710,11 +712,11 @@ def get_custom_albumentations_transforms(self) -> List[dict]: ] return transforms_with_lambdas - def get_bbox_params(self) -> Optional[A.BboxParams]: + def get_bbox_params(self) -> A.BboxParams | None: """Returns BboxParams used by albumentations for data augmentation.""" return None - def get_data_transforms(self) -> Tuple[A.BasicTransform, A.BasicTransform]: + def get_data_transforms(self) -> tuple[A.BasicTransform, A.BasicTransform]: """Get albumentations transform objects for data augmentation. Returns a 2-tuple of a "base" transform and an augmentation transform. @@ -767,21 +769,21 @@ def get_data_transforms(self) -> Tuple[A.BasicTransform, A.BasicTransform]: return base_transform, aug_transform - def build(self, tmp_dir: Optional[str] = None - ) -> Tuple[Dataset, Dataset, Dataset]: + def build(self, + tmp_dir: str | None = None) -> tuple[Dataset, Dataset, Dataset]: """Build and return train, val, and test datasets.""" raise NotImplementedError() def build_dataset(self, split: Literal['train', 'valid', 'test'], - tmp_dir: Optional[str] = None) -> Dataset: + tmp_dir: str | None = None) -> Dataset: """Build and return dataset for a single split.""" raise NotImplementedError() def random_subset_dataset(self, ds: Dataset, - size: Optional[int] = None, - fraction: Optional[Proportion] = None) -> Subset: + size: int | None = None, + fraction: Proportion | None = None) -> Subset: if size is None and fraction is None: return ds if size is not None and fraction is not None: @@ -799,9 +801,9 @@ def random_subset_dataset(self, @register_config('image_data') class ImageDataConfig(DataConfig): """Config related to dataset for training and testing.""" - data_format: Optional[str] = Field( + data_format: str | None = Field( None, description='Name of dataset format.') - uri: Optional[Union[str, List[str]]] = Field( + uri: str | list[str] | None = Field( None, description='One of the following:\n' '(1) a URI of a directory containing "train", "valid", and ' @@ -809,20 +811,20 @@ class ImageDataConfig(DataConfig): '(2) a URI of a zip file containing (1);\n' '(3) a list of (2);\n' '(4) a URI of a directory containing zip files containing (1).') - group_uris: Optional[List[Union[str, List[str]]]] = Field( + group_uris: list[str | list[str]] | None = Field( None, description= 'This can be set instead of uri in order to specify groups of chips. ' 'Each element in the list is expected to be an object of the same ' 'form accepted by the uri field. The purpose of separating chips into ' 'groups is to be able to use the group_train_sz field.') - group_train_sz: Optional[Union[int, List[int]]] = Field( + group_train_sz: int | list[int] | None = Field( None, description='If group_uris is set, this can be used to specify the ' 'number of chips to use per group. Only applies to training chips. ' 'This can either be a single value that will be used for all groups ' 'or a list of values (one for each group).') - group_train_sz_rel: Optional[Union[Proportion, List[Proportion]]] = Field( + group_train_sz_rel: Proportion | list[Proportion] | None = Field( None, description='Relative version of group_train_sz. Must be a float ' 'in [0, 1]. If group_uris is set, this can be used to specify the ' @@ -860,8 +862,8 @@ def validate_group_uris(self) -> Self: def _build_dataset(self, dirs: Iterable[str], - tf: Optional[A.BasicTransform] = None - ) -> Tuple[Dataset, Dataset, Dataset]: + tf: A.BasicTransform | None = None + ) -> tuple[Dataset, Dataset, Dataset]: """Make datasets for a single split. Args: @@ -881,21 +883,21 @@ def _build_datasets(self, train_dirs: Iterable[str], val_dirs: Iterable[str], test_dirs: Iterable[str], - train_tf: Optional[A.BasicTransform] = None, - val_tf: Optional[A.BasicTransform] = None, - test_tf: Optional[A.BasicTransform] = None - ) -> Tuple[Dataset, Dataset, Dataset]: + train_tf: A.BasicTransform | None = None, + val_tf: A.BasicTransform | None = None, + test_tf: A.BasicTransform | None = None + ) -> tuple[Dataset, Dataset, Dataset]: """Make training, validation, and test datasets. Args: train_dirs (str): Directories where training data is located. val_dirs (str): Directories where validation data is located. test_dirs (str): Directories where test data is located. - train_tf (Optional[A.BasicTransform], optional): Transform for the + train_tf (A.BasicTransform | None): Transform for the training dataset. Defaults to None. - val_tf (Optional[A.BasicTransform], optional): Transform for the + val_tf (A.BasicTransform | None): Transform for the validation dataset. Defaults to None. - test_tf (Optional[A.BasicTransform], optional): Transform for the + test_tf (A.BasicTransform | None): Transform for the test dataset. Defaults to None. Returns: @@ -910,7 +912,7 @@ def dir_to_dataset(self, data_dir: str, transform: A.BasicTransform) -> Dataset: raise NotImplementedError() - def build(self, tmp_dir: str) -> Tuple[Dataset, Dataset, Dataset]: + def build(self, tmp_dir: str) -> tuple[Dataset, Dataset, Dataset]: if self.group_uris is None: return self._get_datasets_from_uri(self.uri, tmp_dir=tmp_dir) @@ -930,7 +932,7 @@ def build(self, tmp_dir: str) -> Tuple[Dataset, Dataset, Dataset]: def build_dataset(self, split: Literal['train', 'valid', 'test'], - tmp_dir: Optional[str] = None) -> Dataset: + tmp_dir: str | None = None) -> Dataset: if self.group_uris is None: ds = self._get_dataset_from_uri( @@ -951,12 +953,12 @@ def build_dataset(self, return ds - def _get_datasets_from_uri(self, uri: Union[str, List[str]], tmp_dir: str - ) -> Tuple[Dataset, Dataset, Dataset]: + def _get_datasets_from_uri(self, uri: str | list[str], tmp_dir: str + ) -> tuple[Dataset, Dataset, Dataset]: """Get image train, validation, & test datasets from a single zip file. Args: - uri (Union[str, List[str]]): Uri of a zip file containing the + uri (str | list[str]): Uri of a zip file containing the images. Returns: @@ -985,14 +987,13 @@ def _get_datasets_from_uri(self, uri: Union[str, List[str]], tmp_dir: str test_tf=test_tf) return train_ds, val_ds, test_ds - def _get_dataset_from_uri(self, uri: Union[str, List[str]], + def _get_dataset_from_uri(self, uri: str | list[str], split: Literal['train', 'valid', 'test'], tmp_dir: str) -> Dataset: """Get image dataset from a single zip file. Args: - uri (Union[str, List[str]]): Uri of a zip file containing the - images. + uri: Uri of a zip file containing the images. Returns: Training, validation, and test dataSets. @@ -1011,13 +1012,12 @@ def _get_dataset_from_uri(self, uri: Union[str, List[str]], ds = self._build_dataset(dirs, tf) return ds - def _get_datasets_from_group_uris( - self, - uris: Union[str, List[str]], - tmp_dir: str, - group_train_sz: Optional[int] = None, - group_train_sz_rel: Optional[float] = None - ) -> Tuple[Dataset, Dataset, Dataset]: + def _get_datasets_from_group_uris(self, + uris: str | list[str], + tmp_dir: str, + group_train_sz: int | None = None, + group_train_sz_rel: float | None = None + ) -> tuple[Dataset, Dataset, Dataset]: train_ds_lst, valid_ds_lst, test_ds_lst = [], [], [] group_sizes = None @@ -1050,10 +1050,10 @@ def _get_datasets_from_group_uris( def _get_dataset_from_group_uris( self, split: Literal['train', 'valid', 'test'], - uris: Union[str, List[str]], + uris: str | list[str], tmp_dir: str, - group_sz: Optional[int] = None, - group_sz_rel: Optional[float] = None) -> Dataset: + group_sz: int | None = None, + group_sz_rel: float | None = None) -> Dataset: group_sizes = None if group_sz is not None: @@ -1076,16 +1076,14 @@ def _get_dataset_from_group_uris( combined_dataset = ConcatDataset(per_uri_dataset) return combined_dataset - def get_data_dirs(self, uri: Union[str, List[str]], - unzip_dir: str) -> List[str]: + def get_data_dirs(self, uri: str | list[str], unzip_dir: str) -> list[str]: """Extract data dirs from uri. Data dirs are directories containing "train", "valid", and (optionally) "test" subdirectories. Args: - uri (Union[str, List[str]]): A URI or a list of URIs of one of the - following: + uri: A URI or a list of URIs of one of the following: (1) a URI of a directory containing "train", "valid", and (optionally) "test" subdirectories @@ -1096,7 +1094,7 @@ def get_data_dirs(self, uri: Union[str, List[str]], needed. Returns: - List[str]: Paths to directories that each contain contents of one + list[str]: Paths to directories that each contain contents of one zip file. """ @@ -1136,15 +1134,15 @@ def is_data_dir(uri: str) -> bool: data_dirs = self.unzip_data(zip_uris, unzip_dir) return data_dirs - def unzip_data(self, zip_uris: List[str], unzip_dir: str) -> List[str]: + def unzip_data(self, zip_uris: list[str], unzip_dir: str) -> list[str]: """Unzip dataset zip files. Args: - zip_uris (List[str]): A list of URIs of zip files: + zip_uris (list[str]): A list of URIs of zip files: unzip_dir (str): Directory where zip files will be extracted to. Returns: - List[str]: Paths to directories that each contain contents of one + list[str]: Paths to directories that each contain contents of one zip file. """ data_dirs = [] @@ -1172,10 +1170,9 @@ class GeoDataConfig(DataConfig): See :mod:`rastervision.pytorch_learner.dataset.dataset`. """ - scene_dataset: Optional['SceneDatasetConfig'] = Field(None, description='') - sampling: Union[WindowSamplingConfig, Dict[ - str, WindowSamplingConfig]] = Field( - {}, description='Window sampling config.') + scene_dataset: 'SceneDatasetConfig | None' = Field(None, description='') + sampling: WindowSamplingConfig | dict[str, WindowSamplingConfig] = Field( + {}, description='Window sampling config.') def __repr_args__(self): ds_str = repr(self.scene_dataset) @@ -1214,7 +1211,7 @@ def get_class_config_from_dataset_if_needed(self) -> Self: def build_scenes(self, scene_configs: Iterable['SceneConfig'], - tmp_dir: Optional[str] = None) -> List[Scene]: + tmp_dir: str | None = None) -> list[Scene]: """Build training, validation, and test scenes.""" class_config = self.scene_dataset.class_config scenes = [ @@ -1225,9 +1222,9 @@ def build_scenes(self, def _build_dataset(self, split: Literal['train', 'valid', 'test'], - tf: Optional[A.BasicTransform] = None, - tmp_dir: Optional[str] = None, - **kwargs) -> Tuple[Dataset, Dataset, Dataset]: + tf: A.BasicTransform | None = None, + tmp_dir: str | None = None, + **kwargs) -> tuple[Dataset, Dataset, Dataset]: """Make training, validation, and test datasets. Args: @@ -1263,25 +1260,25 @@ def _build_dataset(self, return combined_dataset def _build_datasets(self, - tmp_dir: Optional[str] = None, - train_tf: Optional[A.BasicTransform] = None, - val_tf: Optional[A.BasicTransform] = None, - test_tf: Optional[A.BasicTransform] = None, - **kwargs) -> Tuple[Dataset, Dataset, Dataset]: + tmp_dir: str | None = None, + train_tf: A.BasicTransform | None = None, + val_tf: A.BasicTransform | None = None, + test_tf: A.BasicTransform | None = None, + **kwargs) -> tuple[Dataset, Dataset, Dataset]: """Make training, validation, and test datasets. Args: tmp_dir (str): Temporary directory to be used for building scenes. - train_tf (Optional[A.BasicTransform], optional): Transform for the + train_tf (A.BasicTransform | None): Transform for the training dataset. Defaults to None. - val_tf (Optional[A.BasicTransform], optional): Transform for the + val_tf (A.BasicTransform | None): Transform for the validation dataset. Defaults to None. - test_tf (Optional[A.BasicTransform], optional): Transform for the + test_tf (A.BasicTransform | None): Transform for the test dataset. Defaults to None. **kwargs: Kwargs to pass to :meth:`.scene_to_dataset`. Returns: - Tuple[Dataset, Dataset, Dataset]: PyTorch-compatiable training, + tuple[Dataset, Dataset, Dataset]: PyTorch-compatiable training, validation, and test datasets. """ train_ds = self._build_dataset('train', train_tf, tmp_dir, **kwargs) @@ -1291,7 +1288,7 @@ def _build_datasets(self, def scene_to_dataset(self, scene: Scene, - transform: Optional[A.BasicTransform] = None, + transform: A.BasicTransform | None = None, for_chipping: bool = False) -> Dataset: """Make a dataset from a single scene. """ @@ -1299,7 +1296,7 @@ def scene_to_dataset(self, def build_dataset(self, split: Literal['train', 'valid', 'test'], - tmp_dir: Optional[str] = None) -> Dataset: + tmp_dir: str | None = None) -> Dataset: base_transform, aug_transform = self.get_data_transforms() if split == 'train': @@ -1316,8 +1313,8 @@ def build_dataset(self, return ds - def build(self, tmp_dir: Optional[str] = None, - for_chipping: bool = False) -> Tuple[Dataset, Dataset, Dataset]: + def build(self, tmp_dir: str | None = None, + for_chipping: bool = False) -> tuple[Dataset, Dataset, Dataset]: base_transform, aug_transform = self.get_data_transforms() if for_chipping: train_tf, val_tf, test_tf = None, None, None @@ -1351,8 +1348,8 @@ def learner_config_upgrader(cfg_dict: dict, version: int) -> dict: @register_config('learner', upgrader=learner_config_upgrader) class LearnerConfig(Config): """Config for Learner.""" - model: Optional[ModelConfig] = None - solver: Optional[SolverConfig] = None + model: ModelConfig | None = None + solver: SolverConfig | None = None data: DataConfig eval_train: bool = Field( @@ -1370,7 +1367,7 @@ class LearnerConfig(Config): description='Save Tensorboard log files at the end of each epoch.') run_tensorboard: bool = Field( False, description='run Tensorboard server during training') - output_uri: Optional[str] = Field( + output_uri: str | None = Field( None, description='URI of where to save output') save_all_checkpoints: bool = Field( False, @@ -1402,24 +1399,24 @@ def validate_class_loss_weights(self) -> Self: return self def build(self, - tmp_dir: Optional[str] = None, - model_weights_path: Optional[str] = None, - model_def_path: Optional[str] = None, - loss_def_path: Optional[str] = None, + tmp_dir: str | None = None, + model_weights_path: str | None = None, + model_def_path: str | None = None, + loss_def_path: str | None = None, training: bool = True) -> 'Learner': """Returns a Learner instantiated using this Config. Args: tmp_dir (str): Root of temp dirs. - model_weights_path (str, optional): A local path to model weights. + model_weights_path (str): A local path to model weights. Defaults to None. - model_def_path (str, optional): A local path to a directory with a + model_def_path (str): A local path to a directory with a hubconf.py. If provided, the model definition is imported from here. Defaults to None. - loss_def_path (str, optional): A local path to a directory with a + loss_def_path (str): A local path to a directory with a hubconf.py. If provided, the loss function definition is imported from here. Defaults to None. - training (bool, optional): Whether the model is to be used for + training (bool): Whether the model is to be used for training or prediction. If False, the model is put in eval mode and the loss function, optimizer, etc. are not initialized. Defaults to True. diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_learner.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_learner.py index d18573867..f2deae442 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_learner.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_learner.py @@ -1,5 +1,4 @@ -from typing import (TYPE_CHECKING, Dict, Iterable, List, Optional, Tuple, - Union) +from typing import TYPE_CHECKING, Iterable import warnings import logging @@ -27,7 +26,7 @@ class ObjectDetectionLearner(Learner): def get_visualizer_class(self): return ObjectDetectionVisualizer - def build_model(self, model_def_path: Optional[str] = None) -> 'nn.Module': + def build_model(self, model_def_path: str | None = None) -> 'nn.Module': """Override to pass img_sz.""" cfg = self.cfg model = cfg.model.build( @@ -40,8 +39,8 @@ def build_model(self, model_def_path: Optional[str] = None) -> 'nn.Module': return model def setup_model(self, - model_weights_path: Optional[str] = None, - model_def_path: Optional[str] = None) -> None: + model_weights_path: str | None = None, + model_def_path: str | None = None) -> None: """Override to apply the TorchVisionODAdapter wrapper.""" if self.model is not None: self.model.to(self.device) @@ -119,35 +118,35 @@ def validate_end(self, outputs): def predict(self, x: 'Tensor', raw_out: bool = False, - out_shape: Optional[Tuple[int, int]] = None) -> BoxList: + out_shape: tuple[int, int] | None = None) -> BoxList: """Make prediction for an image or batch of images. Args: - x (Tensor): Image or batch of images as a float Tensor with pixel - values normalized to [0, 1]. - raw_out (bool, optional): If True, return prediction probabilities. - Defaults to False. - out_shape (Optional[Tuple[int, int]], optional): If provided, - boxes are resized such that they reference pixel coordinates in - an image of this shape. Defaults to None. + x: Image or batch of images as a float Tensor with pixel values + normalized to [0, 1]. + raw_out: If True, return prediction probabilities. + Defaults to ``False``. + out_shape: If provided, boxes are resized such that they reference + pixel coordinates in an image of this shape. + Defaults to ``None``. Returns: BoxList: Predicted boxes. """ - out: List[BoxList] = super().predict(x, raw_out=raw_out) + out: list[BoxList] = super().predict(x, raw_out=raw_out) out = self.postprocess_model_output(x, out, out_shape=out_shape) return out def predict_onnx(self, x: 'Tensor', raw_out: bool = False, - out_shape: Optional[Tuple[int, int]] = None) -> BoxList: - out: List[BoxList] = super().predict(x, raw_out=raw_out) + out_shape: tuple[int, int] | None = None) -> BoxList: + out: list[BoxList] = super().predict(x, raw_out=raw_out) out = self.postprocess_model_output(x, out, out_shape=out_shape) return out def postprocess_model_output(self, x: 'Tensor', out_batch: torch.Tensor, - out_shape: Tuple[int, int]): + out_shape: tuple[int, int]): if out_shape is None: return out_batch h_in, w_in = x.shape[-2:] @@ -160,8 +159,8 @@ def postprocess_model_output(self, x: 'Tensor', out_batch: torch.Tensor, def output_to_numpy( self, out: Iterable[BoxList] - ) -> Union[Dict[str, np.ndarray], List[Dict[str, np.ndarray]]]: - def boxlist_to_numpy(boxlist: BoxList) -> Dict[str, np.ndarray]: + ) -> dict[str, np.ndarray] | list[dict[str, np.ndarray]]: + def boxlist_to_numpy(boxlist: BoxList) -> dict[str, np.ndarray]: return { 'boxes': boxlist.convert_boxes('yxyx').numpy(), 'class_ids': boxlist.get_field('class_ids').numpy(), @@ -178,8 +177,8 @@ def prob_to_pred(self, x): def export_to_onnx(self, path: str, - model: Optional['nn.Module'] = None, - sample_input: Optional[torch.Tensor] = None, + model: 'nn.Module | None' = None, + sample_input: torch.Tensor | None = None, **kwargs) -> None: if model is None and isinstance(self.model, TorchVisionODAdapter): model = self.model.model diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_learner_config.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_learner_config.py index 271c1645f..cb13c1b09 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_learner_config.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_learner_config.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Union +from typing import TYPE_CHECKING from enum import Enum from os.path import join import logging @@ -70,11 +70,10 @@ class ObjectDetectionGeoDataConfig(ObjectDetectionDataConfig, GeoDataConfig): def scene_to_dataset( self, scene: Scene, - transform: Optional[A.BasicTransform] = None, - bbox_params: Optional[A.BboxParams] = DEFAULT_BBOX_PARAMS, + transform: A.BasicTransform | None = None, + bbox_params: A.BboxParams | None = DEFAULT_BBOX_PARAMS, for_chipping: bool = False - ) -> Union[ObjectDetectionSlidingWindowGeoDataset, - ObjectDetectionRandomWindowGeoDataset]: + ) -> ObjectDetectionSlidingWindowGeoDataset | ObjectDetectionRandomWindowGeoDataset: if isinstance(self.sampling, dict): opts = self.sampling[scene.id] else: @@ -204,7 +203,7 @@ def build_default_model(self, num_classes: int, in_channels: int, class ObjectDetectionLearnerConfig(LearnerConfig): """Configure an :class:`.ObjectDetectionLearner`.""" - model: Optional[ObjectDetectionModelConfig] + model: ObjectDetectionModelConfig | None def build(self, tmp_dir=None, diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_utils.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_utils.py index 120a54200..824217e06 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_utils.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/object_detection_utils.py @@ -1,5 +1,5 @@ -from typing import (Any, Callable, Optional, Sequence, Tuple, Iterable, List, - Dict, Union) +from typing import Any, Iterable, Self, Sequence +from collections.abc import Callable from collections import defaultdict from os.path import join from operator import iand @@ -20,8 +20,8 @@ from rastervision.pytorch_learner.utils.utils import ONNXRuntimeAdapter -def get_coco_gt(targets: Iterable['BoxList'], - num_class_ids: int) -> Dict[str, List[dict]]: +def get_coco_gt(targets: Iterable[Self], + num_class_ids: int) -> dict[str, list[dict]]: images = [] annotations = [] ann_id = 1 @@ -60,7 +60,7 @@ def get_coco_gt(targets: Iterable['BoxList'], return coco -def get_coco_preds(outputs: Iterable['BoxList']) -> List[dict]: +def get_coco_preds(outputs: Iterable[Self]) -> list[dict]: preds = [] for img_id, output in enumerate(outputs, 1): boxes = output.convert_boxes('xywh').float().tolist() @@ -144,8 +144,10 @@ def get_field(self, name: str) -> Any: else: return self.extras.get(name) - def _map_extras(self, func: Callable, - cond: Callable = lambda k, v: True) -> dict: + def _map_extras( + self, + func: Callable[[str, Any], Any], + cond: Callable[[str, Any], bool] = lambda k, v: True) -> dict: new_extras = {} for k, v in self.extras.items(): if cond(k, v): @@ -155,13 +157,13 @@ def _map_extras(self, func: Callable, return new_extras - def copy(self) -> 'BoxList': + def copy(self) -> Self: return BoxList( self.boxes.copy(), **self._map_extras(lambda k, v: v.copy()), cond=lambda k, v: torch.is_tensor(v)) - def to(self, *args, **kwargs) -> 'BoxList': + def to(self, *args, **kwargs) -> Self: """Recursively apply :meth:`torch.Tensor.to` to Tensors. Args: @@ -188,7 +190,7 @@ def __len__(self) -> int: return len(self.boxes) @staticmethod - def cat(box_lists: Iterable['BoxList']) -> 'BoxList': + def cat(box_lists: Iterable[Self]) -> Self: boxes = [] extras = defaultdict(list) for bl in box_lists: @@ -200,7 +202,7 @@ def cat(box_lists: Iterable['BoxList']) -> 'BoxList': extras[k] = torch.cat(v) return BoxList(boxes, **extras) - def equal(self, other: 'BoxList') -> bool: + def equal(self, other: Self) -> bool: if len(other) != len(self): return False @@ -216,20 +218,20 @@ def equal(self, other: 'BoxList') -> bool: other_tups = set([tuple([x.item() for x in row]) for row in cat_arr]) return self_tups == other_tups - def ind_filter(self, inds: Sequence[int]) -> 'BoxList': + def ind_filter(self, inds: Sequence[int]) -> Self: boxes = self.boxes[inds] extras = self._map_extras( func=lambda k, v: v[inds], cond=lambda k, v: torch.is_tensor(v)) return BoxList(boxes, **extras) - def score_filter(self, score_thresh: float = 0.25) -> 'BoxList': + def score_filter(self, score_thresh: float = 0.25) -> Self: scores = self.extras.get('scores') if scores is not None: return self.ind_filter(scores > score_thresh) else: raise ValueError('must have scores as key in extras') - def clip_boxes(self, img_height: int, img_width: int) -> 'BoxList': + def clip_boxes(self, img_height: int, img_width: int) -> Self: boxes = clip_boxes_to_image(self.boxes, (img_height, img_width)) return BoxList(boxes, **self.extras) @@ -241,7 +243,7 @@ def nms(self, iou_thresh: float = 0.5) -> torch.Tensor: self.get_field('class_ids'), iou_thresh) return self.ind_filter(good_inds) - def scale(self, yscale: float, xscale: float) -> 'BoxList': + def scale(self, yscale: float, xscale: float) -> Self: """Scale box coords by the given scaling factors.""" dtype = self.boxes.dtype boxes = self.boxes.float() @@ -250,7 +252,7 @@ def scale(self, yscale: float, xscale: float) -> 'BoxList': self.boxes = boxes.to(dtype=dtype) return self - def pin_memory(self) -> 'BoxList': + def pin_memory(self) -> Self: self.boxes = self.boxes.pin_memory() for k, v in self.extras.items(): if torch.is_tensor(v): @@ -261,10 +263,10 @@ def __repr__(self) -> str: # pragma: no cover return pformat(dict(boxes=self.boxes, **self.extras)) -def collate_fn(data: Iterable[Sequence]) -> Tuple[torch.Tensor, List[BoxList]]: +def collate_fn(data: Iterable[Sequence]) -> tuple[torch.Tensor, list[BoxList]]: imgs = [d[0] for d in data] x = torch.stack(imgs) - y: List[BoxList] = [d[1] for d in data] + y: list[BoxList] = [d[1] for d in data] return x, y @@ -274,16 +276,16 @@ def draw_boxes(x: torch.Tensor, y: BoxList, class_names: Sequence[str], image.""" boxes = y.boxes class_ids: np.ndarray = y.get_field('class_ids').numpy() - scores: Optional[torch.Tensor] = y.get_field('scores') + scores: torch.Tensor | None = y.get_field('scores') if len(boxes) > 0: - box_annotations: List[str] = np.array(class_names)[class_ids].tolist() + box_annotations: list[str] = np.array(class_names)[class_ids].tolist() if scores is not None: box_annotations = [ f'{ann} | {score:.2f}' for ann, score in zip(box_annotations, scores) ] - box_colors: List[Union[str, Tuple[int, ...]]] = [ + box_colors: list[str | tuple[int, ...]] = [ tuple(c) if not isinstance(c, str) else c for c in np.array(class_colors)[class_ids] ] @@ -321,7 +323,7 @@ def __init__(self, Args: model (nn.Module): A torchvision object detection model. - ignored_output_inds (Iterable[int], optional): Class labels to exclude. + ignored_output_inds (Iterable[int]): Class labels to exclude. Defaults to [0]. """ super().__init__() @@ -330,17 +332,15 @@ def __init__(self, def forward(self, input: torch.Tensor, - targets: Optional[Iterable[BoxList]] = None - ) -> Union[Dict[str, Any], List[BoxList]]: + targets: Iterable[BoxList] | None = None + ) -> dict[str, Any] | list[BoxList]: """Forward pass. Args: - input (Tensor[batch_size, in_channels, in_height, in_width]): batch - of images. - targets (Optional[Iterable[BoxList]], optional): In training mode, - should be Iterable[BoxList]], with each BoxList having a - 'class_ids' field. In eval mode, should be None. Defaults to - None. + input: batch of images. + targets: In training mode, should be ``Iterable[BoxList]]``, with + each ``BoxList`` having a ``'class_ids'`` field. In eval mode, + should be None. Defaults to ``None``. Returns: In training mode, returns a dict of losses. In eval mode, returns a @@ -404,7 +404,7 @@ def model_output_dict_to_boxlist(self, out: dict) -> BoxList: class ONNXRuntimeAdapterForFasterRCNN(ONNXRuntimeAdapter): """TorchVision Faster RCNN model exported as ONNX""" - def __call__(self, x: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + def __call__(self, x: torch.Tensor | np.ndarray) -> torch.Tensor: N, *_ = x.shape x = x.numpy() outputs = self.ort_session.run(None, dict(x=x)) diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/regression_learner.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/regression_learner.py index 252ef71d2..0eb5cd1c3 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/regression_learner.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/regression_learner.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional +from typing import TYPE_CHECKING import warnings from os.path import join import logging @@ -25,7 +25,7 @@ class RegressionLearner(Learner): def get_visualizer_class(self): return RegressionVisualizer - def build_model(self, model_def_path: Optional[str] = None) -> 'nn.Module': + def build_model(self, model_def_path: str | None = None) -> 'nn.Module': """Override to pass class_names, pos_class_names, and prob_class_names. """ cfg = self.cfg diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/regression_learner_config.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/regression_learner_config.py index 7f8439687..3d118a584 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/regression_learner_config.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/regression_learner_config.py @@ -1,4 +1,5 @@ -from typing import TYPE_CHECKING, Callable, Iterable, List, Optional, Sequence, Union +from typing import TYPE_CHECKING, Iterable, Sequence +from collections.abc import Callable from enum import Enum import logging @@ -45,8 +46,8 @@ def reg_data_config_upgrader(cfg_dict, version): @register_config('regression_data', upgrader=reg_data_config_upgrader) class RegressionDataConfig(Config): - pos_class_names: List[str] = [] - prob_class_names: List[str] = [] + pos_class_names: list[str] = [] + prob_class_names: list[str] = [] @register_config('regression_image_data') @@ -54,7 +55,7 @@ class RegressionImageDataConfig(RegressionDataConfig, ImageDataConfig): """Configure :class:`RegressionImageDatasets <.RegressionImageDataset>`.""" data_format: RegressionDataFormat = RegressionDataFormat.csv - plot_options: Optional[RegressionPlotOptions] = Field( + plot_options: RegressionPlotOptions | None = Field( RegressionPlotOptions(), description='Options to control plotting.') def dir_to_dataset(self, data_dir: str, @@ -71,15 +72,15 @@ class RegressionGeoDataConfig(RegressionDataConfig, GeoDataConfig): See :mod:`rastervision.pytorch_learner.dataset.regression_dataset`. """ - plot_options: Optional[RegressionPlotOptions] = Field( + plot_options: RegressionPlotOptions | None = Field( RegressionPlotOptions(), description='Options to control plotting.') - def scene_to_dataset(self, - scene: Scene, - transform: Optional[A.BasicTransform] = None, - for_chipping: bool = False - ) -> Union[RegressionSlidingWindowGeoDataset, - RegressionRandomWindowGeoDataset]: + def scene_to_dataset( + self, + scene: Scene, + transform: A.BasicTransform | None = None, + for_chipping: bool = False + ) -> RegressionSlidingWindowGeoDataset | RegressionRandomWindowGeoDataset: if isinstance(self.sampling, dict): opts = self.sampling[scene.id] else: @@ -125,8 +126,8 @@ class RegressionModel(nn.Module): def __init__(self, backbone: nn.Module, out_features: int, - pos_out_inds: Optional[Sequence[int]] = None, - prob_out_inds: Optional[Sequence[int]] = None, + pos_out_inds: Sequence[int] | None = None, + prob_out_inds: Sequence[int] | None = None, **kwargs): super().__init__() self.backbone = backbone @@ -160,9 +161,9 @@ def build_default_model( self, num_classes: int, in_channels: int, - class_names: Optional[Sequence[str]] = None, - pos_class_names: Optional[Iterable[str]] = None, - prob_class_names: Optional[Iterable[str]] = None) -> nn.Module: + class_names: Sequence[str] | None = None, + pos_class_names: Iterable[str] | None = None, + prob_class_names: Iterable[str] | None = None) -> nn.Module: backbone_name = self.get_backbone_str() pretrained = self.pretrained out_features = num_classes @@ -216,7 +217,7 @@ def build_default_model( class RegressionLearnerConfig(LearnerConfig): """Configure a :class:`.RegressionLearner`.""" - model: Optional[RegressionModelConfig] + model: RegressionModelConfig | None def build(self, tmp_dir, diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/semantic_segmentation_learner.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/semantic_segmentation_learner.py index 87ca5d32a..9eb4002a3 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/semantic_segmentation_learner.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/semantic_segmentation_learner.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Optional, Tuple +from typing import TYPE_CHECKING import warnings import logging @@ -71,7 +71,7 @@ def post_forward(self, x): def predict(self, x: torch.Tensor, raw_out: bool = False, - out_shape: Optional[Tuple[int, int]] = None) -> torch.Tensor: + out_shape: tuple[int, int] | None = None) -> torch.Tensor: if out_shape is None: out_shape = x.shape[-2:] @@ -85,11 +85,10 @@ def predict(self, out, raw_out=raw_out, out_shape=out_shape) return out - def predict_onnx( - self, - x: torch.Tensor, - raw_out: bool = False, - out_shape: Optional[Tuple[int, int]] = None) -> torch.Tensor: + def predict_onnx(self, + x: torch.Tensor, + raw_out: bool = False, + out_shape: tuple[int, int] | None = None) -> torch.Tensor: if out_shape is None: out_shape = x.shape[-2:] @@ -102,7 +101,7 @@ def predict_onnx( return out def postprocess_model_output(self, out: torch.Tensor, raw_out: bool, - out_shape: Tuple[int, int]): + out_shape: tuple[int, int]): out = out.softmax(dim=1) # ensure correct output shape if out.shape[-2:] != out_shape: @@ -120,8 +119,8 @@ def prob_to_pred(self, x): def export_to_onnx(self, path: str, - model: Optional['nn.Module'] = None, - sample_input: Optional[torch.Tensor] = None, + model: 'nn.Module | None' = None, + sample_input: torch.Tensor | None = None, **kwargs) -> None: args = dict( input_names=['x'], diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/semantic_segmentation_learner_config.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/semantic_segmentation_learner_config.py index a8371a0c8..3be6f9d46 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/semantic_segmentation_learner_config.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/semantic_segmentation_learner_config.py @@ -1,4 +1,4 @@ -from typing import Callable, Optional +from collections.abc import Callable from os.path import join from enum import Enum import logging @@ -113,7 +113,7 @@ def update(self, *args, **kwargs): def scene_to_dataset(self, scene: Scene, - transform: Optional[A.BasicTransform] = None, + transform: A.BasicTransform | None = None, for_chipping: bool = False) -> Dataset: if isinstance(self.sampling, dict): opts = self.sampling[scene.id] @@ -179,8 +179,8 @@ def build_default_model(self, num_classes: int, backbone_name = self.get_backbone_str() pretrained = self.pretrained weights = 'DEFAULT' if pretrained else None - model_factory_func: Callable = getattr(models.segmentation, - f'deeplabv3_{backbone_name}') + model_factory_func: Callable[..., nn.Module] = getattr( + models.segmentation, f'deeplabv3_{backbone_name}') model = model_factory_func( num_classes=num_classes, weights_backbone=weights, @@ -212,7 +212,7 @@ def build_default_model(self, num_classes: int, class SemanticSegmentationLearnerConfig(LearnerConfig): """Configure a :class:`.SemanticSegmentationLearner`.""" - model: Optional[SemanticSegmentationModelConfig] + model: SemanticSegmentationModelConfig | None def build(self, tmp_dir=None, diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/distributed.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/distributed.py index 17c29eb1f..93043cc1c 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/distributed.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/distributed.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Any, Optional +from typing import TYPE_CHECKING, Any import os from contextlib import AbstractContextManager import gc @@ -32,8 +32,8 @@ class DDPContextManager(AbstractContextManager): # pragma: no cover def __init__(self, learner: 'Learner', - rank: Optional[int] = None, - world_size: Optional[int] = None) -> None: + rank: int | None = None, + world_size: int | None = None) -> None: """Constructor. Args: diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/prediction.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/prediction.py index 1c36656d0..568436a6a 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/prediction.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/prediction.py @@ -1,4 +1,4 @@ -from typing import TYPE_CHECKING, Dict, Iterator +from typing import TYPE_CHECKING, Iterator import logging from rastervision.core.data import (ChipClassificationLabels, @@ -34,7 +34,7 @@ def predict_scene_cc( ds = ClassificationSlidingWindowGeoDataset( scene, size=chip_sz, stride=stride, transform=base_tf) - predictions: Iterator['np.array'] = learner.predict_dataset( + predictions: Iterator['np.ndarray'] = learner.predict_dataset( ds, raw_out=True, numpy_out=True, @@ -62,7 +62,7 @@ def predict_scene_od(learner: 'ObjectDetectionLearner', scene: 'Scene', ds = ObjectDetectionSlidingWindowGeoDataset( scene, size=chip_sz, stride=stride, transform=base_tf) - predictions: Iterator[Dict[str, 'np.ndarray']] = learner.predict_dataset( + predictions: Iterator[dict[str, 'np.ndarray']] = learner.predict_dataset( ds, raw_out=True, numpy_out=True, diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/torch_hub.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/torch_hub.py index 7aa6cb7eb..7e78eef27 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/torch_hub.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/torch_hub.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import Any from pathlib import Path from os.path import join, isdir, realpath import shutil @@ -22,10 +22,10 @@ def _repo_name_to_dir_name(repo: str) -> str: Adapted from torch.hub._get_cache_or_reload() Args: - repo (str): /[:tag] + repo: /[:tag] Returns: - str: directory name + Directory name """ from torch.hub import _parse_repo_info repo_owner, repo_name, branch = _parse_repo_info(repo) @@ -39,17 +39,17 @@ def _uri_to_dir_name(uri: str) -> str: return Path(uri).stem -def get_hubconf_dir_from_cfg(cfg, parent: Optional[str] = '') -> str: +def get_hubconf_dir_from_cfg(cfg, parent: str | None = '') -> str: """Determine destination directory name from an ExternalModuleConfig. If a parent path is provided, the dir name is appended to it. Args: cfg (ExternalModuleConfig): an ExternalModuleConfig - parent (str, optional): Parent path. Defaults to ''. + parent: Parent path. Defaults to ''. Returns: - str: directory name or path + Directory name or path """ if cfg.name is not None: dir_name = cfg.name @@ -65,13 +65,13 @@ def get_hubconf_dir_from_cfg(cfg, parent: Optional[str] = '') -> str: def torch_hub_load_github(repo: str, entrypoint: str, *args, - dst_dir: Optional[str] = None, + dst_dir: str | None = None, **kwargs) -> Any: """Load an entrypoint from a github repo using :func:`torch.hub.load`. Args: - repo (str): /[:tag] - entrypoint (str): Name of a callable present in hubconf.py. + repo: /[:tag] + entrypoint: Name of a Callable present in ``hubconf.py``. *args: Args to be passed to the entrypoint. dst_dir: If provided, the contents of the repo are copied there. Defaults to None. @@ -99,7 +99,7 @@ def torch_hub_load_github(repo: str, def torch_hub_load_uri(uri: str, entrypoint: str, *args, - dst_dir: Optional[str] = None, + dst_dir: str | None = None, **kwargs) -> Any: """Load an entrypoint from a uri. @@ -113,8 +113,8 @@ def torch_hub_load_uri(uri: str, latter case, the sub-directory will be copied to dst_dir. Args: - uri (str): A URI. - entrypoint (str): Name of a callable present in hubconf.py. + uri: A URI. + entrypoint: Name of a Callable present in ``hubconf.py``. *args: Args to be passed to the entrypoint. dst_dir: If provided, the contents from the uri are copied there. Defaults to None. diff --git a/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/utils.py b/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/utils.py index 09de0c73b..30c1251de 100644 --- a/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/utils.py +++ b/rastervision_pytorch_learner/rastervision/pytorch_learner/utils/utils.py @@ -1,5 +1,4 @@ -from typing import (TYPE_CHECKING, Any, Dict, Sequence, Tuple, Optional, Union, - List, Iterable, Container) +from typing import TYPE_CHECKING, Any, Container, Iterable, Self, Sequence from os.path import basename, join, isfile import logging @@ -25,7 +24,7 @@ log = logging.getLogger(__name__) -def color_to_triple(color: Optional[str] = None) -> Tuple[int, int, int]: +def color_to_triple(color: str | None = None) -> tuple[int, int, int]: """Given a PIL ImageColor string, return a triple of integers representing the red, green, and blue values. @@ -57,7 +56,7 @@ def compute_conf_mat(out: torch.Tensor, y: torch.Tensor, def compute_conf_mat_metrics(conf_mat: torch.Tensor, label_names: list[str], - ignore_idx: Optional[int] = None, + ignore_idx: int | None = None, eps: float = 1e-6): # eps is to avoid dividing by zero. eps = torch.tensor(eps) @@ -98,7 +97,7 @@ def compute_conf_mat_metrics(conf_mat: torch.Tensor, return metrics -def validate_albumentation_transform(tf_dict: Optional[dict]) -> dict: +def validate_albumentation_transform(tf_dict: dict | None) -> dict: """ Validate a serialized albumentation transform by attempting to deserialize it. """ @@ -122,8 +121,8 @@ def validate_albumentation_transform(tf_dict: Optional[dict]) -> dict: def serialize_albumentation_transform( tf: A.BasicTransform, - lambda_transforms_path: Optional[str] = None, - dst_dir: Optional[str] = None) -> dict: + lambda_transforms_path: str | None = None, + dst_dir: str | None = None) -> dict: """Serialize an albumentations transform to a dict. If the transform includes a Lambda transform, a `lambda_transforms_path` @@ -135,10 +134,10 @@ def serialize_albumentation_transform( Args: tf (A.BasicTransform): The transform to serialize. - lambda_transforms_path (Optional[str], optional): Path to a python file + lambda_transforms_path (str | None): Path to a python file that defines a dict named `lambda_transforms` as required by `A.from_dict()`. Defaults to None. - dst_dir (Optional[str], optional): Directory to copy the transforms + dst_dir (str | None): Directory to copy the transforms file to. Useful for copying the file to S3 when running on Batch. Defaults to None. @@ -291,8 +290,7 @@ def get_transform_init_args_names(self): def adjust_conv_channels(old_conv: nn.Conv2d, in_channels: int, - pretrained: bool = True - ) -> Union[nn.Conv2d, nn.Sequential]: + pretrained: bool = True) -> nn.Conv2d | nn.Sequential: if in_channels == old_conv.in_channels: return old_conv @@ -335,11 +333,11 @@ def adjust_conv_channels(old_conv: nn.Conv2d, new_conv.weight.data[:, :in_channels] = pretrained_kernels return new_conv else: - raise ConfigError(f'Something went wrong.') + raise ConfigError('Something went wrong.') def plot_channel_groups(axs: Iterable, - imgs: Iterable[Union[np.array, torch.Tensor]], + imgs: Iterable[np.ndarray | torch.Tensor], channel_groups: dict, plot_title: bool = True) -> None: for title, ax, img in zip(channel_groups.keys(), axs, imgs): @@ -352,7 +350,7 @@ def plot_channel_groups(axs: Iterable, def channel_groups_to_imgs( x: torch.Tensor, - channel_groups: Dict[str, Sequence[int]]) -> List[torch.Tensor]: + channel_groups: dict[str, Sequence[int]]) -> list[torch.Tensor]: imgs = [] for title, ch_inds in channel_groups.items(): img = x[..., ch_inds] @@ -373,7 +371,7 @@ def channel_groups_to_imgs( return imgs -def log_metrics_to_csv(csv_path: str, metrics: Dict[str, Any]): +def log_metrics_to_csv(csv_path: str, metrics: dict[str, Any]): """Append epoch metrics to CSV file.""" # dict --> single-row DataFrame metrics_df = pd.DataFrame.from_records([metrics]) @@ -384,8 +382,8 @@ def log_metrics_to_csv(csv_path: str, metrics: Dict[str, Any]): def aggregate_metrics( - outputs: List[Dict[str, Union[float, torch.Tensor]]], - exclude_keys: Container[str] = set('conf_mat')) -> Dict[str, float]: + outputs: list[dict[str, float | torch.Tensor]], + exclude_keys: Container[str] = set('conf_mat')) -> dict[str, float]: """Aggregate the output of validate_step at the end of the epoch. Args: @@ -394,7 +392,7 @@ def aggregate_metrics( be included in the output. Defaults to {'conf_mat'}. Returns: - Dict[str, float]: Dict with aggregated values. + dict[str, float]: Dict with aggregated values. """ metrics = {} metric_names = outputs[0].keys() @@ -482,15 +480,13 @@ def __init__(self, ort_session: 'ort.InferenceSession') -> None: self.input_key = inputs[0].name @classmethod - def from_file(cls, path: str, providers: Optional[List[str]] = None - ) -> 'ONNXRuntimeAdapter': + def from_file(cls, path: str, providers: list[str] | None = None) -> Self: """Construct from file. Args: - path (str): Path to a .onnx file. - providers (Optional[List[str]]): ONNX-runtime execution - providers. See onnxruntime documentation for more details. - Defaults to None. + path: Path to a .onnx file. + providers: ONNX-runtime execution providers. See ``onnxruntime`` + documentation for more details. Defaults to ``None``. Returns: ONNXRuntimeAdapter: An ONNXRuntimeAdapter instance. @@ -504,7 +500,7 @@ def from_file(cls, path: str, providers: Optional[List[str]] = None onnx_model = cls(ort_session) return onnx_model - def __call__(self, x: Union[torch.Tensor, np.ndarray]) -> torch.Tensor: + def __call__(self, x: torch.Tensor | np.ndarray) -> torch.Tensor: x = x.numpy() outputs = self.ort_session.run(None, {self.input_key: x}) out = outputs[0] diff --git a/tests/__init__.py b/tests/__init__.py index 75ffbbc9f..17688debc 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import os @@ -8,7 +8,8 @@ def data_file_path(rel_path: str) -> str: def test_config_upgrader(cfg_class: type, old_cfg_dict: dict, - upgrader: Callable, curr_version: int) -> None: + upgrader: Callable[[dict, int], dict], + curr_version: int) -> None: """Try to use upgrader to update cfg dict to curr_version.""" from rastervision.pipeline.config import build_config diff --git a/tests/core/data/label_store/test_chip_classification_geojson_store.py b/tests/core/data/label_store/test_chip_classification_geojson_store.py index 0df8b7b3b..7f2a2db38 100644 --- a/tests/core/data/label_store/test_chip_classification_geojson_store.py +++ b/tests/core/data/label_store/test_chip_classification_geojson_store.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable from os.path import join import unittest diff --git a/tests/core/data/label_store/test_object_detection_geojson_store.py b/tests/core/data/label_store/test_object_detection_geojson_store.py index d1df4c3ff..8bfe617d1 100644 --- a/tests/core/data/label_store/test_object_detection_geojson_store.py +++ b/tests/core/data/label_store/test_object_detection_geojson_store.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable from os.path import join import unittest diff --git a/tests/core/data/label_store/test_semantic_segmentation_label_store.py b/tests/core/data/label_store/test_semantic_segmentation_label_store.py index 8d3638797..8c512b008 100644 --- a/tests/core/data/label_store/test_semantic_segmentation_label_store.py +++ b/tests/core/data/label_store/test_semantic_segmentation_label_store.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable from os.path import join import unittest diff --git a/tests/core/data/raster_source/test_multi_raster_source.py b/tests/core/data/raster_source/test_multi_raster_source.py index 5dbbae8cb..ee0e4a24f 100644 --- a/tests/core/data/raster_source/test_multi_raster_source.py +++ b/tests/core/data/raster_source/test_multi_raster_source.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest import numpy as np diff --git a/tests/core/data/raster_source/test_rasterio_source.py b/tests/core/data/raster_source/test_rasterio_source.py index 6bea8a803..f51f38f53 100644 --- a/tests/core/data/raster_source/test_rasterio_source.py +++ b/tests/core/data/raster_source/test_rasterio_source.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from os.path import join from tempfile import NamedTemporaryFile diff --git a/tests/core/data/raster_source/test_temporal_multi_raster_source.py b/tests/core/data/raster_source/test_temporal_multi_raster_source.py index 22cf04058..752cd0b00 100644 --- a/tests/core/data/raster_source/test_temporal_multi_raster_source.py +++ b/tests/core/data/raster_source/test_temporal_multi_raster_source.py @@ -1,4 +1,4 @@ -from typing import Callable, List +from collections.abc import Callable import unittest import numpy as np @@ -10,7 +10,7 @@ XarraySource) -def make_raster_source(num_channels_raw: int, channel_order: List[int]): +def make_raster_source(num_channels_raw: int, channel_order: list[int]): dtype = np.uint8 arr = np.ones((5, 5, num_channels_raw), dtype=dtype) arr *= np.arange(num_channels_raw, dtype=dtype) diff --git a/tests/core/data/test_class_config.py b/tests/core/data/test_class_config.py index 25cc3bc8d..976bef527 100644 --- a/tests/core/data/test_class_config.py +++ b/tests/core/data/test_class_config.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from rastervision.core.data.class_config import ( diff --git a/tests/core/data/test_dataset.py b/tests/core/data/test_dataset.py index 2cae1e19a..5e3c6d717 100644 --- a/tests/core/data/test_dataset.py +++ b/tests/core/data/test_dataset.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from rastervision.core.data import (DatasetConfig, ClassConfig, SceneConfig, diff --git a/tests/core/data/utils/test_aoi_sampler.py b/tests/core/data/utils/test_aoi_sampler.py index 822adfdfa..e2e5aa9f0 100644 --- a/tests/core/data/utils/test_aoi_sampler.py +++ b/tests/core/data/utils/test_aoi_sampler.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from itertools import product @@ -43,7 +43,7 @@ def test_sampler(self, nsamples: int = 200): use a significance level of 0.05 here. Args: - nsamples (int, optional): Number of points to sample. It is + nsamples (int): Number of points to sample. It is important for the sample size to not be too large or the test will become over-powered. Defaults to 200. """ diff --git a/tests/core/data/utils/test_misc.py b/tests/core/data/utils/test_misc.py index f0c76fd47..da7123f5c 100644 --- a/tests/core/data/utils/test_misc.py +++ b/tests/core/data/utils/test_misc.py @@ -1,4 +1,5 @@ -from typing import Any, Callable, Tuple +from typing import Any +from collections.abc import Callable import unittest from os.path import join import json @@ -101,7 +102,7 @@ def __init__(self, dims: int, bbox: Box) -> None: self.dims = dims self.bbox = bbox - def __getitem__(self, key: Any) -> Tuple[Box, list]: + def __getitem__(self, key: Any) -> tuple[Box, list]: if self.dims == 2: return parse_array_slices_2d(key, self.bbox.extent) return parse_array_slices_Nd(key, self.bbox.extent, dims=self.dims) diff --git a/tests/core/data/vector_source/test_geojson_vector_source.py b/tests/core/data/vector_source/test_geojson_vector_source.py index 2becb1f78..5d9186d47 100644 --- a/tests/core/data/vector_source/test_geojson_vector_source.py +++ b/tests/core/data/vector_source/test_geojson_vector_source.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from os.path import join diff --git a/tests/core/evaluation/test_class_evaluation_item.py b/tests/core/evaluation/test_class_evaluation_item.py index e1f3084b2..731151761 100644 --- a/tests/core/evaluation/test_class_evaluation_item.py +++ b/tests/core/evaluation/test_class_evaluation_item.py @@ -99,7 +99,7 @@ def test_extra_info(self): self.assertEqual(json['extra1'], 'extra1') self.assertEqual(json['extra2'], 'extra2') - def from_multiclass_conf_mat(self): + def test_from_multiclass_conf_mat(self): conf_mat = np.random.randint(100, size=(10, 10)) item = ClassEvaluationItem.from_multiclass_conf_mat( class_id=3, diff --git a/tests/core/rv_pipeline/test_semantic_segmentation_config.py b/tests/core/rv_pipeline/test_semantic_segmentation_config.py index df44f32fb..91379fe1c 100644 --- a/tests/core/rv_pipeline/test_semantic_segmentation_config.py +++ b/tests/core/rv_pipeline/test_semantic_segmentation_config.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest import numpy as np diff --git a/tests/core/test_stats_analyzer.py b/tests/core/test_stats_analyzer.py index 921889d0c..f5ef15074 100644 --- a/tests/core/test_stats_analyzer.py +++ b/tests/core/test_stats_analyzer.py @@ -1,4 +1,3 @@ -from typing import Tuple import unittest from os.path import join @@ -14,7 +13,7 @@ def make_scene(i: int, is_random: bool = False - ) -> Tuple[Scene, MockRasterSource, np.ndarray]: + ) -> tuple[Scene, MockRasterSource, np.ndarray]: rs = MockRasterSource([0, 1, 2], 3) img = np.zeros((600, 600, 3)) img[:, :, 0] = 1 + i diff --git a/tests/core/utils/test_stac.py b/tests/core/utils/test_stac.py index 20d514c8d..844ecbcea 100644 --- a/tests/core/utils/test_stac.py +++ b/tests/core/utils/test_stac.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import os import unittest diff --git a/tests/data_files/lambda_transforms.py b/tests/data_files/lambda_transforms.py index f5ed2be65..c556b48f6 100644 --- a/tests/data_files/lambda_transforms.py +++ b/tests/data_files/lambda_transforms.py @@ -6,14 +6,14 @@ import numpy as np -def ndvi(rgb_nir: 'np.array', **kwargs) -> 'np.array': +def ndvi(rgb_nir: 'np.ndarray', **kwargs) -> 'np.ndarray': red = rgb_nir[..., 0] nir = rgb_nir[..., 3] ndvi = (nir - red) / (nir + red) return ndvi -def swap(image: 'np.array', **kwargs) -> 'np.array': +def swap(image: 'np.ndarray', **kwargs) -> 'np.ndarray': return image[..., [3, 4, 5, 0, 1, 2]] diff --git a/tests/pipeline/test_config.py b/tests/pipeline/test_config.py index 9dc99115b..8b68b73a8 100644 --- a/tests/pipeline/test_config.py +++ b/tests/pipeline/test_config.py @@ -1,4 +1,4 @@ -from typing import Callable, List +from collections.abc import Callable from os.path import join import unittest @@ -45,8 +45,8 @@ def c_upgrader(cfg_dict, version): @register_config('c', plugin='rastervision.c', upgrader=c_upgrader) class CConfig(PipelineConfig): - al: List[AConfig] - bl: List[BConfig] + al: list[AConfig] + bl: list[BConfig] a: AConfig b: BConfig x: str = 'x' diff --git a/tests/pytorch_learner/dataset/test_dataset.py b/tests/pytorch_learner/dataset/test_dataset.py index d09365a7b..a582e1c4c 100644 --- a/tests/pytorch_learner/dataset/test_dataset.py +++ b/tests/pytorch_learner/dataset/test_dataset.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable from os.path import join import unittest diff --git a/tests/pytorch_learner/dataset/visualizer/test_classification_visualizer.py b/tests/pytorch_learner/dataset/visualizer/test_classification_visualizer.py index 8b758caf5..8a55909ba 100644 --- a/tests/pytorch_learner/dataset/visualizer/test_classification_visualizer.py +++ b/tests/pytorch_learner/dataset/visualizer/test_classification_visualizer.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest import torch diff --git a/tests/pytorch_learner/dataset/visualizer/test_object_detection_visualizer.py b/tests/pytorch_learner/dataset/visualizer/test_object_detection_visualizer.py index eb4ff0f6d..5a5662bb3 100644 --- a/tests/pytorch_learner/dataset/visualizer/test_object_detection_visualizer.py +++ b/tests/pytorch_learner/dataset/visualizer/test_object_detection_visualizer.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest import torch diff --git a/tests/pytorch_learner/dataset/visualizer/test_regression_visualizer.py b/tests/pytorch_learner/dataset/visualizer/test_regression_visualizer.py index b299db5f3..2e8ed6ea4 100644 --- a/tests/pytorch_learner/dataset/visualizer/test_regression_visualizer.py +++ b/tests/pytorch_learner/dataset/visualizer/test_regression_visualizer.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest import torch diff --git a/tests/pytorch_learner/dataset/visualizer/test_semantic_segmentation_visualizer.py b/tests/pytorch_learner/dataset/visualizer/test_semantic_segmentation_visualizer.py index 6d48dc1a1..8eb8c03a7 100644 --- a/tests/pytorch_learner/dataset/visualizer/test_semantic_segmentation_visualizer.py +++ b/tests/pytorch_learner/dataset/visualizer/test_semantic_segmentation_visualizer.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest import torch diff --git a/tests/pytorch_learner/test_classification_learner.py b/tests/pytorch_learner/test_classification_learner.py index ba4433d7f..683824c2b 100644 --- a/tests/pytorch_learner/test_classification_learner.py +++ b/tests/pytorch_learner/test_classification_learner.py @@ -1,4 +1,5 @@ -from typing import Any, Callable +from typing import Any +from collections.abc import Callable import unittest from os.path import join from uuid import uuid4 diff --git a/tests/pytorch_learner/test_data_config.py b/tests/pytorch_learner/test_data_config.py index 0fbd578fc..895b7cd34 100644 --- a/tests/pytorch_learner/test_data_config.py +++ b/tests/pytorch_learner/test_data_config.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from rastervision.pipeline.file_system import get_tmp_dir diff --git a/tests/pytorch_learner/test_model_config.py b/tests/pytorch_learner/test_model_config.py index b579b73e5..2098f9d45 100644 --- a/tests/pytorch_learner/test_model_config.py +++ b/tests/pytorch_learner/test_model_config.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest import torch diff --git a/tests/pytorch_learner/test_object_detection_learner.py b/tests/pytorch_learner/test_object_detection_learner.py index 39406f2c9..1414e1947 100644 --- a/tests/pytorch_learner/test_object_detection_learner.py +++ b/tests/pytorch_learner/test_object_detection_learner.py @@ -1,4 +1,5 @@ -from typing import Any, Callable +from typing import Any +from collections.abc import Callable from os.path import join import unittest from uuid import uuid4 diff --git a/tests/pytorch_learner/test_regression_learner.py b/tests/pytorch_learner/test_regression_learner.py index 324f3d537..334c1ebf7 100644 --- a/tests/pytorch_learner/test_regression_learner.py +++ b/tests/pytorch_learner/test_regression_learner.py @@ -1,4 +1,5 @@ -from typing import Any, Callable +from typing import Any +from collections.abc import Callable import unittest from os.path import join from uuid import uuid4 diff --git a/tests/pytorch_learner/test_semantic_segmentation_learner.py b/tests/pytorch_learner/test_semantic_segmentation_learner.py index cb668526b..0ac261116 100644 --- a/tests/pytorch_learner/test_semantic_segmentation_learner.py +++ b/tests/pytorch_learner/test_semantic_segmentation_learner.py @@ -1,4 +1,5 @@ -from typing import Any, Callable +from typing import Any +from collections.abc import Callable import unittest from uuid import uuid4 diff --git a/tests/pytorch_learner/test_solver_config.py b/tests/pytorch_learner/test_solver_config.py index d10030781..5f5bf3515 100644 --- a/tests/pytorch_learner/test_solver_config.py +++ b/tests/pytorch_learner/test_solver_config.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from torch import nn diff --git a/tests/pytorch_learner/utils/test_torch_hub.py b/tests/pytorch_learner/utils/test_torch_hub.py index a05545c42..f58091054 100644 --- a/tests/pytorch_learner/utils/test_torch_hub.py +++ b/tests/pytorch_learner/utils/test_torch_hub.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from os.path import join, exists from os import makedirs diff --git a/tests/pytorch_learner/utils/test_utils.py b/tests/pytorch_learner/utils/test_utils.py index 8953f4061..5ec3b9421 100644 --- a/tests/pytorch_learner/utils/test_utils.py +++ b/tests/pytorch_learner/utils/test_utils.py @@ -1,4 +1,4 @@ -from typing import Callable +from collections.abc import Callable import unittest from os.path import join