Merge pull request #602 from ungarj/mark_tests

add pytest markers 'integration' and 'aws_s3'

pyproject.toml

@@ -46,7 +46,7 @@ dependencies = [
 [project.optional-dependencies]
 complete = [
     "aiohttp",
-    "aiobotocore>=1.1.2",
+    "aiobotocore>=1.1.2,!=2.7.0",
     "boto3>=1.14.44",
     "dask",
     "distributed",
@@ -81,7 +81,7 @@ http = [
     "requests",
 ]
 s3 = [
-    "aiobotocore>=1.1.2",
+    "aiobotocore>=1.1.2,!=2.7.0",
     "boto3>=1.14.44",
     "fsspec[s3]",
     "s3fs!=2023.9.0",

pytest.ini

@@ -1,6 +1,7 @@
 [pytest]
 markers =
-    remote: marks tests which require acces to remote resources (deselect with '-m "not remote"')
+    aws_s3: marks tests which need to acces the AWS S3 test bucket (deselect with '-m "not aws_s3"')
+    integration: marks tests which require the running docker containers from test/docker-compose.yml (deselect with '-m "not integration"')
 env =
     D:AK1=foo
     D:AK2=bar

requirements.txt

@@ -1,4 +1,4 @@
-aiobotocore>=1.1.2
+aiobotocore>=1.1.2,!=2.7.0
 aiohttp>=3.6.2
 aioitertools>=0.7.0
 boto3>=1.14.44

test/test_cli.py

@@ -521,6 +521,7 @@ def test_convert_single_gtiff_overviews(cleantopo_br_tif, mp_tmpdir):
         assert src.overviews(1)
 
 
+@pytest.mark.integration
 def test_convert_remote_single_gtiff(http_raster, mp_tmpdir):
     """Automatic geodetic tile pyramid creation of raster files."""
     single_gtiff = mp_tmpdir / "single_out.tif"
@@ -1365,6 +1366,7 @@ def test_cp(mp_tmpdir, cleantopo_br, wkt_geom):
     )
 
 
+@pytest.mark.integration
 def test_cp_http(mp_tmpdir, http_tiledir):
     # copy tiles and subset by bounds
     run_cli(
@@ -1476,6 +1478,7 @@ def test_stac_mapchete_file(cleantopo_br):
     run_cli(["stac", "create-item", cleantopo_br.path, "-z", "5", "--force"])
 
 
+@pytest.mark.integration
 def test_stac_tiledir(http_tiledir, mp_tmpdir):
     run_cli(
         [

test/test_commands.py

@@ -49,7 +49,7 @@ def test_cp(mp_tmpdir, cleantopo_br, wkt_geom):
     assert len(tiles)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_cp_http(mp_tmpdir, http_tiledir):
     # copy tiles and subset by bounds
     tiles = cp(
@@ -308,7 +308,7 @@ def test_convert_single_gtiff_overviews(cleantopo_br_tif, mp_tmpdir):
         assert src.overviews(1)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_convert_remote_single_gtiff(http_raster, mp_tmpdir):
     """Automatic geodetic tile pyramid creation of raster files."""
     single_gtiff = os.path.join(mp_tmpdir, "single_out.tif")

test/test_formats.py

@@ -146,7 +146,7 @@ def test_base_format_classes():
         tmp.open(None, None)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_http_rasters(files_bounds, http_raster):
     """Raster file on remote server with http:// or https:// URLs."""
     zoom = 13

test/test_formats_flatgeobuf.py

@@ -70,7 +70,7 @@ def test_output_data(mp_tmpdir, flatgeobuf):
         assert len(read_output)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_output_data(flatgeobuf_s3):
     """Check FlatGeobuf as output data."""
     output_params = dict(
@@ -87,7 +87,7 @@ def test_s3_output_data(flatgeobuf_s3):
     assert isinstance(output_params, dict)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_output_data_rw(flatgeobuf_s3):
     with mapchete.open(flatgeobuf_s3.dict) as mp:
         tile = mp.config.process_pyramid.tile(4, 3, 7)

test/test_formats_geobuf.py

@@ -93,7 +93,7 @@ def test_output_data(mp_tmpdir, geobuf):
         assert len(read_output)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_output_data(geobuf_s3):
     """Check Geobuf as output data."""
     output_params = dict(
@@ -110,7 +110,7 @@ def test_s3_output_data(geobuf_s3):
     assert isinstance(output_params, dict)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_output_data_rw(geobuf_s3):
     with mapchete.open(geobuf_s3.dict) as mp:
         tile = mp.config.process_pyramid.tile(4, 3, 7)

test/test_formats_geojson.py

@@ -91,7 +91,7 @@ def test_output_data(mp_tmpdir, geojson):
         assert len(read_output)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_output_data(geojson_s3):
     """Check GeoJSON as output data."""
     output_params = dict(
@@ -108,7 +108,7 @@ def test_s3_output_data(geojson_s3):
     assert isinstance(output_params, dict)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_output_data_rw(geojson_s3):
     with mapchete.open(geojson_s3.dict) as mp:
         tile = mp.config.process_pyramid.tile(4, 3, 7)

test/test_formats_geotiff.py

@@ -183,7 +183,7 @@ def test_write_geotiff_tags(mp_tmpdir, cleantopo_br, write_rasterfile_tags_py):
                 assert src.tags(1)["band_tag"] == "True"
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_write_output_data(gtiff_s3, s3_example_tile):
     """Write and read output."""
     with mapchete.open(gtiff_s3.dict) as mp:
@@ -331,7 +331,7 @@ def test_output_single_gtiff_overviews(output_single_gtiff):
             assert not a.mask.all()
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_output_single_gtiff_s3(output_single_gtiff_s3):
     tile_id = (5, 3, 7)
     with mapchete.open(output_single_gtiff_s3.dict) as mp:
@@ -359,7 +359,7 @@ def test_output_single_gtiff_s3(output_single_gtiff_s3):
     assert path_exists(mp.config.output.path)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_output_single_gtiff_s3_tempfile(output_single_gtiff_s3):
     tile_id = (5, 3, 7)
     with mapchete.open(
@@ -462,7 +462,7 @@ def test_output_single_gtiff_cog_tempfile(output_single_gtiff_cog):
     assert cog_validate(mp.config.output.path, strict=True)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 @pytest.mark.skipif(
     not GDAL_COG_AVAILABLE, reason="GDAL>=3.1 with COG driver is required"
 )

test/test_formats_png.py

@@ -69,7 +69,7 @@ def test_output_data(mp_tmpdir):
         output.write(tile, np.zeros((5,) + tile.shape))
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_write_output_data(mp_s3_tmpdir):
     """Write and read output."""
     output_params = dict(

test/test_formats_png_hillshade.py

@@ -95,7 +95,7 @@ def test_output_data(mp_tmpdir):
     # TODO for_web
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_write_output_data(mp_s3_tmpdir):
     """Write and read output."""
     output_params = dict(

test/test_index.py

@@ -8,7 +8,7 @@
 from mapchete.io import fiona_open, rasterio_open
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_remote_indexes(gtiff_s3):
     zoom = 7
     gtiff_s3.dict.update(zoom_levels=zoom)

test/test_io.py

@@ -28,12 +28,12 @@ def test_best_zoom_level(dummy1_tif):
     assert get_best_zoom_level(dummy1_tif, "mercator")
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_s3_path_exists(raster_4band_s3):
     assert path_exists(raster_4band_s3)
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 def test_remote_path_exists(http_raster):
     assert path_exists(http_raster)
     assert not path_exists(http_raster / "non_existing.tif")
@@ -56,7 +56,7 @@ def test_absolute_path():
     )
 
 
-@pytest.mark.remote
+@pytest.mark.integration
 @pytest.mark.parametrize(
     "path",
     [
@@ -68,6 +68,7 @@ def test_read_remote_json(path):
     assert isinstance(read_json(path), dict)
 
 
+@pytest.mark.integration
 @pytest.mark.parametrize(
     "path",
     [
@@ -242,6 +243,7 @@ def test_tiles_exist_local(example_mapchete):
         assert set(output_tiles) == existing.union(not_existing)
 
 
+@pytest.mark.integration
 def test_tiles_exist_s3(gtiff_s3):
     bounds = (0, 0, 10, 10)
     # bounds = (3, 1, 4, 2)