Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix unprediction of cog files #114

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion aiocogeo/cog.py
Original file line number Diff line number Diff line change
Expand Up @@ -470,7 +470,9 @@ async def get_tile(
self, x: int, y: int, z: int, reducer: Optional[ReduceType] = None
) -> List[np.ndarray]:
"""Fetch a tile from all readers"""
tiles = await self.map(func=lambda r: r.get_tile(x, y, z),)
tiles = await self.map(
func=lambda r: r.get_tile(x, y, z),
)
reducer = reducer or self.default_reducer
return reducer(tiles)

Expand Down
8 changes: 6 additions & 2 deletions aiocogeo/compression.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,12 +60,16 @@ def _decompress_mask(self, tile: bytes) -> np.ndarray:

def _reshape(self, arr: np.ndarray) -> np.ndarray:
"""Internal method to reshape an array to the size expected by the IFD"""
return arr.reshape(self.TileHeight.value, self.TileWidth.value, self.bands,)
return arr.reshape(
self.TileHeight.value,
self.TileWidth.value,
self.bands,
)

def _unpredict(self, arr: np.ndarray) -> None:
"""Internal method to unpredict if there is horizontal differencing"""
if self.Predictor.value == 2:
imagecodecs.delta_decode(arr, out=arr, axis=-1)
imagecodecs.delta_decode(arr, out=arr, axis=-2)

def _jpeg(self, tile: bytes) -> np.ndarray:
"""Internal method to decompress JPEG image bytes and convert to numpy array"""
Expand Down
20 changes: 11 additions & 9 deletions aiocogeo/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,24 +20,26 @@

# https://trac.osgeo.org/gdal/wiki/ConfigOptions#VSI_CACHE
# Determines if in-memory block caching is enabled
ENABLE_BLOCK_CACHE: bool = True if os.getenv(
"ENABLE_BLOCK_CACHE", "TRUE"
).upper() == "TRUE" else False
ENABLE_BLOCK_CACHE: bool = (
True if os.getenv("ENABLE_BLOCK_CACHE", "TRUE").upper() == "TRUE" else False
)


# enable caching of header requests, similar to GDAL's VSI CACHE
# https://trac.osgeo.org/gdal/wiki/ConfigOptions#VSI_CACHE
ENABLE_HEADER_CACHE: bool = True if os.getenv(
"ENABLE_HEADER_CACHE", "TRUE"
).upper() == "TRUE" else False
ENABLE_HEADER_CACHE: bool = (
True if os.getenv("ENABLE_HEADER_CACHE", "TRUE").upper() == "TRUE" else False
)


# https://trac.osgeo.org/gdal/wiki/ConfigOptions#GDAL_HTTP_MERGE_CONSECUTIVE_RANGES
# Determines if consecutive range requests are merged into a single request, reducing the number of HTTP GET range
# requests required to read consecutive internal image tiles
HTTP_MERGE_CONSECUTIVE_RANGES: bool = True if os.getenv(
"HTTP_MERGE_CONSECUTIVE_RANGES", "FALSE"
).upper() == "TRUE" else False
HTTP_MERGE_CONSECUTIVE_RANGES: bool = (
True
if os.getenv("HTTP_MERGE_CONSECUTIVE_RANGES", "FALSE").upper() == "TRUE"
else False
)


# Determines if internal tiles outside the bounds of the IFD are read. When set to ``TRUE`` (default), if a partial read
Expand Down
28 changes: 28 additions & 0 deletions tests/test_reads.py
Original file line number Diff line number Diff line change
Expand Up @@ -404,3 +404,31 @@ async def test_inject_session(create_cog_reader):
# Confirm session is still open
assert not session.closed
assert session._trace_configs


@pytest.mark.asyncio
async def test_cog_read_with_prediction_level_2(create_cog_reader):
infile = "https://sentinel-cogs.s3.us-west-2.amazonaws.com/sentinel-s2-l2a-cogs/22/J/DN/2021/6/S2B_22JDN_20210601_0_L2A/TCI.tif"

async with create_cog_reader(infile) as cog:
# Read top left tile at native resolution
tile = await cog.get_tile(0, 0, 0)
ifd = cog.ifds[0]

# Make sure tile is the right size
assert tile.shape == (
ifd.SamplesPerPixel.value,
ifd.TileHeight.value,
ifd.TileWidth.value,
)

with rasterio.open(infile) as src:
window = Window(0, 0, ifd.TileWidth.value, ifd.TileHeight.value)
rio_tile = src.read(window=window)
# Make sure image data is the same
tile_arr = np.ma.getdata(tile)
import hashlib

rio_tile_hash = hashlib.md5(tile_arr.tobytes()).hexdigest()
tile_arr_hash = hashlib.md5(rio_tile.tobytes()).hexdigest()
assert rio_tile_hash == tile_arr_hash
4 changes: 3 additions & 1 deletion tests/test_stac.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,9 @@

@pytest.mark.asyncio
async def test_stac_reader():
async with STACReader(filepath=STAC_ITEM,) as reader:
async with STACReader(
filepath=STAC_ITEM,
) as reader:
assert len(reader.readers) == 5


Expand Down