changed default download chunk size to 10MB

CHANGELOG.md

@@ -10,8 +10,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 ### Changed
+- Changed default `chunk_size` of various `download` functions from None to 10MB. This improves the handling of large downloads and reduces memory usage.
 - `Connection.execute()` and `DataCube.execute()` now have a `auto_decode` argument. If set to True (default) the response will be decoded as a JSON and throw an exception if this fails, if set to False the raw `requests.Response` object will be returned. ([#499](https://github.com/Open-EO/openeo-python-client/issues/499))
-
 ### Removed
 
 ### Fixed

openeo/rest/__init__.py

@@ -2,6 +2,9 @@
 
 from openeo import BaseOpenEoException
 
+# TODO: get from config file
+DEFAULT_DOWNLOAD_CHUNK_SIZE = 10000000  # 10MB
+
 
 class OpenEoClientException(BaseOpenEoException):
     """Base class for OpenEO client exceptions"""

openeo/rest/connection.py

@@ -51,6 +51,7 @@
     OidcRefreshTokenAuthenticator,
     OidcResourceOwnerPasswordAuthenticator,
 )
+from openeo.rest import DEFAULT_DOWNLOAD_CHUNK_SIZE
 from openeo.rest.datacube import DataCube, InputDate
 from openeo.rest.graph_building import CollectionProperty
 from openeo.rest.job import BatchJob, RESTJob
@@ -1566,7 +1567,7 @@ def download(
 
         if outputfile is not None:
             with Path(outputfile).open(mode="wb") as f:
-                for chunk in response.iter_content(chunk_size=None):
+                for chunk in response.iter_content(chunk_size=DEFAULT_DOWNLOAD_CHUNK_SIZE):
                     f.write(chunk)
         else:
             return response.content

openeo/rest/job.py

@@ -19,7 +19,13 @@
     render_error,
 )
 from openeo.internal.warnings import deprecated, legacy_alias
-from openeo.rest import JobFailedException, OpenEoApiError, OpenEoClientException, OpenEoApiPlainError
+from openeo.rest import (
+    JobFailedException,
+    OpenEoApiError,
+    OpenEoClientException,
+    OpenEoApiPlainError,
+    DEFAULT_DOWNLOAD_CHUNK_SIZE,
+)
 from openeo.util import ensure_dir
 
 if typing.TYPE_CHECKING:
@@ -351,14 +357,17 @@ def __repr__(self):
             n=self.name, t=self.metadata.get("type", "unknown"), h=self.href
         )
 
-    def download(self, target: Optional[Union[Path, str]] = None, chunk_size=None) -> Path:
+    def download(self, target: Optional[Union[Path, str]] = None, chunk_size: Optional[int] = None) -> Path:
         """
         Download asset to given location
 
         :param target: download target path. Can be an existing folder
             (in which case the filename advertised by backend will be used)
             or full file name. By default, the working directory will be used.
+        :param chunk_size: size of chunks to download (in bytes). If None, connection.DEFAULT_DOWNLOAD_CHUNK_SIZE will be used.
         """
+        if not chunk_size:
+            chunk_size = DEFAULT_DOWNLOAD_CHUNK_SIZE
         target = Path(target or Path.cwd())
         if target.is_dir():
             target = target / self.name

openeo/rest/userfile.py

@@ -5,6 +5,7 @@
 from typing import Any, Dict, Optional, Union
 
 from openeo.util import ensure_dir
+from openeo.rest import DEFAULT_DOWNLOAD_CHUNK_SIZE
 
 if typing.TYPE_CHECKING:
     # Imports for type checking only (circular import issue at runtime).
@@ -66,7 +67,7 @@ def download(self, target: Union[Path, str] = None) -> Path:
         ensure_dir(target.parent)
 
         with target.open(mode="wb") as f:
-            for chunk in response.iter_content(chunk_size=None):
+            for chunk in response.iter_content(chunk_size=DEFAULT_DOWNLOAD_CHUNK_SIZE):
                 f.write(chunk)
 
         return target