Configurable HTTP timeout and retry

DOV-Vlaanderen · Jan 31, 2025 · 8b493ca · 8b493ca
1 parent 53fe0f0
commit 8b493ca
Show file tree

Hide file tree

Showing 3 changed files with 27 additions and 8 deletions.
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -9,6 +9,11 @@ on:
   schedule:
     - cron: "0 8 * * 2"
 
+env:
+  HTTP_REQUEST_CONNECT_TIMEOUT: 10
+  HTTP_REQUEST_READ_TIMEOUT: 20
+  HTTP_RETRY_COUNT: 3
+
 jobs:
   tests-linux:
     name: "Tests on ${{ matrix.python-version }} on ${{ matrix.os }}"

diff --git a/pydov/util/net.py b/pydov/util/net.py
@@ -11,8 +11,6 @@
 
 import pydov
 
-request_timeout = (30, 600)
-
 
 def proxy_autoconfiguration():
     """Try proxy autoconfiguration via PAC.
@@ -112,7 +110,21 @@ class TimeoutHTTPAdapter(HTTPAdapter):
 
     def __init__(self, *args, **kwargs):
         """Initialisation."""
-        self.timeout = request_timeout
+
+        request_timeout = os.environ.get('HTTP_REQUEST_TIMEOUT', None)
+        request_connect_timeout = os.environ.get(
+            'HTTP_REQUEST_CONNECT_TIMEOUT', None)
+        request_read_timeout = os.environ.get(
+            'HTTP_REQUEST_READ_TIMEOUT', None)
+
+        if request_connect_timeout is not None and request_read_timeout is not None:
+            self.timeout = (int(request_connect_timeout),
+                            int(request_read_timeout))
+        elif request_timeout is not None:
+            self.timeout = int(request_timeout)
+        else:
+            self.timeout = 600
+
         if "timeout" in kwargs:
             self.timeout = kwargs["timeout"]
             del kwargs["timeout"]
@@ -156,25 +168,27 @@ def get_session():
         """
         session = requests.Session()
 
+        retry_count = int(os.environ.get('HTTP_RETRY_COUNT', 10))
+
         session.headers.update(
             {'user-agent': 'pydov/{}'.format(pydov.__version__)})
 
         try:
             retry = urllib3.util.Retry(
-                total=10, connect=10, read=10, redirect=5, backoff_factor=1,
+                total=retry_count, connect=retry_count, read=retry_count, redirect=5, backoff_factor=1,
                 allowed_methods=set(
                     ['HEAD', 'GET', 'POST', 'PUT', 'OPTIONS']))
         except TypeError:
             # urllib3 < 1.26.0 used method_whitelist instead
             retry = urllib3.util.Retry(
-                total=10, connect=10, read=10, redirect=5, backoff_factor=1,
+                total=retry_count, connect=retry_count, read=retry_count, redirect=5, backoff_factor=1,
                 method_whitelist=set(
                     ['HEAD', 'GET', 'POST', 'PUT', 'OPTIONS']))
 
-        adapter = TimeoutHTTPAdapter(timeout=request_timeout,
-                                     max_retries=retry)
+        adapter = TimeoutHTTPAdapter(max_retries=retry)
         session.mount('http://', adapter)
         session.mount('https://', adapter)
+        session.timeout = adapter.timeout
         return session
 
 

diff --git a/pydov/util/owsutil.py b/pydov/util/owsutil.py
@@ -92,7 +92,7 @@ def get_remote_metadata(contentmetadata):
     """
     with warnings.catch_warnings():
         warnings.filterwarnings("ignore", category=FutureWarning)
-        contentmetadata.parse_remote_metadata(pydov.util.net.request_timeout)
+        contentmetadata.parse_remote_metadata(pydov.session.timeout)
 
     for remote_md in contentmetadata.metadataUrls:
         if 'metadata' in remote_md and remote_md['metadata'] is not None: