diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f769c458..9fa69fa1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,11 @@ on: schedule: - cron: "0 8 * * 2" +env: + HTTP_REQUEST_CONNECT_TIMEOUT: 10 + HTTP_REQUEST_READ_TIMEOUT: 20 + HTTP_RETRY_COUNT: 3 + jobs: tests-linux: name: "Tests on ${{ matrix.python-version }} on ${{ matrix.os }}" diff --git a/pydov/util/net.py b/pydov/util/net.py index 4dbe25dd..5346e5c9 100644 --- a/pydov/util/net.py +++ b/pydov/util/net.py @@ -11,8 +11,6 @@ import pydov -request_timeout = (30, 600) - def proxy_autoconfiguration(): """Try proxy autoconfiguration via PAC. @@ -112,7 +110,21 @@ class TimeoutHTTPAdapter(HTTPAdapter): def __init__(self, *args, **kwargs): """Initialisation.""" - self.timeout = request_timeout + + request_timeout = os.environ.get('HTTP_REQUEST_TIMEOUT', None) + request_connect_timeout = os.environ.get( + 'HTTP_REQUEST_CONNECT_TIMEOUT', None) + request_read_timeout = os.environ.get( + 'HTTP_REQUEST_READ_TIMEOUT', None) + + if request_connect_timeout is not None and request_read_timeout is not None: + self.timeout = (int(request_connect_timeout), + int(request_read_timeout)) + elif request_timeout is not None: + self.timeout = int(request_timeout) + else: + self.timeout = 600 + if "timeout" in kwargs: self.timeout = kwargs["timeout"] del kwargs["timeout"] @@ -156,25 +168,27 @@ def get_session(): """ session = requests.Session() + retry_count = int(os.environ.get('HTTP_RETRY_COUNT', 10)) + session.headers.update( {'user-agent': 'pydov/{}'.format(pydov.__version__)}) try: retry = urllib3.util.Retry( - total=10, connect=10, read=10, redirect=5, backoff_factor=1, + total=retry_count, connect=retry_count, read=retry_count, redirect=5, backoff_factor=1, allowed_methods=set( ['HEAD', 'GET', 'POST', 'PUT', 'OPTIONS'])) except TypeError: # urllib3 < 1.26.0 used method_whitelist instead retry = urllib3.util.Retry( - total=10, connect=10, read=10, redirect=5, backoff_factor=1, + total=retry_count, connect=retry_count, read=retry_count, redirect=5, backoff_factor=1, method_whitelist=set( ['HEAD', 'GET', 'POST', 'PUT', 'OPTIONS'])) - adapter = TimeoutHTTPAdapter(timeout=request_timeout, - max_retries=retry) + adapter = TimeoutHTTPAdapter(max_retries=retry) session.mount('http://', adapter) session.mount('https://', adapter) + session.timeout = adapter.timeout return session diff --git a/pydov/util/owsutil.py b/pydov/util/owsutil.py index 9331bd2b..a33eed11 100644 --- a/pydov/util/owsutil.py +++ b/pydov/util/owsutil.py @@ -92,7 +92,7 @@ def get_remote_metadata(contentmetadata): """ with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=FutureWarning) - contentmetadata.parse_remote_metadata(pydov.util.net.request_timeout) + contentmetadata.parse_remote_metadata(pydov.session.timeout) for remote_md in contentmetadata.metadataUrls: if 'metadata' in remote_md and remote_md['metadata'] is not None: