Merge pull request #334 from ocefpaf/increase_max_itemsPerPage

Increase max ItemsPerPage
ioos · Feb 26, 2024 · 21e3f3c · 21e3f3c
2 parents 6e467c1 + d6911ac
commit 21e3f3c
Show file tree

Hide file tree

Showing 11 changed files with 3,965 additions and 10 deletions.
diff --git a/erddapy/core/url.py b/erddapy/core/url.py
@@ -102,7 +102,7 @@ def _distinct(url: str, distinct: Optional[bool] = False) -> str:
 
 def _format_search_string(server: str, query: str) -> str:
     """Generate a search string for an erddap server with user defined query."""
-    return f'{server}search/index.csv?page=1&itemsPerPage=100000&searchFor="{query}"'
+    return f'{server}search/index.csv?page=1&itemsPerPage=1000000&searchFor="{query}"'
 
 
 def _multi_urlopen(url: str) -> BinaryIO:
@@ -172,7 +172,7 @@ def get_search_url(
     response: str = "html",
     search_for: Optional[str] = None,
     protocol: str = "tabledap",
-    items_per_page: int = 1000,
+    items_per_page: int = 1_000_000,
     page: int = 1,
     **kwargs,
 ):
@@ -198,7 +198,7 @@ def get_search_url(
 
         response: default is HTML.
         items_per_page: how many items per page in the return,
-            default is 1000 for HTML, 1e6 (hopefully all items) for CSV, JSON.
+            default is 1_000_000 for HTML, 1e6 (hopefully all items) for CSV, JSON.
         page: which page to display, default is the first page (1).
         kwargs: extra search constraints based on metadata and/or coordinates ke/value.
             metadata: `cdm_data_type`, `institution`, `ioos_category`,
@@ -263,7 +263,7 @@ def get_search_url(
             kwargs.update({search_term: lowercase})
 
     # These responses should not be paginated b/c that hinders the correct amount of data silently
-    # and can surprise users when the number of items is greater than ERDDAP's defaults (1000 items).
+    # and can surprise users when the number of items is greater than ERDDAP's defaults (1_000_000 items).
     # Ideally there should be no pagination for this on the ERDDAP side but for now we settled for a
     # "really big" `items_per_page` number.
     non_paginated_responses = [

diff --git a/erddapy/erddapy.py b/erddapy/erddapy.py
@@ -181,7 +181,7 @@ def get_search_url(
         response: OptionalStr = None,
         search_for: OptionalStr = None,
         protocol: OptionalStr = None,
-        items_per_page: int = 1000,
+        items_per_page: int = 1_000_000,
         page: int = 1,
         **kwargs,
     ) -> str:
@@ -207,7 +207,7 @@ def get_search_url(
 
             response: default is HTML.
             items_per_page: how many items per page in the return,
-                default is 1000 for HTML, 1e6 (hopefully all items) for CSV, JSON.
+                default is 1_000_000 for HTML, 1e6 (hopefully all items) for CSV, JSON.
             page: which page to display, default is the first page (1).
             kwargs: extra search constraints based on metadata and/or coordinates ke/value.
                 metadata: `cdm_data_type`, `institution`, `ioos_category`,

diff --git a/tests/cassettes/test_fetch_no_results.yaml b/tests/cassettes/test_fetch_no_results.yaml
@@ -124,4 +124,56 @@ interactions:
       - dods/3.7
     http_version: HTTP/1.1
     status_code: 404
+- request:
+    body: ''
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate, br
+      connection:
+      - keep-alive
+      host:
+      - standards.sensors.ioos.us
+      user-agent:
+      - python-httpx/0.27.0
+    method: GET
+    uri: https://standards.sensors.ioos.us/erddap/search/index.csv?page=1&itemsPerPage=1000000&searchFor=%22incredibly_long_string_that_should_never_match_a_real_dataset%22%20
+  response:
+    body:
+      string: "Error {\n    code=404;\n    message=\"Not Found: Resource not found:
+        Your query produced no matching results. Check the spelling of the word(s)
+        you searched for.\";\n}\n"
+    headers:
+      Connection:
+      - keep-alive
+      Content-Description:
+      - dods-error
+      Content-Encoding:
+      - identity
+      Content-Type:
+      - text/plain;charset=UTF-8
+      Date:
+      - Thu, 22 Feb 2024 18:28:29 GMT
+      Last-Modified:
+      - Thu, 22 Feb 2024 18:28:13 GMT
+      Server:
+      - nginx/1.22.1
+      Strict-Transport-Security:
+      - max-age=0
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-XSS-Protection:
+      - 1; mode=block
+      erddap-server:
+      - '2.23'
+      xdods-server:
+      - dods/3.7
+    status:
+      code: 404
+      message: ''
 version: 1
diff --git a/tests/cassettes/test_fetch_results.yaml b/tests/cassettes/test_fetch_results.yaml
@@ -182,4 +182,63 @@ interactions:
       - chunked
     http_version: HTTP/1.1
     status_code: 200
+- request:
+    body: ''
+    headers:
+      accept:
+      - '*/*'
+      accept-encoding:
+      - gzip, deflate, br
+      connection:
+      - keep-alive
+      host:
+      - standards.sensors.ioos.us
+      user-agent:
+      - python-httpx/0.27.0
+    method: GET
+    uri: https://standards.sensors.ioos.us/erddap/search/index.csv?page=1&itemsPerPage=1000000&searchFor=%22sea_water_temperature%22
+  response:
+    body:
+      string: !!binary |
+        H4sIAAAAAAAA/6WWbU/bMBDH3/MpLCZBkUzSMjYEUjWVFlilMVhbbdKEZJn4mlrkaT6Hh336nUPa
+        tWlKWyHlRXIPP//PdnwOjVZKZnyY3yNYbuV9BO77Wj4A67ArI7MJf4qRj3UEyEfaRkDBcSzNC7+8
+        6nV5f3jDWqet1ifeT8YpP5fBQ2jSPFGs+B4Mh/wiljoiN1ptc6vThPeklTQe6/d2+MTa7Mz30cpE
+        SaPQQ0gwNejpNEUvRx+Mk+hPtfmpCUWQmjgTgcyOPHyV/k7Mu2WExVRtjCnms8Lw+XGreXTKfNbt
+        3B6xrszAIPsO0uAkNcB3RzoGBKMBmaIZZGOTxmy/MZ91sJS2zxqLwxzcJXdJoGLhGMK+ZMDazKGH
+        Bfou+dkZ9Dvn3y6GZxRnyGDJyRoIQZooZKiTAGjNT5qHzRY9o2bzrHh+E5kKdyvMGmtUUWgk3X5Q
+        RFYQGgAUSWrsxHnSJKy4QKIlz1/W6ESvaZzFZJDaCAsxwaXNzSxBdCFCneNyhPgTCBmGxNGGjeYy
+        f3QGo5se64SUH0pLhlxG2r6wy0iG9RwLaHElqZ8o/agVUchLgSUjM4BYSD2XtH5gjQ7YbWmkonQU
+        6XuapUr4THdN1kba50Gl8DdQNeINuBV7BDHJY60cvUElB5DYOudM76D0sK/TtHVqa1ml5JW0GsEI
+        UjwR39UuA6sDGQmkkZJC+7B846wFhx/XhM+qmWatLWIdraynynuzjIWN/suZ5vcdZ0t7vzZ1VssS
+        YYui6v+ElcSasp50ogRmAEqkYxHmaCmfbOyKXuk/8PCwVRv2X/80eq3ueshUdBXzpljWqFG2oGhY
+        hG0hqSJlIX+VFnf0C6UNBK/HbXlO1rsX9V26rtGbpW4kdJk3r7ieWCN9d+P+SMeSdP3JH4cq8J/j
+        qNIrhbN7ZN+eqDEtLix11KlvK7Kmi061ldO0wLMX4GNBQYepJH+YCfrcPG6d+GXn3HhUg9X7g0em
+        jdPdtSkwOnNjoi+V8iY2jr6o15tZv9deZO9R1366MIZI7bGMEPbA3eravHszuL7li8E7/wCpE6pq
+        VwoAAA==
+    headers:
+      Connection:
+      - keep-alive
+      Content-Disposition:
+      - attachment;filename=search.csv
+      Content-Encoding:
+      - gzip
+      Content-Type:
+      - text/csv;charset=ISO-8859-1
+      Date:
+      - Thu, 22 Feb 2024 17:19:37 GMT
+      Server:
+      - nginx/1.22.1
+      Strict-Transport-Security:
+      - max-age=0
+      Transfer-Encoding:
+      - chunked
+      X-Content-Type-Options:
+      - nosniff
+      X-Frame-Options:
+      - SAMEORIGIN
+      X-XSS-Protection:
+      - 1; mode=block
+    status:
+      code: 200
+      message: ''
 version: 1