diff --git a/docs/api_reference/core.rst b/docs/api_reference/core.rst index 106fe39dd..64946d647 100644 --- a/docs/api_reference/core.rst +++ b/docs/api_reference/core.rst @@ -81,9 +81,10 @@ Misc EODataAccessGateway.group_by_extent EODataAccessGateway.guess_product_type EODataAccessGateway.list_queryables + EODataAccessGateway.available_sortables .. autoclass:: eodag.api.core.EODataAccessGateway :members: set_preferred_provider, get_preferred_provider, update_providers_config, list_product_types, available_providers, search, search_all, search_iter_page, crunch, download, download_all, serialize, deserialize, deserialize_and_register, load_stac_items, group_by_extent, guess_product_type, get_cruncher, - update_product_types_list, fetch_product_types_list, discover_product_types, list_queryables + update_product_types_list, fetch_product_types_list, discover_product_types, list_queryables, available_sortables diff --git a/docs/notebooks/api_user_guide/4_search.ipynb b/docs/notebooks/api_user_guide/4_search.ipynb index 86c6967b7..ade3ccabb 100644 --- a/docs/notebooks/api_user_guide/4_search.ipynb +++ b/docs/notebooks/api_user_guide/4_search.ipynb @@ -146,10 +146,15 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:03:30,087 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:03:30,089 eodag.plugins.search.qssearch [INFO ] Sending count request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=1&page=1\n", - "2023-01-13 11:03:32,525 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", - "2023-01-13 11:03:33,810 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" + "2024-02-06 16:11:46,094 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:11:46,097 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-06 16:11:48,189 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" ] } ], @@ -183,10 +188,9 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:03:38,323 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:03:38,326 eodag.plugins.search.qssearch [INFO ] Sending count request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=1&page=1\n", - "2023-01-13 11:03:38,824 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=10&page=2\n", - "2023-01-13 11:03:39,756 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" + "2024-02-06 16:11:48,220 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:11:48,224 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=10&page=2\n", + "2024-02-06 16:11:49,625 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" ] } ], @@ -233,7 +237,7 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "The `raise_errors` parameter controls how errors raised internally during a search are propagated to the user. By default this parameter is set to `False`, which means that **errors are not raised**. Instead, errors are logged and a null result is returned (empty [SearchResult](../../api_reference/searchresult.rst#eodag.api.search_result.SearchResult) and 0)." + "The `raise_errors` parameter controls how errors raised internally during a search are propagated to the user. By default this parameter is set to `False`, which means that **errors are not raised**. Instead, errors are logged and a null result is returned (empty [SearchResult](../../api_reference/searchresult.rst#eodag.api.search_result.SearchResult) and 0). The use of the `provider` kwarg and the error raised in the example below are explained in [the id and provider sub-section](#id-and-provider) and [the fallback section](#fallback-in-case-of-error) respectively." ] }, { @@ -255,24 +259,42 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:04:01,626 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:04:01,629 eodag.core [INFO ] No result from provider 'peps' due to an error during search. Raise verbosity of log messages for details\n", - "2023-01-13 11:04:01,631 eodag.core [ERROR ] Error while searching on provider peps (ignored):\n", + "2024-02-06 16:11:49,661 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:11:49,663 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "2024-02-06 16:11:50,430 eodag.search.qssearch [ERROR ] Skipping error while searching for peps QueryStringSearch instance: \n", "Traceback (most recent call last):\n", - " File \"/home/sylvain/workspace/eodag/eodag/api/core.py\", line 1389, in _do_search\n", - " res, nb_res = search_plugin.query(count=count, **kwargs)\n", - " File \"/home/sylvain/workspace/eodag/eodag/plugins/search/qssearch.py\", line 384, in query\n", - " self.search_urls, total_items = self.collect_search_urls(\n", - " File \"/home/sylvain/workspace/eodag/eodag/plugins/search/qssearch.py\", line 582, in collect_search_urls\n", - " for collection in self.get_collections(**kwargs):\n", - " File \"/home/sylvain/workspace/eodag/eodag/plugins/search/qssearch.py\", line 812, in get_collections\n", - " match = re.match(\n", - "AttributeError: 'NoneType' object has no attribute 'groupdict'\n" + " File \"/home/anesson/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py\", line 908, in _request\n", + " response.raise_for_status()\n", + " File \"/home/anesson/.virtualenvs/eodag_docs/lib/python3.8/site-packages/requests/models.py\", line 1021, in raise_for_status\n", + " raise HTTPError(http_error_msg, response=self)\n", + "requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON%20((1.0000%2043.0000,%201.0000%2044.0000,%202.0000%2044.0000,%202.0000%2043.0000,%201.0000%2043.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "2024-02-06 16:11:50,433 eodag.core [INFO ] No result from provider 'peps' due to an error during search. Raise verbosity of log messages for details\n", + "2024-02-06 16:11:50,434 eodag.core [ERROR ] Error while searching on provider peps (ignored):\n", + "Traceback (most recent call last):\n", + " File \"/home/anesson/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py\", line 908, in _request\n", + " response.raise_for_status()\n", + " File \"/home/anesson/.virtualenvs/eodag_docs/lib/python3.8/site-packages/requests/models.py\", line 1021, in raise_for_status\n", + " raise HTTPError(http_error_msg, response=self)\n", + "requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON%20((1.0000%2043.0000,%201.0000%2044.0000,%202.0000%2044.0000,%202.0000%2043.0000,%201.0000%2043.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "\n", + "During handling of the above exception, another exception occurred:\n", + "\n", + "Traceback (most recent call last):\n", + " File \"/home/anesson/workspace/EODAG/dev/eodag/eodag/api/core.py\", line 1685, in _do_search\n", + " res, nb_res = search_plugin.query(count=count, auth=auth_plugin, **kwargs)\n", + " File \"/home/anesson/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py\", line 541, in query\n", + " provider_results = self.do_search(items_per_page=items_per_page, **kwargs)\n", + " File \"/home/anesson/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py\", line 650, in do_search\n", + " response = self._request(\n", + " File \"/home/anesson/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py\", line 923, in _request\n", + " raise RequestError(str(err))\n", + "eodag.utils.exceptions.RequestError: 400 Client Error: Bad Request for url: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON%20((1.0000%2043.0000,%201.0000%2044.0000,%202.0000%2044.0000,%202.0000%2043.0000,%201.0000%2043.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "2024-02-06 16:11:50,437 eodag.core [ERROR ] No result could be obtained from any available provider\n" ] } ], "source": [ - "products_first_page, estimated_total_number = dag.search(**bad_search_criteria)" + "products_first_page, estimated_total_number = dag.search(provider=\"peps\", **bad_search_criteria)" ] }, { @@ -309,24 +331,37 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:04:42,214 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:04:42,217 eodag.core [INFO ] No result from provider 'peps' due to an error during search. Raise verbosity of log messages for details\n" + "2024-02-06 16:11:50,460 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:11:50,465 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "2024-02-06 16:11:51,058 eodag.search.qssearch [ERROR ] Skipping error while searching for peps QueryStringSearch instance: \n", + "Traceback (most recent call last):\n", + " File \"/home/anesson/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py\", line 908, in _request\n", + " response.raise_for_status()\n", + " File \"/home/anesson/.virtualenvs/eodag_docs/lib/python3.8/site-packages/requests/models.py\", line 1021, in raise_for_status\n", + " raise HTTPError(http_error_msg, response=self)\n", + "requests.exceptions.HTTPError: 400 Client Error: Bad Request for url: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON%20((1.0000%2043.0000,%201.0000%2044.0000,%202.0000%2044.0000,%202.0000%2043.0000,%201.0000%2043.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "2024-02-06 16:11:51,059 eodag.core [INFO ] No result from provider 'peps' due to an error during search.\n" ] }, { - "ename": "AttributeError", - "evalue": "'NoneType' object has no attribute 'groupdict'", + "ename": "RequestError", + "evalue": "400 Client Error: Bad Request for url: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON%20((1.0000%2043.0000,%201.0000%2044.0000,%202.0000%2044.0000,%202.0000%2043.0000,%201.0000%2043.0000))&productType=S2MSI1C&maxRecords=20&page=1", "output_type": "error", "traceback": [ "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m", - "\u001b[0;31mAttributeError\u001b[0m Traceback (most recent call last)", - "Input \u001b[0;32mIn [10]\u001b[0m, in \u001b[0;36m\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m products_first_page, estimated_total_number \u001b[38;5;241m=\u001b[39m \u001b[43mdag\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mbad_search_criteria\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mraise_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workspace/eodag/eodag/api/core.py:902\u001b[0m, in \u001b[0;36mEODataAccessGateway.search\u001b[0;34m(self, page, items_per_page, raise_errors, start, end, geom, locations, **kwargs)\u001b[0m\n\u001b[1;32m 897\u001b[0m search_kwargs\u001b[38;5;241m.\u001b[39mupdate(\n\u001b[1;32m 898\u001b[0m page\u001b[38;5;241m=\u001b[39mpage,\n\u001b[1;32m 899\u001b[0m items_per_page\u001b[38;5;241m=\u001b[39mitems_per_page,\n\u001b[1;32m 900\u001b[0m )\n\u001b[1;32m 901\u001b[0m search_plugin\u001b[38;5;241m.\u001b[39mclear()\n\u001b[0;32m--> 902\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_do_search\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 903\u001b[0m \u001b[43m \u001b[49m\u001b[43msearch_plugin\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mraise_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mraise_errors\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msearch_kwargs\u001b[49m\n\u001b[1;32m 904\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n", - "File \u001b[0;32m~/workspace/eodag/eodag/api/core.py:1389\u001b[0m, in \u001b[0;36mEODataAccessGateway._do_search\u001b[0;34m(self, search_plugin, count, raise_errors, **kwargs)\u001b[0m\n\u001b[1;32m 1387\u001b[0m total_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m 1388\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m-> 1389\u001b[0m res, nb_res \u001b[38;5;241m=\u001b[39m \u001b[43msearch_plugin\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1391\u001b[0m \u001b[38;5;66;03m# Only do the pagination computations when it makes sense. For example,\u001b[39;00m\n\u001b[1;32m 1392\u001b[0m \u001b[38;5;66;03m# for a search by id, we can reasonably guess that the provider will return\u001b[39;00m\n\u001b[1;32m 1393\u001b[0m \u001b[38;5;66;03m# At most 1 product, so we don't need such a thing as pagination\u001b[39;00m\n\u001b[1;32m 1394\u001b[0m page \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpage\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", - "File \u001b[0;32m~/workspace/eodag/eodag/plugins/search/qssearch.py:384\u001b[0m, in \u001b[0;36mQueryStringSearch.query\u001b[0;34m(self, items_per_page, page, count, **kwargs)\u001b[0m\n\u001b[1;32m 382\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquery_params \u001b[38;5;241m=\u001b[39m qp\n\u001b[1;32m 383\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquery_string \u001b[38;5;241m=\u001b[39m qs\n\u001b[0;32m--> 384\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msearch_urls, total_items \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcollect_search_urls\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 385\u001b[0m \u001b[43m \u001b[49m\u001b[43mpage\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpage\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mitems_per_page\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mitems_per_page\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\n\u001b[1;32m 386\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 387\u001b[0m provider_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdo_search(items_per_page\u001b[38;5;241m=\u001b[39mitems_per_page, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m 388\u001b[0m eo_products \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mnormalize_results(provider_results, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n", - "File \u001b[0;32m~/workspace/eodag/eodag/plugins/search/qssearch.py:582\u001b[0m, in \u001b[0;36mQueryStringSearch.collect_search_urls\u001b[0;34m(self, page, items_per_page, count, **kwargs)\u001b[0m\n\u001b[1;32m 580\u001b[0m urls \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 581\u001b[0m total_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m count \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 582\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m collection \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_collections\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m:\n\u001b[1;32m 583\u001b[0m \u001b[38;5;66;03m# skip empty collection if one is required in api_endpoint\u001b[39;00m\n\u001b[1;32m 584\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;132;01m{collection}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mapi_endpoint \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m collection:\n\u001b[1;32m 585\u001b[0m \u001b[38;5;28;01mcontinue\u001b[39;00m\n", - "File \u001b[0;32m~/workspace/eodag/eodag/plugins/search/qssearch.py:812\u001b[0m, in \u001b[0;36mQueryStringSearch.get_collections\u001b[0;34m(self, **kwargs)\u001b[0m\n\u001b[1;32m 810\u001b[0m collections \u001b[38;5;241m=\u001b[39m (\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mS2\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mS2ST\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m 811\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 812\u001b[0m match \u001b[38;5;241m=\u001b[39m \u001b[43mre\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmatch\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 813\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43mr\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m(?P\u001b[39;49m\u001b[38;5;124;43m\\\u001b[39;49m\u001b[38;5;124;43md\u001b[39;49m\u001b[38;5;132;43;01m{4}\u001b[39;49;00m\u001b[38;5;124;43m)-(?P\u001b[39;49m\u001b[38;5;124;43m\\\u001b[39;49m\u001b[38;5;124;43md\u001b[39;49m\u001b[38;5;132;43;01m{2}\u001b[39;49;00m\u001b[38;5;124;43m)-(?P\u001b[39;49m\u001b[38;5;124;43m\\\u001b[39;49m\u001b[38;5;124;43md\u001b[39;49m\u001b[38;5;132;43;01m{2}\u001b[39;49;00m\u001b[38;5;124;43m)\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mdate\u001b[49m\n\u001b[1;32m 814\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mgroupdict\u001b[49m()\n\u001b[1;32m 815\u001b[0m year, month, day \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m 816\u001b[0m \u001b[38;5;28mint\u001b[39m(match[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124myear\u001b[39m\u001b[38;5;124m\"\u001b[39m]),\n\u001b[1;32m 817\u001b[0m \u001b[38;5;28mint\u001b[39m(match[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmonth\u001b[39m\u001b[38;5;124m\"\u001b[39m]),\n\u001b[1;32m 818\u001b[0m \u001b[38;5;28mint\u001b[39m(match[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mday\u001b[39m\u001b[38;5;124m\"\u001b[39m]),\n\u001b[1;32m 819\u001b[0m )\n\u001b[1;32m 820\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m year \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m2016\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m (year \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m2016\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m month \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m12\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m day \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m5\u001b[39m):\n", - "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'groupdict'" + "\u001b[0;31mHTTPError\u001b[0m Traceback (most recent call last)", + "File \u001b[0;32m~/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py:908\u001b[0m, in \u001b[0;36mQueryStringSearch._request\u001b[0;34m(self, url, info_message, exception_message)\u001b[0m\n\u001b[1;32m 905\u001b[0m response \u001b[38;5;241m=\u001b[39m requests\u001b[38;5;241m.\u001b[39mget(\n\u001b[1;32m 906\u001b[0m url, timeout\u001b[38;5;241m=\u001b[39mtimeout, headers\u001b[38;5;241m=\u001b[39mUSER_AGENT, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 907\u001b[0m )\n\u001b[0;32m--> 908\u001b[0m \u001b[43mresponse\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraise_for_status\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 909\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m requests\u001b[38;5;241m.\u001b[39mexceptions\u001b[38;5;241m.\u001b[39mTimeout \u001b[38;5;28;01mas\u001b[39;00m exc:\n", + "File \u001b[0;32m~/.virtualenvs/eodag_docs/lib/python3.8/site-packages/requests/models.py:1021\u001b[0m, in \u001b[0;36mResponse.raise_for_status\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 1020\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m http_error_msg:\n\u001b[0;32m-> 1021\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m HTTPError(http_error_msg, response\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m)\n", + "\u001b[0;31mHTTPError\u001b[0m: 400 Client Error: Bad Request for url: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON%20((1.0000%2043.0000,%201.0000%2044.0000,%202.0000%2044.0000,%202.0000%2043.0000,%201.0000%2043.0000))&productType=S2MSI1C&maxRecords=20&page=1", + "\nDuring handling of the above exception, another exception occurred:\n", + "\u001b[0;31mRequestError\u001b[0m Traceback (most recent call last)", + "Cell \u001b[0;32mIn [10], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m products_first_page, estimated_total_number \u001b[38;5;241m=\u001b[39m \u001b[43mdag\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msearch\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mbad_search_criteria\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mraise_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m)\u001b[49m\n", + "File \u001b[0;32m~/workspace/EODAG/dev/eodag/eodag/api/core.py:1056\u001b[0m, in \u001b[0;36mEODataAccessGateway.search\u001b[0;34m(self, page, items_per_page, raise_errors, start, end, geom, locations, provider, **kwargs)\u001b[0m\n\u001b[1;32m 1054\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m i, search_plugin \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28menumerate\u001b[39m(search_plugins):\n\u001b[1;32m 1055\u001b[0m search_plugin\u001b[38;5;241m.\u001b[39mclear()\n\u001b[0;32m-> 1056\u001b[0m search_results, total_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_do_search\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 1057\u001b[0m \u001b[43m \u001b[49m\u001b[43msearch_plugin\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1058\u001b[0m \u001b[43m \u001b[49m\u001b[43mcount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mTrue\u001b[39;49;00m\u001b[43m,\u001b[49m\n\u001b[1;32m 1059\u001b[0m \u001b[43m \u001b[49m\u001b[43mraise_errors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mraise_errors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1060\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43msearch_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 1061\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1062\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(search_results) \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m i \u001b[38;5;241m<\u001b[39m \u001b[38;5;28mlen\u001b[39m(search_plugins) \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m:\n\u001b[1;32m 1063\u001b[0m logger\u001b[38;5;241m.\u001b[39mwarning(\n\u001b[1;32m 1064\u001b[0m \u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo result could be obtained from provider \u001b[39m\u001b[38;5;132;01m{\u001b[39;00msearch_plugin\u001b[38;5;241m.\u001b[39mprovider\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m 1065\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwe will try to get the data from another provider\u001b[39m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 1066\u001b[0m )\n", + "File \u001b[0;32m~/workspace/EODAG/dev/eodag/eodag/api/core.py:1685\u001b[0m, in \u001b[0;36mEODataAccessGateway._do_search\u001b[0;34m(self, search_plugin, count, raise_errors, **kwargs)\u001b[0m\n\u001b[1;32m 1682\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m need_auth \u001b[38;5;129;01mand\u001b[39;00m auth_plugin \u001b[38;5;129;01mand\u001b[39;00m can_authenticate:\n\u001b[1;32m 1683\u001b[0m search_plugin\u001b[38;5;241m.\u001b[39mauth \u001b[38;5;241m=\u001b[39m auth_plugin\u001b[38;5;241m.\u001b[39mauthenticate()\n\u001b[0;32m-> 1685\u001b[0m res, nb_res \u001b[38;5;241m=\u001b[39m \u001b[43msearch_plugin\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mquery\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcount\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcount\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mauth\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mauth_plugin\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 1687\u001b[0m \u001b[38;5;66;03m# Only do the pagination computations when it makes sense. For example,\u001b[39;00m\n\u001b[1;32m 1688\u001b[0m \u001b[38;5;66;03m# for a search by id, we can reasonably guess that the provider will return\u001b[39;00m\n\u001b[1;32m 1689\u001b[0m \u001b[38;5;66;03m# At most 1 product, so we don't need such a thing as pagination\u001b[39;00m\n\u001b[1;32m 1690\u001b[0m page \u001b[38;5;241m=\u001b[39m kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpage\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n", + "File \u001b[0;32m~/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py:541\u001b[0m, in \u001b[0;36mQueryStringSearch.query\u001b[0;34m(self, product_type, items_per_page, page, count, **kwargs)\u001b[0m\n\u001b[1;32m 538\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtotal_items_nb\n\u001b[1;32m 539\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mneed_count\n\u001b[0;32m--> 541\u001b[0m provider_results \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdo_search\u001b[49m\u001b[43m(\u001b[49m\u001b[43mitems_per_page\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mitems_per_page\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 542\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m count \u001b[38;5;129;01mand\u001b[39;00m total_items \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtotal_items_nb\u001b[39m\u001b[38;5;124m\"\u001b[39m):\n\u001b[1;32m 543\u001b[0m total_items \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtotal_items_nb\n", + "File \u001b[0;32m~/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py:650\u001b[0m, in \u001b[0;36mQueryStringSearch.do_search\u001b[0;34m(self, items_per_page, **kwargs)\u001b[0m\n\u001b[1;32m 648\u001b[0m results: List[Any] \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m 649\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m search_url \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msearch_urls:\n\u001b[0;32m--> 650\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_request\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 651\u001b[0m \u001b[43m \u001b[49m\u001b[43msearch_url\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 652\u001b[0m \u001b[43m \u001b[49m\u001b[43minfo_message\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mSending search request: \u001b[39;49m\u001b[38;5;132;43;01m{}\u001b[39;49;00m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat\u001b[49m\u001b[43m(\u001b[49m\u001b[43msearch_url\u001b[49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 653\u001b[0m \u001b[43m \u001b[49m\u001b[43mexception_message\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mSkipping error while searching for \u001b[39;49m\u001b[38;5;132;43;01m{}\u001b[39;49;00m\u001b[38;5;124;43m \u001b[39;49m\u001b[38;5;132;43;01m{}\u001b[39;49;00m\u001b[38;5;124;43m \u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\n\u001b[1;32m 654\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43minstance:\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mformat\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mprovider\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;18;43m__class__\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;18;43m__name__\u001b[39;49m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 655\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 656\u001b[0m next_page_url_key_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mpagination\u001b[38;5;241m.\u001b[39mget(\n\u001b[1;32m 657\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnext_page_url_key_path\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 658\u001b[0m )\n\u001b[1;32m 659\u001b[0m next_page_query_obj_key_path \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mpagination\u001b[38;5;241m.\u001b[39mget(\n\u001b[1;32m 660\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnext_page_query_obj_key_path\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m 661\u001b[0m )\n", + "File \u001b[0;32m~/workspace/EODAG/dev/eodag/eodag/plugins/search/qssearch.py:923\u001b[0m, in \u001b[0;36mQueryStringSearch._request\u001b[0;34m(self, url, info_message, exception_message)\u001b[0m\n\u001b[1;32m 915\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m 916\u001b[0m logger\u001b[38;5;241m.\u001b[39mexception(\n\u001b[1;32m 917\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mSkipping error while requesting: \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m (provider:\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m, plugin:\u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m): \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m\"\u001b[39m,\n\u001b[1;32m 918\u001b[0m url,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 921\u001b[0m err_msg,\n\u001b[1;32m 922\u001b[0m )\n\u001b[0;32m--> 923\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m RequestError(\u001b[38;5;28mstr\u001b[39m(err))\n\u001b[1;32m 924\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n", + "\u001b[0;31mRequestError\u001b[0m: 400 Client Error: Bad Request for url: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=malformed_start_date&completionDate=2021-03-31&geometry=POLYGON%20((1.0000%2043.0000,%201.0000%2044.0000,%202.0000%2044.0000,%202.0000%2043.0000,%201.0000%2043.0000))&productType=S2MSI1C&maxRecords=20&page=1" ] } ], @@ -365,10 +400,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:04:56,848 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:04:56,850 eodag.core [INFO ] Iterate search over multiple pages: page #1\n", - "2023-01-13 11:04:56,853 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=500&page=1\n", - "2023-01-13 11:04:58,343 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" + "2024-02-06 16:12:45,983 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:12:45,987 eodag.core [INFO ] Iterate search over multiple pages: page #1\n", + "2024-02-06 16:12:45,989 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=500&page=1\n", + "2024-02-06 16:12:50,782 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" ] } ], @@ -410,12 +445,12 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:05:03,535 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:05:03,537 eodag.core [INFO ] Iterate search over multiple pages: page #1\n", - "2023-01-13 11:05:03,540 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=30&page=1\n", - "2023-01-13 11:05:04,734 eodag.core [INFO ] Iterate search over multiple pages: page #2\n", - "2023-01-13 11:05:04,736 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=30&page=2\n", - "2023-01-13 11:05:05,550 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" + "2024-02-06 16:12:50,807 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:12:50,810 eodag.core [INFO ] Iterate search over multiple pages: page #1\n", + "2024-02-06 16:12:50,812 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=30&page=1\n", + "2024-02-06 16:12:53,716 eodag.core [INFO ] Iterate search over multiple pages: page #2\n", + "2024-02-06 16:12:53,718 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=30&page=2\n", + "2024-02-06 16:12:55,894 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" ] } ], @@ -458,11 +493,11 @@ "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:05:15,177 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:05:15,180 eodag.core [INFO ] Iterate search over multiple pages: page #1\n", - "2023-01-13 11:05:15,182 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=30&page=1\n", - "2023-01-13 11:05:16,809 eodag.core [INFO ] Iterate search over multiple pages: page #2\n", - "2023-01-13 11:05:16,810 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=30&page=2\n" + "2024-02-06 16:12:55,908 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:12:55,912 eodag.core [INFO ] Iterate search over multiple pages: page #1\n", + "2024-02-06 16:12:55,913 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=30&page=1\n", + "2024-02-06 16:13:00,961 eodag.core [INFO ] Iterate search over multiple pages: page #2\n", + "2024-02-06 16:13:00,962 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=30&page=2\n" ] }, { @@ -522,8 +557,8 @@ { "data": { "text/plain": [ - "SearchResult([EOProduct(id=S2B_MSIL1C_20210328T103629_N0209_R008_T31TCJ_20210328T124650, provider=peps),\n", - " EOProduct(id=S2B_MSIL1C_20210328T103629_N0209_R008_T31TCH_20210328T124650, provider=peps)])" + "SearchResult([EOProduct(id=S2B_MSIL1C_20210328T103629_N0500_R008_T31TDH_20230602T033834, provider=peps),\n", + " EOProduct(id=S2B_MSIL1C_20210328T103629_N0500_R008_T31TCJ_20230602T033834, provider=peps)])" ] }, "execution_count": 16, @@ -551,7 +586,7 @@ { "data": { "text/plain": [ - "EOProduct(id=S2B_MSIL1C_20210328T103629_N0209_R008_T31TCJ_20210328T124650, provider=peps)" + "EOProduct(id=S2B_MSIL1C_20210328T103629_N0500_R008_T31TDH_20230602T033834, provider=peps)" ] }, "execution_count": 17, @@ -588,10 +623,10 @@ { "data": { "image/svg+xml": [ - "" + "" ], "text/plain": [ - "" + "" ] }, "execution_count": 18, @@ -631,11 +666,11 @@ { "data": { "text/plain": [ - "{'auth': GenericAuth(provider=peps, priority=1, topic=Authentication),\n", + "{'auth': GenericAuth(provider=peps, priority=3, topic=Authentication),\n", " 'productType': 'S2_MSI_L1C',\n", " 'startTimeFromAscendingNode': '2021-03-01',\n", " 'completionTimeFromAscendingNode': '2021-03-31',\n", - " 'geometry': }" + " 'geometry': }" ] }, "execution_count": 20, @@ -655,8 +690,8 @@ { "data": { "text/plain": [ - "('https://peps.cnes.fr/resto/collections/S2ST/387c7327-9a71-5a34-9163-0dfdeb024522/download',\n", - " 'https://peps.cnes.fr/resto/collections/S2ST/387c7327-9a71-5a34-9163-0dfdeb024522/download')" + "('https://peps.cnes.fr/resto/collections/S2ST/2d85d4c8-34c1-55dc-b43c-0ab0fe4c2b97/download',\n", + " 'https://peps.cnes.fr/resto/collections/S2ST/2d85d4c8-34c1-55dc-b43c-0ab0fe4c2b97/download')" ] }, "execution_count": 21, @@ -676,7 +711,7 @@ { "data": { "text/plain": [ - "dict_keys(['abstract', 'instrument', 'platform', 'platformSerialIdentifier', 'processingLevel', 'keywords', 'sensorType', 'license', 'missionStartDate', 'title', 'productType', 'uid', 'keyword', 'resolution', 'organisationName', 'publicationDate', 'parentIdentifier', 'orbitNumber', 'orbitDirection', 'cloudCover', 'snowCover', 'creationDate', 'modificationDate', 'sensorMode', 'startTimeFromAscendingNode', 'completionTimeFromAscendingNode', 'id', 'quicklook', 'downloadLink', 'storageStatus', 'thumbnail', 'resourceSize', 'resourceChecksum', 'visible', 'newVersion', 'isNrt', 'realtime', 'relativeOrbitNumber', 'useDatalake', 's2TakeId', 'mgrs', 'bareSoil', 'highProbaClouds', 'mediumProbaClouds', 'lowProbaClouds', 'snowIce', 'vegetation', 'water', 'isRefined', 'nrtResource', 'services', 'links', 'storage'])" + "dict_keys(['abstract', 'instrument', 'platform', 'platformSerialIdentifier', 'processingLevel', 'keywords', 'sensorType', 'license', 'missionStartDate', 'title', 'productType', 'uid', 'keyword', 'resolution', 'organisationName', 'publicationDate', 'parentIdentifier', 'orbitNumber', 'orbitDirection', 'cloudCover', 'snowCover', 'creationDate', 'modificationDate', 'sensorMode', 'startTimeFromAscendingNode', 'completionTimeFromAscendingNode', 'id', 'quicklook', 'downloadLink', 'tileIdentifier', 'storageStatus', 'thumbnail', 'resourceSize', 'resourceChecksum', 'visible', 'newVersion', 'isNrt', 'realtime', 'relativeOrbitNumber', 'useDatalake', 'bucket', 'prefix', 's2TakeId', 'bareSoil', 'highProbaClouds', 'mediumProbaClouds', 'lowProbaClouds', 'snowIce', 'vegetation', 'water', 'isRefined', 'nrtResource', 'services', 'links', 'storage'])" ] }, "execution_count": 22, @@ -704,7 +739,7 @@ { "data": { "text/plain": [ - "EOProduct(id=S2B_MSIL1C_20210328T103629_N0209_R008_T31TCJ_20210328T124650, provider=peps)" + "EOProduct(id=S2B_MSIL1C_20210328T103629_N0500_R008_T31TDH_20230602T033834, provider=peps)" ] }, "execution_count": 23, @@ -735,8 +770,8 @@ { "data": { "text/plain": [ - "SearchResult([EOProduct(id=S2B_MSIL1C_20210328T103629_N0209_R008_T31TCJ_20210328T124650, provider=peps),\n", - " EOProduct(id=S2B_MSIL1C_20210328T103629_N0209_R008_T31TCH_20210328T124650, provider=peps)])" + "SearchResult([EOProduct(id=S2B_MSIL1C_20210328T103629_N0500_R008_T31TDH_20230602T033834, provider=peps),\n", + " EOProduct(id=S2B_MSIL1C_20210328T103629_N0500_R008_T31TCJ_20230602T033834, provider=peps)])" ] }, "execution_count": 24, @@ -1036,7 +1071,9 @@ "data": { "text/html": [ "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + "</script>\n", + "</html>\" style=\"position:absolute;width:100%;height:100%;left:0;top:0;border:none !important;\" allowfullscreen webkitallowfullscreen mozallowfullscreen>" ], "text/plain": [ - "" + "" ] }, "execution_count": 31, @@ -1251,7 +1292,7 @@ "data": { "text/plain": [ "[{'name': 'country',\n", - " 'path': '/home/sylvain/.config/eodag/shp/ne_110m_admin_0_map_units.shp',\n", + " 'path': '/home/anesson/.config/eodag/shp/ne_110m_admin_0_map_units.shp',\n", " 'attr': 'ADM0_A3_US'}]" ] }, @@ -1329,22 +1370,27 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 59, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:06:53,694 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:06:53,696 eodag.plugins.search.qssearch [INFO ] Sending count request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=MULTIPOLYGON (((6.1567 50.8037, 5.6070 51.0373, 4.9740 51.4750, 4.0471 51.2673, 3.3150 51.3458, 3.3150 51.3458, 3.3150 51.3458, 2.5136 51.1485, 2.6584 50.7968, 3.1233 50.7804, 3.5882 50.3790, 4.2860 49.9075, 4.7992 49.9854, 5.6741 49.5295, 5.7824 50.0903, 6.0431 50.1281, 6.1567 50.8037)), ((9.5942 47.5251, 8.5226 47.8308, 8.3173 47.6136, 7.4668 47.6206, 7.1922 47.4498, 6.7366 47.5418, 6.7687 47.2877, 6.0374 46.7258, 6.0226 46.2730, 6.5001 46.4297, 6.8436 45.9911, 7.2739 45.7769, 7.7560 45.8245, 8.3166 46.1636, 8.4900 46.0052, 8.9663 46.0369, 9.1829 46.4402, 9.9228 46.3149, 10.3634 46.4836, 10.4427 46.8935, 9.9324 46.9207, 9.4800 47.1028, 9.6329 47.3476, 9.5942 47.5251)))&productType=S2MSI1C&maxRecords=1&page=1\n", - "2023-01-13 11:06:54,398 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=MULTIPOLYGON (((6.1567 50.8037, 5.6070 51.0373, 4.9740 51.4750, 4.0471 51.2673, 3.3150 51.3458, 3.3150 51.3458, 3.3150 51.3458, 2.5136 51.1485, 2.6584 50.7968, 3.1233 50.7804, 3.5882 50.3790, 4.2860 49.9075, 4.7992 49.9854, 5.6741 49.5295, 5.7824 50.0903, 6.0431 50.1281, 6.1567 50.8037)), ((9.5942 47.5251, 8.5226 47.8308, 8.3173 47.6136, 7.4668 47.6206, 7.1922 47.4498, 6.7366 47.5418, 6.7687 47.2877, 6.0374 46.7258, 6.0226 46.2730, 6.5001 46.4297, 6.8436 45.9911, 7.2739 45.7769, 7.7560 45.8245, 8.3166 46.1636, 8.4900 46.0052, 8.9663 46.0369, 9.1829 46.4402, 9.9228 46.3149, 10.3634 46.4836, 10.4427 46.8935, 9.9324 46.9207, 9.4800 47.1028, 9.6329 47.3476, 9.5942 47.5251)))&productType=S2MSI1C&maxRecords=20&page=1\n", - "2023-01-13 11:06:56,590 eodag.core [INFO ] Found 248 result(s) on provider 'peps'\n" + "2024-02-06 16:14:20,970 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:14:20,974 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=MULTIPOLYGON (((6.1567 50.8037, 6.0431 50.1281, 5.7824 50.0903, 5.6741 49.5295, 4.7992 49.9854, 4.2860 49.9075, 3.5882 50.3790, 3.1233 50.7804, 2.6584 50.7968, 2.5136 51.1485, 3.3150 51.3458, 3.3150 51.3458, 3.3150 51.3458, 4.0471 51.2673, 4.9740 51.4750, 5.6070 51.0373, 6.1567 50.8037)), ((9.5942 47.5251, 9.6329 47.3476, 9.4800 47.1028, 9.9324 46.9207, 10.4427 46.8935, 10.3634 46.4836, 9.9228 46.3149, 9.1829 46.4402, 8.9663 46.0369, 8.4900 46.0052, 8.3166 46.1636, 7.7560 45.8245, 7.2739 45.7769, 6.8436 45.9911, 6.5001 46.4297, 6.0226 46.2730, 6.0374 46.7258, 6.7687 47.2877, 6.7366 47.5418, 7.1922 47.4498, 7.4668 47.6206, 8.3173 47.6136, 8.5226 47.8308, 9.5942 47.5251)))&productType=S2MSI1C&maxRecords=50&page=1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-06 16:14:26,208 eodag.core [INFO ] Found 354 result(s) on provider 'peps'\n" ] } ], "source": [ - "locations_products, estimated_total_number = dag.search(**location_search_criteria)" + "locations_products, estimated_total_number = dag.search(**location_search_criteria, items_per_page=50)" ] }, { @@ -1357,14 +1403,16 @@ }, { "cell_type": "code", - "execution_count": 36, + "execution_count": 60, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + "</script>\n", + "</html>\" style=\"position:absolute;width:100%;height:100%;left:0;top:0;border:none !important;\" allowfullscreen webkitallowfullscreen mozallowfullscreen>" ], "text/plain": [ - "" + "" ] }, - "execution_count": 36, + "execution_count": 60, "metadata": {}, "output_type": "execute_result" } @@ -1610,7 +1662,7 @@ }, { "cell_type": "code", - "execution_count": 37, + "execution_count": 61, "metadata": {}, "outputs": [], "source": [ @@ -1635,7 +1687,7 @@ }, { "cell_type": "code", - "execution_count": 38, + "execution_count": 62, "metadata": {}, "outputs": [ { @@ -1648,7 +1700,7 @@ " 'locations': {'country': 'BEL|CHE'}}" ] }, - "execution_count": 38, + "execution_count": 62, "metadata": {}, "output_type": "execute_result" } @@ -1661,17 +1713,22 @@ }, { "cell_type": "code", - "execution_count": 39, + "execution_count": 63, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:07:18,062 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:07:18,064 eodag.plugins.search.qssearch [INFO ] Sending count request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=MULTIPOLYGON (((6.1567 50.8037, 5.6070 51.0373, 4.9740 51.4750, 4.0471 51.2673, 3.3150 51.3458, 3.3150 51.3458, 3.3150 51.3458, 2.5136 51.1485, 2.6584 50.7968, 3.1233 50.7804, 3.5882 50.3790, 4.2860 49.9075, 4.7992 49.9854, 5.6741 49.5295, 5.7824 50.0903, 6.0431 50.1281, 6.1567 50.8037)), ((9.5942 47.5251, 8.5226 47.8308, 8.3173 47.6136, 7.4668 47.6206, 7.1922 47.4498, 6.7366 47.5418, 6.7687 47.2877, 6.0374 46.7258, 6.0226 46.2730, 6.5001 46.4297, 6.8436 45.9911, 7.2739 45.7769, 7.7560 45.8245, 8.3166 46.1636, 8.4900 46.0052, 8.9663 46.0369, 9.1829 46.4402, 9.9228 46.3149, 10.3634 46.4836, 10.4427 46.8935, 9.9324 46.9207, 9.4800 47.1028, 9.6329 47.3476, 9.5942 47.5251)), ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000)))&productType=S2MSI1C&maxRecords=1&page=1\n", - "2023-01-13 11:07:18,515 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=MULTIPOLYGON (((6.1567 50.8037, 5.6070 51.0373, 4.9740 51.4750, 4.0471 51.2673, 3.3150 51.3458, 3.3150 51.3458, 3.3150 51.3458, 2.5136 51.1485, 2.6584 50.7968, 3.1233 50.7804, 3.5882 50.3790, 4.2860 49.9075, 4.7992 49.9854, 5.6741 49.5295, 5.7824 50.0903, 6.0431 50.1281, 6.1567 50.8037)), ((9.5942 47.5251, 8.5226 47.8308, 8.3173 47.6136, 7.4668 47.6206, 7.1922 47.4498, 6.7366 47.5418, 6.7687 47.2877, 6.0374 46.7258, 6.0226 46.2730, 6.5001 46.4297, 6.8436 45.9911, 7.2739 45.7769, 7.7560 45.8245, 8.3166 46.1636, 8.4900 46.0052, 8.9663 46.0369, 9.1829 46.4402, 9.9228 46.3149, 10.3634 46.4836, 10.4427 46.8935, 9.9324 46.9207, 9.4800 47.1028, 9.6329 47.3476, 9.5942 47.5251)), ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000)))&productType=S2MSI1C&maxRecords=50&page=1\n", - "2023-01-13 11:07:20,210 eodag.core [INFO ] Found 425 result(s) on provider 'peps'\n" + "2024-02-06 16:14:26,774 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:14:26,776 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=MULTIPOLYGON (((6.1567 50.8037, 6.0431 50.1281, 5.7824 50.0903, 5.6741 49.5295, 4.7992 49.9854, 4.2860 49.9075, 3.5882 50.3790, 3.1233 50.7804, 2.6584 50.7968, 2.5136 51.1485, 3.3150 51.3458, 3.3150 51.3458, 3.3150 51.3458, 4.0471 51.2673, 4.9740 51.4750, 5.6070 51.0373, 6.1567 50.8037)), ((9.5942 47.5251, 9.6329 47.3476, 9.4800 47.1028, 9.9324 46.9207, 10.4427 46.8935, 10.3634 46.4836, 9.9228 46.3149, 9.1829 46.4402, 8.9663 46.0369, 8.4900 46.0052, 8.3166 46.1636, 7.7560 45.8245, 7.2739 45.7769, 6.8436 45.9911, 6.5001 46.4297, 6.0226 46.2730, 6.0374 46.7258, 6.7687 47.2877, 6.7366 47.5418, 7.1922 47.4498, 7.4668 47.6206, 8.3173 47.6136, 8.5226 47.8308, 9.5942 47.5251)), ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000)))&productType=S2MSI1C&maxRecords=50&page=1\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-06 16:14:32,033 eodag.core [INFO ] Found 645 result(s) on provider 'peps'\n" ] } ], @@ -1681,14 +1738,16 @@ }, { "cell_type": "code", - "execution_count": 40, + "execution_count": 64, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
Make this Notebook Trusted to load map: File -> Trust Notebook
" + "</script>\n", + "</html>\" style=\"position:absolute;width:100%;height:100%;left:0;top:0;border:none !important;\" allowfullscreen webkitallowfullscreen mozallowfullscreen>" ], "text/plain": [ - "" + "" ] }, - "execution_count": 40, + "execution_count": 64, "metadata": {}, "output_type": "execute_result" } @@ -1948,16 +2011,16 @@ }, { "cell_type": "code", - "execution_count": 41, + "execution_count": 65, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'S2B_MSIL1C_20210328T103629_N0209_R008_T31TCJ_20210328T124650'" + "'S2B_MSIL1C_20210328T103629_N0500_R008_T31TDH_20230602T033834'" ] }, - "execution_count": 41, + "execution_count": 65, "metadata": {}, "output_type": "execute_result" } @@ -1972,38 +2035,102 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "This identifier, if known beforehand, can be used to search for this specific product by passing the `id` kwarg to the [search()](../../api_reference/core.rst#eodag.api.core.EODataAccessGateway.search) method. The `provider` kwarg can also be passed to specify among which provider's catalog the product should be searched for. If `provider` is not provided, `eodag` will iterate over all the providers until it finds the product targeted." + "This identifier, if known beforehand, can be used to search for this specific product by passing the `id` kwarg to the [search()](../../api_reference/core.rst#eodag.api.core.EODataAccessGateway.search) method. This search by identifier can be optimized by including the appropriate `productType` in the kwargs. The `provider` kwarg can also be passed to specify among which provider's catalog the product should be searched for. If `provider` is not provided, `eodag` will iterate over all the providers until it finds the product targeted." ] }, { "cell_type": "code", - "execution_count": 45, + "execution_count": 66, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:09:17,102 eodag.core [INFO ] Searching product with id 'S2B_MSIL1C_20210328T103629_N0209_R008_T31TCJ_20210328T124650' on provider: onda\n", - "2023-01-13 11:09:17,104 eodag.plugins.search.qssearch [INFO ] Sending search request: https://catalogue.onda-dias.eu/dias-catalogue/Products?$format=json&$search=%22S2B_MSIL1C_20210328T103629_N0209_R008_T31TCJ_20210328T124650%22\n", - "2023-01-13 11:09:18,358 eodag.core [INFO ] Found 1 result(s) on provider 'onda'\n" + "2024-02-06 16:14:32,523 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: cop_dataspace\n", + "2024-02-06 16:14:32,528 eodag.core [INFO ] Searching product with id 'S2B_MSIL1C_20210328T103629_N0500_R008_T31TDH_20230602T033834' on provider: cop_dataspace\n", + "2024-02-06 16:14:32,529 eodag.search.base [INFO ] cop_dataspace is configured with default sorting by 'startTimeFromAscendingNode' in ascending order\n", + "2024-02-06 16:14:32,530 eodag.search.qssearch [INFO ] Sending search request: http://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/search.json?productIdentifier=S2B_MSIL1C_20210328T103629_N0500_R008_T31TDH_20230602T033834&productType=S2MSI1C&sortParam=startDate&sortOrder=asc&maxRecords=2&page=1&exactCount=1\n", + "2024-02-06 16:14:55,505 eodag.core [INFO ] Found 1 result(s) on provider 'cop_dataspace'\n" ] }, { "data": { "text/plain": [ - "(SearchResult([EOProduct(id=S2B_MSIL1C_20210328T103629_N0209_R008_T31TCJ_20210328T124650, provider=onda)]),\n", + "(SearchResult([EOProduct(id=S2B_MSIL1C_20210328T103629_N0500_R008_T31TDH_20230602T033834, provider=cop_dataspace)]),\n", " 1)" ] }, - "execution_count": 45, + "execution_count": 66, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "one_product_onda, _ = dag.search(id=product_id, provider=\"onda\")\n", - "one_product_onda, _" + "one_product_cop_dataspace, _ = dag.search(id=product_id, productType=\"S2_MSI_L1C\", provider=\"cop_dataspace\")\n", + "one_product_cop_dataspace, _" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### `sortBy`" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "EO products can be sorted by metadata that the provider used supports as sorting parameters (see the [available_sortables()](../../api_reference/core.rst#eodag.api.core.EODataAccessGateway.available_sortables) method) in the wanted sorting order (`ASC`|`DESC`) by passing the `sortBy` kwarg. If `sortBy` is not passed but the provider has a default sorting parameter, the sort is realized with it. If the number of sorting parameters exceeds the maximum allowed for the provider or if the provider does not support the sorting feature or at least one sorting parameter, an error is returned." + ] + }, + { + "cell_type": "code", + "execution_count": 67, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-02-06 16:14:55,569 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: cop_dataspace\n", + "2024-02-06 16:14:55,575 eodag.search.base [INFO ] cop_dataspace is configured with default sorting by 'startTimeFromAscendingNode' in ascending order\n", + "2024-02-06 16:14:55,579 eodag.search.qssearch [INFO ] Sending search request: http://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&sortParam=startDate&sortOrder=asc&maxRecords=20&page=1&exactCount=1\n", + "2024-02-06 16:15:01,073 eodag.core [INFO ] Found 96 result(s) on provider 'cop_dataspace'\n", + "2024-02-06 16:15:01,074 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: cop_dataspace\n", + "2024-02-06 16:15:01,076 eodag.search.qssearch [INFO ] Sending search request: http://catalogue.dataspace.copernicus.eu/resto/api/collections/Sentinel2/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&sortParam=startDate&sortOrder=desc&maxRecords=20&page=1&exactCount=1\n", + "2024-02-06 16:15:08,425 eodag.core [INFO ] Found 96 result(s) on provider 'cop_dataspace'\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The start time of the first returned product is later than or equal to that of\n", + "the first returned product in the default search which sorts by start date\n", + "in ascending order by default: True\n" + ] + } + ], + "source": [ + "sorted_by_start_date_in_asc_order_products, _ = dag.search(\n", + " provider=\"cop_dataspace\",\n", + " **default_search_criteria\n", + ")\n", + "sorted_by_start_date_in_desc_order_products, _ = dag.search(\n", + " provider=\"cop_dataspace\",\n", + " sortBy=[(\"startTimeFromAscendingNode\", \"DESC\")],\n", + " **default_search_criteria\n", + ")\n", + "print(\n", + " \"The start time of the first returned product is later than or equal to that \"\n", + " \"of\\nthe first returned product in the default search which sorts by start \"\n", + " \"date\\nin ascending order by default:\",\n", + " sorted_by_start_date_in_desc_order_products[0].properties['startTimeFromAscendingNode'] \\\n", + " >= \\\n", + " sorted_by_start_date_in_asc_order_products[0].properties['startTimeFromAscendingNode']\n", + ")" ] }, { @@ -2039,7 +2166,7 @@ }, { "cell_type": "code", - "execution_count": 46, + "execution_count": 68, "metadata": {}, "outputs": [ { @@ -2048,7 +2175,7 @@ "['S1_SAR_GRD', 'S1_SAR_OCN', 'S1_SAR_RAW', 'S1_SAR_SLC']" ] }, - "execution_count": 46, + "execution_count": 68, "metadata": {}, "output_type": "execute_result" } @@ -2067,7 +2194,7 @@ }, { "cell_type": "code", - "execution_count": 47, + "execution_count": 69, "metadata": {}, "outputs": [ { @@ -2087,7 +2214,7 @@ " 'LANDSAT_TM_C2L2']" ] }, - "execution_count": 47, + "execution_count": 69, "metadata": {}, "output_type": "execute_result" } @@ -2106,9 +2233,38 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 70, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['L57_REFLECTANCE',\n", + " 'LANDSAT_C2L1',\n", + " 'LANDSAT_C2L2',\n", + " 'LANDSAT_C2L2ALB_BT',\n", + " 'LANDSAT_C2L2ALB_SR',\n", + " 'LANDSAT_C2L2ALB_ST',\n", + " 'LANDSAT_C2L2ALB_TA',\n", + " 'LANDSAT_C2L2_SR',\n", + " 'LANDSAT_C2L2_ST',\n", + " 'LANDSAT_ETM_C1',\n", + " 'LANDSAT_ETM_C2L1',\n", + " 'LANDSAT_ETM_C2L2',\n", + " 'LANDSAT_TM_C1',\n", + " 'LANDSAT_TM_C2L1',\n", + " 'LANDSAT_TM_C2L2',\n", + " 'S1_SAR_GRD',\n", + " 'S1_SAR_OCN',\n", + " 'S1_SAR_RAW',\n", + " 'S1_SAR_SLC']" + ] + }, + "execution_count": 70, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dag.guess_product_type(platform=\"LANDSAT OR SENTINEL1\")" ] @@ -2125,9 +2281,35 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 71, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['LANDSAT_C2L1',\n", + " 'LANDSAT_C2L2',\n", + " 'LANDSAT_C2L2ALB_BT',\n", + " 'LANDSAT_C2L2ALB_SR',\n", + " 'LANDSAT_C2L2ALB_ST',\n", + " 'LANDSAT_C2L2ALB_TA',\n", + " 'LANDSAT_C2L2_SR',\n", + " 'LANDSAT_C2L2_ST',\n", + " 'LANDSAT_ETM_C2L1',\n", + " 'LANDSAT_ETM_C2L2',\n", + " 'LANDSAT_TM_C2L1',\n", + " 'LANDSAT_TM_C2L2',\n", + " 'S1_SAR_GRD',\n", + " 'S1_SAR_OCN',\n", + " 'S1_SAR_RAW',\n", + " 'S1_SAR_SLC']" + ] + }, + "execution_count": 71, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dag.guess_product_type(keywords=\"(LANDSAT AND collection2) OR SAR\")" ] @@ -2144,9 +2326,37 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 72, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['CLMS_CORINE',\n", + " 'L57_REFLECTANCE',\n", + " 'L8_OLI_TIRS_C1L1',\n", + " 'L8_REFLECTANCE',\n", + " 'LANDSAT_C2L1',\n", + " 'LANDSAT_C2L2',\n", + " 'LANDSAT_C2L2ALB_BT',\n", + " 'LANDSAT_C2L2ALB_SR',\n", + " 'LANDSAT_C2L2ALB_ST',\n", + " 'LANDSAT_C2L2ALB_TA',\n", + " 'LANDSAT_C2L2_SR',\n", + " 'LANDSAT_C2L2_ST',\n", + " 'LANDSAT_ETM_C1',\n", + " 'LANDSAT_ETM_C2L1',\n", + " 'LANDSAT_ETM_C2L2',\n", + " 'LANDSAT_TM_C1',\n", + " 'LANDSAT_TM_C2L1',\n", + " 'LANDSAT_TM_C2L2']" + ] + }, + "execution_count": 72, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dag.guess_product_type(platformSerialIdentifier=\"L?\")" ] @@ -2163,9 +2373,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 73, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['S1_SAR_GRD',\n", + " 'S1_SAR_OCN',\n", + " 'S1_SAR_RAW',\n", + " 'S1_SAR_SLC',\n", + " 'S2_MSI_L1C',\n", + " 'S2_MSI_L2A',\n", + " 'S2_MSI_L2AP',\n", + " 'S2_MSI_L2A_COG',\n", + " 'S2_MSI_L2A_MAJA',\n", + " 'S2_MSI_L2B_MAJA_SNOW',\n", + " 'S2_MSI_L2B_MAJA_WATER',\n", + " 'S2_MSI_L3A_WASP',\n", + " 'S3_EFR',\n", + " 'S3_ERR',\n", + " 'S3_LAN',\n", + " 'S3_OLCI_L2LFR',\n", + " 'S3_OLCI_L2LRR',\n", + " 'S3_OLCI_L2WFR',\n", + " 'S3_OLCI_L2WFR_BC003',\n", + " 'S3_OLCI_L2WRR',\n", + " 'S3_OLCI_L2WRR_BC003',\n", + " 'S3_OLCI_L4BALTIC',\n", + " 'S3_RAC',\n", + " 'S3_SLSTR_L1RBT',\n", + " 'S3_SLSTR_L1RBT_BC004',\n", + " 'S3_SLSTR_L2',\n", + " 'S3_SLSTR_L2AOD',\n", + " 'S3_SLSTR_L2FRP',\n", + " 'S3_SLSTR_L2LST',\n", + " 'S3_SLSTR_L2WST',\n", + " 'S3_SLSTR_L2WST_BC003',\n", + " 'S3_SRA',\n", + " 'S3_SRA_1A_BC004',\n", + " 'S3_SRA_1B_BC004',\n", + " 'S3_SRA_A',\n", + " 'S3_SRA_BS',\n", + " 'S3_SRA_BS_BC004',\n", + " 'S3_SY_AOD',\n", + " 'S3_SY_SYN',\n", + " 'S3_SY_V10',\n", + " 'S3_SY_VG1',\n", + " 'S3_SY_VGP',\n", + " 'S3_WAT',\n", + " 'S3_WAT_BC004']" + ] + }, + "execution_count": 73, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dag.guess_product_type(platform=\"[SENTINEL1 TO SENTINEL3]\")" ] @@ -2190,9 +2454,34 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 74, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "['LANDSAT_C2L1',\n", + " 'L57_REFLECTANCE',\n", + " 'LANDSAT_C2L2',\n", + " 'LANDSAT_C2L2ALB_BT',\n", + " 'LANDSAT_C2L2ALB_SR',\n", + " 'LANDSAT_C2L2ALB_ST',\n", + " 'LANDSAT_C2L2ALB_TA',\n", + " 'LANDSAT_C2L2_SR',\n", + " 'LANDSAT_C2L2_ST',\n", + " 'LANDSAT_ETM_C1',\n", + " 'LANDSAT_ETM_C2L1',\n", + " 'LANDSAT_ETM_C2L2',\n", + " 'LANDSAT_TM_C1',\n", + " 'LANDSAT_TM_C2L1',\n", + " 'LANDSAT_TM_C2L2']" + ] + }, + "execution_count": 74, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "dag.guess_product_type(platform=\"LANDSAT\", platformSerialIdentifier=\"L1\")" ] @@ -2237,7 +2526,7 @@ }, { "cell_type": "code", - "execution_count": 48, + "execution_count": 75, "metadata": {}, "outputs": [ { @@ -2252,7 +2541,7 @@ " 'sensorType': 'OPTICAL'}" ] }, - "execution_count": 48, + "execution_count": 75, "metadata": {}, "output_type": "execute_result" } @@ -2271,7 +2560,7 @@ }, { "cell_type": "code", - "execution_count": 49, + "execution_count": 76, "metadata": {}, "outputs": [ { @@ -2280,7 +2569,7 @@ "'S2_MSI_L1C'" ] }, - "execution_count": 49, + "execution_count": 76, "metadata": {}, "output_type": "execute_result" } @@ -2291,17 +2580,16 @@ }, { "cell_type": "code", - "execution_count": 50, + "execution_count": 77, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:09:49,233 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:09:49,235 eodag.plugins.search.qssearch [INFO ] Sending count request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=1&page=1\n", - "2023-01-13 11:09:49,743 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", - "2023-01-13 11:09:50,514 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" + "2024-02-06 16:15:08,624 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:15:08,626 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "2024-02-06 16:15:10,928 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" ] } ], @@ -2319,7 +2607,7 @@ }, { "cell_type": "code", - "execution_count": 51, + "execution_count": 78, "metadata": {}, "outputs": [ { @@ -2328,7 +2616,7 @@ "'S2_MSI_L1C'" ] }, - "execution_count": 51, + "execution_count": 78, "metadata": {}, "output_type": "execute_result" } @@ -2379,17 +2667,16 @@ }, { "cell_type": "code", - "execution_count": 52, + "execution_count": 79, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:10:05,660 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:10:05,664 eodag.plugins.search.qssearch [INFO ] Sending count request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?cloudCover=[0,10]&startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=1&page=1\n", - "2023-01-13 11:10:06,223 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?cloudCover=[0,10]&startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", - "2023-01-13 11:10:06,922 eodag.core [INFO ] Found 10 result(s) on provider 'peps'\n" + "2024-02-06 16:15:10,956 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:15:10,959 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?cloudCover=[0,10]&startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "2024-02-06 16:15:12,893 eodag.core [INFO ] Found 11 result(s) on provider 'peps'\n" ] } ], @@ -2410,16 +2697,26 @@ }, { "cell_type": "code", - "execution_count": 53, + "execution_count": 80, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "[0, 7.7926, 1.5388000000000002, 0.0185, 0, 0, 4.4443, 1.7617, 6.8998, 0]" + "[0,\n", + " 0.332726867187601,\n", + " 3.4506936264591905,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 0,\n", + " 7.062549162254081,\n", + " 2.06458837230135]" ] }, - "execution_count": 53, + "execution_count": 80, "metadata": {}, "output_type": "execute_result" } @@ -2446,17 +2743,16 @@ }, { "cell_type": "code", - "execution_count": 54, + "execution_count": 81, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "2023-01-13 11:10:20,091 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", - "2023-01-13 11:10:20,094 eodag.plugins.search.qssearch [INFO ] Sending count request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?random_parameter=random_value&startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=1&page=1\n", - "2023-01-13 11:10:20,543 eodag.plugins.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?random_parameter=random_value&startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", - "2023-01-13 11:10:21,423 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" + "2024-02-06 16:15:12,919 eodag.core [INFO ] Searching product type 'S2_MSI_L1C' on provider: peps\n", + "2024-02-06 16:15:12,922 eodag.search.qssearch [INFO ] Sending search request: https://peps.cnes.fr/resto/api/collections/S2ST/search.json?random_parameter=random_value&startDate=2021-03-01&completionDate=2021-03-31&geometry=POLYGON ((1.0000 43.0000, 1.0000 44.0000, 2.0000 44.0000, 2.0000 43.0000, 1.0000 43.0000))&productType=S2MSI1C&maxRecords=20&page=1\n", + "2024-02-06 16:15:15,274 eodag.core [INFO ] Found 48 result(s) on provider 'peps'\n" ] } ], @@ -2482,7 +2778,7 @@ "source": [ "## Fallback in case of error\n", "\n", - "Usually, search is performed on the provider with the highest priority. If the request to this provider fails, i.e. if an error is returned (for example because the provider is currently unavailable) or if no results are returned, a request to the the provider with the next highest priority offering the desired product type will be attempted. In this case a warning message will be shown in the logs. If the request fails for all providers offering the desired product type, an error message will be returned." + "Usually, search is performed on the provider with the highest priority. If the request to this provider fails, i.e. if an error is returned (for example because the provider is currently unavailable) or if no results are returned, a request to the the provider with the next highest priority offering the desired product type will be attempted. In this case a warning message will be shown in the logs. If the request fails for all providers offering the desired product type, if at least one error occured during the fallback, an error will be raised, otherwise only an error message will be returned." ] }, { diff --git a/eodag/api/core.py b/eodag/api/core.py index a7fcb84e4..15e8b2736 100644 --- a/eodag/api/core.py +++ b/eodag/api/core.py @@ -104,6 +104,7 @@ from eodag.plugins.apis.base import Api from eodag.plugins.crunch.base import Crunch from eodag.plugins.search.base import Search + from eodag.types import ProviderSortables from eodag.utils import Annotated, DownloadedCallback, ProgressCallback logger = logging.getLogger("eodag.core") @@ -546,7 +547,7 @@ def list_product_types( product_types.append(product_type) return sorted(product_types, key=itemgetter("ID")) raise UnsupportedProvider( - f"The requested provider is not (yet) supported: {provider}" + f"invalid requested provider: {provider} is not (yet) supported" ) # Only get the product types supported by the available providers for provider in self.available_providers(): @@ -1805,6 +1806,9 @@ def _do_search( if not raise_errors: log_msg += " Raise verbosity of log messages for details" logger.info(log_msg) + # keep only the message from exception args + if len(e.args) > 1: + e.args = (e.args[0],) if raise_errors: # Raise the error, letting the application wrapping eodag know that # something went bad. This way it will be able to decide what to do next @@ -2248,3 +2252,36 @@ def list_queryables( provider_queryables.update(model_fields_to_annotated(common_queryables)) return provider_queryables + + def available_sortables(self) -> Dict[str, Optional[ProviderSortables]]: + """For each provider, gives its available sortable parameter(s) and its maximum + number of them if it supports the sorting feature, otherwise gives None. + + :returns: A dictionnary with providers as keys and dictionnary of sortable parameter(s) and + its (their) maximum number as value(s). + :rtype: dict + :raises: :class:`~eodag.utils.exceptions.UnsupportedProvider` + """ + sortables: Dict[str, Optional[ProviderSortables]] = {} + provider_search_plugins = self._plugins_manager.get_search_plugins() + for provider_search_plugin in provider_search_plugins: + provider = provider_search_plugin.provider + if not hasattr(provider_search_plugin.config, "sort"): + sortables[provider] = None + continue + sortable_params = list( + provider_search_plugin.config.sort["sort_param_mapping"].keys() + ) + if not provider_search_plugin.config.sort.get("max_sort_params"): + sortables[provider] = { + "sortables": sortable_params, + "max_sort_params": None, + } + continue + sortables[provider] = { + "sortables": sortable_params, + "max_sort_params": provider_search_plugin.config.sort[ + "max_sort_params" + ], + } + return sortables diff --git a/eodag/config.py b/eodag/config.py index 9f437fd48..1770f219b 100644 --- a/eodag/config.py +++ b/eodag/config.py @@ -27,6 +27,7 @@ ItemsView, Iterator, List, + Literal, Optional, Tuple, TypedDict, @@ -40,6 +41,7 @@ import yaml import yaml.constructor import yaml.parser +from annotated_types import Gt from jsonpath_ng import JSONPath from pkg_resources import resource_filename from requests.auth import AuthBase @@ -47,6 +49,7 @@ from eodag.utils import ( HTTP_REQ_TIMEOUT, USER_AGENT, + Annotated, cached_yaml_load, cached_yaml_load_all, cast_scalar_value, @@ -228,6 +231,15 @@ class Pagination(TypedDict): count_endpoint: str start_page: int + class Sort(TypedDict): + """Configuration for sort during search""" + + sort_by_default: List[Tuple[str, str]] + sort_by_tpl: str + sort_param_mapping: Dict[str, str] + sort_order_mapping: Dict[Literal["ascending", "descending"], str] + max_sort_params: Annotated[int, Gt(0)] + class OrderStatusOnSuccess(TypedDict): """Configuration for order on-success during download""" @@ -251,6 +263,7 @@ class OrderStatusOnSuccess(TypedDict): result_type: str results_entry: str pagination: PluginConfig.Pagination + sort: PluginConfig.Sort query_params_key: str discover_metadata: Dict[str, str] discover_product_types: Dict[str, Any] diff --git a/eodag/plugins/apis/usgs.py b/eodag/plugins/apis/usgs.py index b306f2e34..46469ace2 100644 --- a/eodag/plugins/apis/usgs.py +++ b/eodag/plugins/apis/usgs.py @@ -52,6 +52,7 @@ NoMatchingProductType, NotAvailableError, RequestError, + ValidationError, ) if TYPE_CHECKING: @@ -117,6 +118,8 @@ def query( raise NoMatchingProductType( "Cannot search on USGS without productType specified" ) + if kwargs.get("sortBy"): + raise ValidationError("USGS does not support sorting feature") self.authenticate() diff --git a/eodag/plugins/search/base.py b/eodag/plugins/search/base.py index b84d2e490..1a2dfacb2 100644 --- a/eodag/plugins/search/base.py +++ b/eodag/plugins/search/base.py @@ -20,6 +20,7 @@ import logging from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple +import orjson from pydantic.fields import Field, FieldInfo from eodag.api.product.metadata_mapping import ( @@ -27,13 +28,16 @@ mtd_cfg_as_conversion_and_querypath, ) from eodag.plugins.base import PluginTopic +from eodag.types.search_args import SortByList from eodag.utils import ( DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, GENERIC_PRODUCT_TYPE, Annotated, format_dict_items, + update_nested_dict, ) +from eodag.utils.exceptions import ValidationError if TYPE_CHECKING: from eodag.api.product import EOProduct @@ -181,3 +185,120 @@ def get_metadata_mapping( return self.config.products.get(product_type, {}).get( "metadata_mapping", self.config.metadata_mapping ) + + def get_sort_by_arg(self, kwargs: Dict[str, Any]) -> Optional[SortByList]: + """Extract the "sortBy" argument from the kwargs or the provider default sort configuration + + :param kwargs: Search arguments + :type kwargs: Dict[str, Any] + :returns: The "sortBy" argument from the kwargs or the provider default sort configuration + :rtype: :class:`~eodag.types.search_args.SortByList` + """ + # remove "sortBy" from search args if exists because it is not part of metadata mapping, + # it will complete the query string or body once metadata mapping will be done + sort_by_arg_tmp = kwargs.pop("sortBy", None) + sort_by_arg = sort_by_arg_tmp or getattr(self.config, "sort", {}).get( + "sort_by_default", None + ) + if not sort_by_arg_tmp and sort_by_arg: + logger.info( + f"{self.provider} is configured with default sorting by '{sort_by_arg[0][0]}' " + f"in {'ascending' if sort_by_arg[0][1] == 'ASC' else 'descending'} order" + ) + return sort_by_arg + + def build_sort_by( + self, sort_by_arg: SortByList + ) -> Tuple[str, Dict[str, List[Dict[str, str]]]]: + """Build the sorting part of the query string or body by transforming + the "sortBy" argument into a provider-specific string or dictionnary + + :param sort_by_arg: the "sortBy" argument in EODAG format + :type sort_by_arg: :class:`~eodag.types.search_args.SortByList` + :returns: The "sortBy" argument in provider-specific format + :rtype: Union[str, Dict[str, List[Dict[str, str]]]] + """ + if not hasattr(self.config, "sort"): + raise ValidationError(f"{self.provider} does not support sorting feature") + # TODO: remove this code block when search args model validation is embeded + # remove duplicates + sort_by_arg = list(set(sort_by_arg)) + + sort_by_qs: str = "" + sort_by_qp: Dict[str, Any] = {} + + provider_sort_by_tuples_used: List[Tuple[str, str]] = [] + for eodag_sort_by_tuple in sort_by_arg: + eodag_sort_param = eodag_sort_by_tuple[0] + provider_sort_param = self.config.sort["sort_param_mapping"].get( + eodag_sort_param, None + ) + if not provider_sort_param: + joined_eodag_params_to_map = ", ".join( + k for k in self.config.sort["sort_param_mapping"].keys() + ) + params = set(self.config.sort["sort_param_mapping"].keys()) + params.add(eodag_sort_param) + raise ValidationError( + f"'{eodag_sort_param}' parameter is not sortable with {self.provider}. " + f"Here is the list of sortable parameter(s) with {self.provider}: {joined_eodag_params_to_map}", + params, + ) + eodag_sort_order = eodag_sort_by_tuple[1] + # TODO: remove this code block when search args model validation is embeded + # Remove leading and trailing whitespace(s) if exist + eodag_sort_order = eodag_sort_order.strip().upper() + if eodag_sort_order[:3] != "ASC" and eodag_sort_order[:3] != "DES": + raise ValidationError( + "Sorting order is invalid: it must be set to 'ASC' (ASCENDING) or " + f"'DESC' (DESCENDING), got '{eodag_sort_order}' with '{eodag_sort_param}' instead" + ) + eodag_sort_order = eodag_sort_order[:3] + + provider_sort_order = ( + self.config.sort["sort_order_mapping"]["ascending"] + if eodag_sort_order == "ASC" + else self.config.sort["sort_order_mapping"]["descending"] + ) + provider_sort_by_tuple: Tuple[str, str] = ( + provider_sort_param, + provider_sort_order, + ) + # TODO: remove this code block when search args model validation is embeded + for provider_sort_by_tuple_used in provider_sort_by_tuples_used: + # since duplicated tuples or dictionnaries have been removed, if two sorting parameters are equal, + # then their sorting order is different and there is a contradiction that would raise an error + if provider_sort_by_tuple[0] == provider_sort_by_tuple_used[0]: + raise ValidationError( + f"'{eodag_sort_param}' parameter is called several times to sort results with different " + "sorting orders. Please set it to only one ('ASC' (ASCENDING) or 'DESC' (DESCENDING))", + set([eodag_sort_param]), + ) + provider_sort_by_tuples_used.append(provider_sort_by_tuple) + + # TODO: move this code block to the top of this method when search args model validation is embeded + # check if the limit number of sorting parameter(s) is respected with this sorting parameter + if ( + self.config.sort.get("max_sort_params", None) + and len(provider_sort_by_tuples_used) + > self.config.sort["max_sort_params"] + ): + raise ValidationError( + f"Search results can be sorted by only {self.config.sort['max_sort_params']} " + f"parameter(s) with {self.provider}" + ) + + parsed_sort_by_tpl: str = self.config.sort["sort_by_tpl"].format( + sort_param=provider_sort_by_tuple[0], + sort_order=provider_sort_by_tuple[1], + ) + try: + parsed_sort_by_tpl_dict: Dict[str, Any] = orjson.loads( + parsed_sort_by_tpl + ) + sort_by_qp = update_nested_dict( + sort_by_qp, parsed_sort_by_tpl_dict, extend_list_values=True + ) + except orjson.JSONDecodeError: + sort_by_qs += parsed_sort_by_tpl + return (sort_by_qs, sort_by_qp) diff --git a/eodag/plugins/search/data_request_search.py b/eodag/plugins/search/data_request_search.py index 1dbfeec2e..4de7583b6 100644 --- a/eodag/plugins/search/data_request_search.py +++ b/eodag/plugins/search/data_request_search.py @@ -41,7 +41,12 @@ deepcopy, string_to_jsonpath, ) -from eodag.utils.exceptions import NotAvailableError, RequestError, TimeOutError +from eodag.utils.exceptions import ( + NotAvailableError, + RequestError, + TimeOutError, + ValidationError, +) if TYPE_CHECKING: from eodag.config import PluginConfig @@ -128,6 +133,9 @@ def query( """ performs the search for a provider where several steps are required to fetch the data """ + if kwargs.get("sortBy"): + raise ValidationError(f"{self.provider} does not support sorting feature") + product_type = kwargs.get("productType", None) # replace "product_type" to "providerProductType" in search args if exists # for compatibility with DataRequestSearch method diff --git a/eodag/plugins/search/qssearch.py b/eodag/plugins/search/qssearch.py index 11f1103e6..f6875ae44 100644 --- a/eodag/plugins/search/qssearch.py +++ b/eodag/plugins/search/qssearch.py @@ -44,6 +44,7 @@ ) from eodag.plugins.search.base import Search from eodag.types import json_field_definition_to_python, model_fields_to_annotated +from eodag.types.search_args import SortByList from eodag.utils import ( DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, @@ -471,6 +472,11 @@ def query( # remove "product_type" from search args if exists for compatibility with QueryStringSearch methods kwargs.pop("product_type", None) + sort_by_arg: Optional[SortByList] = self.get_sort_by_arg(kwargs) + sort_by_qs, _ = ( + ("", {}) if sort_by_arg is None else self.build_sort_by(sort_by_arg) + ) + provider_product_type = self.map_product_type(product_type) keywords = {k: v for k, v in kwargs.items() if k != "auth" and v is not None} keywords["productType"] = ( @@ -510,7 +516,11 @@ def query( self.query_params = qp self.query_string = qs self.search_urls, total_items = self.collect_search_urls( - page=page, items_per_page=items_per_page, count=count, **kwargs + page=page, + items_per_page=items_per_page, + count=count, + sort_by_qs=sort_by_qs, + **kwargs, ) if not count and hasattr(self, "total_items_nb"): # do not try to extract total_items from search results if count is False @@ -559,6 +569,10 @@ def collect_search_urls( urls = [] total_results = 0 if count else None + # use only sort_by parameters for search, not for count + # and remove potential leading '&' + qs_with_sort = (self.query_string + kwargs.get("sort_by_qs", "")).strip("&") + if "count_endpoint" not in self.config.pagination: # if count_endpoint is not set, total_results should be extracted from search result total_results = None @@ -594,14 +608,14 @@ def collect_search_urls( total_results += _total_results or 0 next_url = self.config.pagination["next_page_url_tpl"].format( url=search_endpoint, - search=self.query_string, + search=qs_with_sort, items_per_page=items_per_page, page=page, skip=(page - 1) * items_per_page, skip_base_1=(page - 1) * items_per_page + 1, ) else: - next_url = "{}?{}".format(search_endpoint, self.query_string) + next_url = "{}?{}".format(search_endpoint, qs_with_sort) urls.append(next_url) return urls, total_results @@ -1032,6 +1046,10 @@ def query( product_type = kwargs.get("productType", None) # remove "product_type" from search args if exists for compatibility with QueryStringSearch methods kwargs.pop("product_type", None) + sort_by_arg: Optional[SortByList] = self.get_sort_by_arg(kwargs) + _, sort_by_qp = ( + ("", {}) if sort_by_arg is None else self.build_sort_by(sort_by_arg) + ) provider_product_type = self.map_product_type(product_type) keywords = {k: v for k, v in kwargs.items() if k != "auth" and v is not None} @@ -1116,7 +1134,7 @@ def query( if isinstance(product_type_metadata_mapping.get(k, []), list) ): return [], 0 - self.query_params = qp + self.query_params = dict(qp, **sort_by_qp) self.search_urls, total_items = self.collect_search_urls( page=page, items_per_page=items_per_page, count=count, **kwargs ) diff --git a/eodag/resources/providers.yml b/eodag/resources/providers.yml index 09acbeb3b..6a513b10e 100644 --- a/eodag/resources/providers.yml +++ b/eodag/resources/providers.yml @@ -882,6 +882,18 @@ next_page_url_tpl: '{url}?{search}&maxRecords={items_per_page}&page={page}&exactCount=1' total_items_nb_key_path: '$.properties.totalResults' max_items_per_page: 1_000 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_by_tpl: '&sortParam={sort_param}&sortOrder={sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: startDate + completionTimeFromAscendingNode: completionDate + publicationDate: published + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 discover_metadata: auto_discovery: true metadata_pattern: '^(?!collection)[a-zA-Z0-9]+$' @@ -1239,6 +1251,18 @@ next_page_url_tpl: '{url}?{search}&$top={items_per_page}&$skip={skip}&$expand=Metadata' # 2021/03/19: 2000 is the max, if greater 200 response but contains an error message max_items_per_page: 2_000 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_by_tpl: '&$orderby={sort_param} {sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: beginPosition + uid: id + storageStatus: offline + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 results_entry: 'value' literal_search_params: $format: json @@ -1529,6 +1553,13 @@ # This provider doesn't implement any pagination, let's just try to get the maximum number of # products available at once then, so we stick to 10_000. max_items_per_page: 10_000 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_param_mapping: + id: id + startTimeFromAscendingNode: properties.datetime + creationDate: properties.created metadata_mapping: # redefine the following mapppings as the provider does not support advanced queries/filtering, # these parameters will not be queryable @@ -1647,6 +1678,18 @@ # but in practive if an Internal Server Error is returned for more than # about 500 products. max_items_per_page: 500 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_param_mapping: + id: id + startTimeFromAscendingNode: properties.datetime + creationDate: properties.created + modificationDate: properties.updated + platformSerialIdentifier: properties.platform + illuminationElevationAngle: properties.view:sun_elevation + illuminationAzimuthAngle: properties.view:sun_azimuth + cloudCover: properties.eo:cloud_cover metadata_mapping: assets: '{$.assets#recursive_sub_str(r"https?(.*)landsatlook.usgs.gov/data/",r"s3\1usgs-landsat/")}' awsProductId: '{$.assets.thumbnail.href#replace_str(r".+/([A-Z0-9_]+)/[\w.]+$",r"\1")}' @@ -1701,6 +1744,17 @@ # say the max is 10_000. In practice a too high number (e.g. 5_000) returns a 502 error ({"message": "Internal server error"}). # Let's set it to a more robust number: 500 max_items_per_page: 500 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_param_mapping: + id: id + startTimeFromAscendingNode: properties.datetime + creationDate: properties.created + modificationDate: properties.updated + platformSerialIdentifier: properties.platform + resolution: properties.gsd + cloudCover: properties.eo:cloud_cover metadata_mapping: utmZone: - '{{"query":{{"mgrs:utm_zone":{{"eq":"{utmZone}"}}}}}}' @@ -1790,6 +1844,18 @@ # say the max is 10_000. In practice a too high number (e.g. 5_000) returns a 502 error ({"message": "Internal server error"}). # Let's set it to a more robust number: 500 max_items_per_page: 500 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_param_mapping: + id: id + startTimeFromAscendingNode: properties.datetime + creationDate: properties.created + modificationDate: properties.updated + platform: properties.constellation + platformSerialIdentifier: properties.platform + resolution: properties.gsd + cloudCover: properties.eo:cloud_cover metadata_mapping: platformSerialIdentifier: '$.id.`split(_, 0, -1)`' polarizationMode: '$.id.`sub(/.{14}([A-Z]{2}).*/, \\1)`' @@ -1829,6 +1895,17 @@ # say the max is 10_000. In practice a too high number (e.g. 5_000) returns a 502 error ({"message": "Internal server error"}). # Let's set it to a more robust number: 500 max_items_per_page: 500 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_param_mapping: + id: id + startTimeFromAscendingNode: properties.datetime + creationDate: properties.created + modificationDate: properties.updated + platformSerialIdentifier: properties.platform + resolution: properties.gsd + cloudCover: properties.eo:cloud_cover products: S2_MSI_L1C: productType: sentinel-s2-l1c @@ -3084,6 +3161,18 @@ total_items_nb_key_path: '$.properties.totalResults' # 2021/03/19: 500 is the max, no error if greater max_items_per_page: 500 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_by_tpl: '&sortParam={sort_param}&sortOrder={sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: startDate + completionTimeFromAscendingNode: completionDate + sensorMode: sensorMode + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 discover_metadata: auto_discovery: true metadata_pattern: '^(?!collection)[a-zA-Z0-9_]+$' @@ -3435,6 +3524,19 @@ next_page_url_tpl: '{url}?{search}&maxRecords={items_per_page}&page={page}&exactCount=1' total_items_nb_key_path: '$.properties.totalResults' max_items_per_page: 1_000 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_by_tpl: '&sortParam={sort_param}&sortOrder={sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: startDate + completionTimeFromAscendingNode: completionDate + publicationDate: published + modificationDate: updated + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 discover_metadata: auto_discovery: true metadata_pattern: '^(?!collection)[a-zA-Z0-9]+$' @@ -3723,6 +3825,11 @@ need_auth: false pagination: max_items_per_page: 1000 + sort: + sort_param_mapping: + id: id + startTimeFromAscendingNode: properties.datetime + platformSerialIdentifier: properties.platform metadata_mapping: tileIdentifier: - '{{"query":{{"s2:mgrs_tile":{{"eq":"{tileIdentifier}"}}}}}}' @@ -3775,6 +3882,15 @@ product_type_fetch_url: null pagination: max_items_per_page: 10_000 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_param_mapping: + id: id + startTimeFromAscendingNode: properties.start_datetime + completionTimeFromAscendingNode: properties.end_datetime + productVersion: properties.version + processingLevel: processing:level metadata_mapping: startTimeFromAscendingNode: - '{{"query":{{"end_datetime":{{"gte":"{startTimeFromAscendingNode#to_iso_utc_datetime}"}}}}}}' @@ -6745,6 +6861,18 @@ next_page_url_tpl: '{url}?{search}&maxRecords={items_per_page}&page={page}&exactCount=1' total_items_nb_key_path: '$.properties.totalResults' max_items_per_page: 1_000 + sort: + sort_by_default: + - !!python/tuple [startTimeFromAscendingNode, ASC] + sort_by_tpl: '&sortParam={sort_param}&sortOrder={sort_order}' + sort_param_mapping: + startTimeFromAscendingNode: startDate + completionTimeFromAscendingNode: completionDate + publicationDate: published + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 discover_metadata: auto_discovery: true metadata_pattern: '^(?!collection)[a-zA-Z0-9]+$' diff --git a/eodag/resources/stac_api.yml b/eodag/resources/stac_api.yml index 5415be3d8..9e9ade9de 100644 --- a/eodag/resources/stac_api.yml +++ b/eodag/resources/stac_api.yml @@ -1470,10 +1470,11 @@ components: - 34 collections: - S2_MSI_L1C - platform: - eq: S2A - eo:cloud_cover: - lte: 80 + query: + platform: + eq: S2A + eo:cloud_cover: + lte: 80 limit: 10 limit: type: integer diff --git a/eodag/resources/stac_provider.yml b/eodag/resources/stac_provider.yml index a6acbeff0..cd175373b 100644 --- a/eodag/resources/stac_provider.yml +++ b/eodag/resources/stac_provider.yml @@ -24,6 +24,11 @@ search: next_page_url_key_path: '$.links[?(@.rel="next")].href' next_page_query_obj_key_path: '$.links[?(@.rel="next")].body' next_page_merge_key_path: '$.links[?(@.rel="next")].merge' + sort: + sort_by_tpl: '{{"sortby": [ {{"field": "{sort_param}", "direction": "{sort_order}" }} ] }}' + sort_order_mapping: + ascending: asc + descending: desc discover_metadata: auto_discovery: true metadata_pattern: '^[a-zA-Z0-9_:-]+$' diff --git a/eodag/rest/server.py b/eodag/rest/server.py index b2b9b48bf..7700e25a6 100755 --- a/eodag/rest/server.py +++ b/eodag/rest/server.py @@ -44,6 +44,7 @@ from starlette.exceptions import HTTPException as StarletteHTTPException from eodag.config import load_stac_api_config +from eodag.rest.types.eodag_search import EODAGSearch from eodag.rest.types.stac_queryables import StacQueryables from eodag.rest.utils import ( download_stac_item_by_id_stream, @@ -79,6 +80,12 @@ logger = logging.getLogger("eodag.rest.server") +ERRORS_WITH_500_STATUS_CODE = { + "MisconfiguredError", + "AuthenticationError", + "DownloadError", + "RequestError", +} class APIRouter(FastAPIRouter): @@ -221,10 +228,28 @@ async def default_exception_handler( ) +@app.exception_handler(ValidationError) +async def handle_invalid_usage_with_validation_error( + request: Request, error: ValidationError +) -> ORJSONResponse: + """Invalid usage [400] ValidationError handle""" + if error.parameters: + for error_param in error.parameters: + stac_param = EODAGSearch.to_stac(error_param) + error.message = error.message.replace(error_param, stac_param) + logger.warning(traceback.format_exc()) + return await default_exception_handler( + request, + HTTPException( + status_code=400, + detail=f"{type(error).__name__}: {str(error.message)}", + ), + ) + + @app.exception_handler(NoMatchingProductType) @app.exception_handler(UnsupportedProductType) @app.exception_handler(UnsupportedProvider) -@app.exception_handler(ValidationError) async def handle_invalid_usage(request: Request, error: Exception) -> ORJSONResponse: """Invalid usage [400] errors handle""" logger.warning(traceback.format_exc()) @@ -254,7 +279,7 @@ async def handle_resource_not_found( @app.exception_handler(MisconfiguredError) @app.exception_handler(AuthenticationError) async def handle_auth_error(request: Request, error: Exception) -> ORJSONResponse: - """AuthenticationError should be sent as internal server error to the client""" + """These errors should be sent as internal server error to the client""" logger.error(f"{type(error).__name__}: {str(error)}") return await default_exception_handler( request, @@ -266,9 +291,36 @@ async def handle_auth_error(request: Request, error: Exception) -> ORJSONRespons @app.exception_handler(DownloadError) +async def handle_download_error(request: Request, error: Exception) -> ORJSONResponse: + """DownloadError should be sent as internal server error with details to the client""" + logger.error(f"{type(error).__name__}: {str(error)}") + return await default_exception_handler( + request, + HTTPException( + status_code=500, + detail=f"{type(error).__name__}: {str(error)}", + ), + ) + + @app.exception_handler(RequestError) -async def handle_server_error(request: Request, error: Exception) -> ORJSONResponse: - """These errors should be sent as internal server error with details to the client""" +async def handle_request_error(request: Request, error: RequestError) -> ORJSONResponse: + """RequestError should be sent as internal server error with details to the client""" + if getattr(error, "history", None): + error_history_tmp = list(error.history) + for i, search_error in enumerate(error_history_tmp): + if search_error[1].__class__.__name__ in ERRORS_WITH_500_STATUS_CODE: + search_error[1].args = ("an internal error occured",) + error_history_tmp[i] = search_error + continue + if getattr(error, "parameters", None): + for error_param in error.parameters: + stac_param = EODAGSearch.to_stac(error_param) + search_error[1].args = ( + search_error[1].args[0].replace(error_param, stac_param), + ) + error_history_tmp[i] = search_error + error.history = set(error_history_tmp) logger.error(f"{type(error).__name__}: {str(error)}") return await default_exception_handler( request, diff --git a/eodag/rest/types/eodag_search.py b/eodag/rest/types/eodag_search.py index 937a64f31..70d5305a8 100644 --- a/eodag/rest/types/eodag_search.py +++ b/eodag/rest/types/eodag_search.py @@ -15,7 +15,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Union from pydantic import ( BaseModel, @@ -90,7 +90,6 @@ class EODAGSearch(BaseModel): illuminationAzimuthAngle: Optional[float] = Field(None, alias="view:sun_azimuth") page: Optional[int] = Field(1) items_per_page: int = Field(DEFAULT_ITEMS_PER_PAGE, alias="limit") - sortBy: Optional[List[Tuple[str, str]]] = Field(None, alias="sortby") @model_validator(mode="before") @classmethod @@ -125,21 +124,6 @@ def join_instruments(cls, v: Union[str, List[str]]) -> str: return ",".join(v) return v - @field_validator("sortBy", mode="before") - @classmethod - def convert_stac_to_eodag_sortby( - cls, - sortby_post_params: List[Dict[str, str]], - ) -> List[Tuple[str, str]]: - """ - Convert STAC POST sortby to EODAG sortby - """ - eodag_sortby: List[Tuple[str, str]] = [] - for sortby_post_param in sortby_post_params: - field = cls.snake_to_camel(cls.to_eodag(sortby_post_param["field"])) - eodag_sortby.append((field, sortby_post_param["direction"])) - return eodag_sortby - @field_validator("productType") @classmethod def verify_producttype_is_present( diff --git a/eodag/rest/utils.py b/eodag/rest/utils.py index 524a52edd..922823db4 100644 --- a/eodag/rest/utils.py +++ b/eodag/rest/utils.py @@ -510,9 +510,12 @@ def search_products( if not products and eodag_api.search_errors: search_error = RequestError( "No result could be obtained from any available provider and following " - "error(s) appeared while searching:" + "error(s) occured while searching:" ) search_error.history = eodag_api.search_errors + for one_search_error in eodag_api.search_errors: + if getattr(one_search_error[1], "parameters", None): + search_error.parameters.update(one_search_error[1].parameters) raise search_error products = filter_products(products, arguments, **criterias) diff --git a/eodag/types/__init__.py b/eodag/types/__init__.py index 7d950bfc0..90c298d17 100644 --- a/eodag/types/__init__.py +++ b/eodag/types/__init__.py @@ -18,8 +18,9 @@ """EODAG types""" from __future__ import annotations -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, Dict, List, Literal, Optional, TypedDict, Union +from annotated_types import Gt from pydantic import Field from pydantic.fields import FieldInfo @@ -217,3 +218,17 @@ def model_fields_to_annotated( new_field_info.annotation = None annotated_model_fields[param] = Annotated[field_type, new_field_info] return annotated_model_fields + + +class ProviderSortables(TypedDict): + """A class representing sortable parameter(s) of a provider and the allowed + maximum number of used sortable(s) in a search request with the provider + + :param sortables: The list of sortable parameter(s) of a provider + :type sortables: list[str] + :param max_sort_params: (optional) The allowed maximum number of sortable(s) in a search request with the provider + :type max_sort_params: int + """ + + sortables: List[str] + max_sort_params: Annotated[Optional[int], Gt(0)] diff --git a/eodag/types/search_args.py b/eodag/types/search_args.py index ea2ca24d5..ffb27d6fe 100644 --- a/eodag/types/search_args.py +++ b/eodag/types/search_args.py @@ -15,9 +15,11 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import re from datetime import datetime from typing import Dict, List, Optional, Tuple, Union, cast +from annotated_types import MinLen from pydantic import BaseModel, ConfigDict, Field, conint, field_validator from shapely import wkt from shapely.errors import GEOSException @@ -25,12 +27,14 @@ from shapely.geometry.base import GEOMETRY_TYPES, BaseGeometry from eodag.types.bbox import BBox -from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE +from eodag.utils import DEFAULT_ITEMS_PER_PAGE, DEFAULT_PAGE, Annotated +from eodag.utils.exceptions import ValidationError NumType = Union[float, int] GeomArgs = Union[List[NumType], Tuple[NumType], Dict[str, NumType], str, BaseGeometry] PositiveInt = conint(gt=0) +SortByList = Annotated[List[Tuple[str, str]], MinLen(1)] class SearchArgs(BaseModel): @@ -47,6 +51,7 @@ class SearchArgs(BaseModel): locations: Optional[Dict[str, str]] = Field(None) page: Optional[int] = Field(DEFAULT_PAGE, gt=0) # type: ignore items_per_page: Optional[PositiveInt] = Field(DEFAULT_ITEMS_PER_PAGE) # type: ignore + sortBy: Optional[SortByList] = Field(None) # type: ignore @field_validator("start", "end", mode="before") @classmethod @@ -81,3 +86,49 @@ def check_geom(cls, v: GeomArgs) -> BaseGeometry: return v raise TypeError(f"Invalid geometry type: {type(v)}") + + @field_validator("sortBy", mode="before") + @classmethod + def check_sort_by_arg( + cls, sort_by_arg: Optional[SortByList] # type: ignore + ) -> Optional[SortByList]: # type: ignore + """Check if the sortBy argument is correct + + :param sort_by_arg: The sortBy argument + :type sort_by_arg: str + :returns: The sortBy argument with sorting order parsed (whitespace(s) are + removed and only the 3 first letters in uppercase are kept) + :rtype: str + """ + if sort_by_arg is None: + return None + + assert isinstance( + sort_by_arg, list + ), f"Sort argument must be a list of tuple(s), got a '{type(sort_by_arg)}' instead" + sort_order_pattern = r"^(ASC|DES)[a-zA-Z]*$" + for i, sort_by_tuple in enumerate(sort_by_arg): + assert isinstance( + sort_by_tuple, tuple + ), f"Sort argument must be a list of tuple(s), got a list of '{type(sort_by_tuple)}' instead" + # get sorting elements by removing leading and trailing whitespace(s) if exist + sort_param = sort_by_tuple[0].strip() + sort_order = sort_by_tuple[1].strip().upper() + assert re.match(sort_order_pattern, sort_order) is not None, ( + "Sorting order must be set to 'ASC' (ASCENDING) or 'DESC' (DESCENDING), " + f"got '{sort_order}' with '{sort_param}' instead" + ) + sort_by_arg[i] = (sort_param, sort_order[:3]) + # remove duplicates + pruned_sort_by_arg: SortByList = list(set(sort_by_arg)) # type: ignore + for i, sort_by_tuple in enumerate(pruned_sort_by_arg): + for j, sort_by_tuple_tmp in enumerate(pruned_sort_by_arg): + # since duplicated tuples or dictionnaries have been removed, if two sorting parameters are equal, + # then their sorting order is different and there is a contradiction that would raise an error + if i != j and sort_by_tuple[0] == sort_by_tuple_tmp[0]: + raise ValidationError( + f"'{sort_by_tuple[0]}' parameter is called several times to sort results with different " + "sorting orders. Please set it to only one ('ASC' (ASCENDING) or 'DESC' (DESCENDING))", + set([sort_by_tuple[0]]), + ) + return pruned_sort_by_arg diff --git a/eodag/utils/exceptions.py b/eodag/utils/exceptions.py index d12f18a9a..cd8d4af13 100644 --- a/eodag/utils/exceptions.py +++ b/eodag/utils/exceptions.py @@ -26,8 +26,9 @@ class ValidationError(Exception): """Error validating data""" - def __init__(self, message: str) -> None: + def __init__(self, message: str, parameters: Set[str] = set()) -> None: self.message = message + self.parameters = parameters class PluginNotFoundError(Exception): @@ -79,12 +80,13 @@ class RequestError(Exception): """An error indicating that a request has failed. Usually eodag functions and methods should catch and skip this""" - history: Set[Tuple[Exception, str]] = set() + history: Set[Tuple[str, Exception]] = set() + parameters: Set[str] = set() def __str__(self): repr = super().__str__() for err_tuple in self.history: - repr += f"\n- {str(err_tuple)}" + repr += f"- {str(err_tuple)}" return repr diff --git a/tests/units/test_core.py b/tests/units/test_core.py index e083a27bc..1692e7c5a 100644 --- a/tests/units/test_core.py +++ b/tests/units/test_core.py @@ -1085,10 +1085,10 @@ def test_list_queryables( ): """list_queryables must return queryables list adapted to provider and product-type""" with self.assertRaises(UnsupportedProvider): - self.dag.list_queryables(provider="not_existing_provider") + self.dag.list_queryables(provider="not_supported_provider") with self.assertRaises(UnsupportedProductType): - self.dag.list_queryables(productType="not_existing_product_type") + self.dag.list_queryables(productType="not_supported_product_type") queryables_none_none = self.dag.list_queryables() expected_result = model_fields_to_annotated(CommonQueryables.model_fields) @@ -1159,6 +1159,158 @@ def test_list_queryables_with_constraints(self, mock_discover_queryables): } mock_discover_queryables.assert_called_once_with(plugin, **defaults) + def test_available_sortables(self): + """available_sortables must return available sortable(s) and its (their) + maximum number dict for providers which support the sorting feature""" + expected_result = { + "peps": None, + "usgs": None, + "creodias": { + "sortables": [ + "startTimeFromAscendingNode", + "completionTimeFromAscendingNode", + "publicationDate", + ], + "max_sort_params": 1, + }, + "aws_eos": None, + "theia": None, + "onda": { + "sortables": ["startTimeFromAscendingNode", "uid", "storageStatus"], + "max_sort_params": 1, + }, + "astraea_eod": { + "sortables": ["id", "startTimeFromAscendingNode", "creationDate"], + "max_sort_params": None, + }, + "usgs_satapi_aws": { + "sortables": [ + "id", + "startTimeFromAscendingNode", + "creationDate", + "modificationDate", + "platformSerialIdentifier", + "illuminationElevationAngle", + "illuminationAzimuthAngle", + "cloudCover", + ], + "max_sort_params": None, + }, + "earth_search": { + "sortables": [ + "id", + "startTimeFromAscendingNode", + "creationDate", + "modificationDate", + "platformSerialIdentifier", + "resolution", + "cloudCover", + ], + "max_sort_params": None, + }, + "earth_search_cog": { + "sortables": [ + "id", + "startTimeFromAscendingNode", + "creationDate", + "modificationDate", + "platform", + "platformSerialIdentifier", + "resolution", + "cloudCover", + ], + "max_sort_params": None, + }, + "earth_search_gcs": { + "sortables": [ + "id", + "startTimeFromAscendingNode", + "creationDate", + "modificationDate", + "platformSerialIdentifier", + "resolution", + "cloudCover", + ], + "max_sort_params": None, + }, + "ecmwf": None, + "cop_ads": None, + "cop_cds": None, + "sara": { + "sortables": [ + "startTimeFromAscendingNode", + "completionTimeFromAscendingNode", + "sensorMode", + ], + "max_sort_params": 1, + }, + "meteoblue": None, + "cop_dataspace": { + "sortables": [ + "startTimeFromAscendingNode", + "completionTimeFromAscendingNode", + "publicationDate", + "modificationDate", + ], + "max_sort_params": 1, + }, + "planetary_computer": { + "sortables": [ + "id", + "startTimeFromAscendingNode", + "platformSerialIdentifier", + ], + "max_sort_params": None, + }, + "hydroweb_next": { + "sortables": [ + "id", + "startTimeFromAscendingNode", + "completionTimeFromAscendingNode", + "productVersion", + "processingLevel", + ], + "max_sort_params": None, + }, + "wekeo": None, + "creodias_s3": { + "sortables": [ + "startTimeFromAscendingNode", + "completionTimeFromAscendingNode", + "publicationDate", + ], + "max_sort_params": 1, + }, + } + sortables = self.dag.available_sortables() + self.assertDictEqual(sortables, expected_result) + + # check if sortables are set to None when the provider does not support the sorting feature + self.assertFalse(hasattr(self.dag.providers_config["peps"].search, "sort")) + self.assertEqual(sortables["peps"], None) + + # check if sortable parameter(s) and its (their) maximum number of a provider are set + # to their value when the provider supports the sorting feature and has a maximum number of sortables + self.assertTrue(hasattr(self.dag.providers_config["creodias"].search, "sort")) + self.assertTrue( + self.dag.providers_config["creodias"].search.sort.get("max_sort_params") + ) + if sortables["creodias"]: + self.assertIsNotNone(sortables["creodias"]["max_sort_params"]) + + # check if sortable parameter(s) of a provider is set to its value and its (their) maximum number is set + # to None when the provider supports the sorting feature and does not have a maximum number of sortables + self.assertTrue( + hasattr(self.dag.providers_config["planetary_computer"].search, "sort") + ) + self.assertFalse( + self.dag.providers_config["planetary_computer"].search.sort.get( + "max_sort_params" + ) + ) + if sortables["planetary_computer"]: + self.assertIsNone(sortables["planetary_computer"]["max_sort_params"]) + class TestCoreConfWithEnvVar(TestCoreBase): @classmethod @@ -2151,6 +2303,222 @@ def test_search_iter_page_must_reset_next_attrs_if_next_mechanism( "dummy_next_page_url_tpl", ) + @mock.patch("eodag.plugins.search.qssearch.PostJsonSearch._request", autospec=True) + @mock.patch( + "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True + ) + @mock.patch( + "eodag.plugins.search.qssearch.QueryStringSearch.normalize_results", + autospec=True, + ) + def test_search_sort_by( + self, + mock_normalize_results, + mock_qssearch__request, + mock_postjsonsearch__request, + ): + """search must sort results by sorting parameter(s) in their sorting order + from the "sortBy" argument or by default sorting parameter if exists""" + mock_qssearch__request.return_value.json.return_value = { + "properties": {"totalResults": 2}, + "features": [], + "links": [{"rel": "next", "href": "url/to/next/page"}], + } + mock_postjsonsearch__request.return_value.json.return_value = { + "meta": {"found": 2}, + "features": [], + "links": [{"rel": "next", "href": "url/to/next/page"}], + } + + p1 = EOProduct( + "dummy", dict(geometry="POINT (0 0)", id="1", eodagSortParam="1") + ) + p1.search_intersection = None + p2 = EOProduct( + "dummy", dict(geometry="POINT (0 0)", id="2", eodagSortParam="2") + ) + p2.search_intersection = None + mock_normalize_results.return_value = [p2, p1] + + dag = EODataAccessGateway() + + # with a GET mode search + dummy_provider_config = """ + dummy_provider: + search: + type: QueryStringSearch + api_endpoint: https://api.my_new_provider/search + pagination: + next_page_url_tpl: '{url}?{search}' + total_items_nb_key_path: '$.properties.totalResults' + sort: + sort_by_tpl: '&sortParam={sort_param}&sortOrder={sort_order}' + sort_param_mapping: + eodagSortParam: providerSortParam + sort_order_mapping: + ascending: asc + descending: desc + metadata_mapping: + dummy: 'dummy' + products: + S2_MSI_L1C: + productType: '{productType}' + """ + dag.update_providers_config(dummy_provider_config) + + dag.search( + provider="dummy_provider", + productType="S2_MSI_L1C", + sortBy=[("eodagSortParam", "DESC")], + ) + + # a provider-specific string has been created to sort by + self.assertIn( + "sortParam=providerSortParam&sortOrder=desc", + mock_qssearch__request.call_args[0][1], + ) + + # with a POST mode search + dummy_provider_config = """ + other_dummy_provider: + search: + type: PostJsonSearch + api_endpoint: https://api.my_new_provider/search + pagination: + next_page_query_obj: '{{"limit":{items_per_page},"page":{page}}}' + total_items_nb_key_path: '$.meta.found' + sort: + sort_by_tpl: '{{"sortby": [ {{"field": "{sort_param}", "direction": "{sort_order}" }} ] }}' + sort_param_mapping: + eodagSortParam: providerSortParam + sort_order_mapping: + ascending: asc + descending: desc + metadata_mapping: + dummy: 'dummy' + products: + S2_MSI_L1C: + productType: '{productType}' + """ + dag.update_providers_config(dummy_provider_config) + dag.search( + provider="other_dummy_provider", + productType="S2_MSI_L1C", + sortBy=[("eodagSortParam", "DESC")], + ) + + # a provider-specific dictionnary has been created to sort by + self.assertIn( + "sortby", mock_postjsonsearch__request.call_args[0][0].query_params.keys() + ) + self.assertEqual( + [{"field": "providerSortParam", "direction": "desc"}], + mock_postjsonsearch__request.call_args[0][0].query_params["sortby"], + ) + + # TODO: sort by default sorting parameter and sorting order + + def test_search_sort_by_raise_errors(self): + """search used with "sortBy" argument must raise errors if the argument is incorrect or if the provider does + not support a maximum number of sorting parameter, one sorting parameter or the sorting feature""" + dag = EODataAccessGateway() + dummy_provider_config = """ + dummy_provider: + search: + type: QueryStringSearch + api_endpoint: https://api.my_new_provider/search + pagination: + next_page_url_tpl: '{url}?{search}{sort_by}&maxRecords={items_per_page}&page={page}&exactCount=1' + total_items_nb_key_path: '$.properties.totalResults' + metadata_mapping: + dummy: 'dummy' + products: + S2_MSI_L1C: + productType: '{productType}' + """ + dag.update_providers_config(dummy_provider_config) + # raise an error with a provider which does not support sorting feature + with self.assertLogs(level="ERROR") as cm_logs: + dag.search( + provider="dummy_provider", + productType="S2_MSI_L1C", + sortBy=[("eodagSortParam", "ASC")], + ) + self.assertIn( + "dummy_provider does not support sorting feature", str(cm_logs.output) + ) + + dummy_provider_config = """ + dummy_provider: + search: + type: QueryStringSearch + api_endpoint: https://api.my_new_provider/search + pagination: + next_page_url_tpl: '{url}?{search}{sort_by}&maxRecords={items_per_page}&page={page}&exactCount=1' + total_items_nb_key_path: '$.properties.totalResults' + sort: + sort_by_tpl: '&sortParam={sort_param}&sortOrder={sort_order}' + sort_param_mapping: + eodagSortParam: providerSortParam + sort_order_mapping: + ascending: asc + descending: desc + metadata_mapping: + dummy: 'dummy' + products: + S2_MSI_L1C: + productType: '{productType}' + """ + dag.update_providers_config(dummy_provider_config) + # raise an error with a parameter not sortable with a provider + with self.assertLogs(level="ERROR") as cm_logs: + dag.search( + provider="dummy_provider", + productType="S2_MSI_L1C", + sortBy=[("otherEodagSortParam", "ASC")], + ) + self.assertIn( + "\\'otherEodagSortParam\\' parameter is not sortable with dummy_provider. " + "Here is the list of sortable parameter(s) with dummy_provider: eodagSortParam", + str(cm_logs.output), + ) + + dummy_provider_config = """ + dummy_provider: + search: + type: QueryStringSearch + api_endpoint: https://api.my_new_provider/search + pagination: + next_page_url_tpl: '{url}?{search}{sort_by}&maxRecords={items_per_page}&page={page}&exactCount=1' + total_items_nb_key_path: '$.properties.totalResults' + sort: + sort_by_tpl: '&sortParam={sort_param}&sortOrder={sort_order}' + sort_param_mapping: + eodagSortParam: providerSortParam + otherEodagSortParam: otherProviderSortParam + sort_order_mapping: + ascending: asc + descending: desc + max_sort_params: 1 + metadata_mapping: + dummy: 'dummy' + products: + S2_MSI_L1C: + productType: '{productType}' + """ + dag.update_providers_config(dummy_provider_config) + # raise an error with more sorting parameters than supported by the provider + with self.assertLogs(level="ERROR") as cm_logs: + dag.search( + provider="dummy_provider", + productType="S2_MSI_L1C", + sortBy=[("eodagSortParam", "ASC"), ("otherEodagSortParam", "ASC")], + ) + self.assertIn( + "Search results can be sorted by only 1 parameter(s) with dummy_provider", + str(cm_logs.output), + ) + @mock.patch("eodag.api.core.EODataAccessGateway._prepare_search", autospec=True) @mock.patch("eodag.plugins.search.qssearch.QueryStringSearch", autospec=True) def test_search_all_must_collect_them_all(self, search_plugin, prepare_seach): diff --git a/tests/units/test_http_server.py b/tests/units/test_http_server.py index a7eb76027..9ebd8a0c7 100644 --- a/tests/units/test_http_server.py +++ b/tests/units/test_http_server.py @@ -1093,6 +1093,7 @@ def test_queryables_with_provider(self, mock_requests_get): timeout=HTTP_REQ_TIMEOUT, headers=USER_AGENT, ) + # TODO: with an unsupported provider def test_product_type_queryables(self): """Request to /collections/{collection_id}/queryables should return a valid response.""" @@ -1111,6 +1112,7 @@ def test_product_type_queryables(self): "additionalProperties", ], ) + # TODO: with an unsupported product type @mock.patch("eodag.plugins.search.qssearch.requests.get", autospec=True) def test_product_type_queryables_with_provider(self, mock_requests_get): @@ -1121,8 +1123,8 @@ def test_product_type_queryables_with_provider(self, mock_requests_get): mock_requests_get.return_value = MockResponse( provider_queryables, status_code=200 ) - # no provider specified (only 1 available for the moment) : queryables intresection returned + # no provider specified (only 1 available for the moment) : queryables intersection returned res_no_provider = self._request_valid( "collections/S1_SAR_GRD/queryables", check_links=False, diff --git a/tests/units/test_search_plugins.py b/tests/units/test_search_plugins.py index 7b9e80a6f..711979c64 100644 --- a/tests/units/test_search_plugins.py +++ b/tests/units/test_search_plugins.py @@ -161,7 +161,7 @@ def setUp(self): @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_xml_count_and_search_mundi( + def test_plugins_search_querystringsearch_xml_count_and_search_mundi( self, mock__request ): """A query with a QueryStringSearch (mundi here) must return tuple with a list of EOProduct and a number of available products""" # noqa @@ -205,7 +205,7 @@ def test_plugins_search_querystringseach_xml_count_and_search_mundi( @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_xml_no_count_and_search_mundi( + def test_plugins_search_querystringsearch_xml_no_count_and_search_mundi( self, mock__request, mock_count_hits ): """A query with a QueryStringSearch (here mundi) without a count""" @@ -250,7 +250,7 @@ def test_plugins_search_querystringseach_xml_no_count_and_search_mundi( @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_xml_distinct_product_type_mtd_mapping( + def test_plugins_search_querystringsearch_xml_distinct_product_type_mtd_mapping( self, mock__request, mock_count_hits ): """The metadata mapping for XML QueryStringSearch should not mix specific product-types metadata-mapping""" @@ -295,7 +295,9 @@ def setUp(self): @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_count_and_search_peps(self, mock__request): + def test_plugins_search_querystringsearch_count_and_search_peps( + self, mock__request + ): """A query with a QueryStringSearch (peps here) must return tuple with a list of EOProduct and a number of available products""" # noqa with open(self.provider_resp_dir / "peps_search.json") as f: peps_resp_search = json.load(f) @@ -337,7 +339,7 @@ def test_plugins_search_querystringseach_count_and_search_peps(self, mock__reque @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_no_count_and_search_peps( + def test_plugins_search_querystringsearch_no_count_and_search_peps( self, mock__request, mock_count_hits ): """A query with a QueryStringSearch (here peps) without a count""" @@ -381,7 +383,7 @@ def test_plugins_search_querystringseach_no_count_and_search_peps( @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_search_cloudcover_peps( + def test_plugins_search_querystringsearch_search_cloudcover_peps( self, mock__request, mock_normalize_results ): """A query with a QueryStringSearch (here peps) must only use cloudCover filtering for non-radar product types""" # noqa @@ -398,7 +400,7 @@ def test_plugins_search_querystringseach_search_cloudcover_peps( @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_discover_product_types( + def test_plugins_search_querystringsearch_discover_product_types( self, mock__request ): """QueryStringSearch.discover_product_types must return a well formatted dict""" @@ -446,7 +448,7 @@ def test_plugins_search_querystringseach_discover_product_types( @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_discover_product_types_keywords( + def test_plugins_search_querystringsearch_discover_product_types_keywords( self, mock__request ): """QueryStringSearch.discover_product_types must return a dict with well formatted keywords""" @@ -491,7 +493,7 @@ def test_plugins_search_querystringseach_discover_product_types_keywords( @mock.patch( "eodag.plugins.search.qssearch.QueryStringSearch._request", autospec=True ) - def test_plugins_search_querystringseach_distinct_product_type_mtd_mapping( + def test_plugins_search_querystringsearch_distinct_product_type_mtd_mapping( self, mock__request ): """The metadata mapping for QueryStringSearch should not mix specific product-types metadata-mapping""" @@ -882,7 +884,8 @@ def test_plugins_search_odatav4search_count_and_search_onda( 'https://catalogue.onda-dias.eu/dias-catalogue/Products?$format=json&$search="' 'footprint:"Intersects(POLYGON ((137.7729 13.1342, 137.7729 23.8860, 153.7491 23.8860, 153.7491 13.1342, ' '137.7729 13.1342)))" AND productType:S2MSI1C AND beginPosition:[2020-08-08T00:00:00.000Z TO *] ' - 'AND endPosition:[* TO 2020-08-16T00:00:00.000Z] AND foo:bar"&$top=2&$skip=0&$expand=Metadata' + "AND endPosition:[* TO 2020-08-16T00:00:00.000Z] " + 'AND foo:bar"&$orderby=beginPosition asc&$top=2&$skip=0&$expand=Metadata' ) mock__request.assert_any_call( @@ -967,7 +970,8 @@ def test_plugins_search_odatav4search_count_and_search_onda_per_product_metadata 'https://catalogue.onda-dias.eu/dias-catalogue/Products?$format=json&$search="' 'footprint:"Intersects(POLYGON ((137.7729 13.1342, 137.7729 23.8860, 153.7491 23.8860, 153.7491 13.1342, ' '137.7729 13.1342)))" AND productType:S2MSI1C AND beginPosition:[2020-08-08T00:00:00.000Z TO *] ' - 'AND endPosition:[* TO 2020-08-16T00:00:00.000Z] AND foo:bar"&$top=2&$skip=0&$expand=Metadata' + "AND endPosition:[* TO 2020-08-16T00:00:00.000Z] " + 'AND foo:bar"&$orderby=beginPosition asc&$top=2&$skip=0&$expand=Metadata' ) mock__request.assert_any_call( diff --git a/tests/units/test_search_types.py b/tests/units/test_search_types.py new file mode 100644 index 000000000..945de57c2 --- /dev/null +++ b/tests/units/test_search_types.py @@ -0,0 +1,94 @@ +# -*- coding: utf-8 -*- +# Copyright 2024, CS GROUP - France, https://www.csgroup.eu/ +# +# This file is part of EODAG project +# https://www.github.com/CS-SI/EODAG +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from pydantic_core import ValidationError + +from eodag.types import search_args +from eodag.utils.exceptions import ValidationError as EodagValidationError + + +class TestStacSearch(unittest.TestCase): + def test_search_sort_by_arg(self): + """search used with "sortBy" argument must not raise errors if the argument is correct""" + # "sortBy" argument must be a list of tuples of two elements and the second element must be "ASC" or "DESC" + search_args.SearchArgs.model_validate( + {"productType": "dummy_product_type", "sortBy": [("eodagSortParam", "ASC")]} + ) + search_args.SearchArgs.model_validate( + { + "productType": "dummy_product_type", + "sortBy": [("eodagSortParam", "DESC")], + } + ) + + def test_search_sort_by_arg_with_errors(self): + """search used with "sortBy" argument must raise errors if the argument is incorrect""" + # raise a Pydantic error with an empty list + with self.assertRaises(ValidationError) as context: + search_args.SearchArgs.model_validate( + {"productType": "dummy_product_type", "sortBy": []} + ) + self.assertIn( + "List should have at least 1 item after validation, not 0", + str(context.exception), + ) + # raise a Pydantic error with syntax errors + with self.assertRaises(ValidationError) as context: + search_args.SearchArgs.model_validate( + {"productType": "dummy_product_type", "sortBy": "eodagSortParam ASC"} + ) + self.assertIn( + "Sort argument must be a list of tuple(s), got a '' instead", + str(context.exception), + ) + with self.assertRaises(ValidationError) as context: + search_args.SearchArgs.model_validate( + {"productType": "dummy_product_type", "sortBy": ["eodagSortParam ASC"]} + ) + self.assertIn( + "Sort argument must be a list of tuple(s), got a list of '' instead", + str(context.exception), + ) + # raise a Pydantic error with a wrong sorting order + with self.assertRaises(ValidationError) as context: + search_args.SearchArgs.model_validate( + { + "productType": "dummy_product_type", + "sortBy": [("eodagSortParam", " wrong_order ")], + } + ) + self.assertIn( + "Sorting order must be set to 'ASC' (ASCENDING) or 'DESC' (DESCENDING), " + "got 'WRONG_ORDER' with 'eodagSortParam' instead", + str(context.exception), + ) + # raise an EODAG error with a sorting order called with different values for a same sorting parameter + with self.assertRaises(EodagValidationError) as e: + search_args.SearchArgs.model_validate( + { + "productType": "dummy_product_type", + "sortBy": [("eodagSortParam", "ASC"), ("eodagSortParam", "DESC")], + } + ) + self.assertIn( + "'eodagSortParam' parameter is called several times to sort results with different sorting " + "orders. Please set it to only one ('ASC' (ASCENDING) or 'DESC' (DESCENDING))", + str(e.exception.message), + )