From 19e2e99a26d4fab504cdd21cf6b7e83fa34434e7 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 17:31:48 -0600 Subject: [PATCH 01/27] Add typechecking and action config --- .github/workflows/typecheck.yml | 29 +++++++++++++++++++++++++++++ pyproject.toml | 27 +++++++++++++++++++++++++++ requirements-dev.txt | 5 +++++ 3 files changed, 61 insertions(+) create mode 100644 .github/workflows/typecheck.yml diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml new file mode 100644 index 000000000..68a5de19a --- /dev/null +++ b/.github/workflows/typecheck.yml @@ -0,0 +1,29 @@ +name: Typecheck + +on: + pull_request: + push: + branches: + - main + - development + + +jobs: + test: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - uses: actions/setup-python@v5 + with: + python-version: "3.10" + + - name: Install package and test dependencies + run: | + python -m pip install . + python -m pip install -r requirements-dev.txt + + - uses: jakebailey/pyright-action@v2 diff --git a/pyproject.toml b/pyproject.toml index e1ea8ed90..e0179da59 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -126,3 +126,30 @@ ignore = [ [tool.ruff.lint.isort] force-sort-within-sections = true + + +[tool.pyright] +pythonVersion = "3.10" +typeCheckingMode = "standard" +include = [ + "icepyx", +] +exclude = [ + "**/__pycache__", + "icepyx/tests", +] +ignore = [ + "icepyx/quest/*", + "icepyx/core/APIformatting.py", + "icepyx/core/auth.py", + "icepyx/core/exceptions.py", + "icepyx/core/icesat2data.py", + "icepyx/core/is2ref.py", + "icepyx/core/query.py", + "icepyx/core/read.py", + "icepyx/core/spatial.py", + "icepyx/core/temporal.py", + "icepyx/core/validate_inputs.py", + "icepyx/core/variables.py", + "icepyx/core/visualization.py", +] diff --git a/requirements-dev.txt b/requirements-dev.txt index 66106dab8..238785444 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,10 @@ +pandas-stubs pre-commit pypistats +pyright pytest>=4.6 pytest-cov responses +types-docutils +types-requests +types-tqdm From 4cc2eb9c41d86953b9f0086e261e235633dc90d0 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 17:34:00 -0600 Subject: [PATCH 02/27] Typecheck granules module --- icepyx/core/granules.py | 35 ++++++++++++++++------ icepyx/core/types.py | 64 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 90 insertions(+), 9 deletions(-) create mode 100644 icepyx/core/types.py diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index b29a147e1..c02283530 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime import io import json @@ -10,10 +12,12 @@ import numpy as np import requests +from requests.compat import unquote import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin import icepyx.core.exceptions +from icepyx.core.types import CMRParams, EGISpecificParams from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL, ORDER_BASE_URL @@ -170,14 +174,19 @@ def __init__( # ---------------------------------------------------------------------- # Methods - def get_avail(self, CMRparams, reqparams, cloud=False): + def get_avail( + self, + CMRparams: CMRParams | None, + reqparams: EGISpecificParams | None, + cloud=False, + ): """ Get a list of available granules for the query object's parameters. Generates the `avail` attribute of the granules object. Parameters ---------- - CMRparams : dictionary + CMRparams : Dictionary of properly formatted CMR search parameters. reqparams : dictionary Dictionary of properly formatted parameters required for searching, ordering, @@ -261,13 +270,13 @@ def get_avail(self, CMRparams, reqparams, cloud=False): # DevGoal: add kwargs to allow subsetting and more control over request options. def place_order( self, - CMRparams, - reqparams, + CMRparams: CMRParams, + reqparams: EGISpecificParams, subsetparams, verbose, subset=True, geom_filepath=None, - ): # , **kwargs): + ): """ Place an order for the available granules for the query object. Adds the list of zipped files (orders) to the granules data object (which is @@ -276,11 +285,11 @@ def place_order( Parameters ---------- - CMRparams : dictionary + CMRparams : Dictionary of properly formatted CMR search parameters. - reqparams : dictionary + reqparams : Dictionary of properly formatted parameters required for searching, ordering, - or downloading from NSIDC. + or downloading from EGI. subsetparams : dictionary Dictionary of properly formatted subsetting parameters. An empty dictionary is passed as input here when subsetting is set to False in query methods. @@ -359,7 +368,7 @@ def place_order( request.raise_for_status() esir_root = ET.fromstring(request.content) if verbose is True: - print("Order request URL: ", requests.utils.unquote(request.url)) + print("Order request URL: ", unquote(request.url)) print( "Order request response XML content: ", request.content.decode("utf-8"), @@ -402,6 +411,7 @@ def place_order( loop_root = ET.fromstring(loop_response.content) # Continue loop while request is still processing + loop_root = None while status == "pending" or status == "processing": print( "Your order status is still ", @@ -425,6 +435,13 @@ def place_order( if status == "pending" or status == "processing": continue + if not isinstance(loop_root, ET.Element): + # The typechecker determined that loop_root could be unbound at this + # point. We know for sure this shouldn't be possible, though, because + # the while loop should run once. + # See: https://github.com/microsoft/pyright/discussions/2033 + raise RuntimeError("Programmer error!") + # Order can either complete, complete_with_errors, or fail: # Provide complete_with_errors error message: if status == "complete_with_errors" or status == "failed": diff --git a/icepyx/core/types.py b/icepyx/core/types.py new file mode 100644 index 000000000..580039739 --- /dev/null +++ b/icepyx/core/types.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +from typing import Literal, TypedDict + + +class CMRParamsBase(TypedDict): + temporal: str + + +class CMRParamsWithBbox(CMRParamsBase): + bounding_box: str + + +class CMRParamsWithPolygon(CMRParamsBase): + polygon: str + + +CMRParams = CMRParamsWithBbox | CMRParamsWithPolygon + + +class EGISpecificParamsBase(TypedDict): + """Common parameters for searching, ordering, or downloading from EGI. + + See: https://wiki.earthdata.nasa.gov/display/SDPSDOCS/EGI+Programmatic+Access+Documentation + + EGI shares parameters with CMR, so this data is used in conjunction with CMRParams + to build EGI requests. + + TODO: Validate more strongly (with Pydantic and its annotated types? + https://docs.pydantic.dev/latest/concepts/types/#composing-types-via-annotated): + + * short_name is `ATL##` (or Literal list of values?) + * version is 1-3 digits + * 0 < page_size <= 2000 + """ + + short_name: str # alias: "product" + version: str + page_size: int # default 2000 + page_num: int # default 0 + + +class EGISpecificParamsSearch(EGISpecificParamsBase): + """Parameters for searching through EGI.""" + + +class EGISpecificParamsOrder(EGISpecificParamsBase): + """Parameters for ordering through EGI.""" + + # TODO: Does this type need page_* attributes? + + +class EGISpecificParamsDownload(EGISpecificParamsBase): + """Parameters for ordering from EGI. + + TODO: Validate more strongly (with Pydantic?): page_num >=0. + """ + + request_mode: Literal["sync", "async", "stream"] # default "async" + include_meta: Literal["Y", "N"] # default "Y" + client_string: Literal["icepyx"] # default "icepyx" + + +EGISpecificParams = EGISpecificParamsSearch | EGISpecificParamsDownload From 041c5baae09110331c1ba4580102dea1b479c72b Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 14 Aug 2024 17:18:13 -0600 Subject: [PATCH 03/27] Correct invalid type annotation --- icepyx/core/visualization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icepyx/core/visualization.py b/icepyx/core/visualization.py index 0ddb9fd40..0f983b5e1 100644 --- a/icepyx/core/visualization.py +++ b/icepyx/core/visualization.py @@ -466,7 +466,7 @@ def parallel_request_OA(self) -> da.array: OA_data_da = da.concatenate(requested_OA_data, axis=0) return OA_data_da - def viz_elevation(self) -> (hv.DynamicMap, hv.Layout): + def viz_elevation(self) -> tuple[hv.DynamicMap, hv.Layout]: """ Visualize elevation requested from OpenAltimetry API using datashader based on cycles https://holoviz.org/tutorial/Large_Data.html From a43a25efc5917e12bda3f881530deb95bb5e9d5d Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 26 Aug 2024 16:03:55 -0600 Subject: [PATCH 04/27] Flesh out CMR params & EGI subset types --- icepyx/core/types.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/icepyx/core/types.py b/icepyx/core/types.py index 580039739..374e1da5a 100644 --- a/icepyx/core/types.py +++ b/icepyx/core/types.py @@ -1,10 +1,18 @@ from __future__ import annotations from typing import Literal, TypedDict +from typing_extensions import NotRequired -class CMRParamsBase(TypedDict): - temporal: str +CMRParamsBase = TypedDict( + "CMRParamsBase", + { + "temporal": NotRequired[str], + "options[readable_granule_name][pattern]": NotRequired[str], + "options[spatial][or]": NotRequired[str], + "readable_granule_name[]": NotRequired[str], + }, +) class CMRParamsWithBbox(CMRParamsBase): @@ -59,6 +67,13 @@ class EGISpecificParamsDownload(EGISpecificParamsBase): request_mode: Literal["sync", "async", "stream"] # default "async" include_meta: Literal["Y", "N"] # default "Y" client_string: Literal["icepyx"] # default "icepyx" + # token, email -EGISpecificParams = EGISpecificParamsSearch | EGISpecificParamsDownload +class EGISpecificParamsSubset(EGISpecificParamsBase): + """Parameters for subsetting with EGI.""" + + +EGISpecificParams = ( + EGISpecificParamsSearch | EGISpecificParamsDownload | EGISpecificParamsSubset +) From 3fc95f1f4ec61f01f1db49f506e8886a16bccc56 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 26 Aug 2024 16:04:48 -0600 Subject: [PATCH 05/27] Define generic type annotation for Parameters class --- icepyx/core/APIformatting.py | 77 ++++++++++++++++++++++++++++++------ 1 file changed, 64 insertions(+), 13 deletions(-) diff --git a/icepyx/core/APIformatting.py b/icepyx/core/APIformatting.py index 4b1966910..b4975f569 100644 --- a/icepyx/core/APIformatting.py +++ b/icepyx/core/APIformatting.py @@ -1,6 +1,9 @@ -# Generate and format information for submitting to API (CMR and NSIDC) +"""Generate and format information for submitting to API (CMR and NSIDC).""" import datetime as dt +from typing import Any, Generic, Literal, TypeVar, overload + +from icepyx.core.types import CMRParams, EGISpecificParams, EGISpecificParamsSubset # ---------------------------------------------------------------------- # parameter-specific formatting for display @@ -183,12 +186,56 @@ def to_string(params): return "&".join(param_list) +ParameterType = Literal["CMR", "required", "subset"] +# DevGoal: When Python 3.12 is minimum supported version, migrate to PEP695 style +T = TypeVar("T", bound=ParameterType) + + +class _FmtedKeysDescriptor: + """Enable the Parameters class' fmted_keys property to be typechecked correctly. + + See: https://github.com/microsoft/pyright/issues/3071#issuecomment-1043978070 + """ + + @overload + def __get__( # type: ignore + self, + instance: 'Parameters[Literal["CMR"]]', + owner: Any, + ) -> CMRParams: ... + + @overload + def __get__( + self, + instance: 'Parameters[Literal["required"]]', + owner: Any, + ) -> EGISpecificParams: ... + + @overload + def __get__( + self, + instance: 'Parameters[Literal["subset"]]', + owner: Any, + ) -> EGISpecificParamsSubset: ... + + def __get__( + self, + instance: "Parameters", + owner: Any, + ) -> CMRParams | EGISpecificParams: + """ + Returns the dictionary of formatted keys associated with the + parameter object. + """ + return instance._fmted_keys + + # ---------------------------------------------------------------------- # DevNote: Currently, this class is not tested!! # DevGoal: this could be expanded, similar to the variables class, to provide users with valid options if need be # DevGoal: currently this does not do much by way of checking/formatting of other subsetting options (reprojection or formats) # it would be great to incorporate that so that people can't just feed any keywords in... -class Parameters: +class Parameters(Generic[T]): """ Build and update the parameter lists needed to submit a data order @@ -206,7 +253,14 @@ class Parameters: on the type of query. Must be one of ['search','download'] """ - def __init__(self, partype, values=None, reqtype=None): + fmted_keys = _FmtedKeysDescriptor() + + def __init__( + self, + partype: T, + values=None, + reqtype=None, + ): assert partype in [ "CMR", "required", @@ -242,15 +296,7 @@ def poss_keys(self): # return self._wanted - @property - def fmted_keys(self): - """ - Returns the dictionary of formatted keys associated with the - parameter object. - """ - return self._fmted_keys - - def _get_possible_keys(self): + def _get_possible_keys(self) -> dict[str, list[str]]: """ Use the parameter type to get a list of possible parameter keys. """ @@ -347,7 +393,7 @@ def check_values(self): else: return False - def build_params(self, **kwargs): + def build_params(self, **kwargs) -> None: """ Build the parameter dictionary of formatted key:value pairs for submission to NSIDC in the data request. @@ -443,3 +489,8 @@ def build_params(self, **kwargs): k = "Boundingshape" self._fmted_keys.update({k: kwargs["spatial_extent"]}) + + +CMRParameters = Parameters[Literal["CMR"]] +RequiredParameters = Parameters[Literal["required"]] +SubsetParameters = Parameters[Literal["subset"]] From b9c77e8a6b39e33e4df3f1e4423337fc0c393fb7 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 17:35:43 -0600 Subject: [PATCH 06/27] Partially annotate the Query class --- icepyx/core/query.py | 20 +++++++++++++------- pyproject.toml | 1 - 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/icepyx/core/query.py b/icepyx/core/query.py index d547a959f..ac9231ce5 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -2,6 +2,7 @@ import geopandas as gpd import matplotlib.pyplot as plt +from typing_extensions import Never import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin @@ -11,6 +12,7 @@ import icepyx.core.is2ref as is2ref import icepyx.core.spatial as spat import icepyx.core.temporal as tp +from icepyx.core.types import CMRParams, EGISpecificParams, EGISpecificParamsSubset import icepyx.core.validate_inputs as val from icepyx.core.variables import Variables as Variables from icepyx.core.visualization import Visualize @@ -393,6 +395,10 @@ class Query(GenQuery, EarthdataAuthMixin): GenQuery """ + _CMRparams: apifmt.CMRParameters + _reqparams: apifmt.RequiredParameters + _subsetparams: apifmt.SubsetParameters | None + # ---------------------------------------------------------------------- # Constructors @@ -532,7 +538,7 @@ def tracks(self): return sorted(set(self._tracks)) @property - def CMRparams(self): + def CMRparams(self) -> CMRParams: """ Display the CMR key:value pairs that will be submitted. It generates the dictionary if it does not already exist. @@ -573,7 +579,7 @@ def CMRparams(self): return self._CMRparams.fmted_keys @property - def reqparams(self): + def reqparams(self) -> EGISpecificParams: """ Display the required key:value pairs that will be submitted. It generates the dictionary if it does not already exist. @@ -599,7 +605,7 @@ def reqparams(self): # @property # DevQuestion: if I make this a property, I get a "dict" object is not callable # when I try to give input kwargs... what approach should I be taking? - def subsetparams(self, **kwargs): + def subsetparams(self, **kwargs) -> EGISpecificParamsSubset | dict[Never, Never]: """ Display the subsetting key:value pairs that will be submitted. It generates the dictionary if it does not already exist @@ -1001,7 +1007,7 @@ def order_granules(self, verbose=False, subset=True, email=False, **kwargs): if "email" in self._reqparams.fmted_keys or email is False: self._reqparams.build_params(**self._reqparams.fmted_keys) elif email is True: - user_profile = self.auth.get_user_profile() + user_profile = self.auth.get_user_profile() # pyright: ignore[reportAttributeAccessIssue] self._reqparams.build_params( **self._reqparams.fmted_keys, email=user_profile["email_address"] ) @@ -1135,14 +1141,14 @@ def visualize_spatial_extent( import geoviews as gv from shapely.geometry import Polygon # noqa: F401 - gv.extension("bokeh") + gv.extension("bokeh") # pyright: ignore[reportCallIssue] bbox_poly = gv.Path(gdf["geometry"]).opts(color="red", line_color="red") tile = gv.tile_sources.EsriImagery.opts(width=500, height=500) - return tile * bbox_poly + return tile * bbox_poly # pyright: ignore[reportOperatorIssue] except ImportError: - world = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) + world = gpd.read_file(gpd.datasets.get_path("naturalearth_lowres")) # pyright: ignore[reportAttributeAccessIssue] f, ax = plt.subplots(1, figsize=(12, 6)) world.plot(ax=ax, facecolor="lightgray", edgecolor="gray") gdf.plot(ax=ax, color="#FF8C00", alpha=0.7) diff --git a/pyproject.toml b/pyproject.toml index e0179da59..288910c9d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -145,7 +145,6 @@ ignore = [ "icepyx/core/exceptions.py", "icepyx/core/icesat2data.py", "icepyx/core/is2ref.py", - "icepyx/core/query.py", "icepyx/core/read.py", "icepyx/core/spatial.py", "icepyx/core/temporal.py", From da851aa4255f295ccac3b226b11f8722ddcb4a1e Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 26 Aug 2024 16:31:03 -0600 Subject: [PATCH 07/27] Add miscellaneous annotation --- icepyx/core/auth.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icepyx/core/auth.py b/icepyx/core/auth.py index 9f12fbecf..71b4393e5 100644 --- a/icepyx/core/auth.py +++ b/icepyx/core/auth.py @@ -54,7 +54,7 @@ def __init__(self, auth=None): self._s3login_credentials = None self._s3_initial_ts = None # timer for 1h expiration on s3 credentials - def __str__(self): + def __str__(self) -> str: if self.session: repr_string = "EarthdataAuth obj with session initialized" else: From 2da7a0bde8ddb1551c3d3df8f605014fda287a4e Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 26 Aug 2024 19:05:23 -0600 Subject: [PATCH 08/27] Add all dependencies to typecheck environment --- .github/workflows/typecheck.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/typecheck.yml b/.github/workflows/typecheck.yml index 68a5de19a..cfe884258 100644 --- a/.github/workflows/typecheck.yml +++ b/.github/workflows/typecheck.yml @@ -23,7 +23,7 @@ jobs: - name: Install package and test dependencies run: | - python -m pip install . + python -m pip install .[complete] python -m pip install -r requirements-dev.txt - uses: jakebailey/pyright-action@v2 From c8ba01a376ede22c0064a075db2bdc25767005c3 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Mon, 26 Aug 2024 19:51:15 -0600 Subject: [PATCH 09/27] Add notes to pyright config --- pyproject.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pyproject.toml b/pyproject.toml index 288910c9d..96048615d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -130,6 +130,7 @@ force-sort-within-sections = true [tool.pyright] pythonVersion = "3.10" +# DevGoal: "strict" typeCheckingMode = "standard" include = [ "icepyx", @@ -138,6 +139,7 @@ exclude = [ "**/__pycache__", "icepyx/tests", ] +# DevGoal: Remove all ignores ignore = [ "icepyx/quest/*", "icepyx/core/APIformatting.py", From 27f4641f9a49e92c25c8dd151c3f2297971d1197 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 17:42:42 -0600 Subject: [PATCH 10/27] Remove pyright ignores for modules without any errors Co-Authored-By: Jessica Scheick --- pyproject.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 96048615d..3e243bf76 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -144,13 +144,9 @@ ignore = [ "icepyx/quest/*", "icepyx/core/APIformatting.py", "icepyx/core/auth.py", - "icepyx/core/exceptions.py", - "icepyx/core/icesat2data.py", "icepyx/core/is2ref.py", "icepyx/core/read.py", "icepyx/core/spatial.py", - "icepyx/core/temporal.py", - "icepyx/core/validate_inputs.py", "icepyx/core/variables.py", "icepyx/core/visualization.py", ] From 181099fa809d9569f5e374085de296b4b050bead Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 4 Sep 2024 23:44:45 +0000 Subject: [PATCH 11/27] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- icepyx/core/types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icepyx/core/types.py b/icepyx/core/types.py index 374e1da5a..89f7ca02e 100644 --- a/icepyx/core/types.py +++ b/icepyx/core/types.py @@ -1,8 +1,8 @@ from __future__ import annotations from typing import Literal, TypedDict -from typing_extensions import NotRequired +from typing_extensions import NotRequired CMRParamsBase = TypedDict( "CMRParamsBase", From e6e7fe2cc82cae6f282154081b614555af41d723 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 17:49:24 -0600 Subject: [PATCH 12/27] Clarify use of term "EGI" in docstring Co-authored-by: Jessica Scheick --- icepyx/core/granules.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index c02283530..22cc8d140 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -289,7 +289,7 @@ def place_order( Dictionary of properly formatted CMR search parameters. reqparams : Dictionary of properly formatted parameters required for searching, ordering, - or downloading from EGI. + or downloading from NSIDC (via their EGI system). subsetparams : dictionary Dictionary of properly formatted subsetting parameters. An empty dictionary is passed as input here when subsetting is set to False in query methods. From 0440fa99fc322ce854aa41440060cdba5e536cae Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 17:52:31 -0600 Subject: [PATCH 13/27] Type valid data product short names --- icepyx/core/types.py | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/icepyx/core/types.py b/icepyx/core/types.py index 89f7ca02e..b97a410ec 100644 --- a/icepyx/core/types.py +++ b/icepyx/core/types.py @@ -4,6 +4,31 @@ from typing_extensions import NotRequired +IcesatProductShortName = Literal[ + "ATL01", + "ATL02", + "ATL03", + "ATL04", + "ATL06", + "ATL07", + "ATL07QL", + "ATL08", + "ATL09", + "ATL09QL", + "ATL10", + "ATL11", + "ATL12", + "ATL13", + "ATL14", + "ATL15", + "ATL16", + "ATL17", + "ATL19", + "ATL20", + "ATL21", + "ATL23", +] + CMRParamsBase = TypedDict( "CMRParamsBase", { @@ -37,12 +62,11 @@ class EGISpecificParamsBase(TypedDict): TODO: Validate more strongly (with Pydantic and its annotated types? https://docs.pydantic.dev/latest/concepts/types/#composing-types-via-annotated): - * short_name is `ATL##` (or Literal list of values?) - * version is 1-3 digits + * version is 3 digits * 0 < page_size <= 2000 """ - short_name: str # alias: "product" + short_name: IcesatProductShortName # alias: "product" version: str page_size: int # default 2000 page_num: int # default 0 From df78ca2a2d342d3886eba6c37327f3b7d76410db Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 18:29:41 -0600 Subject: [PATCH 14/27] Fixup API parameter types Co-Authored-By: Jessica Scheick --- icepyx/core/granules.py | 6 +++--- icepyx/core/query.py | 6 +++--- icepyx/core/types.py | 30 +++++++++++++++++++----------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index 22cc8d140..bcaaef596 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -17,7 +17,7 @@ import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin import icepyx.core.exceptions -from icepyx.core.types import CMRParams, EGISpecificParams +from icepyx.core.types import CMRParams, EGISpecificRequiredParams from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL, ORDER_BASE_URL @@ -177,7 +177,7 @@ def __init__( def get_avail( self, CMRparams: CMRParams | None, - reqparams: EGISpecificParams | None, + reqparams: EGISpecificRequiredParams | None, cloud=False, ): """ @@ -271,7 +271,7 @@ def get_avail( def place_order( self, CMRparams: CMRParams, - reqparams: EGISpecificParams, + reqparams: EGISpecificRequiredParams, subsetparams, verbose, subset=True, diff --git a/icepyx/core/query.py b/icepyx/core/query.py index ac9231ce5..6fa1ecd77 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -12,7 +12,7 @@ import icepyx.core.is2ref as is2ref import icepyx.core.spatial as spat import icepyx.core.temporal as tp -from icepyx.core.types import CMRParams, EGISpecificParams, EGISpecificParamsSubset +from icepyx.core.types import CMRParams, EGIParamsSubset, EGISpecificRequiredParams import icepyx.core.validate_inputs as val from icepyx.core.variables import Variables as Variables from icepyx.core.visualization import Visualize @@ -579,7 +579,7 @@ def CMRparams(self) -> CMRParams: return self._CMRparams.fmted_keys @property - def reqparams(self) -> EGISpecificParams: + def reqparams(self) -> EGISpecificRequiredParams: """ Display the required key:value pairs that will be submitted. It generates the dictionary if it does not already exist. @@ -605,7 +605,7 @@ def reqparams(self) -> EGISpecificParams: # @property # DevQuestion: if I make this a property, I get a "dict" object is not callable # when I try to give input kwargs... what approach should I be taking? - def subsetparams(self, **kwargs) -> EGISpecificParamsSubset | dict[Never, Never]: + def subsetparams(self, **kwargs) -> EGIParamsSubset | dict[Never, Never]: """ Display the subsetting key:value pairs that will be submitted. It generates the dictionary if it does not already exist diff --git a/icepyx/core/types.py b/icepyx/core/types.py index b97a410ec..e01e6bd05 100644 --- a/icepyx/core/types.py +++ b/icepyx/core/types.py @@ -73,13 +73,7 @@ class EGISpecificParamsBase(TypedDict): class EGISpecificParamsSearch(EGISpecificParamsBase): - """Parameters for searching through EGI.""" - - -class EGISpecificParamsOrder(EGISpecificParamsBase): - """Parameters for ordering through EGI.""" - - # TODO: Does this type need page_* attributes? + """Parameters for interacting with EGI.""" class EGISpecificParamsDownload(EGISpecificParamsBase): @@ -94,10 +88,24 @@ class EGISpecificParamsDownload(EGISpecificParamsBase): # token, email -class EGISpecificParamsSubset(EGISpecificParamsBase): +class EGIParamsSubsetBase(TypedDict): """Parameters for subsetting with EGI.""" + time: NotRequired[str] + format: NotRequired[str] + projection: NotRequired[str] + projection_parameters: NotRequired[str] + Coverage: NotRequired[str] -EGISpecificParams = ( - EGISpecificParamsSearch | EGISpecificParamsDownload | EGISpecificParamsSubset -) + +class EGIParamsSubsetBbox(EGIParamsSubsetBase): + bbox: str + + +class EGIParamsSubsetBoundingShape(EGIParamsSubsetBase): + Boundingshape: str + + +EGIParamsSubset = EGIParamsSubsetBbox | EGIParamsSubsetBoundingShape + +EGISpecificRequiredParams = EGISpecificParamsSearch | EGISpecificParamsDownload From ee70196a1f089acb3b7a529b36462f1ad04c80b8 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 18:37:01 -0600 Subject: [PATCH 15/27] Fixup missed name change --- icepyx/core/APIformatting.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/icepyx/core/APIformatting.py b/icepyx/core/APIformatting.py index b4975f569..d409b19d1 100644 --- a/icepyx/core/APIformatting.py +++ b/icepyx/core/APIformatting.py @@ -3,7 +3,11 @@ import datetime as dt from typing import Any, Generic, Literal, TypeVar, overload -from icepyx.core.types import CMRParams, EGISpecificParams, EGISpecificParamsSubset +from icepyx.core.types import ( + CMRParams, + EGIParamsSubset, + EGISpecificRequiredParams, +) # ---------------------------------------------------------------------- # parameter-specific formatting for display @@ -198,7 +202,7 @@ class _FmtedKeysDescriptor: """ @overload - def __get__( # type: ignore + def __get__( self, instance: 'Parameters[Literal["CMR"]]', owner: Any, @@ -209,20 +213,20 @@ def __get__( self, instance: 'Parameters[Literal["required"]]', owner: Any, - ) -> EGISpecificParams: ... + ) -> EGISpecificRequiredParams: ... @overload def __get__( self, instance: 'Parameters[Literal["subset"]]', owner: Any, - ) -> EGISpecificParamsSubset: ... + ) -> EGIParamsSubset: ... def __get__( self, instance: "Parameters", owner: Any, - ) -> CMRParams | EGISpecificParams: + ) -> CMRParams | EGISpecificRequiredParams | EGIParamsSubset: """ Returns the dictionary of formatted keys associated with the parameter object. From 434c28a64dcd7648684f4b30b170fde67146b843 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Wed, 4 Sep 2024 18:50:19 -0600 Subject: [PATCH 16/27] Use Python 3.9-compatible annotation style --- icepyx/core/APIformatting.py | 4 ++-- icepyx/core/granules.py | 5 +++-- icepyx/core/query.py | 5 +++-- icepyx/core/types.py | 8 ++++---- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/icepyx/core/APIformatting.py b/icepyx/core/APIformatting.py index d409b19d1..bdb9b0f06 100644 --- a/icepyx/core/APIformatting.py +++ b/icepyx/core/APIformatting.py @@ -1,7 +1,7 @@ """Generate and format information for submitting to API (CMR and NSIDC).""" import datetime as dt -from typing import Any, Generic, Literal, TypeVar, overload +from typing import Any, Generic, Literal, TypeVar, Union, overload from icepyx.core.types import ( CMRParams, @@ -226,7 +226,7 @@ def __get__( self, instance: "Parameters", owner: Any, - ) -> CMRParams | EGISpecificRequiredParams | EGIParamsSubset: + ) -> Union[CMRParams, EGISpecificRequiredParams, EGIParamsSubset]: """ Returns the dictionary of formatted keys associated with the parameter object. diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index bcaaef596..0de4edda7 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -7,6 +7,7 @@ import pprint import re import time +from typing import Optional from xml.etree import ElementTree as ET import zipfile @@ -176,8 +177,8 @@ def __init__( def get_avail( self, - CMRparams: CMRParams | None, - reqparams: EGISpecificRequiredParams | None, + CMRparams: Optional[CMRParams], + reqparams: Optional[EGISpecificRequiredParams], cloud=False, ): """ diff --git a/icepyx/core/query.py b/icepyx/core/query.py index 6fa1ecd77..5cf1f8cbd 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -1,4 +1,5 @@ import pprint +from typing import Optional, Union import geopandas as gpd import matplotlib.pyplot as plt @@ -397,7 +398,7 @@ class Query(GenQuery, EarthdataAuthMixin): _CMRparams: apifmt.CMRParameters _reqparams: apifmt.RequiredParameters - _subsetparams: apifmt.SubsetParameters | None + _subsetparams: Optional[apifmt.SubsetParameters] # ---------------------------------------------------------------------- # Constructors @@ -605,7 +606,7 @@ def reqparams(self) -> EGISpecificRequiredParams: # @property # DevQuestion: if I make this a property, I get a "dict" object is not callable # when I try to give input kwargs... what approach should I be taking? - def subsetparams(self, **kwargs) -> EGIParamsSubset | dict[Never, Never]: + def subsetparams(self, **kwargs) -> Union[EGIParamsSubset, dict[Never, Never]]: """ Display the subsetting key:value pairs that will be submitted. It generates the dictionary if it does not already exist diff --git a/icepyx/core/types.py b/icepyx/core/types.py index e01e6bd05..4df2ac012 100644 --- a/icepyx/core/types.py +++ b/icepyx/core/types.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import Literal, TypedDict +from typing import Literal, TypedDict, Union from typing_extensions import NotRequired @@ -48,7 +48,7 @@ class CMRParamsWithPolygon(CMRParamsBase): polygon: str -CMRParams = CMRParamsWithBbox | CMRParamsWithPolygon +CMRParams = Union[CMRParamsWithBbox, CMRParamsWithPolygon] class EGISpecificParamsBase(TypedDict): @@ -106,6 +106,6 @@ class EGIParamsSubsetBoundingShape(EGIParamsSubsetBase): Boundingshape: str -EGIParamsSubset = EGIParamsSubsetBbox | EGIParamsSubsetBoundingShape +EGIParamsSubset = Union[EGIParamsSubsetBbox, EGIParamsSubsetBoundingShape] -EGISpecificRequiredParams = EGISpecificParamsSearch | EGISpecificParamsDownload +EGISpecificRequiredParams = Union[EGISpecificParamsSearch, EGISpecificParamsDownload] From a36d17c5eb55449c60c8ef2dccee303cc21b4016 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 10:16:23 -0600 Subject: [PATCH 17/27] Correct product name type name Co-Authored-By: Jessica Scheick --- icepyx/core/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepyx/core/types.py b/icepyx/core/types.py index 4df2ac012..e600b78cb 100644 --- a/icepyx/core/types.py +++ b/icepyx/core/types.py @@ -4,7 +4,7 @@ from typing_extensions import NotRequired -IcesatProductShortName = Literal[ +ICESat2ProductShortName = Literal[ "ATL01", "ATL02", "ATL03", @@ -66,7 +66,7 @@ class EGISpecificParamsBase(TypedDict): * 0 < page_size <= 2000 """ - short_name: IcesatProductShortName # alias: "product" + short_name: ICESat2ProductShortName # alias: "product" version: str page_size: int # default 2000 page_num: int # default 0 From a8996d1636ab7f50df6f8c1d5b6933ce40f236e6 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 10:21:00 -0600 Subject: [PATCH 18/27] Narrow argument types Co-Authored-By: Jessica Scheick --- icepyx/core/granules.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index 0de4edda7..8b1d82264 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -7,7 +7,6 @@ import pprint import re import time -from typing import Optional from xml.etree import ElementTree as ET import zipfile @@ -18,7 +17,7 @@ import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin import icepyx.core.exceptions -from icepyx.core.types import CMRParams, EGISpecificRequiredParams +from icepyx.core.types import CMRParams, EGISpecificParamsSearch from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL, ORDER_BASE_URL @@ -177,8 +176,8 @@ def __init__( def get_avail( self, - CMRparams: Optional[CMRParams], - reqparams: Optional[EGISpecificRequiredParams], + CMRparams: CMRParams, + reqparams: EGISpecificParamsSearch, cloud=False, ): """ @@ -272,7 +271,7 @@ def get_avail( def place_order( self, CMRparams: CMRParams, - reqparams: EGISpecificRequiredParams, + reqparams: EGISpecificParamsSearch, subsetparams, verbose, subset=True, From dfe4c096750621307560cbd2f1504d39529579b1 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 10:43:18 -0600 Subject: [PATCH 19/27] Set pyright's Python version to 3.9 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 3e243bf76..d06230f3d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -129,7 +129,7 @@ force-sort-within-sections = true [tool.pyright] -pythonVersion = "3.10" +pythonVersion = "3.9" # DevGoal: "strict" typeCheckingMode = "standard" include = [ From fddf23a6f8116e211f0df4a88068d0ac9030a978 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 12:54:48 -0600 Subject: [PATCH 20/27] Unblock upgrading sphinx extensions by replacing sphinx-panels --- doc/source/conf.py | 2 +- doc/source/index.rst | 133 +++++++++++++++++++++++------------------- requirements-docs.txt | 2 +- 3 files changed, 75 insertions(+), 62 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 0303f72ca..e6231f616 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -38,7 +38,7 @@ "myst_nb", "contributors", # custom extension, from pandas "sphinxcontrib.bibtex", - "sphinx_panels", + "sphinx_design", # "sphinx.ext.imgconverter", # this extension should help the latex svg warning, but results in an error instead ] myst_enable_extensions = [ diff --git a/doc/source/index.rst b/doc/source/index.rst index e73818942..cb8180dc4 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -31,86 +31,99 @@ ICESat-2 datasets to enable scientific discovery. To further enhance data discovery, we have developed the QUEST module to facilitate querying of ICESat-2 data and complimentary Argo oceanographic data, with additional dataset support expected in the future. -.. panels:: - :card: + intro-card text-center - :column: col-lg-4 col-md-4 col-sm-6 col-xs-12 p-2 - :img-top-cls: pl-2 pr-2 pt-2 pb-2 +.. grid:: 1 2 2 3 + :gutter: 2 - --- - :img-top: https://cdn-icons-png.flaticon.com/128/2498/2498074.png + .. grid-item-card:: + :img-top: https://cdn-icons-png.flaticon.com/128/2498/2498074.png - **Getting Started** - ^^^^^^^^^^^^^^^^^^^ + **Getting Started** + ^^^^^^^^^^^^^^^^^^^ - New to ICESat-2 or icepyx? - Learn how to install icepyx and use it to jumpstart your project today. - Check out our gallery of examples, too! + New to ICESat-2 or icepyx? + Learn how to install icepyx and use it to jumpstart your project today. + Check out our gallery of examples, too! - .. link-button:: install_ref - :type: ref - :text: Installation Instructions - :classes: stretched-link btn-outline-primary btn-block + .. button-ref:: install_ref + :ref-type: ref + :color: primary + :outline: + :expand: - --- - :img-top: https://cdn-icons-png.flaticon.com/128/3730/3730041.png + Installation Instructions - **User Guide** - ^^^^^^^^^^^^^^ + .. grid-item-card:: + :img-top: https://cdn-icons-png.flaticon.com/128/3730/3730041.png - The user guide provides in-depth information on the tools and functionality - available for obtaining and interacting with ICESat-2 data products. + **User Guide** + ^^^^^^^^^^^^^^ - .. link-button:: api_doc_ref - :type: ref - :text: Software Docs - :classes: stretched-link btn-outline-primary btn-block + The user guide provides in-depth information on the tools and functionality + available for obtaining and interacting with ICESat-2 data products. - --- - :img-top: https://cdn-icons-png.flaticon.com/512/4230/4230997.png + .. button-ref:: api_doc_ref + :ref-type: ref + :color: primary + :outline: + :expand: - **Development Guide** - ^^^^^^^^^^^^^^^^^^^^^ + Software Docs - Have an idea or an ancillary dataset to contribute to icepyx? Go here for information on best practices - for developing and contributing to icepyx. + .. grid-item-card:: + :img-top: https://cdn-icons-png.flaticon.com/512/4230/4230997.png - .. link-button:: dev_guide_label - :type: ref - :text: Development Guide - :classes: stretched-link btn-outline-primary btn-block + **Development Guide** + ^^^^^^^^^^^^^^^^^^^^^ - --- - :img-top: https://cdn-icons-png.flaticon.com/128/1283/1283342.png + Have an idea or an ancillary dataset to contribute to icepyx? Go here for information on best practices + for developing and contributing to icepyx. - **Get in Touch** - ^^^^^^^^^^^^^^^^ + .. button-ref:: dev_guide_label + :ref-type: ref + :color: primary + :outline: + :expand: - icepyx is more than just software! - We're a community of data producers, managers, and users - who collaborate openly and share code and skills - for every step along the entire data pipeline. Find resources for - your questions here! + Development Guide - .. link-button:: contact_ref_label - :type: ref - :text: Get Involved! - :classes: stretched-link btn-outline-primary btn-block + .. grid-item-card:: + :img-top: https://cdn-icons-png.flaticon.com/128/1283/1283342.png - --- - :img-top: https://icesat-2.gsfc.nasa.gov/sites/default/files/MissionLogo_0.png - :img-top-cls: pl-2 pr-2 pt-4 pb-4 + **Get in Touch** + ^^^^^^^^^^^^^^^^ - **ICESat-2 Resources** - ^^^^^^^^^^^^^^^^^^^^^^ + icepyx is more than just software! + We're a community of data producers, managers, and users + who collaborate openly and share code and skills + for every step along the entire data pipeline. Find resources for + your questions here! - Curious about other tools for working with ICESat-2 data? - Want to share your resource? - Check out the amazing work already in progress! + .. button-ref:: contact_ref_label + :ref-type: ref + :color: primary + :outline: + :expand: - .. link-button:: resource_ref_label - :type: ref - :text: ICESat-2 Resource Guide - :classes: stretched-link btn-outline-primary btn-block + Get Involved! + + .. grid-item-card:: + :img-top: https://icesat-2.gsfc.nasa.gov/sites/default/files/MissionLogo_0.png + :class-img-top: pl-2 pr-2 pt-4 pb-4 + + **ICESat-2 Resources** + ^^^^^^^^^^^^^^^^^^^^^^ + + Curious about other tools for working with ICESat-2 data? + Want to share your resource? + Check out the amazing work already in progress! + + .. button-ref:: resource_ref_label + :ref-type: ref + :color: primary + :outline: + :expand: + + ICESat-2 Resource Guide .. toctree:: diff --git a/requirements-docs.txt b/requirements-docs.txt index 51dc9ff5a..df666133a 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -6,6 +6,6 @@ numpydoc pybtex pygithub sphinx>=4.3 -sphinx-panels +sphinx-design sphinx_rtd_theme>=1.0 sphinxcontrib-bibtex From e155a9bd44b72fe7a94dfaace55999388bae3c87 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 13:31:05 -0600 Subject: [PATCH 21/27] Parse and correctly generate docs from numpy-style docstrings --- doc/source/conf.py | 29 +++++++++++++++++++++++++++++ icepyx/core/granules.py | 11 +++++++---- requirements-docs.txt | 1 + 3 files changed, 37 insertions(+), 4 deletions(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index e6231f616..18b4141ed 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -32,6 +32,10 @@ # ones. extensions = [ "sphinx.ext.autodoc", + # IMPORTANT: napoleon must be loaded before sphinx_autodoc_typehints + # https://github.com/tox-dev/sphinx-autodoc-typehints/issues/15 + "sphinx.ext.napoleon", + "sphinx_autodoc_typehints", "sphinx.ext.autosectionlabel", "numpydoc", # "sphinx.ext.autosummary", @@ -79,6 +83,31 @@ nb_execution_mode = "off" suppress_warnings = ["myst.header"] # suppress non-consecutive header warning + +# -- Options for Napoleon docstrign parsing ---------------------------------- +napoleon_google_docstring = False +napoleon_numpy_docstring = True +napoleon_use_admonition_for_examples = True +napoleon_use_admonition_for_notes = True + + +# -- Options for autodoc ----------------------------------------------------- + +# Show the typehints in the description of each object instead of the signature. +autodoc_typehints = "description" + + +# -- Options for autodoc typehints-------------------------------------------- + +# Replace Union annotations with union operator "|" +always_use_bars_union = True +# always_document_param_types = True + +# Show the default value for a parameter after its type +typehints_defaults = "comma" +typehints_use_return = True + + # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index 8b1d82264..7b26d6eac 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -178,7 +178,7 @@ def get_avail( self, CMRparams: CMRParams, reqparams: EGISpecificParamsSearch, - cloud=False, + cloud: bool = False, ): """ Get a list of available granules for the query object's parameters. @@ -188,15 +188,18 @@ def get_avail( ---------- CMRparams : Dictionary of properly formatted CMR search parameters. - reqparams : dictionary + reqparams : Dictionary of properly formatted parameters required for searching, ordering, or downloading from NSIDC. - cloud : deprecated, boolean, default False + cloud : CMR metadata is always collected for the cloud system. + .. deprecated:: 1.2 + This parameter is ignored. + Notes ----- - This function is used by query.Query.avail_granules(), which automatically + This function is used by ``query.Query.avail_granules()``, which automatically feeds in the required parameters. See Also diff --git a/requirements-docs.txt b/requirements-docs.txt index df666133a..554b4d2b8 100644 --- a/requirements-docs.txt +++ b/requirements-docs.txt @@ -6,6 +6,7 @@ numpydoc pybtex pygithub sphinx>=4.3 +sphinx-autodoc-typehints>=2.0 sphinx-design sphinx_rtd_theme>=1.0 sphinxcontrib-bibtex From 51b34923f834b2e20472c85d8f64a72f5a377c8e Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 18:00:31 -0600 Subject: [PATCH 22/27] Fix typo Co-authored-by: Jessica Scheick --- doc/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/source/conf.py b/doc/source/conf.py index 18b4141ed..d51c371ee 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -84,7 +84,7 @@ suppress_warnings = ["myst.header"] # suppress non-consecutive header warning -# -- Options for Napoleon docstrign parsing ---------------------------------- +# -- Options for Napoleon docstring parsing ---------------------------------- napoleon_google_docstring = False napoleon_numpy_docstring = True napoleon_use_admonition_for_examples = True From 7f608499e3168e39e5fd188c293e0426d5c69c84 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 19:13:29 -0600 Subject: [PATCH 23/27] Autogenerate docs for types Excludes CMR typeddicts which contain symbols in some keys. --- doc/source/user_guide/documentation/icepyx.rst | 1 + doc/source/user_guide/documentation/types.rst | 11 +++++++++++ 2 files changed, 12 insertions(+) create mode 100644 doc/source/user_guide/documentation/types.rst diff --git a/doc/source/user_guide/documentation/icepyx.rst b/doc/source/user_guide/documentation/icepyx.rst index eec823e10..bb71b63e0 100644 --- a/doc/source/user_guide/documentation/icepyx.rst +++ b/doc/source/user_guide/documentation/icepyx.rst @@ -19,3 +19,4 @@ Diagrams are updated automatically after a pull request (PR) is approved and bef read variables components + types diff --git a/doc/source/user_guide/documentation/types.rst b/doc/source/user_guide/documentation/types.rst new file mode 100644 index 000000000..991a80431 --- /dev/null +++ b/doc/source/user_guide/documentation/types.rst @@ -0,0 +1,11 @@ +Types +===== + +.. automodule:: icepyx.core.types + :members: + :undoc-members: + :exclude-members: CMRParamsBase,CMRParamsWithBbox,CMRParamsWithPolygon + +.. COMMENT. `exclude-members` specified above is required because those models + contain symbols ('[', ']') in some keys, which sphinx doesn't like. + See: https://github.com/sphinx-doc/sphinx/issues/11039 From 4cb595609056a7659214a7867850405d7a97a14e Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 19:25:18 -0600 Subject: [PATCH 24/27] Tell the typechecker to trust us on the type of reqparams Is this the right way to handle this? Perhaps we need a typeguard. Co-Authored-By: Jessica Scheick --- icepyx/core/granules.py | 8 ++++++-- icepyx/core/query.py | 13 +++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index 7b26d6eac..adf30e14c 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -17,7 +17,11 @@ import icepyx.core.APIformatting as apifmt from icepyx.core.auth import EarthdataAuthMixin import icepyx.core.exceptions -from icepyx.core.types import CMRParams, EGISpecificParamsSearch +from icepyx.core.types import ( + CMRParams, + EGISpecificParamsDownload, + EGISpecificParamsSearch, +) from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL, ORDER_BASE_URL @@ -274,7 +278,7 @@ def get_avail( def place_order( self, CMRparams: CMRParams, - reqparams: EGISpecificParamsSearch, + reqparams: EGISpecificParamsDownload, subsetparams, verbose, subset=True, diff --git a/icepyx/core/query.py b/icepyx/core/query.py index 5cf1f8cbd..dbc8ce0bb 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -1,5 +1,5 @@ import pprint -from typing import Optional, Union +from typing import Optional, Union, cast import geopandas as gpd import matplotlib.pyplot as plt @@ -13,7 +13,12 @@ import icepyx.core.is2ref as is2ref import icepyx.core.spatial as spat import icepyx.core.temporal as tp -from icepyx.core.types import CMRParams, EGIParamsSubset, EGISpecificRequiredParams +from icepyx.core.types import ( + CMRParams, + EGIParamsSubset, + EGISpecificParamsDownload, + EGISpecificRequiredParams, +) import icepyx.core.validate_inputs as val from icepyx.core.variables import Variables as Variables from icepyx.core.visualization import Visualize @@ -1039,7 +1044,7 @@ def order_granules(self, verbose=False, subset=True, email=False, **kwargs): tempCMRparams["readable_granule_name[]"] = gran self._granules.place_order( tempCMRparams, - self.reqparams, + cast(EGISpecificParamsDownload, self.reqparams), self.subsetparams(**kwargs), verbose, subset, @@ -1049,7 +1054,7 @@ def order_granules(self, verbose=False, subset=True, email=False, **kwargs): else: self._granules.place_order( self.CMRparams, - self.reqparams, + cast(EGISpecificParamsDownload, self.reqparams), self.subsetparams(**kwargs), verbose, subset, From 7a423a8f45f5f9085c7f1327de78c4aa44c7c7ea Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 20:04:32 -0600 Subject: [PATCH 25/27] Tweak style of panels in index page --- doc/source/index.rst | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/doc/source/index.rst b/doc/source/index.rst index cb8180dc4..1630006c7 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -32,10 +32,13 @@ To further enhance data discovery, we have developed the QUEST module to facilit .. grid:: 1 2 2 3 - :gutter: 2 + :gutter: 3 + :class-container: sd-text-center .. grid-item-card:: :img-top: https://cdn-icons-png.flaticon.com/128/2498/2498074.png + :class-img-top: sd-p-2 + :class-card: sd-shadow-md **Getting Started** ^^^^^^^^^^^^^^^^^^^ @@ -54,6 +57,8 @@ To further enhance data discovery, we have developed the QUEST module to facilit .. grid-item-card:: :img-top: https://cdn-icons-png.flaticon.com/128/3730/3730041.png + :class-img-top: sd-p-2 + :class-card: sd-shadow-md **User Guide** ^^^^^^^^^^^^^^ @@ -71,6 +76,8 @@ To further enhance data discovery, we have developed the QUEST module to facilit .. grid-item-card:: :img-top: https://cdn-icons-png.flaticon.com/512/4230/4230997.png + :class-img-top: sd-p-2 + :class-card: sd-shadow-md **Development Guide** ^^^^^^^^^^^^^^^^^^^^^ @@ -88,6 +95,8 @@ To further enhance data discovery, we have developed the QUEST module to facilit .. grid-item-card:: :img-top: https://cdn-icons-png.flaticon.com/128/1283/1283342.png + :class-img-top: sd-p-2 + :class-card: sd-shadow-md **Get in Touch** ^^^^^^^^^^^^^^^^ @@ -108,7 +117,8 @@ To further enhance data discovery, we have developed the QUEST module to facilit .. grid-item-card:: :img-top: https://icesat-2.gsfc.nasa.gov/sites/default/files/MissionLogo_0.png - :class-img-top: pl-2 pr-2 pt-4 pb-4 + :class-img-top: sd-p-2 + :class-card: sd-shadow-md **ICESat-2 Resources** ^^^^^^^^^^^^^^^^^^^^^^ From 6a4acd91257128a928dca34a29afb9a82edab67e Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 20:26:43 -0600 Subject: [PATCH 26/27] Rename "Specific" classes to "Required" --- icepyx/core/APIformatting.py | 6 +++--- icepyx/core/granules.py | 8 ++++---- icepyx/core/query.py | 10 +++++----- icepyx/core/types.py | 8 ++++---- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/icepyx/core/APIformatting.py b/icepyx/core/APIformatting.py index bdb9b0f06..1c225d2ef 100644 --- a/icepyx/core/APIformatting.py +++ b/icepyx/core/APIformatting.py @@ -6,7 +6,7 @@ from icepyx.core.types import ( CMRParams, EGIParamsSubset, - EGISpecificRequiredParams, + EGIRequiredParams, ) # ---------------------------------------------------------------------- @@ -213,7 +213,7 @@ def __get__( self, instance: 'Parameters[Literal["required"]]', owner: Any, - ) -> EGISpecificRequiredParams: ... + ) -> EGIRequiredParams: ... @overload def __get__( @@ -226,7 +226,7 @@ def __get__( self, instance: "Parameters", owner: Any, - ) -> Union[CMRParams, EGISpecificRequiredParams, EGIParamsSubset]: + ) -> Union[CMRParams, EGIRequiredParams, EGIParamsSubset]: """ Returns the dictionary of formatted keys associated with the parameter object. diff --git a/icepyx/core/granules.py b/icepyx/core/granules.py index adf30e14c..080d8b19c 100644 --- a/icepyx/core/granules.py +++ b/icepyx/core/granules.py @@ -19,8 +19,8 @@ import icepyx.core.exceptions from icepyx.core.types import ( CMRParams, - EGISpecificParamsDownload, - EGISpecificParamsSearch, + EGIRequiredParamsDownload, + EGIRequiredParamsSearch, ) from icepyx.core.urls import DOWNLOAD_BASE_URL, GRANULE_SEARCH_BASE_URL, ORDER_BASE_URL @@ -181,7 +181,7 @@ def __init__( def get_avail( self, CMRparams: CMRParams, - reqparams: EGISpecificParamsSearch, + reqparams: EGIRequiredParamsSearch, cloud: bool = False, ): """ @@ -278,7 +278,7 @@ def get_avail( def place_order( self, CMRparams: CMRParams, - reqparams: EGISpecificParamsDownload, + reqparams: EGIRequiredParamsDownload, subsetparams, verbose, subset=True, diff --git a/icepyx/core/query.py b/icepyx/core/query.py index dbc8ce0bb..71df8723e 100644 --- a/icepyx/core/query.py +++ b/icepyx/core/query.py @@ -16,8 +16,8 @@ from icepyx.core.types import ( CMRParams, EGIParamsSubset, - EGISpecificParamsDownload, - EGISpecificRequiredParams, + EGIRequiredParams, + EGIRequiredParamsDownload, ) import icepyx.core.validate_inputs as val from icepyx.core.variables import Variables as Variables @@ -585,7 +585,7 @@ def CMRparams(self) -> CMRParams: return self._CMRparams.fmted_keys @property - def reqparams(self) -> EGISpecificRequiredParams: + def reqparams(self) -> EGIRequiredParams: """ Display the required key:value pairs that will be submitted. It generates the dictionary if it does not already exist. @@ -1044,7 +1044,7 @@ def order_granules(self, verbose=False, subset=True, email=False, **kwargs): tempCMRparams["readable_granule_name[]"] = gran self._granules.place_order( tempCMRparams, - cast(EGISpecificParamsDownload, self.reqparams), + cast(EGIRequiredParamsDownload, self.reqparams), self.subsetparams(**kwargs), verbose, subset, @@ -1054,7 +1054,7 @@ def order_granules(self, verbose=False, subset=True, email=False, **kwargs): else: self._granules.place_order( self.CMRparams, - cast(EGISpecificParamsDownload, self.reqparams), + cast(EGIRequiredParamsDownload, self.reqparams), self.subsetparams(**kwargs), verbose, subset, diff --git a/icepyx/core/types.py b/icepyx/core/types.py index e600b78cb..3bd89be7c 100644 --- a/icepyx/core/types.py +++ b/icepyx/core/types.py @@ -51,7 +51,7 @@ class CMRParamsWithPolygon(CMRParamsBase): CMRParams = Union[CMRParamsWithBbox, CMRParamsWithPolygon] -class EGISpecificParamsBase(TypedDict): +class EGIRequiredParamsBase(TypedDict): """Common parameters for searching, ordering, or downloading from EGI. See: https://wiki.earthdata.nasa.gov/display/SDPSDOCS/EGI+Programmatic+Access+Documentation @@ -72,11 +72,11 @@ class EGISpecificParamsBase(TypedDict): page_num: int # default 0 -class EGISpecificParamsSearch(EGISpecificParamsBase): +class EGIRequiredParamsSearch(EGIRequiredParamsBase): """Parameters for interacting with EGI.""" -class EGISpecificParamsDownload(EGISpecificParamsBase): +class EGIRequiredParamsDownload(EGIRequiredParamsBase): """Parameters for ordering from EGI. TODO: Validate more strongly (with Pydantic?): page_num >=0. @@ -108,4 +108,4 @@ class EGIParamsSubsetBoundingShape(EGIParamsSubsetBase): EGIParamsSubset = Union[EGIParamsSubsetBbox, EGIParamsSubsetBoundingShape] -EGISpecificRequiredParams = Union[EGISpecificParamsSearch, EGISpecificParamsDownload] +EGIRequiredParams = Union[EGIRequiredParamsSearch, EGIRequiredParamsDownload] From e41aed016417700f0ef90a9c5f7cddc593a20556 Mon Sep 17 00:00:00 2001 From: Matt Fisher Date: Thu, 12 Sep 2024 20:30:54 -0600 Subject: [PATCH 27/27] Mark EGI subset shape params NotRequired Co-authored-by: Jessica Scheick --- icepyx/core/types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/icepyx/core/types.py b/icepyx/core/types.py index 3bd89be7c..e85f8696f 100644 --- a/icepyx/core/types.py +++ b/icepyx/core/types.py @@ -99,11 +99,11 @@ class EGIParamsSubsetBase(TypedDict): class EGIParamsSubsetBbox(EGIParamsSubsetBase): - bbox: str + bbox: NotRequired[str] class EGIParamsSubsetBoundingShape(EGIParamsSubsetBase): - Boundingshape: str + Boundingshape: NotRequired[str] EGIParamsSubset = Union[EGIParamsSubsetBbox, EGIParamsSubsetBoundingShape]