From 53a5b311b932150da1b91e6f84233fee1879c46f Mon Sep 17 00:00:00 2001 From: uriii3 Date: Fri, 4 Oct 2024 11:59:04 +0200 Subject: [PATCH 01/14] Trying to see if this will work check back some updates finalising the dictionary should work for the subset all python api interface documentation updated Trying to see if this will work check back some updates finalising the dictionary should work for the subset all python api interface documentation updated all cli documentation update all cli documentation update order of decorators add description and response parts, version updates delete example ready to rebase coords selection method docs last changes mainly and will correct tests and main revision some updates on the docstrings documentation updated Some tests on the matter add numpydoc updating snapshots description changes Error in the describe delete one file first batch of changes check with r changes after comments on pr --- conda_environment_test.yaml | 1 + .../command_line_interface/group_describe.py | 38 +-- .../command_line_interface/group_get.py | 99 +++---- .../command_line_interface/group_login.py | 35 +-- .../command_line_interface/group_subset.py | 143 +++------- .../command_line_interface/utils.py | 8 +- .../core_functions/documentation_utils.py | 249 ++++++++++++++++++ copernicusmarine/core_functions/utils.py | 6 - copernicusmarine/download_functions/utils.py | 6 +- copernicusmarine/python_interface/describe.py | 23 +- copernicusmarine/python_interface/get.py | 48 ++-- copernicusmarine/python_interface/login.py | 24 +- .../python_interface/open_dataset.py | 45 ++-- .../python_interface/read_dataframe.py | 55 ++-- copernicusmarine/python_interface/subset.py | 61 ++--- doc/command-line-interface.rst | 2 +- doc/usage/subset-usage.rst | 26 +- .../test_help_command_interface.ambr | 220 +++++++--------- tests/test_documentation.py | 121 +++++++++ 19 files changed, 718 insertions(+), 492 deletions(-) create mode 100644 copernicusmarine/core_functions/documentation_utils.py create mode 100644 tests/test_documentation.py diff --git a/conda_environment_test.yaml b/conda_environment_test.yaml index f09f406e..227a8992 100644 --- a/conda_environment_test.yaml +++ b/conda_environment_test.yaml @@ -7,6 +7,7 @@ dependencies: - netcdf4==1.6.5 - syrupy==4.6.1 - compliance-checker==5.1.1 + - numpydoc==1.8.0 - pip: - pytest-order==1.2.1 - freezegun==1.5.1 diff --git a/copernicusmarine/command_line_interface/group_describe.py b/copernicusmarine/command_line_interface/group_describe.py index 4ec4e773..b5138f91 100644 --- a/copernicusmarine/command_line_interface/group_describe.py +++ b/copernicusmarine/command_line_interface/group_describe.py @@ -6,6 +6,7 @@ log_exception_and_exit, ) from copernicusmarine.command_line_interface.utils import tqdm_disable_option +from copernicusmarine.core_functions import documentation_utils from copernicusmarine.core_functions.click_custom_class import ( CustomClickOptionsCommand, ) @@ -24,13 +25,7 @@ def cli_describe() -> None: "describe", cls=CustomClickOptionsCommand, short_help="Print Copernicus Marine catalogue as JSON.", - help=""" - Print Copernicus Marine catalogue as JSON. - - The default display contains information on the products, and more data can be displayed using the ``--include-`` flags. - - The ``--contains`` option allows the user to specify one or several strings to filter through the catalogue display. The search is performed recursively on all attributes of the catalog, and the tokens only need to be contained in one of the attributes (i.e., not an exact match). - """, # noqa + help=documentation_utils.DESCRIBE["DESCRIBE_DESCRIPTION_HELP"], # noqa epilog=""" Examples: @@ -40,37 +35,37 @@ def cli_describe() -> None: .. code-block:: bash - copernicusmarine describe -c METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2 - """, # noqa + copernicusmarine describe -c METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2 \n + """ # noqa + + documentation_utils.DESCRIBE["DESCRIBE_RESPONSE_HELP"], # noqa ) @click.option( "--include-description", type=bool, is_flag=True, default=False, - help="Include product description in output.", + help=documentation_utils.DESCRIBE["INCLUDE_DESCRIPTION_HELP"], ) @click.option( "--include-datasets", type=bool, is_flag=True, default=False, - help="Include product dataset details in output.", + help=documentation_utils.DESCRIBE["INCLUDE_DATASETS_HELP"], ) @click.option( "--include-keywords", type=bool, is_flag=True, default=False, - help="Include product keyword details in output.", + help=documentation_utils.DESCRIBE["INCLUDE_KEYWORDS_HELP"], ) @click.option( "--include-versions", type=bool, is_flag=True, default=False, - help="Include dataset versions in output. " - "By default, shows only the default version.", + help=documentation_utils.DESCRIBE["INCLUDE_VERSIONS_HELP"], ) @click.option( "-a", @@ -78,34 +73,27 @@ def cli_describe() -> None: type=bool, is_flag=True, default=False, - help="Include all the possible data in output: " - "description, datasets, keywords, and versions.", + help=documentation_utils.DESCRIBE["INCLUDE_ALL_HELP"], ) @click.option( "--contains", "-c", type=str, multiple=True, - help="Filter catalogue output. Returns products with attributes " - "matching a string token.", + help=documentation_utils.DESCRIBE["CONTAINS_HELP"], ) @click.option( "--max-concurrent-requests", type=int, default=15, - help="Maximum number of concurrent requests. " - "Default 15. The describe command uses a thread " - "pool executor to manage concurrent requests.", + help=documentation_utils.DESCRIBE["MAX_CONCURRENT_REQUESTS_HELP"], ) @tqdm_disable_option @click.option( "--log-level", type=click.Choice(["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL", "QUIET"]), default="INFO", - help=( - "Set the details printed to console by the command " - "(based on standard logging library)." - ), + help=documentation_utils.DESCRIBE["LOG_LEVEL_HELP"], ) @click.option( "--staging", diff --git a/copernicusmarine/command_line_interface/group_get.py b/copernicusmarine/command_line_interface/group_get.py index dce20769..c95bc239 100644 --- a/copernicusmarine/command_line_interface/group_get.py +++ b/copernicusmarine/command_line_interface/group_get.py @@ -14,6 +14,7 @@ force_dataset_version_option, tqdm_disable_option, ) +from copernicusmarine.core_functions import documentation_utils from copernicusmarine.core_functions.click_custom_class import ( CustomClickOptionsCommand, ) @@ -21,11 +22,6 @@ create_get_template, get_function, ) -from copernicusmarine.core_functions.utils import ( - OVERWRITE_LONG_OPTION, - OVERWRITE_OPTION_HELP_TEXT, - OVERWRITE_SHORT_OPTION, -) logger = logging.getLogger("copernicusmarine") blank_logger = logging.getLogger("copernicusmarine_blank_logger") @@ -40,25 +36,26 @@ def cli_get() -> None: "get", cls=CustomClickOptionsCommand, short_help="Download originally produced data files.", - help=""" - Download originally produced data files. - - The ``--dataset-id`` is required (can be found via the "describe" command). The function fetches the files recursively if a folder path is passed as a URL. When provided a dataset ID, all the files in the corresponding folder will be downloaded if none of the ``--filter`` or ``--regex`` options is specified. - """, # noqa + help=documentation_utils.GET["GET_DESCRIPTION_HELP"] + + " See :ref:`describe `." + " When provided a datasetID, all the files " + "in the corresponding folder will be downloaded. Filters can be applied " + "with ``--filter`` or ``--regex`` options.", # noqa epilog=""" Example to download all the files from a given dataset: .. code-block:: bash - copernicusmarine get -i cmems_mod_nws_bgc-pft_myint_7km-3D-diato_P1M-m - """, # noqa + copernicusmarine get -i cmems_mod_nws_bgc-pft_myint_7km-3D-diato_P1M-m \n + """ + + documentation_utils.GET["GET_RESPONSE_HELP"], # noqa ) @click.option( "--dataset-id", "-i", type=str, default=None, - help="The datasetID.", + help=documentation_utils.GET["DATASET_ID_HELP"], ) @force_dataset_version_option @force_dataset_part_option @@ -66,123 +63,97 @@ def cli_get() -> None: "--username", type=str, default=None, - help="If not set, search for environment variable" - + " COPERNICUSMARINE_SERVICE_USERNAME" - + ", or else look for configuration files, or else ask for user input.", + help=documentation_utils.GET["USERNAME_HELP"], ) @click.option( "--password", type=str, default=None, - help="If not set, search for environment variable" - + " COPERNICUSMARINE_SERVICE_PASSWORD" - + ", or else look for configuration files, or else ask for user input.", + help=documentation_utils.GET["PASSWORD_HELP"], ) @click.option( "--no-directories", "-nd", cls=MutuallyExclusiveOption, is_flag=True, - help="Option to not recreate folder hierarchy in ouput directory.", + help=documentation_utils.GET["NO_DIRECTORIES_HELP"], default=False, mutually_exclusive=["sync"], ) @click.option( "--show-outputnames", is_flag=True, - help="Option to display the names of the" - + " output files before download.", + help=documentation_utils.GET["SHOW_OUTPUTNAMES_HELP"], default=False, ) @click.option( "--output-directory", "-o", type=click.Path(path_type=pathlib.Path), - help="The destination directory for the downloaded files." - + " Default is the current directory.", + help=documentation_utils.GET["OUTPUT_DIRECTORY_HELP"], ) @click.option( "--credentials-file", type=click.Path(path_type=pathlib.Path), - help=( - "Path to a credentials file if not in its default directory. " - "Accepts .copernicusmarine-credentials / .netrc or _netrc / " - "motuclient-python.ini files." - ), + help=documentation_utils.GET["CREDENTIALS_FILE_HELP"], ) @click.option( "--force-download", is_flag=True, default=False, - help="Flag to skip confirmation before download.", + help=documentation_utils.GET["FORCE_DOWNLOAD_HELP"], ) @click.option( - OVERWRITE_LONG_OPTION, - OVERWRITE_SHORT_OPTION, + documentation_utils.GET["OVERWRITE_LONG_OPTION"], + documentation_utils.GET["OVERWRITE_SHORT_OPTION"], is_flag=True, default=False, - help=OVERWRITE_OPTION_HELP_TEXT, + help=documentation_utils.GET["OVERWRITE_OUTPUT_DATA_HELP"], ) @click.option( "--create-template", type=bool, is_flag=True, default=False, - help="Option to create a file get_template.json in your current directory " - "containing CLI arguments. If specified, no other action will be performed.", + help=documentation_utils.GET["CREATE_TEMPLATE_HELP"], ) @click.option( "--request-file", type=click.Path(exists=True, path_type=pathlib.Path), - help="Option to pass a file containing CLI arguments. " - "The file MUST follow the structure of dataclass 'GetRequest'." - " For more information please refer to the README.", + help=documentation_utils.GET["REQUEST_FILE_HELP"], ) @click.option( "--filter", "--filter-with-globbing-pattern", type=str, default=None, - help="A pattern that must match the absolute paths of " - "the files to download.", + help=documentation_utils.GET["FILTER_HELP"], ) @click.option( "--regex", "--filter-with-regular-expression", type=str, default=None, - help="The regular expression that must match the absolute paths of " - "the files to download.", + help=documentation_utils.GET["REGEX_HELP"], ) @click.option( "--file-list", type=pathlib.Path, default=None, - help="Path to a .txt file containing a list of file paths," - " line by line, that will be downloaded directly." - " These files must be from the specified dataset using the --dataset-id." - " If no files can be found, the Toolbox will list all" - " files on the remote server and attempt to find a match.", + help=documentation_utils.GET["FILE_LIST_HELP"], ) @click.option( "--create-file-list", type=str, default=None, - help="Option to only create a file containing " - "the names of the targeted files instead of downloading them. " - "It writes the file in the directory specified with the " - "--output-directory option (default to current directory). " - "The file name specified should end with '.txt' or '.csv' " - "If specified, no other action will be performed. " - "Please find more information in the README.", + help=documentation_utils.GET["CREATE_FILE_LIST_HELP"], ) @click.option( "--sync", cls=MutuallyExclusiveOption, is_flag=True, default=False, - help="Option to synchronize the local directory with " - "the remote directory. See the documentation for more details.", + help=documentation_utils.GET["SYNC_HELP"], mutually_exclusive=["no-directories"], ) @click.option( @@ -190,8 +161,7 @@ def cli_get() -> None: cls=MutuallyExclusiveOption, is_flag=True, default=False, - help="Option to delete local files that are not present on " - "the remote server while applying sync.", + help=documentation_utils.GET["SYNC_DELETE_HELP"], mutually_exclusive=["no-directories"], ) @click.option( @@ -199,32 +169,27 @@ def cli_get() -> None: type=bool, is_flag=True, default=False, - help="Option to get the index files of an INSITU dataset. Temporary option.", + help=documentation_utils.GET["INDEX_PARTS_HELP"], ) @click.option( "--dry-run", type=bool, is_flag=True, default=False, - help="Runs query without downloading data.", + help=documentation_utils.GET["DRY_RUN_HELP"], ) @click.option( "--max-concurrent-requests", type=int, default=15, - help="Maximum number of concurrent requests. " - "Default 15. The get command uses a thread " - "pool executor to manage concurrent requests.", + help=documentation_utils.GET["MAX_CONCURRENT_REQUESTS_HELP"], ) @tqdm_disable_option @click.option( "--log-level", type=click.Choice(["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL", "QUIET"]), default="INFO", - help=( - "Set the details printed to console by the command " - "(based on standard logging library)." - ), + help=documentation_utils.GET["LOG_LEVEL_HELP"], ) @click.option( "--staging", diff --git a/copernicusmarine/command_line_interface/group_login.py b/copernicusmarine/command_line_interface/group_login.py index d7c2c0f4..9615199e 100644 --- a/copernicusmarine/command_line_interface/group_login.py +++ b/copernicusmarine/command_line_interface/group_login.py @@ -10,6 +10,7 @@ from copernicusmarine.core_functions.click_custom_class import ( CustomClickOptionsCommand, ) +from copernicusmarine.core_functions import documentation_utils from copernicusmarine.core_functions.credentials_utils import ( DEFAULT_CLIENT_BASE_DIRECTORY, ) @@ -27,11 +28,7 @@ def cli_login() -> None: "login", cls=CustomClickOptionsCommand, short_help="Create a configuration file with your Copernicus Marine credentials.", - help=""" - Create a configuration file with your Copernicus Marine credentials. - - Create a configuration file under the ``$HOME/.copernicusmarine`` directory (overwritable with the ``--credentials-file`` option). - """, # noqa + help=documentation_utils.LOGIN["LOGIN_DESCRIPTION_HELP"], # noqa epilog=""" Examples: @@ -54,55 +51,43 @@ def cli_login() -> None: copernicusmarine login > Username: [USER-INPUT] > Password: [USER-INPUT] - """, # noqa + """ # noqa + + documentation_utils.LOGIN["LOGIN_RESPONSE_HELP"], ) @click.option( "--username", hide_input=False, - help="If not set, search for environment variable" - + " COPERNICUSMARINE_SERVICE_USERNAME" - + ", or else ask for user input.", + help=documentation_utils.LOGIN["USERNAME_HELP"], ) @click.option( "--password", hide_input=True, - help="If not set, search for environment variable" - + " COPERNICUSMARINE_SERVICE_PASSWORD" - + ", or else ask for user input.", + help=documentation_utils.LOGIN["PASSWORD_HELP"], ) @click.option( "--configuration-file-directory", type=click.Path(path_type=pathlib.Path), default=DEFAULT_CLIENT_BASE_DIRECTORY, - help="Path to the directory where the configuration file is stored.", + help=documentation_utils.LOGIN["CONFIGURATION_FILE_DIRECTORY_HELP"], ) @click.option( "--overwrite-configuration-file", "-overwrite", is_flag=True, default=False, - help="Flag to skip confirmation before overwriting configuration file.", + help=documentation_utils.LOGIN["OVERWRITE_CONFIGURATION_FILE_HELP"], ) @click.option( "--check-credentials-valid", is_flag=True, default=False, - help="Flag to check if the credentials are valid. " - "No other action will be performed. " - "The validity will be check in this order: " - "1. Check if the credentials are valid with the provided username and password." - "2. Check if the credentials are valid in the configuration file." - "3. Check if the credentials are valid in the environment variables." - "When any is found not valid, will return False immediately.", + help=documentation_utils.LOGIN["CHECK_CREDENTIALS_VALID_HELP"], ) @click.option( "--log-level", type=click.Choice(["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL", "QUIET"]), default="INFO", - help=( - "Set the details printed to console by the command " - "(based on standard logging library)." - ), + help=documentation_utils.LOGIN["LOG_LEVEL_HELP"], ) @log_exception_and_exit def login( diff --git a/copernicusmarine/command_line_interface/group_subset.py b/copernicusmarine/command_line_interface/group_subset.py index 962bea5c..05aff925 100644 --- a/copernicusmarine/command_line_interface/group_subset.py +++ b/copernicusmarine/command_line_interface/group_subset.py @@ -13,6 +13,7 @@ force_dataset_version_option, tqdm_disable_option, ) +from copernicusmarine.core_functions import documentation_utils from copernicusmarine.core_functions.click_custom_class import ( CustomClickOptionsCommand, ) @@ -27,17 +28,11 @@ FileFormat, VerticalDimensionOutput, ) -from copernicusmarine.core_functions.services_utils import CommandType from copernicusmarine.core_functions.subset import ( create_subset_template, subset_function, ) -from copernicusmarine.core_functions.utils import ( - OVERWRITE_LONG_OPTION, - OVERWRITE_OPTION_HELP_TEXT, - OVERWRITE_SHORT_OPTION, - datetime_parser, -) +from copernicusmarine.core_functions.utils import datetime_parser logger = logging.getLogger("copernicusmarine") blank_logger = logging.getLogger("copernicusmarine_blank_logger") @@ -52,11 +47,8 @@ def cli_subset() -> None: "subset", cls=CustomClickOptionsCommand, short_help="Download subsets of datasets as NetCDF files or Zarr stores.", - help=""" - Download subsets of datasets as NetCDF files or Zarr stores. - - The ``--dataset-id`` is required (can be found via the "describe" command). The argument values passed individually through the CLI take precedence over the values from the ``--motu-api-request`` option, which takes precedence over the ones from the ``--request-file`` option. - """, # noqa + help=documentation_utils.SUBSET["SUBSET_DESCRIPTION_HELP"] + + "See :ref:`describe `.", epilog=""" Examples: @@ -68,15 +60,16 @@ def cli_subset() -> None: .. code-block:: bash - copernicusmarine subset -i cmems_mod_ibi_phy_my_0.083deg-3D_P1D-m -v thetao -v so -t 2021-01-01 -T 2021-01-03 -x 0.0 -X 0.1 -y 28.0 -Y 28.1 -z 1 -Z 2 - """, # noqa + copernicusmarine subset -i cmems_mod_ibi_phy_my_0.083deg-3D_P1D-m -v thetao -v so -t 2021-01-01 -T 2021-01-03 -x 0.0 -X 0.1 -y 28.0 -Y 28.1 -z 1 -Z 2 \n + """ # noqa + + documentation_utils.SUBSET["SUBSET_RESPONSE_HELP"], ) @click.option( "--dataset-id", "-i", type=str, default=None, - help="The datasetID.", + help=documentation_utils.SUBSET["DATASET_ID_HELP"], ) @force_dataset_version_option @force_dataset_part_option @@ -84,212 +77,155 @@ def cli_subset() -> None: "--username", type=str, default=None, - help="If not set, search for environment variable" - + " COPERNICUSMARINE_SERVICE_USERNAME" - + ", or else look for configuration files, or else ask for user input.", + help=documentation_utils.SUBSET["USERNAME_HELP"], ) @click.option( "--password", type=str, default=None, - help="If not set, search for environment variable" - + " COPERNICUSMARINE_SERVICE_PASSWORD" - + ", or else look for configuration files, or else ask for user input.", + help=documentation_utils.SUBSET["PASSWORD_HELP"], ) @click.option( "--variable", "-v", "variables", type=str, - help="Specify dataset variable. Can be used multiple times.", + help=documentation_utils.SUBSET["VARIABLES_HELP"], multiple=True, ) @click.option( "--minimum-longitude", "-x", type=float, - help=( - "Minimum longitude for the subset. " - "The value will be reduced to the interval [-180; 360[." - ), + help=documentation_utils.SUBSET["MINIMUM_LONGITUDE_HELP"], ) @click.option( "--maximum-longitude", "-X", type=float, - help=( - "Maximum longitude for the subset. " - "The value will be reduced to the interval [-180; 360[." - ), + help=documentation_utils.SUBSET["MAXIMUM_LONGITUDE_HELP"], ) @click.option( "--minimum-latitude", "-y", type=click.FloatRange(min=-90, max=90), - help="Minimum latitude for the subset." - " Requires a float within this range:", + help=documentation_utils.SUBSET["MINIMUM_LATITUDE_HELP"], ) @click.option( "--maximum-latitude", "-Y", type=click.FloatRange(min=-90, max=90), - help="Maximum latitude for the subset." - " Requires a float within this range:", + help=documentation_utils.SUBSET["MAXIMUM_LATITUDE_HELP"], ) @click.option( "--minimum-depth", "-z", type=click.FloatRange(min=0), - help="Minimum depth for the subset. Requires a float within this range:", + help=documentation_utils.SUBSET["MINIMUM_DEPTH_HELP"], ) @click.option( "--maximum-depth", "-Z", type=click.FloatRange(min=0), - help="Maximum depth for the subset. Requires a float within this range:", + help=documentation_utils.SUBSET["MAXIMUM_DEPTH_HELP"], ) @click.option( "--vertical-dimension-output", "-V", type=click.Choice(DEFAULT_VERTICAL_DIMENSION_OUTPUTS), default=DEFAULT_VERTICAL_DIMENSION_OUTPUT, - help=( - "Consolidate the vertical dimension (the z-axis) as requested:" - " `depth` with descending positive values." - " `elevation` with ascending positive values." - " Default is `depth`." - ), + help=documentation_utils.SUBSET["VERTICAL_DIMENSION_OUTPUT_HELP"], ) @click.option( "--start-datetime", "-t", type=str, - help="The start datetime of the temporal subset. " - "Caution: encapsulate date " - + 'with " " to ensure valid expression for format "%Y-%m-%d %H:%M:%S". ' - + "Supports common format parsed by pendulum. " - + "See https://pendulum.eustace.io/docs/#parsing", + help=documentation_utils.SUBSET["START_DATETIME_HELP"], ) @click.option( "--end-datetime", "-T", type=str, - help="The end datetime of the temporal subset. Caution: encapsulate date " - + 'with " " to ensure valid expression for format "%Y-%m-%d %H:%M:%S". ' - + "Supports common format parsed by pendulum. " - + "See https://pendulum.eustace.io/docs/#parsing", + help=documentation_utils.SUBSET["END_DATETIME_HELP"], ) @click.option( "--coordinates-selection-method", type=click.Choice(DEFAULT_COORDINATES_SELECTION_METHODS), default=DEFAULT_COORDINATES_SELECTION_METHOD, - help=( - "The method in which the coordinates will be retrieved." - " If 'inside', the retrieved selection will be inside the requested" - " interval." - " If 'strict-inside', the retrieved selection will be inside the requested" - " interval and an error will raise if there doesn't exist the values." - " If 'nearest', the returned interval extremes will be the closest to what" - " has been asked for. A warning will be displayed if outside of bounds." - " If 'outisde', the extremes will be taken to contain all the requested" - " interval. A warning will also be displayed if the subset is " - "outside of the dataset bounds." - ), + help=documentation_utils.SUBSET["COORDINATES_SELECTION_METHOD_HELP"], ) @click.option( "--output-directory", "-o", type=click.Path(path_type=pathlib.Path), - help="The destination folder for the downloaded files." - + " Default is the current directory.", + help=documentation_utils.SUBSET["OUTPUT_DIRECTORY_HELP"], ) @click.option( "--credentials-file", type=click.Path(path_type=pathlib.Path), - help=( - "Path to a credentials file if not in its default directory. " - "Accepts .copernicusmarine-credentials / .netrc or _netrc / " - "motuclient-python.ini files." - ), + help=documentation_utils.SUBSET["CREDENTIALS_FILE_HELP"], ) @click.option( "--output-filename", "-f", type=str, - help=( - "Concatenate the downloaded data in the given file name " - "(under the output directory)." - ), + help=documentation_utils.SUBSET["OUTPUT_FILENAME_HELP"], ) @click.option( "--file-format", type=click.Choice(DEFAULT_FILE_FORMATS), default=DEFAULT_FILE_FORMAT, - help=("Format of the downloaded dataset. Default to NetCDF (.nc)."), + help=documentation_utils.SUBSET["FILE_FORMAT_HELP"], ) @click.option( "--force-download", is_flag=True, default=False, - help="Flag to skip confirmation before download.", + help=documentation_utils.SUBSET["FORCE_DOWNLOAD_HELP"], ) @click.option( - OVERWRITE_LONG_OPTION, - OVERWRITE_SHORT_OPTION, + documentation_utils.SUBSET["OVERWRITE_LONG_OPTION"], + documentation_utils.SUBSET["OVERWRITE_SHORT_OPTION"], is_flag=True, default=False, - help=OVERWRITE_OPTION_HELP_TEXT, + help=documentation_utils.SUBSET["OVERWRITE_OUTPUT_DATA_HELP"], ) @click.option( "--service", "-s", type=str, - help=( - "Force download through one of the available services " - f"using the service name among {CommandType.SUBSET.service_names()} " - f"or its short name among {CommandType.SUBSET.service_short_names()}." - ), + help=documentation_utils.SUBSET["SERVICE_HELP"], ) @click.option( "--create-template", type=bool, is_flag=True, default=False, - help="Option to create a file subset_template.json in your current directory " - "containing CLI arguments. If specified, no other action will be performed.", + help=documentation_utils.SUBSET["CREATE_TEMPLATE_HELP"], ) @click.option( "--request-file", type=click.Path(exists=True, path_type=pathlib.Path), - help="Option to pass a file containing CLI arguments. " - "The file MUST follow the structure of dataclass 'SubsetRequest'." - " For more information please refer to the README.", + help=documentation_utils.SUBSET["REQUEST_FILE_HELP"], ) @click.option( "--motu-api-request", type=str, - help=( - "Option to pass a complete MOTU API request as a string. " - 'Caution, user has to replace double quotes " with single ' - "quotes ' in the request." - ), + help=documentation_utils.SUBSET["MOTU_API_REQUEST_HELP"], ) @click.option( "--dry-run", type=bool, is_flag=True, default=False, - help="Runs query without downloading data.", + help=documentation_utils.SUBSET["DRY_RUN_HELP"], ) @tqdm_disable_option @click.option( "--log-level", type=click.Choice(["DEBUG", "INFO", "WARN", "ERROR", "CRITICAL", "QUIET"]), default="INFO", - help=( - "Set the details printed to console by the command " - "(based on standard logging library)." - ), + help=documentation_utils.SUBSET["LOG_LEVEL_HELP"], ) @click.option( "--staging", @@ -304,18 +240,15 @@ def cli_subset() -> None: is_flag=False, flag_value=1, default=0, - help=( - "Specify a compression level to apply on the NetCDF output file. " - "A value of 0 means no compression, and 9 is the highest level of " - "compression available" - ), + help=documentation_utils.SUBSET["NETCDF_COMPRESSION_LEVEL_HELP"] + + " If used as a flag, the assigned value will be 1.", ) @click.option( "--netcdf3-compatible", type=bool, default=False, is_flag=True, - help=("Enable downloading the dataset in a netCDF 3 compatible format."), + help=documentation_utils.SUBSET["NETCDF3_COMPATIBLE_HELP"], ) @log_exception_and_exit def subset( diff --git a/copernicusmarine/command_line_interface/utils.py b/copernicusmarine/command_line_interface/utils.py index dc0d60da..7d1a3d1c 100644 --- a/copernicusmarine/command_line_interface/utils.py +++ b/copernicusmarine/command_line_interface/utils.py @@ -2,6 +2,8 @@ from click import Context, Option, UsageError from click.core import ParameterSource +from copernicusmarine.core_functions import documentation_utils + class MutuallyExclusiveOption(Option): def __init__(self, *args, **kwargs): @@ -55,19 +57,19 @@ def assert_cli_args_are_not_set_except_create_template( "--disable-progress-bar", is_flag=True, default=False, - help="Flag to hide progress bar.", + help=documentation_utils.SUBSET["DISABLE_PROGRESS_BAR_HELP"], ) force_dataset_version_option = click.option( "--dataset-version", type=str, default=None, - help="Force the selection of a specific dataset version.", + help=documentation_utils.SUBSET["DATASET_VERSION_HELP"], ) force_dataset_part_option = click.option( "--dataset-part", type=str, default=None, - help="Force the selection of a specific dataset part.", + help=documentation_utils.SUBSET["DATASET_PART_HELP"], ) diff --git a/copernicusmarine/core_functions/documentation_utils.py b/copernicusmarine/core_functions/documentation_utils.py new file mode 100644 index 00000000..80b37998 --- /dev/null +++ b/copernicusmarine/core_functions/documentation_utils.py @@ -0,0 +1,249 @@ +from copernicusmarine.core_functions.services_utils import CommandType + +SHARED: dict[str, str] = { + "OVERWRITE_SHORT_OPTION": "--overwrite", + "OVERWRITE_LONG_OPTION": "--overwrite-output-data", + "OVERWRITE_OUTPUT_DATA_HELP": ( + "If specified and if the file already exists on destination, then it will be " + "overwritten instead of creating new one with unique index." + ), + "USERNAME_HELP": ( + "The username for authentication." + ), # a little hardcoding in Python API + "PASSWORD_HELP": ( + "The password for authentication." + ), # a little hardcoding in Python API + "LOG_LEVEL_HELP": ( + "Set the details printed to console by the command " + "(based on standard logging library)." + ), + "CREATE_TEMPLATE_HELP": ( + "Option to create a file _template.json in your current directory " + "containing the arguments. If specified, no other action will be performed." + ), + "CREDENTIALS_FILE_HELP": ( + "Path to a credentials file if not in its default directory" + " (which would be $HOME/.copernicusmarine). Accepts " + ".copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini " + "files." + ), + "DATASET_VERSION_HELP": "Force the selection of a specific dataset version.", + "DATASET_PART_HELP": "Force the selection of a specific dataset part.", + "DATASET_ID_HELP": ( + "The datasetID, required either as an argument or in the request_file option." + ), + "DISABLE_PROGRESS_BAR_HELP": "Flag to hide progress bar.", + "FORCE_DOWNLOAD_HELP": "Flag to skip confirmation before download.", + "DRY_RUN_HELP": "If True, runs query without downloading data.", + "OUTPUT_DIRECTORY_HELP": ( + "The destination folder for the downloaded files. Default is the current " + "directory." + ), + "REQUEST_FILE_HELP": ( + "Option to pass a file containing the arguments." + " For more information please refer to" + " the documentation or use option ``--create-template`` " + "for an example template." + ), +} + +LOGIN: dict[str, str] = { + "LOGIN_DESCRIPTION_HELP": ( + "Create a configuration file with your Copernicus Marine credentials" + " under the ``$HOME/.copernicusmarine`` directory" + " (overwritable with the ``--credentials-file`` option)." + ), + "LOGIN_RESPONSE_HELP": ( + "Error code\n 0 error code if the login was successfully " + "completed, 1 otherwise." + ), + "CONFIGURATION_FILE_DIRECTORY_HELP": ( + "Path to the directory where the configuration file is stored." + ), + "OVERWRITE_CONFIGURATION_FILE_HELP": ( + "Flag to skip confirmation before overwriting configuration file." + ), + "CHECK_CREDENTIALS_VALID_HELP": ( + "Flag to check if the credentials are valid. " + "No other action will be performed. " + "The validity will be check in this order: " + "1. Check if the credentials are valid" + " with the provided username and password. " + "2. Check if the credentials are valid in the environment variables. " + "3. Check if the credentials are valid in the configuration file. " + "When any is found not valid, will return False immediately." + ), +} + +DESCRIBE: dict[str, str] = { + "DESCRIBE_DESCRIPTION_HELP": ( + "Retrieve and parse the metadata information " + " from the Copernicus Marine catalogue." + ), + "DESCRIBE_RESPONSE_HELP": ( + "dict[str, Any]\n" + "A dictionary containing the retrieved metadata information." + ), + "MAX_CONCURRENT_REQUESTS_HELP": ( + "Maximum number of concurrent requests (>=1). Default 15. The command uses " + "a thread pool executor to manage concurrent requests." + ), + "INCLUDE_DESCRIPTION_HELP": "Include product description in output.", + "INCLUDE_DATASETS_HELP": "Include product dataset details in output.", + "INCLUDE_KEYWORDS_HELP": "Include product keyword details in output.", + "INCLUDE_VERSIONS_HELP": ( + "Include dataset versions in output. By default, shows only the default " + "version." + ), + "INCLUDE_ALL_HELP": ( + "Include all the possible data in output: description, datasets, keywords, " + "and versions." + ), + "CONTAINS_HELP": ( + "Filter catalogue output. Returns products with attributes matching a string " + "token." + ), +} + +SUBSET: dict[str, str] = { + "SUBSET_DESCRIPTION_HELP": ( + "Extracts a subset of data from a specified dataset using given parameters." + " \n \n The ``--dataset-id`` is required and can be found via the ``describe`` " + "command. " # has some hardcoding in CLI and python API + ), + "SUBSET_RESPONSE_HELP": ( + "ResponseSubset \n" + "A description of the downloaded data and its destination." + ), + "SERVICE_HELP": ( + f"Force download through one of the available services using the service name " + f"among {CommandType.SUBSET.service_names()} or " + f"its short name among {CommandType.SUBSET.service_names()}." + ), + "VARIABLES_HELP": "Specify dataset variable. Can be used multiple times.", + "MINIMUM_LONGITUDE_HELP": ( + "Minimum longitude for the subset. The value will be transposed " + "to the interval [-180; 360[." + ), + "MAXIMUM_LONGITUDE_HELP": ( + "Maximum longitude for the subset. The value will be transposed" + " to the interval [-180; 360[." + ), + "MINIMUM_LATITUDE_HELP": ( + "Minimum latitude for the subset. Requires a float from -90 " + "degrees to +90." + ), + "MAXIMUM_LATITUDE_HELP": ( + "Maximum latitude for the subset. Requires a float from -90 degrees " + "to +90." + ), + "MINIMUM_DEPTH_HELP": ( + "Minimum depth for the subset. Requires a positive float (or 0)." + ), + "MAXIMUM_DEPTH_HELP": ( + "Maximum depth for the subset. Requires a positive float (or 0)." + ), + "VERTICAL_DIMENSION_OUTPUT_HELP": ( + "Consolidate the vertical dimension (the z-axis) as requested: depth with " + "descending positive values, elevation with ascending positive values. " + "Default is depth." + ), + "START_DATETIME_HELP": ( + "The start datetime of the temporal subset. Caution: encapsulate date with " + "“ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports " + "common format parsed by pendulum." + ), + "END_DATETIME_HELP": ( + "The end datetime of the temporal subset. Caution: encapsulate date with “ “ " + "to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common " + "format parsed by pendulum." + ), + "COORDINATES_SELECTION_METHOD_HELP": ( + "If ‘inside´, the " + "selection retrieved will be inside the requested range. If ‘strict-" + "inside’, the selection retrieved will be inside the requested range, " + "and an error will be raised if the values don't exist. " + "If ‘nearest’, the extremes closest to the requested values will " + "be returned. If ‘outside’," + " the extremes will be taken to contain all the requested interval." + " The methods ‘inside´, ‘nearest’ and ‘outside’ will display" + " a warning if the request is out of bounds." + ), + "OUTPUT_FILENAME_HELP": ( + "Concatenate the downloaded data in the given file name (under the output " + "directory)." + ), + "FILE_FORMAT_HELP": "Format of the downloaded dataset. Default to NetCDF (.nc).", + "MOTU_API_REQUEST_HELP": ( + "Option to pass a complete MOTU API request as a string. Caution, user has to " + "replace double quotes “ with single quotes ‘ in the request." + ), + "NETCDF_COMPRESSION_LEVEL_HELP": ( + "Specify a compression level to apply on the NetCDF output file. A value of 0 " + "means no compression, and 9 is the highest level of compression available." + ), # some hardcoding in CLI to add the flag value + "NETCDF3_COMPATIBLE_HELP": ( + "Enable downloading the dataset in a netCDF3 compatible format." + ), +} + +GET: dict[str, str] = { + "GET_DESCRIPTION_HELP": ( + "Download originally produced data files.\n \n " + " The ``--dataset-id`` is required (either as an " + "argument or in a request file) and can be found via the ``describe``" + " command." + ), # has some hardcoding in CLI + "MAX_CONCURRENT_REQUESTS_HELP": ( + "Maximum number of concurrent requests. Default 15. The command uses a thread " + "pool executor to manage concurrent requests. If set to 0, no parallel" + " executions are used." + ), + "GET_RESPONSE_HELP": ( + "ResponseGet \n" + "A list of files that were downloaded and some metadata." + ), + "SHOW_OUTPUTNAMES_HELP": ( + "Option to display the names of the output files before download." + ), + "FILTER_HELP": ( + "A pattern that must match the absolute paths of the files to download." + ), + "REGEX_HELP": ( + "The regular expression that must match the absolute paths of the files to " + "download." + ), + "FILE_LIST_HELP": ( + "Path to a .txt file containing a list of file paths, line by line, that will " + "be downloaded directly. These files must be from the specified dataset using " + "the --dataset-id. If no files can be found, the Toolbox will list all files " + "on the remote server and attempt to find a match." + ), + "CREATE_FILE_LIST_HELP": ( + "Option to only create a file containing the names of the targeted files " + "instead of downloading them. It writes the file in the directory specified " + "with the --output-directory option (default to current directory). The file " + "name specified should end with ‘.txt’ or ‘.csv’. If specified, no other " + "action will be performed." + ), + "SYNC_HELP": ( + "Option to synchronize the local directory with the remote directory. See the " + "documentation for more details." + ), + "SYNC_DELETE_HELP": ( + "Option to delete local files that are not present on the remote server while " + "applying sync." + ), + "INDEX_PARTS_HELP": ( + "Option to get the index files of an INSITU dataset." + ), + "NO_DIRECTORIES_HELP": ( + "If True, downloaded files will not be organized into directories." + ), +} + + +SUBSET.update(SHARED) +GET.update(SHARED) +LOGIN.update(SHARED) +DESCRIBE.update(SHARED) diff --git a/copernicusmarine/core_functions/utils.py b/copernicusmarine/core_functions/utils.py index 767c3b9d..3e08e290 100644 --- a/copernicusmarine/core_functions/utils.py +++ b/copernicusmarine/core_functions/utils.py @@ -31,12 +31,6 @@ logger = logging.getLogger("copernicusmarine") -OVERWRITE_SHORT_OPTION = "--overwrite" -OVERWRITE_LONG_OPTION = "--overwrite-output-data" -OVERWRITE_OPTION_HELP_TEXT = ( - "If specified and if the file already exists on destination, then it will be " - "overwritten instead of creating new one with unique index." -) FORCE_DOWNLOAD_CLI_PROMPT_MESSAGE = "Do you want to proceed with download?" diff --git a/copernicusmarine/download_functions/utils.py b/copernicusmarine/download_functions/utils.py index 2fe87099..986f6cb1 100644 --- a/copernicusmarine/download_functions/utils.py +++ b/copernicusmarine/download_functions/utils.py @@ -229,9 +229,9 @@ def get_message_formatted_dataset_size_estimation( f"\nEstimated size of the data that needs " f"to be downloaded to obtain the result:" f" {estimation_data_downloaded:.0f} MB" - "\nThis a very rough estimation and usually" - " its higher than the actual size of the" - " data that needs to be downloaded." + "\nThis is a very rough estimate that is" + " generally higher than the actual size of the" + " data that needs to be downloaded." ) diff --git a/copernicusmarine/python_interface/describe.py b/copernicusmarine/python_interface/describe.py index 03a5a83c..44b62e28 100644 --- a/copernicusmarine/python_interface/describe.py +++ b/copernicusmarine/python_interface/describe.py @@ -25,35 +25,32 @@ def describe( staging: bool = False, ) -> dict[str, Any]: """ - Retrieve metadata information from the Copernicus Marine catalogue. - - This function fetches metadata information from the Copernicus Marine catalogue - based on specified parameters and options. + Retrieve metadata information from the Copernicus Marine catalogue based on specified parameters and options. Parameters ---------- include_description : bool, optional - Whether to include description for each product. Defaults to False. + Include product description in output. include_datasets : bool, optional - Whether to include dataset information. Defaults to False. + Include product dataset details in output. include_keywords : bool, optional - Whether to include keywords for each product. Defaults to False. + Include product keyword details in output. include_versions : bool, optional - Whether to include all versions of each dataset. Defaults to False. + Include dataset versions in output. By default, shows only the default version. include_all : bool, optional - Whether to include all metadata information. Defaults to False. + Include all the possible data in output: description, datasets, keywords, and versions. contains : list[str], optional - List of strings to filter items containing these values. Defaults to []. + Filter catalogue output. Returns products with attributes matching a string token. max_concurrent_requests : int, optional - Maximum number of concurrent requests. Defaults to 15. The describe command - uses a thread pool executor to manage concurrent requests. + Maximum number of concurrent requests. Default 15. The command uses a thread pool executor to manage concurrent requests. disable_progress_bar : bool, optional - Whether to disable the progress bar. Defaults to False. + Flag to hide progress bar. Returns ------- dict[str, Any] A dictionary containing the retrieved metadata information. + """ # noqa if not isinstance(contains, list): diff --git a/copernicusmarine/python_interface/get.py b/copernicusmarine/python_interface/get.py index 9646560f..14cc9bb0 100644 --- a/copernicusmarine/python_interface/get.py +++ b/copernicusmarine/python_interface/get.py @@ -40,57 +40,54 @@ def get( staging: bool = False, ) -> ResponseGet: """ - Fetches data from the Copernicus Marine server based on the provided parameters. + Download originally produced data files. + + The ``dataset_id`` is required and can be found via the ``describe`` command. Parameters ---------- dataset_id : str, optional - The unique identifier of the dataset. + The datasetID, required either as an argument or in the request_file option. dataset_version : str, optional - Force the use of a specific dataset version. + Force the selection of a specific dataset version. dataset_part : str, optional - Force the use of a specific dataset part. + Force the selection of a specific dataset part. username : str, optional - The username for authentication. See also :func:`~copernicusmarine.login`. + The username for authentication. password : str, optional - The password for authentication. See also :func:`~copernicusmarine.login`. + The password for authentication. output_directory : Union[pathlib.Path, str], optional - The directory where downloaded files will be saved. + The destination folder for the downloaded files. Default is the current directory. credentials_file : Union[pathlib.Path, str], optional - Path to a file containing authentication credentials. + Path to a credentials file if not in its default directory. Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. force_download : bool, optional - Skip confirmation before download. + Flag to skip confirmation before download. overwrite_output_data : bool, optional - If True, overwrite existing output files. + If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Path to a file containing request parameters. For more information, please refer to the README. + Option to pass a file containing the arguments. The file MUST follow the structure of dataclass ‘GetRequest’. For more information please refer to the documentation or use option ``--create-template`` for an example template. no_directories : bool, optional If True, downloaded files will not be organized into directories. show_outputnames : bool, optional - If True, display the names of the downloaded files. + Option to display the names of the output files before download. filter : str, optional - Apply a filter to the downloaded data. + A pattern that must match the absolute paths of the files to download. regex : str, optional - Apply a regular expression filter to the downloaded data. + The regular expression that must match the absolute paths of the files to download. file_list : Union[pathlib.Path, str], optional - Path to a .txt file containing a list of file paths, line by line, that will be downloaded directly. - These files must be from the specified dataset using the --dataset-id. If no files can be found, - the Toolbox will list all files on the remote server and attempt to find a match. + Path to a .txt file containing a list of file paths, line by line, that will be downloaded directly. These files must be from the specified dataset using the –dataset-id. If no files can be found, the Toolbox will list all files on the remote server and attempt to find a match. create_file_list : str, optional - Option to only create a file containing the names of the targeted files instead of downloading them. - It writes the file in the directory specified with the --output-directory option (default to current directory). - If specified, no other action will be performed. + Option to only create a file containing the names of the targeted files instead of downloading them. It writes the file in the directory specified with the –output-directory option (default to current directory). The file name specified should end with ‘.txt’ or ‘.csv’. If specified, no other action will be performed. index_parts : bool, optional - If True, download index files. Only for INSITU datasets. Temporary option. + Option to get the index files of an INSITU dataset. Temporary option. sync : bool, optional - If True, synchronize the local directory with the remote directory. + Option to synchronize the local directory with the remote directory. See the documentation for more details. sync_delete : bool, optional - If True, delete local files that are not present on the remote server while applying sync. + Option to delete local files that are not present on the remote server while applying sync. dry_run : bool, optional If True, runs query without downloading data. max_concurrent_requests : int, optional - Maximum number of concurrent requests. Defaults to 15. The get command - uses a thread pool executor to manage concurrent requests. + Maximum number of concurrent requests. Default 15. The command uses a thread pool executor to manage concurrent requests. If set to 0, no parallel executions are used. disable_progress_bar : bool, optional Flag to hide progress bar. @@ -98,6 +95,7 @@ def get( ------- ResponseGet A list of files that were downloaded and some metadata. + """ # noqa output_directory = ( pathlib.Path(output_directory) if output_directory else None diff --git a/copernicusmarine/python_interface/login.py b/copernicusmarine/python_interface/login.py index 54d063ab..dcaa871f 100644 --- a/copernicusmarine/python_interface/login.py +++ b/copernicusmarine/python_interface/login.py @@ -15,28 +15,30 @@ def login( check_credentials_valid: bool = False, ) -> bool: """ - Create a configuration file with your Copernicus Marine credentials. + Create a configuration file with your Copernicus Marine credentials under the ``$HOME/.copernicusmarine`` directory (overwritable with the ``overwrite_configuration_file`` option). + Parameters ---------- username : str, optional - If not set, searches for the environment variable `COPERNICUSMARINE_SERVICE_USERNAME`, - or else asks for user input. + The username for authentication. password : str, optional - If not set, searches for the environment variable `COPERNICUSMARINE_SERVICE_PASSWORD`, - or else asks for user input. + The password for authentication. configuration_file_directory : Union[pathlib.Path, str] Path to the directory where the configuration file is stored. overwrite_configuration_file : bool - Flag to skip confirmation before overwriting the configuration file. - skip_if_user_logged_in : bool - Flag to check if the credentials are valid. - No other action will be performed. - The validity will be check in this order: + Flag to skip confirmation before overwriting configuration file. + check_credentials_valid : bool + Flag to check if the credentials are valid. No other action will be performed. The validity will be check in this order: 1. Check if the credentials are valid with the provided username and password. 2. Check if the credentials are valid in the configuration file. 3. Check if the credentials are valid in the environment variables. - When any is found not valid, will return False immediately., + When any is found not valid, will return False immediately. + + Returns + ------- + bool + True value if the login was successfully completed, False otherwise. """ # noqa return login_function( username=username, diff --git a/copernicusmarine/python_interface/open_dataset.py b/copernicusmarine/python_interface/open_dataset.py index aa6d334a..5439ebf8 100644 --- a/copernicusmarine/python_interface/open_dataset.py +++ b/copernicusmarine/python_interface/open_dataset.py @@ -59,50 +59,51 @@ def open_dataset( credentials_file: Optional[Union[pathlib.Path, str]] = None, ) -> xarray.Dataset: """ - Load an xarray dataset using "lazy-loading" mode from a Copernicus Marine data source using either the ARCO series protocol. + Load an xarray dataset using 'lazy-loading' mode from a Copernicus Marine data source. This means that data is only loaded into memory when a computation is called, optimizing RAM usage by avoiding immediate loading. It supports various parameters for customization, such as specifying geographical bounds, temporal range, depth range, and more. + Parameters ---------- dataset_id : str - The ID of the dataset. `dataset_id` is mandatory. + The datasetID, required. dataset_version : str, optional - Force the use of a specific dataset version. + Force the selection of a specific dataset version. dataset_part : str, optional - Force the use of a specific dataset part. + Force the selection of a specific dataset part. username : str, optional - Username for authentication, if required. + The username for authentication. password : str, optional - Password for authentication, if required. + The password for authentication. variables : List[str], optional - List of variable names to be loaded from the dataset. + List of variable names to extract. minimum_longitude : float, optional - The minimum longitude for subsetting the data. + Minimum longitude for the subset. The value will be transposed to the interval [-180; 360[. maximum_longitude : float, optional - The maximum longitude for subsetting the data. + Maximum longitude for the subset. The value will be transposed to the interval [-180; 360[. minimum_latitude : float, optional - The minimum latitude for subsetting the data. + Minimum latitude for the subset. Requires a float from -90 degrees to +90. maximum_latitude : float, optional - The maximum latitude for subsetting the data. + Maximum latitude for the subset. Requires a float from -90 degrees to +90. minimum_depth : float, optional - The minimum depth for subsetting the data. + Minimum depth for the subset. Requires a positive float (or 0). maximum_depth : float, optional - The maximum depth for subsetting the data. + Maximum depth for the subset. Requires a positive float (or 0). coordinates_selection_method : str, optional - The method in which the coordinates will be retrieved. If 'strict', the retrieved selection will be inside the requested interval. If 'strict', the retrieved selection will be inside the requested interval and an error will raise if there doesn't exist the values. If 'nearest', the returned interval extremes will be the closest to what has been asked for. A warning will be displayed if outside of bounds. If 'outisde', the extremes will be taken to contain all the requested interval. A warning will also be displayed if the subset is outside of the dataset bounds. + The method in which the coordinates will be retrieved. If ‘inside´, the selection retrieved will be inside the requested range. If ‘strict-inside’, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ‘nearest’, the extremes closest to the requested values will be returned. A warning will be displayed if outside of bounds. If ‘outside’, the extremes will be taken to contain all the requested interval. vertical_dimension_output : str, optional - Consolidate the vertical dimension (the z-axis) as requested: 'depth' with descending positive values. - 'elevation' with ascending positive values. Default is 'depth'. - start_datetime : datetime, optional - The start datetime for temporal subsetting. - end_datetime : datetime, optional - The end datetime for temporal subsetting. + Consolidate the vertical dimension (the z-axis) as requested: depth with descending positive values, elevation with ascending positive values. Default is depth. + start_datetime : Union[datetime, str], optional + The start datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + end_datetime : Union[datetime, str], optional + The end datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. service : str, optional - Force the use of a specific service (ARCO geo series or time series). + Force download through one of the available services using the service name among [‘arco-geo-series’, ‘arco-time-series’, ‘omi-arco’, ‘static-arco’] or its short name among [‘geoseries’, ‘timeseries’, ‘omi-arco’, ‘static-arco’]. credentials_file : Union[pathlib.Path, str], optional - Path to a file containing authentication credentials. + Path to a credentials file if not in its default directory. Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. + Returns ------- diff --git a/copernicusmarine/python_interface/read_dataframe.py b/copernicusmarine/python_interface/read_dataframe.py index 70c0d365..796f926e 100644 --- a/copernicusmarine/python_interface/read_dataframe.py +++ b/copernicusmarine/python_interface/read_dataframe.py @@ -55,55 +55,54 @@ def read_dataframe( coordinates_selection_method: CoordinatesSelectionMethod = ( DEFAULT_COORDINATES_SELECTION_METHOD ), - force_service: Optional[str] = None, + service: Optional[str] = None, credentials_file: Optional[Union[pathlib.Path, str]] = None, ) -> pandas.DataFrame: """ Immediately loads a Pandas DataFrame into memory from a specified dataset. - Unlike "lazy-loading," the data is loaded as soon as this function is executed, - which may be preferable when rapid access to the entire dataset is required, - but may require careful memory management. + Unlike 'lazy-loading,' the data is loaded as soon as this function is executed, which may be preferable when rapid access to the entire dataset is required, but may require careful memory management. + Parameters ---------- - dataset_id : str, optional - The identifier of the dataset. + dataset_id : str + The datasetID, required. dataset_version : str, optional - Force a specific dataset version. + Force the selection of a specific dataset version. dataset_part : str, optional - Force a specific dataset part. + Force the selection of a specific dataset part. username : str, optional - Username for authentication. + The username for authentication. password : str, optional - Password for authentication. + The password for authentication. variables : List[str], optional - List of variable names to load. + List of variable names to extract. minimum_longitude : float, optional - Minimum longitude for spatial subset. + Minimum longitude for the subset. The value will be transposed to the interval [-180; 360[. maximum_longitude : float, optional - Maximum longitude for spatial subset. + Maximum longitude for the subset. The value will be transposed to the interval [-180; 360[. minimum_latitude : float, optional - Minimum latitude for spatial subset. + Minimum latitude for the subset. Requires a float from -90 degrees to +90. maximum_latitude : float, optional - Maximum latitude for spatial subset. + Maximum latitude for the subset. Requires a float from -90 degrees to +90. minimum_depth : float, optional - Minimum depth for vertical subset. + Minimum depth for the subset. Requires a positive float (or 0). maximum_depth : float, optional - Maximum depth for vertical subset. + Maximum depth for the subset. Requires a positive float (or 0). vertical_dimension_output : str, optional - Consolidate the vertical dimension (the z-axis) as requested: 'depth' with descending positive values. - 'elevation' with ascending positive values. Default is 'depth'. - start_datetime : datetime, optional - Start datetime for temporal subset. - end_datetime : datetime, optional - End datetime for temporal subset. + Consolidate the vertical dimension (the z-axis) as requested: depth with descending positive values, elevation with ascending positive values. Default is depth. + start_datetime : Union[datetime, str], optional + The start datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + end_datetime : Union[datetime, str], optional + The end datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. coordinates_selection_method : str, optional - The method in which the coordinates will be retrieved.If 'strict', the retrieved selection will be inside the requested interval. If 'strict', the retrieved selection will be inside the requested interval and an error will raise if there doesn't exist the values. If 'nearest', the returned interval extremes will be the closest to what has been asked for. A warning will be displayed if outside of bounds. If 'outisde', the extremes will be taken to contain all the requested interval. A warning will also be displayed if the subset is outside of the dataset bounds. - force_service : str, optional - Force a specific service for data download. + The method in which the coordinates will be retrieved. If ‘inside´, the selection retrieved will be inside the requested range. If ‘strict-inside’, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ‘nearest’, the extremes closest to the requested values will be returned. A warning will be displayed if outside of bounds. If ‘outside’, the extremes will be taken to contain all the requested interval. + service : str, optional + Force download through one of the available services using the service name among [‘arco-geo-series’, ‘arco-time-series’, ‘omi-arco’, ‘static-arco’] or its short name among [‘geoseries’, ‘timeseries’, ‘omi-arco’, ‘static-arco’]. credentials_file : Union[pathlib.Path, str], optional - Path to a credentials file for authentication. + Path to a credentials file if not in its default directory. Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. + Returns ------- @@ -143,7 +142,7 @@ def read_dataframe( maximum_depth=maximum_depth, vertical_dimension_output=vertical_dimension_output, ), - force_service=force_service, + force_service=service, credentials_file=credentials_file, ) dataset = load_data_object_from_load_request( diff --git a/copernicusmarine/python_interface/subset.py b/copernicusmarine/python_interface/subset.py index 21e9f96f..82caccbc 100644 --- a/copernicusmarine/python_interface/subset.py +++ b/copernicusmarine/python_interface/subset.py @@ -61,71 +61,72 @@ def subset( """ Extracts a subset of data from a specified dataset using given parameters. + The ``dataset_id`` is required and can be found via the ``describe`` command. + Parameters ---------- dataset_id : str, optional - The unique identifier of the dataset. + The datasetID, required either as an argument or in the request_file option. dataset_version : str, optional - Force the use of a specific dataset version. + Force the selection of a specific dataset version. dataset_part : str, optional - Force the use of a specific dataset part. + Force the selection of a specific dataset part. username : str, optional - The username for authentication. See also :func:`~copernicusmarine.login`. + The username for authentication. See also :func:`~copernicusmarine.login` password : str, optional - The password for authentication. See also :func:`~copernicusmarine.login`. + The password for authentication. See also :func:`~copernicusmarine.login` output_directory : Union[pathlib.Path, str], optional - The directory where downloaded files will be saved. + The destination folder for the downloaded files. Default is the current directory. credentials_file : Union[pathlib.Path, str], optional - Path to a file containing authentication credentials. + Path to a credentials file if not in its default directory. Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. force_download : bool, optional - Skip confirmation before download. + Flag to skip confirmation before download. overwrite_output_data : bool, optional - If True, overwrite existing output files. + If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Path to a file containing request parameters. For more information, please refer to the README. + Option to pass a file containing the arguments. The file MUST follow the structure of dataclass ‘SubsetRequest’. For more information please refer to the documentation or use option ``--create-template`` for an example template. service : str, optional - Force the use of a specific service. + Force download through one of the available services using the service name among [‘arco-geo-series’, ‘arco-time-series’, ‘omi-arco’, ‘static-arco’] or its short name among [‘geoseries’, ‘timeseries’, ‘omi-arco’, ‘static-arco’]. variables : List[str], optional List of variable names to extract. minimum_longitude : float, optional - Minimum longitude value for spatial subset. + Minimum longitude for the subset. The value will be transposed to the interval [-180; 360[. maximum_longitude : float, optional - Maximum longitude value for spatial subset. + Maximum longitude for the subset. The value will be transposed to the interval [-180; 360[. minimum_latitude : float, optional - Minimum latitude value for spatial subset. + Minimum latitude for the subset. Requires a float from -90 degrees to +90. maximum_latitude : float, optional - Maximum latitude value for spatial subset. + Maximum latitude for the subset. Requires a float from -90 degrees to +90. minimum_depth : float, optional - Minimum depth value for vertical subset. + Minimum depth for the subset. Requires a positive float (or 0). maximum_depth : float, optional - Maximum depth value for vertical subset. + Maximum depth for the subset. Requires a positive float (or 0). vertical_dimension_output : str, optional - Consolidate the vertical dimension (the z-axis) as requested: 'depth' with descending positive values. - 'elevation' with ascending positive values. Default is 'depth'. - start_datetime : datetime, optional - Start datetime for temporal subset. - end_datetime : datetime, optional - End datetime for temporal subset. + Consolidate the vertical dimension (the z-axis) as requested: depth with descending positive values, elevation with ascending positive values. Default is depth. + start_datetime : Union[datetime, str], optional + The start datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + end_datetime : Union[datetime, str], optional + The end datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. coordinates_selection_method : str, optional - The method in which the coordinates will be retrieved. If 'strict', the retrieved selection will be inside the requested interval. If 'strict', the retrieved selection will be inside the requested interval and an error will raise if there doesn't exist the values. If 'nearest', the returned interval extremes will be the closest to what has been asked for. A warning will be displayed if outside of bounds. If 'outisde', the extremes will be taken to contain all the requested interval. A warning will also be displayed if the subset is outside of the dataset bounds. + The method in which the coordinates will be retrieved. If ‘inside´, the selection retrieved will be inside the requested range. If ‘strict-inside’, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ‘nearest’, the extremes closest to the requested values will be returned. A warning will be displayed if outside of bounds. If ‘outside’, the extremes will be taken to contain all the requested interval. output_filename : str, optional - Output filename for the subsetted data. + Concatenate the downloaded data in the given file name (under the output directory). file_format : str, optional - Extension format for the filename. + Format of the downloaded dataset. Default to NetCDF (.nc). motu_api_request : str, optional - MOTU API request string. + Option to pass a complete MOTU API request as a string. Caution, user has to replace double quotes “ with single quotes ‘ in the request. dry_run : bool, optional If True, runs query without downloading data. netcdf_compression_level : int, optional - Specify a compression level to apply on the NetCDF output file. A value of 0 means no compression, and 9 is the - highest level of compression available. + Specify a compression level to apply on the NetCDF output file. A value of 0 means no compression, and 9 is the highest level of compression available. The default level is 1. netcdf3_compatible : bool, optional - Enable downloading the dataset in a netCDF 3 compatible format. + Enable downloading the dataset in a netCDF3 compatible format. Returns ------- ResponseSubset A description of the downloaded data and its destination. + """ # noqa request_file = pathlib.Path(request_file) if request_file else None output_directory = ( diff --git a/doc/command-line-interface.rst b/doc/command-line-interface.rst index 18968277..4654ff11 100644 --- a/doc/command-line-interface.rst +++ b/doc/command-line-interface.rst @@ -8,7 +8,7 @@ Command line interface :nested: short :commands: login, get, subset, describe - +.. _cli-describe: .. click:: copernicusmarine.command_line_interface.group_describe:describe :prog: describe :nested: short diff --git a/doc/usage/subset-usage.rst b/doc/usage/subset-usage.rst index 6b4fb27a..02cda682 100644 --- a/doc/usage/subset-usage.rst +++ b/doc/usage/subset-usage.rst @@ -71,26 +71,34 @@ About ``--netcdf3-compatible`` option The ``--netcdf3-compatible`` option enables compatibility with the netCDF3 format. This uses the ``format="NETCDF3_CLASSIC"`` setting in the xarray `to_netcdf` method. (cf. `xarray documentation `_.) -About ``--bounding-box-method`` option -"""""""""""""""""""""""""""""""""""""""" +About ``--coordinates-selection-method`` option +"""""""""""""""""""""""""""""""""""""""""""""""""" -The ``--bounding-box-method`` option lets you specify how the requested interval selects data points: +The ``--coordinates-selection-method`` option lets you specify how the requested interval selects data points: - **inside** (default): Returns points strictly within the requested area. +- **strict-inside**: Returns points strictly within the requested area. Fails with an error if the requested area is outside the dataset. - **nearest**: Returns points closest to the requested interval. - **outside**: Returns all points covering the requested area. This applies to all dimensions: longitude, latitude, time, and depth. **Example of longitude requests:** +Imagine a dataset with longitudes from 5.0 to 36.5, with steps of 0.5. + +- ``--coordinates-selection-method``= **inside**, with requested interval = [0.4, 35.9]: + - Returns longitudes within the range: [0.5, 35.5] + +- ``--coordinates-selection-method``= **strict-inside**, with requested interval = [0.4, 35.9]: + - Returns longitudes within the range: [0.5, 35.5] -- ``--bounding-box-method inside``: - - Returns longitudes within the range: [0.08334, 0.917] +- ``--coordinates-selection-method``= **strict-inside**, with requested interval = [0.0, 40]: + - Returns an error -- ``--bounding-box-method nearest``: - - Returns longitudes: [0.0, 2.917] +- ``--coordinates-selection-method``= **outside**, with requested interval = [0.4, 35.9]: + - Returns longitudes within the range: [0.0, 36.0] -- ``--bounding-box-method outside``: - - Outputs longitudes from: [0.0, 3.0] +- ``--coordinates-selection-method``= **nearest**, with requested interval = [0.4, 35.9]: + - Returns longitudes within the range: [0.5, 36.0] If you request a single point, the nearest point in that dimension will be returned. diff --git a/tests/__snapshots__/test_help_command_interface.ambr b/tests/__snapshots__/test_help_command_interface.ambr index 92c3fc2a..1f984a0f 100644 --- a/tests/__snapshots__/test_help_command_interface.ambr +++ b/tests/__snapshots__/test_help_command_interface.ambr @@ -20,15 +20,8 @@ list([ "CompletedProcess(args=['copernicusmarine', 'describe', '--help'], returncode=0, stdout=b'Usage: copernicusmarine describe [OPTIONS]", '', - ' Print Copernicus Marine catalogue as JSON.', - '', - ' The default display contains information on the products, and more data can', - ' be displayed using the ``--include-`` flags.', - '', - ' The ``--contains`` option allows the user to specify one or several strings', - ' to filter through the catalogue display. The search is performed recursively', - ' on all attributes of the catalog, and the tokens only need to be contained', - ' in one of the attributes (i.e., not an exact match).', + ' Retrieve metadata information from the Copernicus Marine catalogue based on', + ' specified parameters and options.', '', 'Options:', ' --include-description Include product description in output.', @@ -43,9 +36,8 @@ ' with attributes matching a string token.', ' --max-concurrent-requests INTEGER', ' Maximum number of concurrent requests.', - ' Default 15. The describe command uses a', - ' thread pool executor to manage concurrent', - ' requests.', + ' Default 15. The command uses a thread pool', + ' executor to manage concurrent requests.', ' --disable-progress-bar Flag to hide progress bar.', ' --log-level [DEBUG|INFO|WARN|ERROR|CRITICAL|QUIET]', ' Set the details printed to console by the', @@ -53,8 +45,11 @@ ' -h, --help Show this message and exit.', '', '', - ' Examples:', + ' Examples:', + '', '', + ' copernicusmarine describe --contains METOFFICE-GLO-SST-L4-NRT-OBS-', + ' SST-V2 --include-datasets', '', ' copernicusmarine describe --contains METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2 --include-datasets', '', @@ -70,11 +65,11 @@ '', ' Download originally produced data files.', '', - ' The ``--dataset-id`` is required (can be found via the "describe" command).', - ' The function fetches the files recursively if a folder path is passed as a', - ' URL. When provided a dataset ID, all the files in the corresponding folder', - ' will be downloaded if none of the ``--filter`` or ``--regex`` options is', - ' specified.', + ' The ``--dataset-id`` is required and can be found via the ``describe``', + ' command. See :ref:`describe `. The function fetches the files', + ' recursively if a folder path is passed as a URL. When provided a datasetID,', + ' all the files in the corresponding folder will be downloaded if none of the', + ' ``--filter`` or ``--regex`` options is specified.', '', 'Options:', ' -i, --dataset-id TEXT The datasetID.', @@ -82,21 +77,15 @@ ' version.', ' --dataset-part TEXT Force the selection of a specific dataset', ' part.', - ' --username TEXT If not set, search for environment variable', - ' COPERNICUSMARINE_SERVICE_USERNAME, or else', - ' look for configuration files, or else ask', - ' for user input.', - ' --password TEXT If not set, search for environment variable', - ' COPERNICUSMARINE_SERVICE_PASSWORD, or else', - ' look for configuration files, or else ask', - ' for user input.', - ' -nd, --no-directories Option to not recreate folder hierarchy in', - ' ouput directory. \\x08 NOTE:', - ' This argument is mutually exclusive with', - ' arguments: [sync].', + ' --username TEXT The username for authentication.', + ' --password TEXT The password for authentication.', + ' -nd, --no-directories If True, downloaded files will not be', + ' organized into directories.', + ' \\x08 NOTE: This argument is mutually exclusive', + ' with arguments: [sync].', ' --show-outputnames Option to display the names of the output', ' files before download.', - ' -o, --output-directory PATH The destination directory for the downloaded', + ' -o, --output-directory PATH The destination folder for the downloaded', ' files. Default is the current directory.', ' --credentials-file PATH Path to a credentials file if not in its', ' default directory. Accepts', @@ -108,14 +97,17 @@ ' on destination, then it will be overwritten', ' instead of creating new one with unique', ' index.', - ' --create-template Option to create a file get_template.json in', - ' your current directory containing CLI', - ' arguments. If specified, no other action', - ' will be performed.', - ' --request-file PATH Option to pass a file containing CLI', + ' --create-template Option to create a file', + ' _template.json in your current', + ' directory containing the arguments. If', + ' specified, no other action will be', + ' performed.', + ' --request-file PATH Option to pass a file containing the', ' arguments. The file MUST follow the', - " structure of dataclass \\'GetRequest\\'. For", - ' more information please refer to the README.', + ' structure of dataclass \\xe2\\x80\\x98GetRequest\\xe2\\x80\\x99. For', + ' more information please refer to the', + ' documentation or use option ``--create-', + ' template`` for an example template.', ' --filter, --filter-with-globbing-pattern TEXT', ' A pattern that must match the absolute paths', ' of the files to download.', @@ -126,18 +118,17 @@ ' file paths, line by line, that will be', ' downloaded directly. These files must be', ' from the specified dataset using the', - ' --dataset-id. If no files can be found, the', + ' \\xe2\\x80\\x93dataset-id. If no files can be found, the', ' Toolbox will list all files on the remote', ' server and attempt to find a match.', ' --create-file-list TEXT Option to only create a file containing the', ' names of the targeted files instead of', ' downloading them. It writes the file in the', - ' directory specified with the --output-', + ' directory specified with the \\xe2\\x80\\x93output-', ' directory option (default to current', ' directory). The file name specified should', - " end with \\'.txt\\' or \\'.csv\\' If specified, no", - ' other action will be performed. Please find', - ' more information in the README.', + ' end with \\xe2\\x80\\x98.txt\\xe2\\x80\\x99 or \\xe2\\x80\\x98.csv\\xe2\\x80\\x99. If specified, no', + ' other action will be performed.', ' --sync Option to synchronize the local directory', ' with the remote directory. See the', ' documentation for more details.', @@ -150,11 +141,12 @@ ' directories].', ' --index-parts Option to get the index files of an INSITU', ' dataset. Temporary option.', - ' --dry-run Runs query without downloading data.', + ' --dry-run If True, runs query without downloading', + ' data.', ' --max-concurrent-requests INTEGER', ' Maximum number of concurrent requests.', - ' Default 15. The get command uses a thread', - ' pool executor to manage concurrent requests.', + ' Default 15. The command uses a thread pool', + ' executor to manage concurrent requests.', ' --disable-progress-bar Flag to hide progress bar.', ' --log-level [DEBUG|INFO|WARN|ERROR|CRITICAL|QUIET]', ' Set the details printed to console by the', @@ -174,18 +166,13 @@ list([ "CompletedProcess(args=['copernicusmarine', 'login', '--help'], returncode=0, stdout=b'Usage: copernicusmarine login [OPTIONS]", '', - ' Create a configuration file with your Copernicus Marine credentials.', - '', - ' Create a configuration file under the ``$HOME/.copernicusmarine`` directory', - ' (overwritable with the ``--credentials-file`` option).', + ' Create a configuration file with your Copernicus Marine credentials under', + ' the ``$HOME/.copernicusmarine`` directory (overwritable with the', + ' ``--credentials-file`` option).', '', 'Options:', - ' --username TEXT If not set, search for environment variable', - ' COPERNICUSMARINE_SERVICE_USERNAME, or else', - ' ask for user input.', - ' --password TEXT If not set, search for environment variable', - ' COPERNICUSMARINE_SERVICE_PASSWORD, or else', - ' ask for user input.', + ' --username TEXT The username for authentication.', + ' --password TEXT The password for authentication.', ' --configuration-file-directory PATH', ' Path to the directory where the', ' configuration file is stored.', @@ -196,11 +183,11 @@ ' No other action will be performed. The', ' validity will be check in this order: 1.', ' Check if the credentials are valid with the', - ' provided username and password.2. Check if', + ' provided username and password. 2. Check if', ' the credentials are valid in the', - ' configuration file.3. Check if the', + ' configuration file. 3. Check if the', ' credentials are valid in the environment', - ' variables.When any is found not valid, will', + ' variables. When any is found not valid, will', ' return False immediately.', ' --log-level [DEBUG|INFO|WARN|ERROR|CRITICAL|QUIET]', ' Set the details printed to console by the', @@ -232,14 +219,15 @@ # --- # name: TestHelpCommandLineInterface.test_help_from_subset_is_as_expected list([ - "CompletedProcess(args=['copernicusmarine', 'subset', '--help'], returncode=0, stdout=b'Usage: copernicusmarine subset [OPTIONS]", + 'CompletedProcess(args=[\'copernicusmarine\', \'subset\', \'--help\'], returncode=0, stdout=b"Usage: copernicusmarine subset [OPTIONS]', '', - ' Download subsets of datasets as NetCDF files or Zarr stores.', + ' Extracts a subset of data from a specified dataset using given parameters.', '', - ' The ``--dataset-id`` is required (can be found via the "describe" command).', - ' The argument values passed individually through the CLI take precedence over', - ' the values from the ``--motu-api-request`` option, which takes precedence', - ' over the ones from the ``--request-file`` option.', + ' The ``--dataset-id`` is required and can be found via the ``describe``', + ' command. See :ref:`describe `. The argument values passed', + ' individually through the CLI take precedence over the values from the', + ' ``--motu-api-request`` option, which takes precedence over the ones from the', + ' ``--request-file`` option.', '', 'Options:', ' -i, --dataset-id TEXT The datasetID.', @@ -247,14 +235,8 @@ ' version.', ' --dataset-part TEXT Force the selection of a specific dataset', ' part.', - ' --username TEXT If not set, search for environment variable', - ' COPERNICUSMARINE_SERVICE_USERNAME, or else', - ' look for configuration files, or else ask', - ' for user input.', - ' --password TEXT If not set, search for environment variable', - ' COPERNICUSMARINE_SERVICE_PASSWORD, or else', - ' look for configuration files, or else ask', - ' for user input.', + ' --username TEXT The username for authentication.', + ' --password TEXT The password for authentication.', ' -v, --variable TEXT Specify dataset variable. Can be used', ' multiple times.', ' -x, --minimum-longitude FLOAT Minimum longitude for the subset. The value', @@ -263,49 +245,44 @@ ' will be reduced to the interval [-180; 360[.', ' -y, --minimum-latitude FLOAT RANGE', ' Minimum latitude for the subset. Requires a', - ' float within this range: [-90<=x<=90]', + ' float from -90 degrees to +90. [-90<=x<=90]', ' -Y, --maximum-latitude FLOAT RANGE', ' Maximum latitude for the subset. Requires a', - ' float within this range: [-90<=x<=90]', + ' float from -90 degrees to +90. [-90<=x<=90]', ' -z, --minimum-depth FLOAT RANGE', ' Minimum depth for the subset. Requires a', - ' float within this range: [x>=0]', + ' positive float (or 0). [x>=0]', ' -Z, --maximum-depth FLOAT RANGE', ' Maximum depth for the subset. Requires a', - ' float within this range: [x>=0]', + ' positive float (or 0). [x>=0]', ' -V, --vertical-dimension-output [depth|elevation]', ' Consolidate the vertical dimension (the', - ' z-axis) as requested: `depth` with', - ' descending positive values. `elevation` with', - ' ascending positive values. Default is', - ' `depth`.', + ' z-axis) as requested: depth with descending', + ' positive values, elevation with ascending', + ' positive values. Default is depth.', ' -t, --start-datetime TEXT The start datetime of the temporal subset.', - ' Caution: encapsulate date with " " to ensure', - ' valid expression for format "%Y-%m-%d', - ' %H:%M:%S". Supports common format parsed by', - ' pendulum. See', - ' https://pendulum.eustace.io/docs/#parsing', + ' Caution: encapsulate date with \\xe2\\x80\\x9c \\xe2\\x80\\x9c to ensure', + ' valid expression for format \\xe2\\x80\\x9c%Y-%m-%d', + ' %H:%M:%S\\xe2\\x80\\x9d. Supports common format parsed by', + ' pendulum.', ' -T, --end-datetime TEXT The end datetime of the temporal subset.', - ' Caution: encapsulate date with " " to ensure', - ' valid expression for format "%Y-%m-%d', - ' %H:%M:%S". Supports common format parsed by', - ' pendulum. See', - ' https://pendulum.eustace.io/docs/#parsing', + ' Caution: encapsulate date with \\xe2\\x80\\x9c \\xe2\\x80\\x9c to ensure', + ' valid expression for format \\xe2\\x80\\x9c%Y-%m-%d', + ' %H:%M:%S\\xe2\\x80\\x9d. Supports common format parsed by', + ' pendulum.', ' --coordinates-selection-method [inside|strict-inside|nearest|outside]', ' The method in which the coordinates will be', - " retrieved. If \\'inside\\', the retrieved", - ' selection will be inside the requested', - " interval. If \\'strict-inside\\', the retrieved", - ' selection will be inside the requested', - ' interval and an error will raise if there', - " doesn\\'t exist the values. If \\'nearest\\', the", - ' returned interval extremes will be the', - ' closest to what has been asked for. A', - ' warning will be displayed if outside of', - " bounds. If \\'outisde\\', the extremes will be", - ' taken to contain all the requested interval.', - ' A warning will also be displayed if the', - ' subset is outside of the dataset bounds.', + ' retrieved. If \\xe2\\x80\\x98inside\\xc2\\xb4, the selection', + ' retrieved will be inside the requested', + ' range. If \\xe2\\x80\\x98strict-inside\\xe2\\x80\\x99, the selection', + ' retrieved will be inside the requested', + ' range, and an error will be raised if the', + " values don't exist. If \\xe2\\x80\\x98nearest\\xe2\\x80\\x99, the", + ' extremes closest to the requested values', + ' will be returned. A warning will be', + ' displayed if outside of bounds. If', + ' \\xe2\\x80\\x98outside\\xe2\\x80\\x99, the extremes will be taken to', + ' contain all the requested interval.', ' -o, --output-directory PATH The destination folder for the downloaded', ' files. Default is the current directory.', ' --credentials-file PATH Path to a credentials file if not in its', @@ -324,23 +301,27 @@ ' index.', ' -s, --service TEXT Force download through one of the available', ' services using the service name among', - " [\\'arco-geo-series\\', \\'arco-time-series\\',", - " \\'omi-arco\\', \\'static-arco\\'] or its short name", - " among [\\'geoseries\\', \\'timeseries\\', \\'omi-", - " arco\\', \\'static-arco\\'].", - ' --create-template Option to create a file subset_template.json', - ' in your current directory containing CLI', - ' arguments. If specified, no other action', - ' will be performed.', - ' --request-file PATH Option to pass a file containing CLI', + ' [\\xe2\\x80\\x98arco-geo-series\\xe2\\x80\\x99, \\xe2\\x80\\x98arco-time-series\\xe2\\x80\\x99,', + ' \\xe2\\x80\\x98omi-arco\\xe2\\x80\\x99, \\xe2\\x80\\x98static-arco\\xe2\\x80\\x99] or its short name', + ' among [\\xe2\\x80\\x98geoseries\\xe2\\x80\\x99, \\xe2\\x80\\x98timeseries\\xe2\\x80\\x99, \\xe2\\x80\\x98omi-', + ' arco\\xe2\\x80\\x99, \\xe2\\x80\\x98static-arco\\xe2\\x80\\x99].', + ' --create-template Option to create a file', + ' _template.json in your current', + ' directory containing the arguments. If', + ' specified, no other action will be', + ' performed.', + ' --request-file PATH Option to pass a file containing the', ' arguments. The file MUST follow the', - " structure of dataclass \\'SubsetRequest\\'. For", - ' more information please refer to the README.', + ' structure of dataclass \\xe2\\x80\\x98SubsetRequest\\xe2\\x80\\x99. For', + ' more information please refer to the', + ' documentation or use option ``--create-', + ' template`` for an example template.', ' --motu-api-request TEXT Option to pass a complete MOTU API request', ' as a string. Caution, user has to replace', - ' double quotes " with single quotes \\\' in the', + ' double quotes \\xe2\\x80\\x9c with single quotes \\xe2\\x80\\x98 in the', ' request.', - ' --dry-run Runs query without downloading data.', + ' --dry-run If True, runs query without downloading', + ' data.', ' --disable-progress-bar Flag to hide progress bar.', ' --log-level [DEBUG|INFO|WARN|ERROR|CRITICAL|QUIET]', ' Set the details printed to console by the', @@ -349,8 +330,9 @@ ' Specify a compression level to apply on the', ' NetCDF output file. A value of 0 means no', ' compression, and 9 is the highest level of', - ' compression available [0<=x<=9]', - ' --netcdf3-compatible Enable downloading the dataset in a netCDF 3', + ' compression available. The default level is', + ' 1. [0<=x<=9]', + ' --netcdf3-compatible Enable downloading the dataset in a netCDF3', ' compatible format.', ' -h, --help Show this message and exit.', '', diff --git a/tests/test_documentation.py b/tests/test_documentation.py new file mode 100644 index 00000000..f554b682 --- /dev/null +++ b/tests/test_documentation.py @@ -0,0 +1,121 @@ +from numpydoc.docscrape import FunctionDoc + +import copernicusmarine +from copernicusmarine.core_functions import documentation_utils + +LIST_OF_EXCEPTIONS = ["username", "password"] + + +class TestDocumentation: + def test_subset(self): + text_subset = FunctionDoc(copernicusmarine.subset) + + for i in range(len(text_subset["Parameters"])): + name_of_variable = text_subset["Parameters"][i].name + if name_of_variable == "variables": + assert ( + text_subset["Parameters"][i].desc + == "List of variable names to extract." + ) + continue + if name_of_variable == "netcdf_compression_level": + assert text_subset["Parameters"][ + i + ].desc + " If used as a flag, the assigned value will be 1." == [ + [ + documentation_utils.SUBSET[ + name_of_variable.upper() + "_HELP" + ] + ] + ] + continue + if name_of_variable in LIST_OF_EXCEPTIONS: + continue + assert text_subset["Parameters"][i].desc == [ + documentation_utils.SUBSET[name_of_variable.upper() + "_HELP"] + ] + + def test_get(self): + text_get = FunctionDoc(copernicusmarine.get) + + for i in range(len(text_get["Parameters"])): + name_of_variable = text_get["Parameters"][i].name + if name_of_variable in LIST_OF_EXCEPTIONS: + continue + assert text_get["Parameters"][i].desc == [ + documentation_utils.GET[name_of_variable.upper() + "_HELP"] + ] + + def test_login(self): + text_login = FunctionDoc(copernicusmarine.login) + + for i in range(len(text_login["Parameters"])): + name_of_variable = text_login["Parameters"][i].name + if len(text_login["Parameters"][i].desc) > 1: + assert ( + " ".join(text_login["Parameters"][i].desc) + == documentation_utils.LOGIN[ + name_of_variable.upper() + "_HELP" + ] + ) + continue + if name_of_variable in LIST_OF_EXCEPTIONS: + continue + assert text_login["Parameters"][i].desc == [ + documentation_utils.LOGIN[name_of_variable.upper() + "_HELP"] + ] + + def test_describe(self): + text_describe = FunctionDoc(copernicusmarine.describe) + + for i in range(len(text_describe["Parameters"])): + name_of_variable = text_describe["Parameters"][i].name + assert text_describe["Parameters"][i].desc == [ + documentation_utils.DESCRIBE[ + name_of_variable.upper() + "_HELP" + ] + ] + + def test_open_dataset(self): + text_open_dataset = FunctionDoc(copernicusmarine.open_dataset) + + for i in range(len(text_open_dataset["Parameters"])): + name_of_variable = text_open_dataset["Parameters"][i].name + if name_of_variable == "variables": + assert ( + text_open_dataset["Parameters"][i].desc + == "List of variable names to extract." + ) + continue + if name_of_variable == "dataset_id": + assert text_open_dataset["Parameters"][i].desc == [ + "The datasetID, required." + ] + continue + if name_of_variable in LIST_OF_EXCEPTIONS: + continue + assert text_open_dataset["Parameters"][i].desc == [ + documentation_utils.SUBSET[name_of_variable.upper() + "_HELP"] + ] + + def test_read_dataframe(self): + text_read_dataframe = FunctionDoc(copernicusmarine.read_dataframe) + + for i in range(len(text_read_dataframe["Parameters"])): + name_of_variable = text_read_dataframe["Parameters"][i].name + if name_of_variable == "variables": + assert ( + text_read_dataframe["Parameters"][i].desc + == "List of variable names to extract." + ) + continue + if name_of_variable == "dataset_id": + assert text_read_dataframe["Parameters"][i].desc == [ + "The datasetID, required." + ] + continue + if name_of_variable in LIST_OF_EXCEPTIONS: + continue + assert text_read_dataframe["Parameters"][i].desc == [ + documentation_utils.SUBSET[name_of_variable.upper() + "_HELP"] + ] From 764d45b33a10a77383e1d6c1770f793067862f61 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Wed, 16 Oct 2024 17:30:32 +0200 Subject: [PATCH 02/14] running precommit --- copernicusmarine/command_line_interface/group_login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copernicusmarine/command_line_interface/group_login.py b/copernicusmarine/command_line_interface/group_login.py index 9615199e..6a25d75b 100644 --- a/copernicusmarine/command_line_interface/group_login.py +++ b/copernicusmarine/command_line_interface/group_login.py @@ -7,10 +7,10 @@ from copernicusmarine.command_line_interface.exception_handler import ( log_exception_and_exit, ) +from copernicusmarine.core_functions import documentation_utils from copernicusmarine.core_functions.click_custom_class import ( CustomClickOptionsCommand, ) -from copernicusmarine.core_functions import documentation_utils from copernicusmarine.core_functions.credentials_utils import ( DEFAULT_CLIENT_BASE_DIRECTORY, ) From 6d2d3a67bd7a01a089d5fc62467e5877892f129a Mon Sep 17 00:00:00 2001 From: uriii3 Date: Thu, 17 Oct 2024 15:27:33 +0200 Subject: [PATCH 03/14] results after another iteration of checks --- .../command_line_interface/group_describe.py | 6 +- .../command_line_interface/group_get.py | 7 +- .../command_line_interface/group_login.py | 6 +- .../command_line_interface/group_subset.py | 12 ++-- .../core_functions/documentation_utils.py | 64 +++++++++---------- copernicusmarine/python_interface/get.py | 8 +-- copernicusmarine/python_interface/subset.py | 4 +- tests/test_documentation.py | 30 +++++++++ 8 files changed, 83 insertions(+), 54 deletions(-) diff --git a/copernicusmarine/command_line_interface/group_describe.py b/copernicusmarine/command_line_interface/group_describe.py index b5138f91..6c18d746 100644 --- a/copernicusmarine/command_line_interface/group_describe.py +++ b/copernicusmarine/command_line_interface/group_describe.py @@ -25,7 +25,8 @@ def cli_describe() -> None: "describe", cls=CustomClickOptionsCommand, short_help="Print Copernicus Marine catalogue as JSON.", - help=documentation_utils.DESCRIBE["DESCRIBE_DESCRIPTION_HELP"], # noqa + help=documentation_utils.DESCRIBE["DESCRIBE_DESCRIPTION_HELP"] + + documentation_utils.DESCRIBE["DESCRIBE_RESPONSE_HELP"], # noqa epilog=""" Examples: @@ -36,8 +37,7 @@ def cli_describe() -> None: .. code-block:: bash copernicusmarine describe -c METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2 \n - """ # noqa - + documentation_utils.DESCRIBE["DESCRIBE_RESPONSE_HELP"], # noqa + """, # noqa ) @click.option( "--include-description", diff --git a/copernicusmarine/command_line_interface/group_get.py b/copernicusmarine/command_line_interface/group_get.py index c95bc239..125489ec 100644 --- a/copernicusmarine/command_line_interface/group_get.py +++ b/copernicusmarine/command_line_interface/group_get.py @@ -38,17 +38,14 @@ def cli_get() -> None: short_help="Download originally produced data files.", help=documentation_utils.GET["GET_DESCRIPTION_HELP"] + " See :ref:`describe `." - " When provided a datasetID, all the files " - "in the corresponding folder will be downloaded. Filters can be applied " - "with ``--filter`` or ``--regex`` options.", # noqa + + documentation_utils.GET["GET_RESPONSE_HELP"], epilog=""" Example to download all the files from a given dataset: .. code-block:: bash copernicusmarine get -i cmems_mod_nws_bgc-pft_myint_7km-3D-diato_P1M-m \n - """ - + documentation_utils.GET["GET_RESPONSE_HELP"], # noqa + """, # noqa ) @click.option( "--dataset-id", diff --git a/copernicusmarine/command_line_interface/group_login.py b/copernicusmarine/command_line_interface/group_login.py index 6a25d75b..3a50a96a 100644 --- a/copernicusmarine/command_line_interface/group_login.py +++ b/copernicusmarine/command_line_interface/group_login.py @@ -28,7 +28,8 @@ def cli_login() -> None: "login", cls=CustomClickOptionsCommand, short_help="Create a configuration file with your Copernicus Marine credentials.", - help=documentation_utils.LOGIN["LOGIN_DESCRIPTION_HELP"], # noqa + help=documentation_utils.LOGIN["LOGIN_DESCRIPTION_HELP"] + + documentation_utils.LOGIN["LOGIN_RESPONSE_HELP"], # noqa epilog=""" Examples: @@ -51,8 +52,7 @@ def cli_login() -> None: copernicusmarine login > Username: [USER-INPUT] > Password: [USER-INPUT] - """ # noqa - + documentation_utils.LOGIN["LOGIN_RESPONSE_HELP"], + """, # noqa ) @click.option( "--username", diff --git a/copernicusmarine/command_line_interface/group_subset.py b/copernicusmarine/command_line_interface/group_subset.py index 05aff925..8b663ee9 100644 --- a/copernicusmarine/command_line_interface/group_subset.py +++ b/copernicusmarine/command_line_interface/group_subset.py @@ -48,6 +48,7 @@ def cli_subset() -> None: cls=CustomClickOptionsCommand, short_help="Download subsets of datasets as NetCDF files or Zarr stores.", help=documentation_utils.SUBSET["SUBSET_DESCRIPTION_HELP"] + + documentation_utils.SUBSET["SUBSET_RESPONSE_HELP"] + "See :ref:`describe `.", epilog=""" Examples: @@ -61,8 +62,7 @@ def cli_subset() -> None: .. code-block:: bash copernicusmarine subset -i cmems_mod_ibi_phy_my_0.083deg-3D_P1D-m -v thetao -v so -t 2021-01-01 -T 2021-01-03 -x 0.0 -X 0.1 -y 28.0 -Y 28.1 -z 1 -Z 2 \n - """ # noqa - + documentation_utils.SUBSET["SUBSET_RESPONSE_HELP"], + """, # noqa ) @click.option( "--dataset-id", @@ -140,13 +140,17 @@ def cli_subset() -> None: "--start-datetime", "-t", type=str, - help=documentation_utils.SUBSET["START_DATETIME_HELP"], + help=documentation_utils.SUBSET["START_DATETIME_HELP"] + + "Caution: encapsulate date with “ “ to ensure valid " + "expression for format “%Y-%m-%d %H:%M:%S”.", ) @click.option( "--end-datetime", "-T", type=str, - help=documentation_utils.SUBSET["END_DATETIME_HELP"], + help=documentation_utils.SUBSET["END_DATETIME_HELP"] + + "Caution: encapsulate date with “ “ to ensure valid " + "expression for format “%Y-%m-%d %H:%M:%S”.", ) @click.option( "--coordinates-selection-method", diff --git a/copernicusmarine/core_functions/documentation_utils.py b/copernicusmarine/core_functions/documentation_utils.py index 80b37998..002ab65f 100644 --- a/copernicusmarine/core_functions/documentation_utils.py +++ b/copernicusmarine/core_functions/documentation_utils.py @@ -23,7 +23,7 @@ ), "CREDENTIALS_FILE_HELP": ( "Path to a credentials file if not in its default directory" - " (which would be $HOME/.copernicusmarine). Accepts " + " (``$HOME/.copernicusmarine``). Accepts " ".copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini " "files." ), @@ -71,7 +71,7 @@ " with the provided username and password. " "2. Check if the credentials are valid in the environment variables. " "3. Check if the credentials are valid in the configuration file. " - "When any is found not valid, will return False immediately." + "When any is found (valid or not valid), will return immediately." ), } @@ -81,8 +81,7 @@ " from the Copernicus Marine catalogue." ), "DESCRIBE_RESPONSE_HELP": ( - "dict[str, Any]\n" - "A dictionary containing the retrieved metadata information." + "JSON\n A dictionary containing the retrieved metadata information." ), "MAX_CONCURRENT_REQUESTS_HELP": ( "Maximum number of concurrent requests (>=1). Default 15. The command uses " @@ -108,12 +107,11 @@ SUBSET: dict[str, str] = { "SUBSET_DESCRIPTION_HELP": ( "Extracts a subset of data from a specified dataset using given parameters." - " \n \n The ``--dataset-id`` is required and can be found via the ``describe`` " + " \n \n The datasetID is required and can be found via the ``describe`` " "command. " # has some hardcoding in CLI and python API ), "SUBSET_RESPONSE_HELP": ( - "ResponseSubset \n" - "A description of the downloaded data and its destination." + "JSON \n A description of the downloaded data and its destination." ), "SERVICE_HELP": ( f"Force download through one of the available services using the service name " @@ -149,34 +147,34 @@ "Default is depth." ), "START_DATETIME_HELP": ( - "The start datetime of the temporal subset. Caution: encapsulate date with " - "“ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports " - "common format parsed by pendulum." - ), + "The start datetime of the temporal subset. Supports common " + "format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing)." + ), # hardocded in cli: Caution: encapsulate date with “ “ to ensure valid + # expression for format “%Y-%m-%d %H:%M:%S”. "END_DATETIME_HELP": ( - "The end datetime of the temporal subset. Caution: encapsulate date with “ “ " - "to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common " - "format parsed by pendulum." - ), + "The end datetime of the temporal subset. Supports common " + "format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing)." + ), # hardocded in cli: Caution: encapsulate date with “ “ + # to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. "COORDINATES_SELECTION_METHOD_HELP": ( - "If ‘inside´, the " - "selection retrieved will be inside the requested range. If ‘strict-" - "inside’, the selection retrieved will be inside the requested range, " + "If ``inside``, the " + "selection retrieved will be inside the requested range. If ``strict-" + "inside``, the selection retrieved will be inside the requested range, " "and an error will be raised if the values don't exist. " - "If ‘nearest’, the extremes closest to the requested values will " - "be returned. If ‘outside’," + "If ``nearest``, the extremes closest to the requested values will " + "be returned. If ``outside``," " the extremes will be taken to contain all the requested interval." - " The methods ‘inside´, ‘nearest’ and ‘outside’ will display" + " The methods ``inside``, ``nearest`` and ``outside`` will display" " a warning if the request is out of bounds." ), "OUTPUT_FILENAME_HELP": ( - "Concatenate the downloaded data in the given file name (under the output " + "Save the downloaded data with the given file name (under the output " "directory)." ), "FILE_FORMAT_HELP": "Format of the downloaded dataset. Default to NetCDF (.nc).", "MOTU_API_REQUEST_HELP": ( "Option to pass a complete MOTU API request as a string. Caution, user has to " - "replace double quotes “ with single quotes ‘ in the request." + "replace double quotes “ with single quotes ' in the request." ), "NETCDF_COMPRESSION_LEVEL_HELP": ( "Specify a compression level to apply on the NetCDF output file. A value of 0 " @@ -190,7 +188,7 @@ GET: dict[str, str] = { "GET_DESCRIPTION_HELP": ( "Download originally produced data files.\n \n " - " The ``--dataset-id`` is required (either as an " + " The datasetID is required (either as an " "argument or in a request file) and can be found via the ``describe``" " command." ), # has some hardcoding in CLI @@ -200,8 +198,7 @@ " executions are used." ), "GET_RESPONSE_HELP": ( - "ResponseGet \n" - "A list of files that were downloaded and some metadata." + "JSON \n A list of files that were downloaded and some metadata." ), "SHOW_OUTPUTNAMES_HELP": ( "Option to display the names of the output files before download." @@ -214,16 +211,17 @@ "download." ), "FILE_LIST_HELP": ( - "Path to a .txt file containing a list of file paths, line by line, that will " - "be downloaded directly. These files must be from the specified dataset using " - "the --dataset-id. If no files can be found, the Toolbox will list all files " - "on the remote server and attempt to find a match." + "Path to a '.txt' file containing a " + "list of file paths, line by line, that will " + "be downloaded directly. These files must be from the same dataset as the one s" + "pecified dataset with the datasetID option. If no files can be found, the " + "Toolbox will list all files on the remote server and attempt to find a match." ), "CREATE_FILE_LIST_HELP": ( "Option to only create a file containing the names of the targeted files " - "instead of downloading them. It writes the file in the directory specified " - "with the --output-directory option (default to current directory). The file " - "name specified should end with ‘.txt’ or ‘.csv’. If specified, no other " + "instead of downloading them. It writes the file to the specified output" + " directory (default to current directory). The file " + "name specified should end with '.txt' or '.csv'. If specified, no other " "action will be performed." ), "SYNC_HELP": ( diff --git a/copernicusmarine/python_interface/get.py b/copernicusmarine/python_interface/get.py index 14cc9bb0..5e9fd70a 100644 --- a/copernicusmarine/python_interface/get.py +++ b/copernicusmarine/python_interface/get.py @@ -42,7 +42,7 @@ def get( """ Download originally produced data files. - The ``dataset_id`` is required and can be found via the ``describe`` command. + The `datasetID is required and can be found via the ``describe`` command. Parameters ---------- @@ -65,7 +65,7 @@ def get( overwrite_output_data : bool, optional If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Option to pass a file containing the arguments. The file MUST follow the structure of dataclass ‘GetRequest’. For more information please refer to the documentation or use option ``--create-template`` for an example template. + Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` for an example template. no_directories : bool, optional If True, downloaded files will not be organized into directories. show_outputnames : bool, optional @@ -75,9 +75,9 @@ def get( regex : str, optional The regular expression that must match the absolute paths of the files to download. file_list : Union[pathlib.Path, str], optional - Path to a .txt file containing a list of file paths, line by line, that will be downloaded directly. These files must be from the specified dataset using the –dataset-id. If no files can be found, the Toolbox will list all files on the remote server and attempt to find a match. + Path to a .txt file containing a list of file paths, line by line, that will be downloaded directly. These files must be from the specified dataset using the datasetID. If no files can be found, the Toolbox will list all files on the remote server and attempt to find a match. create_file_list : str, optional - Option to only create a file containing the names of the targeted files instead of downloading them. It writes the file in the directory specified with the –output-directory option (default to current directory). The file name specified should end with ‘.txt’ or ‘.csv’. If specified, no other action will be performed. + Option to only create a file containing the names of the targeted files instead of downloading them. It writes the file to the specified output directory (default to current directory). The file name specified should end with '.txt' or '.csv'. If specified, no other action will be performed. index_parts : bool, optional Option to get the index files of an INSITU dataset. Temporary option. sync : bool, optional diff --git a/copernicusmarine/python_interface/subset.py b/copernicusmarine/python_interface/subset.py index 82caccbc..140c58f6 100644 --- a/copernicusmarine/python_interface/subset.py +++ b/copernicusmarine/python_interface/subset.py @@ -84,7 +84,7 @@ def subset( overwrite_output_data : bool, optional If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Option to pass a file containing the arguments. The file MUST follow the structure of dataclass ‘SubsetRequest’. For more information please refer to the documentation or use option ``--create-template`` for an example template. + Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` for an example template. service : str, optional Force download through one of the available services using the service name among [‘arco-geo-series’, ‘arco-time-series’, ‘omi-arco’, ‘static-arco’] or its short name among [‘geoseries’, ‘timeseries’, ‘omi-arco’, ‘static-arco’]. variables : List[str], optional @@ -108,7 +108,7 @@ def subset( end_datetime : Union[datetime, str], optional The end datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. coordinates_selection_method : str, optional - The method in which the coordinates will be retrieved. If ‘inside´, the selection retrieved will be inside the requested range. If ‘strict-inside’, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ‘nearest’, the extremes closest to the requested values will be returned. A warning will be displayed if outside of bounds. If ‘outside’, the extremes will be taken to contain all the requested interval. + The method in which the coordinates will be retrieved. If ``inside``, the selection retrieved will be inside the requested range. If ``strict-inside``, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ``nearest``, the extremes closest to the requested values will be returned. A warning will be displayed if outside of bounds. If ``outside``, the extremes will be taken to contain all the requested interval. output_filename : str, optional Concatenate the downloaded data in the given file name (under the output directory). file_format : str, optional diff --git a/tests/test_documentation.py b/tests/test_documentation.py index f554b682..a28c6fcd 100644 --- a/tests/test_documentation.py +++ b/tests/test_documentation.py @@ -12,6 +12,16 @@ def test_subset(self): for i in range(len(text_subset["Parameters"])): name_of_variable = text_subset["Parameters"][i].name + if name_of_variable in ["start_datetime", "end_datetime"]: + assert ( + text_subset["Parameters"][i].desc + == documentation_utils.SUBSET[ + name_of_variable.upper() + "_HELP" + ] + + "Caution: encapsulate date with “ “ to ensure valid " + "expression for format “%Y-%m-%d %H:%M:%S”." + ) + continue if name_of_variable == "variables": assert ( text_subset["Parameters"][i].desc @@ -81,6 +91,16 @@ def test_open_dataset(self): for i in range(len(text_open_dataset["Parameters"])): name_of_variable = text_open_dataset["Parameters"][i].name + if name_of_variable in ["start_datetime", "end_datetime"]: + assert ( + text_open_dataset["Parameters"][i].desc + == documentation_utils.SUBSET[ + name_of_variable.upper() + "_HELP" + ] + + "Caution: encapsulate date with “ “ to ensure valid " + "expression for format “%Y-%m-%d %H:%M:%S”." + ) + continue if name_of_variable == "variables": assert ( text_open_dataset["Parameters"][i].desc @@ -103,6 +123,16 @@ def test_read_dataframe(self): for i in range(len(text_read_dataframe["Parameters"])): name_of_variable = text_read_dataframe["Parameters"][i].name + if name_of_variable in ["start_datetime", "end_datetime"]: + assert ( + text_read_dataframe["Parameters"][i].desc + == documentation_utils.SUBSET[ + name_of_variable.upper() + "_HELP" + ] + + "Caution: encapsulate date with “ “ to ensure valid " + "expression for format “%Y-%m-%d %H:%M:%S”." + ) + continue if name_of_variable == "variables": assert ( text_read_dataframe["Parameters"][i].desc From 3f69949ce0481f390e898f9b086f9167596ba6d8 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Thu, 17 Oct 2024 16:15:26 +0200 Subject: [PATCH 04/14] Update to enjoy the results --- .../core_functions/documentation_utils.py | 2 +- copernicusmarine/core_functions/get.py | 4 +- copernicusmarine/python_interface/describe.py | 2 +- copernicusmarine/python_interface/get.py | 6 +- copernicusmarine/python_interface/login.py | 6 +- .../python_interface/open_dataset.py | 10 +- .../python_interface/read_dataframe.py | 10 +- copernicusmarine/python_interface/subset.py | 18 +- .../test_help_command_interface.ambr | 177 +++++++++--------- tests/test_documentation.py | 60 +++--- 10 files changed, 143 insertions(+), 152 deletions(-) diff --git a/copernicusmarine/core_functions/documentation_utils.py b/copernicusmarine/core_functions/documentation_utils.py index 002ab65f..95c3df92 100644 --- a/copernicusmarine/core_functions/documentation_utils.py +++ b/copernicusmarine/core_functions/documentation_utils.py @@ -171,7 +171,7 @@ "Save the downloaded data with the given file name (under the output " "directory)." ), - "FILE_FORMAT_HELP": "Format of the downloaded dataset. Default to NetCDF (.nc).", + "FILE_FORMAT_HELP": "Format of the downloaded dataset. Default to NetCDF '.nc'.", "MOTU_API_REQUEST_HELP": ( "Option to pass a complete MOTU API request as a string. Caution, user has to " "replace double quotes “ with single quotes ' in the request." diff --git a/copernicusmarine/core_functions/get.py b/copernicusmarine/core_functions/get.py index 86b86e02..aa1b4405 100644 --- a/copernicusmarine/core_functions/get.py +++ b/copernicusmarine/core_functions/get.py @@ -108,7 +108,7 @@ def get_function( if create_file_list is not None: assert create_file_list.endswith(".txt") or create_file_list.endswith( ".csv" - ), "Download file list must be a .txt or .csv file. " + ), "Download file list must be a '.txt' or '.csv' file. " f"Got '{create_file_list}' instead." if file_list_path: direct_download_files = get_direct_download_files(file_list_path) @@ -211,7 +211,7 @@ def get_direct_download_files( if not os.path.exists(file_list_path): raise FileNotFoundError( f"File {file_list_path} does not exist." - " Please provide a valid path to a .txt file." + " Please provide a valid path to a '.txt' file." ) with open(file_list_path) as f: direct_download_files = [line.strip() for line in f.readlines()] diff --git a/copernicusmarine/python_interface/describe.py b/copernicusmarine/python_interface/describe.py index 44b62e28..0c0b8ba2 100644 --- a/copernicusmarine/python_interface/describe.py +++ b/copernicusmarine/python_interface/describe.py @@ -42,7 +42,7 @@ def describe( contains : list[str], optional Filter catalogue output. Returns products with attributes matching a string token. max_concurrent_requests : int, optional - Maximum number of concurrent requests. Default 15. The command uses a thread pool executor to manage concurrent requests. + Maximum number of concurrent requests (>=1). Default 15. The command uses a thread pool executor to manage concurrent requests. disable_progress_bar : bool, optional Flag to hide progress bar. diff --git a/copernicusmarine/python_interface/get.py b/copernicusmarine/python_interface/get.py index 5e9fd70a..3a58c263 100644 --- a/copernicusmarine/python_interface/get.py +++ b/copernicusmarine/python_interface/get.py @@ -59,7 +59,7 @@ def get( output_directory : Union[pathlib.Path, str], optional The destination folder for the downloaded files. Default is the current directory. credentials_file : Union[pathlib.Path, str], optional - Path to a credentials file if not in its default directory. Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. + Path to a credentials file if not in its default directory (``$HOME/.copernicusmarine``). Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. force_download : bool, optional Flag to skip confirmation before download. overwrite_output_data : bool, optional @@ -75,11 +75,11 @@ def get( regex : str, optional The regular expression that must match the absolute paths of the files to download. file_list : Union[pathlib.Path, str], optional - Path to a .txt file containing a list of file paths, line by line, that will be downloaded directly. These files must be from the specified dataset using the datasetID. If no files can be found, the Toolbox will list all files on the remote server and attempt to find a match. + Path to a '.txt' file containing a list of file paths, line by line, that will be downloaded directly. These files must be from the same dataset as the one specified dataset with the datasetID option. If no files can be found, the Toolbox will list all files on the remote server and attempt to find a match. create_file_list : str, optional Option to only create a file containing the names of the targeted files instead of downloading them. It writes the file to the specified output directory (default to current directory). The file name specified should end with '.txt' or '.csv'. If specified, no other action will be performed. index_parts : bool, optional - Option to get the index files of an INSITU dataset. Temporary option. + Option to get the index files of an INSITU dataset. sync : bool, optional Option to synchronize the local directory with the remote directory. See the documentation for more details. sync_delete : bool, optional diff --git a/copernicusmarine/python_interface/login.py b/copernicusmarine/python_interface/login.py index dcaa871f..81baced9 100644 --- a/copernicusmarine/python_interface/login.py +++ b/copernicusmarine/python_interface/login.py @@ -31,9 +31,9 @@ def login( check_credentials_valid : bool Flag to check if the credentials are valid. No other action will be performed. The validity will be check in this order: 1. Check if the credentials are valid with the provided username and password. - 2. Check if the credentials are valid in the configuration file. - 3. Check if the credentials are valid in the environment variables. - When any is found not valid, will return False immediately. + 2. Check if the credentials are valid in the environment variables. + 3. Check if the credentials are valid in the configuration file. + When any is found (valid or not valid), will return immediately. Returns ------- diff --git a/copernicusmarine/python_interface/open_dataset.py b/copernicusmarine/python_interface/open_dataset.py index 5439ebf8..b15fb669 100644 --- a/copernicusmarine/python_interface/open_dataset.py +++ b/copernicusmarine/python_interface/open_dataset.py @@ -92,17 +92,17 @@ def open_dataset( maximum_depth : float, optional Maximum depth for the subset. Requires a positive float (or 0). coordinates_selection_method : str, optional - The method in which the coordinates will be retrieved. If ‘inside´, the selection retrieved will be inside the requested range. If ‘strict-inside’, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ‘nearest’, the extremes closest to the requested values will be returned. A warning will be displayed if outside of bounds. If ‘outside’, the extremes will be taken to contain all the requested interval. + If ``inside``, the selection retrieved will be inside the requested range. If ``strict-inside``, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ``nearest``, the extremes closest to the requested values will be returned. If ``outside``, the extremes will be taken to contain all the requested interval. The methods ``inside``, ``nearest`` and ``outside`` will display a warning if the request is out of bounds. vertical_dimension_output : str, optional Consolidate the vertical dimension (the z-axis) as requested: depth with descending positive values, elevation with ascending positive values. Default is depth. start_datetime : Union[datetime, str], optional - The start datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + The start datetime of the temporal subset. Supports common format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing). end_datetime : Union[datetime, str], optional - The end datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + The end datetime of the temporal subset. Supports common format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing). service : str, optional - Force download through one of the available services using the service name among [‘arco-geo-series’, ‘arco-time-series’, ‘omi-arco’, ‘static-arco’] or its short name among [‘geoseries’, ‘timeseries’, ‘omi-arco’, ‘static-arco’]. + Force download through one of the available services using the service name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco'] or its short name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco']. credentials_file : Union[pathlib.Path, str], optional - Path to a credentials file if not in its default directory. Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. + Path to a credentials file if not in its default directory (``$HOME/.copernicusmarine``). Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. Returns diff --git a/copernicusmarine/python_interface/read_dataframe.py b/copernicusmarine/python_interface/read_dataframe.py index 796f926e..322618bb 100644 --- a/copernicusmarine/python_interface/read_dataframe.py +++ b/copernicusmarine/python_interface/read_dataframe.py @@ -93,15 +93,15 @@ def read_dataframe( vertical_dimension_output : str, optional Consolidate the vertical dimension (the z-axis) as requested: depth with descending positive values, elevation with ascending positive values. Default is depth. start_datetime : Union[datetime, str], optional - The start datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + The start datetime of the temporal subset. Supports common format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing). end_datetime : Union[datetime, str], optional - The end datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + The end datetime of the temporal subset. Supports common format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing). coordinates_selection_method : str, optional - The method in which the coordinates will be retrieved. If ‘inside´, the selection retrieved will be inside the requested range. If ‘strict-inside’, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ‘nearest’, the extremes closest to the requested values will be returned. A warning will be displayed if outside of bounds. If ‘outside’, the extremes will be taken to contain all the requested interval. + If ``inside``, the selection retrieved will be inside the requested range. If ``strict-inside``, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ``nearest``, the extremes closest to the requested values will be returned. If ``outside``, the extremes will be taken to contain all the requested interval. The methods ``inside``, ``nearest`` and ``outside`` will display a warning if the request is out of bounds. service : str, optional - Force download through one of the available services using the service name among [‘arco-geo-series’, ‘arco-time-series’, ‘omi-arco’, ‘static-arco’] or its short name among [‘geoseries’, ‘timeseries’, ‘omi-arco’, ‘static-arco’]. + Force download through one of the available services using the service name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco'] or its short name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco']. credentials_file : Union[pathlib.Path, str], optional - Path to a credentials file if not in its default directory. Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. + Path to a credentials file if not in its default directory (``$HOME/.copernicusmarine``). Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. Returns diff --git a/copernicusmarine/python_interface/subset.py b/copernicusmarine/python_interface/subset.py index 140c58f6..f040a1b3 100644 --- a/copernicusmarine/python_interface/subset.py +++ b/copernicusmarine/python_interface/subset.py @@ -78,7 +78,7 @@ def subset( output_directory : Union[pathlib.Path, str], optional The destination folder for the downloaded files. Default is the current directory. credentials_file : Union[pathlib.Path, str], optional - Path to a credentials file if not in its default directory. Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. + Path to a credentials file if not in its default directory (``$HOME/.copernicusmarine``). Accepts .copernicusmarine-credentials / .netrc or _netrc / motuclient-python.ini files. force_download : bool, optional Flag to skip confirmation before download. overwrite_output_data : bool, optional @@ -86,7 +86,7 @@ def subset( request_file : Union[pathlib.Path, str], optional Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` for an example template. service : str, optional - Force download through one of the available services using the service name among [‘arco-geo-series’, ‘arco-time-series’, ‘omi-arco’, ‘static-arco’] or its short name among [‘geoseries’, ‘timeseries’, ‘omi-arco’, ‘static-arco’]. + Force download through one of the available services using the service name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco'] or its short name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco']. variables : List[str], optional List of variable names to extract. minimum_longitude : float, optional @@ -104,21 +104,21 @@ def subset( vertical_dimension_output : str, optional Consolidate the vertical dimension (the z-axis) as requested: depth with descending positive values, elevation with ascending positive values. Default is depth. start_datetime : Union[datetime, str], optional - The start datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + The start datetime of the temporal subset. Supports common format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing). end_datetime : Union[datetime, str], optional - The end datetime of the temporal subset. Caution: encapsulate date with “ “ to ensure valid expression for format “%Y-%m-%d %H:%M:%S”. Supports common format parsed by pendulum. + The end datetime of the temporal subset. Supports common format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing). coordinates_selection_method : str, optional - The method in which the coordinates will be retrieved. If ``inside``, the selection retrieved will be inside the requested range. If ``strict-inside``, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ``nearest``, the extremes closest to the requested values will be returned. A warning will be displayed if outside of bounds. If ``outside``, the extremes will be taken to contain all the requested interval. + If ``inside``, the selection retrieved will be inside the requested range. If ``strict-inside``, the selection retrieved will be inside the requested range, and an error will be raised if the values don't exist. If ``nearest``, the extremes closest to the requested values will be returned. If ``outside``, the extremes will be taken to contain all the requested interval. The methods ``inside``, ``nearest`` and ``outside`` will display a warning if the request is out of bounds. output_filename : str, optional - Concatenate the downloaded data in the given file name (under the output directory). + Save the downloaded data with the given file name (under the output directory). file_format : str, optional - Format of the downloaded dataset. Default to NetCDF (.nc). + Format of the downloaded dataset. Default to NetCDF '.nc'. motu_api_request : str, optional - Option to pass a complete MOTU API request as a string. Caution, user has to replace double quotes “ with single quotes ‘ in the request. + Option to pass a complete MOTU API request as a string. Caution, user has to replace double quotes “ with single quotes ' in the request. dry_run : bool, optional If True, runs query without downloading data. netcdf_compression_level : int, optional - Specify a compression level to apply on the NetCDF output file. A value of 0 means no compression, and 9 is the highest level of compression available. The default level is 1. + Specify a compression level to apply on the NetCDF output file. A value of 0 means no compression, and 9 is the highest level of compression available. netcdf3_compatible : bool, optional Enable downloading the dataset in a netCDF3 compatible format. diff --git a/tests/__snapshots__/test_help_command_interface.ambr b/tests/__snapshots__/test_help_command_interface.ambr index 1f984a0f..28678dc1 100644 --- a/tests/__snapshots__/test_help_command_interface.ambr +++ b/tests/__snapshots__/test_help_command_interface.ambr @@ -20,8 +20,8 @@ list([ "CompletedProcess(args=['copernicusmarine', 'describe', '--help'], returncode=0, stdout=b'Usage: copernicusmarine describe [OPTIONS]", '', - ' Retrieve metadata information from the Copernicus Marine catalogue based on', - ' specified parameters and options.', + ' Retrieve and parse the metadata information from the Copernicus Marine', + ' catalogue.JSON A dictionary containing the retrieved metadata information.', '', 'Options:', ' --include-description Include product description in output.', @@ -35,7 +35,7 @@ ' -c, --contains TEXT Filter catalogue output. Returns products', ' with attributes matching a string token.', ' --max-concurrent-requests INTEGER', - ' Maximum number of concurrent requests.', + ' Maximum number of concurrent requests (>=1).', ' Default 15. The command uses a thread pool', ' executor to manage concurrent requests.', ' --disable-progress-bar Flag to hide progress bar.', @@ -45,34 +45,31 @@ ' -h, --help Show this message and exit.', '', '', - ' Examples:', - '', + ' Examples:', '', - ' copernicusmarine describe --contains METOFFICE-GLO-SST-L4-NRT-OBS-', - ' SST-V2 --include-datasets', '', ' copernicusmarine describe --contains METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2 --include-datasets', '', '', - ' copernicusmarine describe -c METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2', + ' copernicusmarine describe -c METOFFICE-GLO-SST-L4-NRT-OBS-SST-V2 ', + '', ' ', "', stderr=b'')", ]) # --- # name: TestHelpCommandLineInterface.test_help_from_get_is_as_expected list([ - "CompletedProcess(args=['copernicusmarine', 'get', '--help'], returncode=0, stdout=b'Usage: copernicusmarine get [OPTIONS]", + 'CompletedProcess(args=[\'copernicusmarine\', \'get\', \'--help\'], returncode=0, stdout=b"Usage: copernicusmarine get [OPTIONS]', '', ' Download originally produced data files.', '', - ' The ``--dataset-id`` is required and can be found via the ``describe``', - ' command. See :ref:`describe `. The function fetches the files', - ' recursively if a folder path is passed as a URL. When provided a datasetID,', - ' all the files in the corresponding folder will be downloaded if none of the', - ' ``--filter`` or ``--regex`` options is specified.', + ' The datasetID is required (either as an argument or in a request file) and', + ' can be found via the ``describe`` command. See :ref:`describe `.JSON A list of files that were downloaded and some metadata.', '', 'Options:', - ' -i, --dataset-id TEXT The datasetID.', + ' -i, --dataset-id TEXT The datasetID, required either as an', + ' argument or in the request_file option.', ' --dataset-version TEXT Force the selection of a specific dataset', ' version.', ' --dataset-part TEXT Force the selection of a specific dataset', @@ -88,7 +85,8 @@ ' -o, --output-directory PATH The destination folder for the downloaded', ' files. Default is the current directory.', ' --credentials-file PATH Path to a credentials file if not in its', - ' default directory. Accepts', + ' default directory', + ' (``$HOME/.copernicusmarine``). Accepts', ' .copernicusmarine-credentials / .netrc or', ' _netrc / motuclient-python.ini files.', ' --force-download Flag to skip confirmation before download.', @@ -103,32 +101,32 @@ ' specified, no other action will be', ' performed.', ' --request-file PATH Option to pass a file containing the', - ' arguments. The file MUST follow the', - ' structure of dataclass \\xe2\\x80\\x98GetRequest\\xe2\\x80\\x99. For', - ' more information please refer to the', - ' documentation or use option ``--create-', - ' template`` for an example template.', + ' arguments. For more information please refer', + ' to the documentation or use option', + ' ``--create-template`` for an example', + ' template.', ' --filter, --filter-with-globbing-pattern TEXT', ' A pattern that must match the absolute paths', ' of the files to download.', ' --regex, --filter-with-regular-expression TEXT', ' The regular expression that must match the', ' absolute paths of the files to download.', - ' --file-list PATH Path to a .txt file containing a list of', + " --file-list PATH Path to a '.txt' file containing a list of", ' file paths, line by line, that will be', ' downloaded directly. These files must be', - ' from the specified dataset using the', - ' \\xe2\\x80\\x93dataset-id. If no files can be found, the', - ' Toolbox will list all files on the remote', - ' server and attempt to find a match.', + ' from the same dataset as the one specified', + ' dataset with the datasetID option. If no', + ' files can be found, the Toolbox will list', + ' all files on the remote server and attempt', + ' to find a match.', ' --create-file-list TEXT Option to only create a file containing the', ' names of the targeted files instead of', - ' downloading them. It writes the file in the', - ' directory specified with the \\xe2\\x80\\x93output-', - ' directory option (default to current', - ' directory). The file name specified should', - ' end with \\xe2\\x80\\x98.txt\\xe2\\x80\\x99 or \\xe2\\x80\\x98.csv\\xe2\\x80\\x99. If specified, no', - ' other action will be performed.', + ' downloading them. It writes the file to the', + ' specified output directory (default to', + ' current directory). The file name specified', + " should end with '.txt' or '.csv'. If", + ' specified, no other action will be', + ' performed.', ' --sync Option to synchronize the local directory', ' with the remote directory. See the', ' documentation for more details.', @@ -140,13 +138,14 @@ ' is mutually exclusive with arguments: [no-', ' directories].', ' --index-parts Option to get the index files of an INSITU', - ' dataset. Temporary option.', + ' dataset.', ' --dry-run If True, runs query without downloading', ' data.', ' --max-concurrent-requests INTEGER', ' Maximum number of concurrent requests.', ' Default 15. The command uses a thread pool', - ' executor to manage concurrent requests.', + ' executor to manage concurrent requests. If', + ' set to 0, no parallel executions are used.', ' --disable-progress-bar Flag to hide progress bar.', ' --log-level [DEBUG|INFO|WARN|ERROR|CRITICAL|QUIET]', ' Set the details printed to console by the', @@ -157,9 +156,10 @@ ' Example to download all the files from a given dataset:', '', '', - ' copernicusmarine get -i cmems_mod_nws_bgc-pft_myint_7km-3D-diato_P1M-m', + ' copernicusmarine get -i cmems_mod_nws_bgc-pft_myint_7km-3D-diato_P1M-m ', + '', ' ', - "', stderr=b'')", + '", stderr=b\'\')', ]) # --- # name: TestHelpCommandLineInterface.test_help_from_login_is_as_expected @@ -168,7 +168,8 @@ '', ' Create a configuration file with your Copernicus Marine credentials under', ' the ``$HOME/.copernicusmarine`` directory (overwritable with the', - ' ``--credentials-file`` option).', + ' ``--credentials-file`` option).Error code 0 error code if the login was', + ' successfully completed, 1 otherwise.', '', 'Options:', ' --username TEXT The username for authentication.', @@ -184,11 +185,11 @@ ' validity will be check in this order: 1.', ' Check if the credentials are valid with the', ' provided username and password. 2. Check if', - ' the credentials are valid in the', - ' configuration file. 3. Check if the', - ' credentials are valid in the environment', - ' variables. When any is found not valid, will', - ' return False immediately.', + ' the credentials are valid in the environment', + ' variables. 3. Check if the credentials are', + ' valid in the configuration file. When any is', + ' found (valid or not valid), will return', + ' immediately.', ' --log-level [DEBUG|INFO|WARN|ERROR|CRITICAL|QUIET]', ' Set the details printed to console by the', ' command (based on standard logging library).', @@ -223,14 +224,13 @@ '', ' Extracts a subset of data from a specified dataset using given parameters.', '', - ' The ``--dataset-id`` is required and can be found via the ``describe``', - ' command. See :ref:`describe `. The argument values passed', - ' individually through the CLI take precedence over the values from the', - ' ``--motu-api-request`` option, which takes precedence over the ones from the', - ' ``--request-file`` option.', + ' The datasetID is required and can be found via the ``describe`` command.', + ' JSON A description of the downloaded data and its destination.See', + ' :ref:`describe `.', '', 'Options:', - ' -i, --dataset-id TEXT The datasetID.', + ' -i, --dataset-id TEXT The datasetID, required either as an', + ' argument or in the request_file option.', ' --dataset-version TEXT Force the selection of a specific dataset', ' version.', ' --dataset-part TEXT Force the selection of a specific dataset', @@ -240,9 +240,11 @@ ' -v, --variable TEXT Specify dataset variable. Can be used', ' multiple times.', ' -x, --minimum-longitude FLOAT Minimum longitude for the subset. The value', - ' will be reduced to the interval [-180; 360[.', + ' will be transposed to the interval [-180;', + ' 360[.', ' -X, --maximum-longitude FLOAT Maximum longitude for the subset. The value', - ' will be reduced to the interval [-180; 360[.', + ' will be transposed to the interval [-180;', + ' 360[.', ' -y, --minimum-latitude FLOAT RANGE', ' Minimum latitude for the subset. Requires a', ' float from -90 degrees to +90. [-90<=x<=90]', @@ -261,38 +263,41 @@ ' positive values, elevation with ascending', ' positive values. Default is depth.', ' -t, --start-datetime TEXT The start datetime of the temporal subset.', - ' Caution: encapsulate date with \\xe2\\x80\\x9c \\xe2\\x80\\x9c to ensure', + ' Supports common format parsed by pendulum (h', + ' ttps://pendulum.eustace.io/docs/#parsing).Ca', + ' ution: encapsulate date with \\xe2\\x80\\x9c \\xe2\\x80\\x9c to ensure', ' valid expression for format \\xe2\\x80\\x9c%Y-%m-%d', - ' %H:%M:%S\\xe2\\x80\\x9d. Supports common format parsed by', - ' pendulum.', + ' %H:%M:%S\\xe2\\x80\\x9d.', ' -T, --end-datetime TEXT The end datetime of the temporal subset.', - ' Caution: encapsulate date with \\xe2\\x80\\x9c \\xe2\\x80\\x9c to ensure', + ' Supports common format parsed by pendulum (h', + ' ttps://pendulum.eustace.io/docs/#parsing).Ca', + ' ution: encapsulate date with \\xe2\\x80\\x9c \\xe2\\x80\\x9c to ensure', ' valid expression for format \\xe2\\x80\\x9c%Y-%m-%d', - ' %H:%M:%S\\xe2\\x80\\x9d. Supports common format parsed by', - ' pendulum.', + ' %H:%M:%S\\xe2\\x80\\x9d.', ' --coordinates-selection-method [inside|strict-inside|nearest|outside]', - ' The method in which the coordinates will be', - ' retrieved. If \\xe2\\x80\\x98inside\\xc2\\xb4, the selection', - ' retrieved will be inside the requested', - ' range. If \\xe2\\x80\\x98strict-inside\\xe2\\x80\\x99, the selection', - ' retrieved will be inside the requested', - ' range, and an error will be raised if the', - " values don't exist. If \\xe2\\x80\\x98nearest\\xe2\\x80\\x99, the", - ' extremes closest to the requested values', - ' will be returned. A warning will be', - ' displayed if outside of bounds. If', - ' \\xe2\\x80\\x98outside\\xe2\\x80\\x99, the extremes will be taken to', - ' contain all the requested interval.', + ' If ``inside``, the selection retrieved will', + ' be inside the requested range. If ``strict-', + ' inside``, the selection retrieved will be', + ' inside the requested range, and an error', + " will be raised if the values don't exist. If", + ' ``nearest``, the extremes closest to the', + ' requested values will be returned. If', + ' ``outside``, the extremes will be taken to', + ' contain all the requested interval. The', + ' methods ``inside``, ``nearest`` and', + ' ``outside`` will display a warning if the', + ' request is out of bounds.', ' -o, --output-directory PATH The destination folder for the downloaded', ' files. Default is the current directory.', ' --credentials-file PATH Path to a credentials file if not in its', - ' default directory. Accepts', + ' default directory', + ' (``$HOME/.copernicusmarine``). Accepts', ' .copernicusmarine-credentials / .netrc or', ' _netrc / motuclient-python.ini files.', - ' -f, --output-filename TEXT Concatenate the downloaded data in the given', - ' file name (under the output directory).', + ' -f, --output-filename TEXT Save the downloaded data with the given file', + ' name (under the output directory).', ' --file-format [netcdf|zarr] Format of the downloaded dataset. Default to', - ' NetCDF (.nc).', + " NetCDF '.nc'.", ' --force-download Flag to skip confirmation before download.', ' --overwrite-output-data, --overwrite', ' If specified and if the file already exists', @@ -301,24 +306,23 @@ ' index.', ' -s, --service TEXT Force download through one of the available', ' services using the service name among', - ' [\\xe2\\x80\\x98arco-geo-series\\xe2\\x80\\x99, \\xe2\\x80\\x98arco-time-series\\xe2\\x80\\x99,', - ' \\xe2\\x80\\x98omi-arco\\xe2\\x80\\x99, \\xe2\\x80\\x98static-arco\\xe2\\x80\\x99] or its short name', - ' among [\\xe2\\x80\\x98geoseries\\xe2\\x80\\x99, \\xe2\\x80\\x98timeseries\\xe2\\x80\\x99, \\xe2\\x80\\x98omi-', - ' arco\\xe2\\x80\\x99, \\xe2\\x80\\x98static-arco\\xe2\\x80\\x99].', + " ['arco-geo-series', 'arco-time-series',", + " 'omi-arco', 'static-arco'] or its short name", + " among ['arco-geo-series', 'arco-time-", + " series', 'omi-arco', 'static-arco'].", ' --create-template Option to create a file', ' _template.json in your current', ' directory containing the arguments. If', ' specified, no other action will be', ' performed.', ' --request-file PATH Option to pass a file containing the', - ' arguments. The file MUST follow the', - ' structure of dataclass \\xe2\\x80\\x98SubsetRequest\\xe2\\x80\\x99. For', - ' more information please refer to the', - ' documentation or use option ``--create-', - ' template`` for an example template.', + ' arguments. For more information please refer', + ' to the documentation or use option', + ' ``--create-template`` for an example', + ' template.', ' --motu-api-request TEXT Option to pass a complete MOTU API request', ' as a string. Caution, user has to replace', - ' double quotes \\xe2\\x80\\x9c with single quotes \\xe2\\x80\\x98 in the', + " double quotes \\xe2\\x80\\x9c with single quotes ' in the", ' request.', ' --dry-run If True, runs query without downloading', ' data.', @@ -330,8 +334,8 @@ ' Specify a compression level to apply on the', ' NetCDF output file. A value of 0 means no', ' compression, and 9 is the highest level of', - ' compression available. The default level is', - ' 1. [0<=x<=9]', + ' compression available. If used as a flag,', + ' the assigned value will be 1. [0<=x<=9]', ' --netcdf3-compatible Enable downloading the dataset in a netCDF3', ' compatible format.', ' -h, --help Show this message and exit.', @@ -345,8 +349,9 @@ ' Equivalent to:', '', '', - ' copernicusmarine subset -i cmems_mod_ibi_phy_my_0.083deg-3D_P1D-m -v thetao -v so -t 2021-01-01 -T 2021-01-03 -x 0.0 -X 0.1 -y 28.0 -Y 28.1 -z 1 -Z 2', + ' copernicusmarine subset -i cmems_mod_ibi_phy_my_0.083deg-3D_P1D-m -v thetao -v so -t 2021-01-01 -T 2021-01-03 -x 0.0 -X 0.1 -y 28.0 -Y 28.1 -z 1 -Z 2 ', + '', ' ', - "', stderr=b'')", + '", stderr=b\'\')', ]) # --- diff --git a/tests/test_documentation.py b/tests/test_documentation.py index a28c6fcd..27a4cb6e 100644 --- a/tests/test_documentation.py +++ b/tests/test_documentation.py @@ -13,29 +13,21 @@ def test_subset(self): for i in range(len(text_subset["Parameters"])): name_of_variable = text_subset["Parameters"][i].name if name_of_variable in ["start_datetime", "end_datetime"]: - assert ( - text_subset["Parameters"][i].desc - == documentation_utils.SUBSET[ + assert text_subset["Parameters"][i].desc == [ + documentation_utils.SUBSET[ name_of_variable.upper() + "_HELP" ] - + "Caution: encapsulate date with “ “ to ensure valid " - "expression for format “%Y-%m-%d %H:%M:%S”." - ) + ] continue if name_of_variable == "variables": - assert ( - text_subset["Parameters"][i].desc - == "List of variable names to extract." - ) + assert text_subset["Parameters"][i].desc == [ + "List of variable names to extract." + ] continue if name_of_variable == "netcdf_compression_level": - assert text_subset["Parameters"][ - i - ].desc + " If used as a flag, the assigned value will be 1." == [ - [ - documentation_utils.SUBSET[ - name_of_variable.upper() + "_HELP" - ] + assert text_subset["Parameters"][i].desc == [ + documentation_utils.SUBSET[ + name_of_variable.upper() + "_HELP" ] ] continue @@ -92,20 +84,16 @@ def test_open_dataset(self): for i in range(len(text_open_dataset["Parameters"])): name_of_variable = text_open_dataset["Parameters"][i].name if name_of_variable in ["start_datetime", "end_datetime"]: - assert ( - text_open_dataset["Parameters"][i].desc - == documentation_utils.SUBSET[ + assert text_open_dataset["Parameters"][i].desc == [ + documentation_utils.SUBSET[ name_of_variable.upper() + "_HELP" ] - + "Caution: encapsulate date with “ “ to ensure valid " - "expression for format “%Y-%m-%d %H:%M:%S”." - ) + ] continue if name_of_variable == "variables": - assert ( - text_open_dataset["Parameters"][i].desc - == "List of variable names to extract." - ) + assert text_open_dataset["Parameters"][i].desc == [ + "List of variable names to extract." + ] continue if name_of_variable == "dataset_id": assert text_open_dataset["Parameters"][i].desc == [ @@ -124,20 +112,16 @@ def test_read_dataframe(self): for i in range(len(text_read_dataframe["Parameters"])): name_of_variable = text_read_dataframe["Parameters"][i].name if name_of_variable in ["start_datetime", "end_datetime"]: - assert ( - text_read_dataframe["Parameters"][i].desc - == documentation_utils.SUBSET[ + assert text_read_dataframe["Parameters"][i].desc == [ + documentation_utils.SUBSET[ name_of_variable.upper() + "_HELP" ] - + "Caution: encapsulate date with “ “ to ensure valid " - "expression for format “%Y-%m-%d %H:%M:%S”." - ) + ] continue if name_of_variable == "variables": - assert ( - text_read_dataframe["Parameters"][i].desc - == "List of variable names to extract." - ) + assert text_read_dataframe["Parameters"][i].desc == [ + "List of variable names to extract." + ] continue if name_of_variable == "dataset_id": assert text_read_dataframe["Parameters"][i].desc == [ @@ -146,6 +130,8 @@ def test_read_dataframe(self): continue if name_of_variable in LIST_OF_EXCEPTIONS: continue + print(name_of_variable) + assert text_read_dataframe["Parameters"][i].desc == [ documentation_utils.SUBSET[name_of_variable.upper() + "_HELP"] ] From db70e5774c1b921ac78e0c2072d0254251f24da3 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Thu, 17 Oct 2024 16:40:01 +0200 Subject: [PATCH 05/14] changes after checking the website --- .../command_line_interface/group_describe.py | 1 + .../command_line_interface/group_get.py | 2 +- .../command_line_interface/group_login.py | 1 + .../command_line_interface/group_subset.py | 5 +++-- .../core_functions/documentation_utils.py | 21 +++++++++++-------- copernicusmarine/core_functions/models.py | 2 +- copernicusmarine/python_interface/get.py | 4 ++-- copernicusmarine/python_interface/subset.py | 2 +- 8 files changed, 22 insertions(+), 16 deletions(-) diff --git a/copernicusmarine/command_line_interface/group_describe.py b/copernicusmarine/command_line_interface/group_describe.py index 6c18d746..7290c128 100644 --- a/copernicusmarine/command_line_interface/group_describe.py +++ b/copernicusmarine/command_line_interface/group_describe.py @@ -26,6 +26,7 @@ def cli_describe() -> None: cls=CustomClickOptionsCommand, short_help="Print Copernicus Marine catalogue as JSON.", help=documentation_utils.DESCRIBE["DESCRIBE_DESCRIPTION_HELP"] + + " \n\nReturns\n " + documentation_utils.DESCRIBE["DESCRIBE_RESPONSE_HELP"], # noqa epilog=""" Examples: diff --git a/copernicusmarine/command_line_interface/group_get.py b/copernicusmarine/command_line_interface/group_get.py index 125489ec..06d58cf4 100644 --- a/copernicusmarine/command_line_interface/group_get.py +++ b/copernicusmarine/command_line_interface/group_get.py @@ -37,7 +37,7 @@ def cli_get() -> None: cls=CustomClickOptionsCommand, short_help="Download originally produced data files.", help=documentation_utils.GET["GET_DESCRIPTION_HELP"] - + " See :ref:`describe `." + + " See :ref:`describe `. \n\nReturns\n " + documentation_utils.GET["GET_RESPONSE_HELP"], epilog=""" Example to download all the files from a given dataset: diff --git a/copernicusmarine/command_line_interface/group_login.py b/copernicusmarine/command_line_interface/group_login.py index 3a50a96a..8df8c1d8 100644 --- a/copernicusmarine/command_line_interface/group_login.py +++ b/copernicusmarine/command_line_interface/group_login.py @@ -29,6 +29,7 @@ def cli_login() -> None: cls=CustomClickOptionsCommand, short_help="Create a configuration file with your Copernicus Marine credentials.", help=documentation_utils.LOGIN["LOGIN_DESCRIPTION_HELP"] + + " \n\nReturns\n " + documentation_utils.LOGIN["LOGIN_RESPONSE_HELP"], # noqa epilog=""" Examples: diff --git a/copernicusmarine/command_line_interface/group_subset.py b/copernicusmarine/command_line_interface/group_subset.py index 8b663ee9..a06b6146 100644 --- a/copernicusmarine/command_line_interface/group_subset.py +++ b/copernicusmarine/command_line_interface/group_subset.py @@ -48,8 +48,9 @@ def cli_subset() -> None: cls=CustomClickOptionsCommand, short_help="Download subsets of datasets as NetCDF files or Zarr stores.", help=documentation_utils.SUBSET["SUBSET_DESCRIPTION_HELP"] - + documentation_utils.SUBSET["SUBSET_RESPONSE_HELP"] - + "See :ref:`describe `.", + + "See :ref:`describe `." + + " \n\nReturns\n " + + documentation_utils.SUBSET["SUBSET_RESPONSE_HELP"], epilog=""" Examples: diff --git a/copernicusmarine/core_functions/documentation_utils.py b/copernicusmarine/core_functions/documentation_utils.py index 95c3df92..4cdc597b 100644 --- a/copernicusmarine/core_functions/documentation_utils.py +++ b/copernicusmarine/core_functions/documentation_utils.py @@ -39,12 +39,6 @@ "The destination folder for the downloaded files. Default is the current " "directory." ), - "REQUEST_FILE_HELP": ( - "Option to pass a file containing the arguments." - " For more information please refer to" - " the documentation or use option ``--create-template`` " - "for an example template." - ), } LOGIN: dict[str, str] = { @@ -107,7 +101,7 @@ SUBSET: dict[str, str] = { "SUBSET_DESCRIPTION_HELP": ( "Extracts a subset of data from a specified dataset using given parameters." - " \n \n The datasetID is required and can be found via the ``describe`` " + "\n\nThe datasetID is required and can be found via the ``describe`` " "command. " # has some hardcoding in CLI and python API ), "SUBSET_RESPONSE_HELP": ( @@ -146,6 +140,12 @@ "descending positive values, elevation with ascending positive values. " "Default is depth." ), + "REQUEST_FILE_HELP": ( + "Option to pass a file containing the arguments." + " For more information please refer to" + " the documentation or use option ``--create-template`` " + "for an example template." + ), "START_DATETIME_HELP": ( "The start datetime of the temporal subset. Supports common " "format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing)." @@ -187,8 +187,8 @@ GET: dict[str, str] = { "GET_DESCRIPTION_HELP": ( - "Download originally produced data files.\n \n " - " The datasetID is required (either as an " + "Download originally produced data files.\n\n" + "The datasetID is required (either as an " "argument or in a request file) and can be found via the ``describe``" " command." ), # has some hardcoding in CLI @@ -210,6 +210,9 @@ "The regular expression that must match the absolute paths of the files to " "download." ), + "REQUEST_FILE_HELP": ( + "Option to pass a file containing the arguments " "..." + ), "FILE_LIST_HELP": ( "Path to a '.txt' file containing a " "list of file paths, line by line, that will " diff --git a/copernicusmarine/core_functions/models.py b/copernicusmarine/core_functions/models.py index 2ae33679..3671083b 100644 --- a/copernicusmarine/core_functions/models.py +++ b/copernicusmarine/core_functions/models.py @@ -91,4 +91,4 @@ class ResponseSubset(BaseModel): #: get the final result in MB. data_needed: Optional[float] #: The bounds of the subsetted dataset. - coodinates_extent: DatasetCoordinatesExtent + coordinates_extent: DatasetCoordinatesExtent diff --git a/copernicusmarine/python_interface/get.py b/copernicusmarine/python_interface/get.py index 3a58c263..508b94cd 100644 --- a/copernicusmarine/python_interface/get.py +++ b/copernicusmarine/python_interface/get.py @@ -42,7 +42,7 @@ def get( """ Download originally produced data files. - The `datasetID is required and can be found via the ``describe`` command. + The datasetID is required and can be found via the ``describe`` command. Parameters ---------- @@ -65,7 +65,7 @@ def get( overwrite_output_data : bool, optional If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` for an example template. + Option to pass a file containing the arguments ... no_directories : bool, optional If True, downloaded files will not be organized into directories. show_outputnames : bool, optional diff --git a/copernicusmarine/python_interface/subset.py b/copernicusmarine/python_interface/subset.py index f040a1b3..c37811c1 100644 --- a/copernicusmarine/python_interface/subset.py +++ b/copernicusmarine/python_interface/subset.py @@ -61,7 +61,7 @@ def subset( """ Extracts a subset of data from a specified dataset using given parameters. - The ``dataset_id`` is required and can be found via the ``describe`` command. + The datasetID is required and can be found via the ``describe`` command. Parameters ---------- From f6829a826f12a88fc66d6278426f7dd5b8a9fb06 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Thu, 17 Oct 2024 16:46:33 +0200 Subject: [PATCH 06/14] changes in the first tests that i already know --- tests/test_get_create_file_list.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/test_get_create_file_list.py b/tests/test_get_create_file_list.py index 22c80d75..56bdbbbd 100644 --- a/tests/test_get_create_file_list.py +++ b/tests/test_get_create_file_list.py @@ -16,7 +16,7 @@ def test_get_create_file_list_without_extension_raises(self): ] self.output = execute_in_terminal(self.command) assert ( - b"Assertion error: Download file list must be a .txt or .csv file." + b"Assertion error: Download file list must be a '.txt' or '.csv' file." in self.output.stderr ) @@ -58,7 +58,10 @@ def test_get_create_file_list_without_extension_raises_python(self): create_file_list="hello", ) except AssertionError as e: - assert str(e) == "Download file list must be a .txt or .csv file. " + assert ( + str(e) + == "Download file list must be a '.txt' or '.csv' file. " + ) def test_get_create_file_list_python(self): get( From 3beb225eeb40ff21c88eda43d3b436163abe865f Mon Sep 17 00:00:00 2001 From: uriii3 Date: Thu, 17 Oct 2024 17:05:25 +0200 Subject: [PATCH 07/14] coodinates --- copernicusmarine/download_functions/download_arco_series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copernicusmarine/download_functions/download_arco_series.py b/copernicusmarine/download_functions/download_arco_series.py index 066f4bae..703b4301 100644 --- a/copernicusmarine/download_functions/download_arco_series.py +++ b/copernicusmarine/download_functions/download_arco_series.py @@ -134,7 +134,7 @@ def download_dataset( output=output_path, size=final_result_size_estimation, data_needed=data_needed_approximation, - coodinates_extent=get_dataset_coordinates_extent(dataset), + coordinates_extent=get_dataset_coordinates_extent(dataset), ) if dry_run: From 9c241bab10eb87b6ee9412efc2404123244eaa1b Mon Sep 17 00:00:00 2001 From: uriii3 Date: Thu, 17 Oct 2024 17:19:55 +0200 Subject: [PATCH 08/14] some changes in the descriptions --- copernicusmarine/core_functions/documentation_utils.py | 6 +++--- copernicusmarine/python_interface/describe.py | 2 +- copernicusmarine/python_interface/get.py | 2 +- copernicusmarine/python_interface/login.py | 3 +-- copernicusmarine/python_interface/subset.py | 2 +- 5 files changed, 7 insertions(+), 8 deletions(-) diff --git a/copernicusmarine/core_functions/documentation_utils.py b/copernicusmarine/core_functions/documentation_utils.py index 4cdc597b..17d9365c 100644 --- a/copernicusmarine/core_functions/documentation_utils.py +++ b/copernicusmarine/core_functions/documentation_utils.py @@ -45,7 +45,7 @@ "LOGIN_DESCRIPTION_HELP": ( "Create a configuration file with your Copernicus Marine credentials" " under the ``$HOME/.copernicusmarine`` directory" - " (overwritable with the ``--credentials-file`` option)." + "." ), "LOGIN_RESPONSE_HELP": ( "Error code\n 0 error code if the login was successfully " @@ -72,7 +72,7 @@ DESCRIBE: dict[str, str] = { "DESCRIBE_DESCRIPTION_HELP": ( "Retrieve and parse the metadata information " - " from the Copernicus Marine catalogue." + "from the Copernicus Marine catalogue." ), "DESCRIBE_RESPONSE_HELP": ( "JSON\n A dictionary containing the retrieved metadata information." @@ -211,7 +211,7 @@ "download." ), "REQUEST_FILE_HELP": ( - "Option to pass a file containing the arguments " "..." + "Option to pass a file containing the arguments. " "..." ), "FILE_LIST_HELP": ( "Path to a '.txt' file containing a " diff --git a/copernicusmarine/python_interface/describe.py b/copernicusmarine/python_interface/describe.py index 0c0b8ba2..e367ff98 100644 --- a/copernicusmarine/python_interface/describe.py +++ b/copernicusmarine/python_interface/describe.py @@ -25,7 +25,7 @@ def describe( staging: bool = False, ) -> dict[str, Any]: """ - Retrieve metadata information from the Copernicus Marine catalogue based on specified parameters and options. + Retrieve and parse the metadata information from the Copernicus Marine catalogue. Parameters ---------- diff --git a/copernicusmarine/python_interface/get.py b/copernicusmarine/python_interface/get.py index 508b94cd..ce84e7b1 100644 --- a/copernicusmarine/python_interface/get.py +++ b/copernicusmarine/python_interface/get.py @@ -42,7 +42,7 @@ def get( """ Download originally produced data files. - The datasetID is required and can be found via the ``describe`` command. + The datasetID is required (either as an argument or in a request file) and can be found via the ``describe`` command. Parameters ---------- diff --git a/copernicusmarine/python_interface/login.py b/copernicusmarine/python_interface/login.py index 81baced9..81d70af3 100644 --- a/copernicusmarine/python_interface/login.py +++ b/copernicusmarine/python_interface/login.py @@ -15,8 +15,7 @@ def login( check_credentials_valid: bool = False, ) -> bool: """ - Create a configuration file with your Copernicus Marine credentials under the ``$HOME/.copernicusmarine`` directory (overwritable with the ``overwrite_configuration_file`` option). - + Create a configuration file with your Copernicus Marine credentials under the ``$HOME/.copernicusmarine`` directory. Parameters ---------- diff --git a/copernicusmarine/python_interface/subset.py b/copernicusmarine/python_interface/subset.py index c37811c1..211c8d8f 100644 --- a/copernicusmarine/python_interface/subset.py +++ b/copernicusmarine/python_interface/subset.py @@ -59,7 +59,7 @@ def subset( netcdf3_compatible: bool = False, ) -> ResponseSubset: """ - Extracts a subset of data from a specified dataset using given parameters. + Extracts a subset of data from a specified dataset using given parameters." The datasetID is required and can be found via the ``describe`` command. From c2fe04a31b97ba40c9a0d39007554250f7670bc9 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Fri, 18 Oct 2024 10:32:34 +0200 Subject: [PATCH 09/14] final commit documentation --- .../core_functions/documentation_utils.py | 16 ++++----- copernicusmarine/python_interface/get.py | 2 +- copernicusmarine/python_interface/login.py | 2 +- copernicusmarine/python_interface/subset.py | 2 +- .../test_help_command_interface.ambr | 33 +++++++++++-------- 5 files changed, 29 insertions(+), 26 deletions(-) diff --git a/copernicusmarine/core_functions/documentation_utils.py b/copernicusmarine/core_functions/documentation_utils.py index 17d9365c..3a5e5583 100644 --- a/copernicusmarine/core_functions/documentation_utils.py +++ b/copernicusmarine/core_functions/documentation_utils.py @@ -39,13 +39,18 @@ "The destination folder for the downloaded files. Default is the current " "directory." ), + "REQUEST_FILE_HELP": ( + "Option to pass a file containing the arguments. For more information " + "please refer to the documentation or use option ``--create-template`` " + "of the command line interface for an example template." + ), } LOGIN: dict[str, str] = { "LOGIN_DESCRIPTION_HELP": ( "Create a configuration file with your Copernicus Marine credentials" " under the ``$HOME/.copernicusmarine`` directory" - "." + " (overwritable with the ``overwrite_configuration_file`` option)." ), "LOGIN_RESPONSE_HELP": ( "Error code\n 0 error code if the login was successfully " @@ -140,12 +145,6 @@ "descending positive values, elevation with ascending positive values. " "Default is depth." ), - "REQUEST_FILE_HELP": ( - "Option to pass a file containing the arguments." - " For more information please refer to" - " the documentation or use option ``--create-template`` " - "for an example template." - ), "START_DATETIME_HELP": ( "The start datetime of the temporal subset. Supports common " "format parsed by pendulum (https://pendulum.eustace.io/docs/#parsing)." @@ -210,9 +209,6 @@ "The regular expression that must match the absolute paths of the files to " "download." ), - "REQUEST_FILE_HELP": ( - "Option to pass a file containing the arguments. " "..." - ), "FILE_LIST_HELP": ( "Path to a '.txt' file containing a " "list of file paths, line by line, that will " diff --git a/copernicusmarine/python_interface/get.py b/copernicusmarine/python_interface/get.py index ce84e7b1..ff6b5092 100644 --- a/copernicusmarine/python_interface/get.py +++ b/copernicusmarine/python_interface/get.py @@ -65,7 +65,7 @@ def get( overwrite_output_data : bool, optional If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Option to pass a file containing the arguments ... + Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` of the command line interface for an example template. no_directories : bool, optional If True, downloaded files will not be organized into directories. show_outputnames : bool, optional diff --git a/copernicusmarine/python_interface/login.py b/copernicusmarine/python_interface/login.py index 81d70af3..23007141 100644 --- a/copernicusmarine/python_interface/login.py +++ b/copernicusmarine/python_interface/login.py @@ -15,7 +15,7 @@ def login( check_credentials_valid: bool = False, ) -> bool: """ - Create a configuration file with your Copernicus Marine credentials under the ``$HOME/.copernicusmarine`` directory. + Create a configuration file with your Copernicus Marine credentials under the ``$HOME/.copernicusmarine`` directory (overwritable with the ``overwrite_configuration_file`` option). Parameters ---------- diff --git a/copernicusmarine/python_interface/subset.py b/copernicusmarine/python_interface/subset.py index 211c8d8f..5a48700b 100644 --- a/copernicusmarine/python_interface/subset.py +++ b/copernicusmarine/python_interface/subset.py @@ -84,7 +84,7 @@ def subset( overwrite_output_data : bool, optional If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` for an example template. + Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` of the command line interface for an example template. service : str, optional Force download through one of the available services using the service name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco'] or its short name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco']. variables : List[str], optional diff --git a/tests/__snapshots__/test_help_command_interface.ambr b/tests/__snapshots__/test_help_command_interface.ambr index 28678dc1..3ba33fbd 100644 --- a/tests/__snapshots__/test_help_command_interface.ambr +++ b/tests/__snapshots__/test_help_command_interface.ambr @@ -20,8 +20,10 @@ list([ "CompletedProcess(args=['copernicusmarine', 'describe', '--help'], returncode=0, stdout=b'Usage: copernicusmarine describe [OPTIONS]", '', - ' Retrieve and parse the metadata information from the Copernicus Marine', - ' catalogue.JSON A dictionary containing the retrieved metadata information.', + ' Retrieve and parse the metadata information from the Copernicus Marine', + ' catalogue.', + '', + ' Returns JSON A dictionary containing the retrieved metadata information.', '', 'Options:', ' --include-description Include product description in output.', @@ -63,9 +65,11 @@ '', ' Download originally produced data files.', '', - ' The datasetID is required (either as an argument or in a request file) and', - ' can be found via the ``describe`` command. See :ref:`describe `.JSON A list of files that were downloaded and some metadata.', + ' The datasetID is required (either as an argument or in a request file) and', + ' can be found via the ``describe`` command. See :ref:`describe `.', + '', + ' Returns JSON A list of files that were downloaded and some metadata.', '', 'Options:', ' -i, --dataset-id TEXT The datasetID, required either as an', @@ -103,8 +107,8 @@ ' --request-file PATH Option to pass a file containing the', ' arguments. For more information please refer', ' to the documentation or use option', - ' ``--create-template`` for an example', - ' template.', + ' ``--create-template`` of the command line', + ' interface for an example template.', ' --filter, --filter-with-globbing-pattern TEXT', ' A pattern that must match the absolute paths', ' of the files to download.', @@ -168,8 +172,10 @@ '', ' Create a configuration file with your Copernicus Marine credentials under', ' the ``$HOME/.copernicusmarine`` directory (overwritable with the', - ' ``--credentials-file`` option).Error code 0 error code if the login was', - ' successfully completed, 1 otherwise.', + ' ``overwrite_configuration_file`` option).', + '', + ' Returns Error code 0 error code if the login was successfully completed, 1', + ' otherwise.', '', 'Options:', ' --username TEXT The username for authentication.', @@ -224,10 +230,11 @@ '', ' Extracts a subset of data from a specified dataset using given parameters.', '', - ' The datasetID is required and can be found via the ``describe`` command.', - ' JSON A description of the downloaded data and its destination.See', + ' The datasetID is required and can be found via the ``describe`` command. See', ' :ref:`describe `.', '', + ' Returns JSON A description of the downloaded data and its destination.', + '', 'Options:', ' -i, --dataset-id TEXT The datasetID, required either as an', ' argument or in the request_file option.', @@ -318,8 +325,8 @@ ' --request-file PATH Option to pass a file containing the', ' arguments. For more information please refer', ' to the documentation or use option', - ' ``--create-template`` for an example', - ' template.', + ' ``--create-template`` of the command line', + ' interface for an example template.', ' --motu-api-request TEXT Option to pass a complete MOTU API request', ' as a string. Caution, user has to replace', " double quotes \\xe2\\x80\\x9c with single quotes ' in the", From bd4093383ec9a35d44ba25c8957c6b4d378321d4 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Fri, 18 Oct 2024 10:35:58 +0200 Subject: [PATCH 10/14] of to from --- copernicusmarine/core_functions/documentation_utils.py | 2 +- copernicusmarine/python_interface/get.py | 2 +- copernicusmarine/python_interface/subset.py | 2 +- tests/__snapshots__/test_help_command_interface.ambr | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/copernicusmarine/core_functions/documentation_utils.py b/copernicusmarine/core_functions/documentation_utils.py index 3a5e5583..a929c0dc 100644 --- a/copernicusmarine/core_functions/documentation_utils.py +++ b/copernicusmarine/core_functions/documentation_utils.py @@ -42,7 +42,7 @@ "REQUEST_FILE_HELP": ( "Option to pass a file containing the arguments. For more information " "please refer to the documentation or use option ``--create-template`` " - "of the command line interface for an example template." + "from the command line interface for an example template." ), } diff --git a/copernicusmarine/python_interface/get.py b/copernicusmarine/python_interface/get.py index ff6b5092..0dc67109 100644 --- a/copernicusmarine/python_interface/get.py +++ b/copernicusmarine/python_interface/get.py @@ -65,7 +65,7 @@ def get( overwrite_output_data : bool, optional If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` of the command line interface for an example template. + Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` from the command line interface for an example template. no_directories : bool, optional If True, downloaded files will not be organized into directories. show_outputnames : bool, optional diff --git a/copernicusmarine/python_interface/subset.py b/copernicusmarine/python_interface/subset.py index 5a48700b..5f836c20 100644 --- a/copernicusmarine/python_interface/subset.py +++ b/copernicusmarine/python_interface/subset.py @@ -84,7 +84,7 @@ def subset( overwrite_output_data : bool, optional If specified and if the file already exists on destination, then it will be overwritten instead of creating new one with unique index. request_file : Union[pathlib.Path, str], optional - Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` of the command line interface for an example template. + Option to pass a file containing the arguments. For more information please refer to the documentation or use option ``--create-template`` from the command line interface for an example template. service : str, optional Force download through one of the available services using the service name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco'] or its short name among ['arco-geo-series', 'arco-time-series', 'omi-arco', 'static-arco']. variables : List[str], optional diff --git a/tests/__snapshots__/test_help_command_interface.ambr b/tests/__snapshots__/test_help_command_interface.ambr index 3ba33fbd..5a6976bc 100644 --- a/tests/__snapshots__/test_help_command_interface.ambr +++ b/tests/__snapshots__/test_help_command_interface.ambr @@ -107,7 +107,7 @@ ' --request-file PATH Option to pass a file containing the', ' arguments. For more information please refer', ' to the documentation or use option', - ' ``--create-template`` of the command line', + ' ``--create-template`` from the command line', ' interface for an example template.', ' --filter, --filter-with-globbing-pattern TEXT', ' A pattern that must match the absolute paths', @@ -325,7 +325,7 @@ ' --request-file PATH Option to pass a file containing the', ' arguments. For more information please refer', ' to the documentation or use option', - ' ``--create-template`` of the command line', + ' ``--create-template`` from the command line', ' interface for an example template.', ' --motu-api-request TEXT Option to pass a complete MOTU API request', ' as a string. Caution, user has to replace', From dfafbf48abb1b79ce4f53a4e8b25fd39e94aa7d0 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Fri, 18 Oct 2024 12:00:57 +0200 Subject: [PATCH 11/14] only one change remaining --- doc/usage/subset-usage.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/usage/subset-usage.rst b/doc/usage/subset-usage.rst index 02cda682..ad024f8b 100644 --- a/doc/usage/subset-usage.rst +++ b/doc/usage/subset-usage.rst @@ -55,7 +55,7 @@ Note that any longitudes can be requested. The system applies a modulus operatio About ``--netcdf-compression-level`` options """""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" -If writing data to a NetCDF file (the default format), the ``--netcdf-compression-level`` option can be set to compress the downloaded file. This reduces file size but increases writing time. Without this option, the file is written faster but with a larger size. For Zarr format (`.zarr` extension), the default compression of the Copernicus Marine Data Store is applied, making the download fast and compressed without using ``--netcdf-compression-level``. +If writing data to a NetCDF file (the default format), the ``--netcdf-compression-level`` option can be set to compress the downloaded file. This reduces file size but increases writing time. Without this option, the file is written faster but with a larger size. For Zarr format ('.zarr' extension), the default compression of the Copernicus Marine Data Store is applied, making the download fast and compressed without using ``--netcdf-compression-level``. Default NetCDF compression settings for xarray: From 14c7a54e3be7ad7c5f7bcd84c3e4fdc2e0afd9ae Mon Sep 17 00:00:00 2001 From: "Oriol Ricart Vilarrubias." <45894267+uriii3@users.noreply.github.com> Date: Fri, 18 Oct 2024 12:34:36 +0200 Subject: [PATCH 12/14] Update copernicusmarine/core_functions/documentation_utils.py Co-authored-by: Renaud <38732257+renaudjester@users.noreply.github.com> --- copernicusmarine/core_functions/documentation_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/copernicusmarine/core_functions/documentation_utils.py b/copernicusmarine/core_functions/documentation_utils.py index a929c0dc..798b2f5e 100644 --- a/copernicusmarine/core_functions/documentation_utils.py +++ b/copernicusmarine/core_functions/documentation_utils.py @@ -53,7 +53,7 @@ " (overwritable with the ``overwrite_configuration_file`` option)." ), "LOGIN_RESPONSE_HELP": ( - "Error code\n 0 error code if the login was successfully " + "Exit code\n 0 exit code if the login was successfully " "completed, 1 otherwise." ), "CONFIGURATION_FILE_DIRECTORY_HELP": ( From fb774a3a4d3c39be409087ed5ee5cee8e778ef11 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Fri, 18 Oct 2024 12:34:18 +0200 Subject: [PATCH 13/14] rebase subset coordinates --- copernicusmarine/core_functions/exceptions.py | 2 +- doc/usage/subset-usage.rst | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/copernicusmarine/core_functions/exceptions.py b/copernicusmarine/core_functions/exceptions.py index 5affdb8f..58e396da 100644 --- a/copernicusmarine/core_functions/exceptions.py +++ b/copernicusmarine/core_functions/exceptions.py @@ -33,7 +33,7 @@ class CoordinatesOutOfDatasetBounds(Exception): Exception raised when the coordinates are out of the dataset bounds. Please make sure the coordinates are within the dataset bounds. If you are using the - strict mode, the coordinates must be within the dataset bounds. + strict-inside mode, the coordinates must be within the dataset bounds. """ def __init__(self, message: str): diff --git a/doc/usage/subset-usage.rst b/doc/usage/subset-usage.rst index ad024f8b..d652bcfc 100644 --- a/doc/usage/subset-usage.rst +++ b/doc/usage/subset-usage.rst @@ -86,19 +86,24 @@ This applies to all dimensions: longitude, latitude, time, and depth. **Example of longitude requests:** Imagine a dataset with longitudes from 5.0 to 36.5, with steps of 0.5. -- ``--coordinates-selection-method``= **inside**, with requested interval = [0.4, 35.9]: +- ``--coordinates-selection-method`` = **inside**, with requested interval = [0.4, 35.9]: + - Returns longitudes within the range: [0.5, 35.5] -- ``--coordinates-selection-method``= **strict-inside**, with requested interval = [0.4, 35.9]: +- ``--coordinates-selection-method`` = **strict-inside**, with requested interval = [0.4, 35.9]: + - Returns longitudes within the range: [0.5, 35.5] -- ``--coordinates-selection-method``= **strict-inside**, with requested interval = [0.0, 40]: - - Returns an error +- ``--coordinates-selection-method`` = **strict-inside**, with requested interval = [0.0, 40]: + + - Returns an error, see :class:`copernicusmarine.CoordinatesOutOfDatasetBounds`. + +- ``--coordinates-selection-method`` = **outside**, with requested interval = [0.4, 35.9]: -- ``--coordinates-selection-method``= **outside**, with requested interval = [0.4, 35.9]: - Returns longitudes within the range: [0.0, 36.0] -- ``--coordinates-selection-method``= **nearest**, with requested interval = [0.4, 35.9]: +- ``--coordinates-selection-method`` = **nearest**, with requested interval = [0.4, 35.9]: + - Returns longitudes within the range: [0.5, 36.0] If you request a single point, the nearest point in that dimension will be returned. From 89ca3844f5e6d8d649c94ad73e16e3f16e71e563 Mon Sep 17 00:00:00 2001 From: uriii3 Date: Fri, 18 Oct 2024 13:24:37 +0200 Subject: [PATCH 14/14] update snapshots --- tests/__snapshots__/test_help_command_interface.ambr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/__snapshots__/test_help_command_interface.ambr b/tests/__snapshots__/test_help_command_interface.ambr index 5a6976bc..629e69a7 100644 --- a/tests/__snapshots__/test_help_command_interface.ambr +++ b/tests/__snapshots__/test_help_command_interface.ambr @@ -174,7 +174,7 @@ ' the ``$HOME/.copernicusmarine`` directory (overwritable with the', ' ``overwrite_configuration_file`` option).', '', - ' Returns Error code 0 error code if the login was successfully completed, 1', + ' Returns Exit code 0 exit code if the login was successfully completed, 1', ' otherwise.', '', 'Options:',