diff --git a/README.md b/README.md index b5fa69f..8ffde4e 100644 --- a/README.md +++ b/README.md @@ -4,20 +4,31 @@ [![Built Status](https://api.cirrus-ci.com/github//gypsum-client.svg?branch=main)](https://cirrus-ci.com/github//gypsum-client) [![ReadTheDocs](https://readthedocs.org/projects/gypsum-client/badge/?version=latest)](https://gypsum-client.readthedocs.io/en/stable/) [![Coveralls](https://img.shields.io/coveralls/github//gypsum-client/main.svg)](https://coveralls.io/r//gypsum-client) -[![PyPI-Server](https://img.shields.io/pypi/v/gypsum-client.svg)](https://pypi.org/project/gypsum-client/) + [![Conda-Forge](https://img.shields.io/conda/vn/conda-forge/gypsum-client.svg)](https://anaconda.org/conda-forge/gypsum-client) -[![Monthly Downloads](https://pepy.tech/badge/gypsum-client/month)](https://pepy.tech/project/gypsum-client) + [![Twitter](https://img.shields.io/twitter/url/http/shields.io.svg?style=social&label=Twitter)](https://twitter.com/gypsum-client) --> [![Project generated with PyScaffold](https://img.shields.io/badge/-PyScaffold-005CA0?logo=pyscaffold)](https://pyscaffold.org/) +[![PyPI-Server](https://img.shields.io/pypi/v/gypsum-client.svg)](https://pypi.org/project/gypsum-client/) + +# Python client to the gypsum REST API + + +Provides Python client for the [**gypsum** REST API](https://github.com/ArtifactDB/gypsum-worker). + +Readers are referred to the [API's documentation](https://gypsum-test.aaron-lun.workers.dev) or the [user guide](https://bioconductor.org/packages/devel/bioc/vignettes/gypsum/inst/doc/userguide.html) from its R equivalent for more details. -# gypsum-client +***Note: check out the R/Bioconductor package for the gypsum client [here](https://github.com/ArtifactDB/gypsum-R).*** -> Add a short description here! +## Installation -A longer description of your project goes here... +Package is published to [PyPI](https://pypi.org/project/gypsum-client/), +```sh +pip install gypsum_client +``` diff --git a/docs/conf.py b/docs/conf.py index d7ae76a..4ca59d6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -72,6 +72,7 @@ "sphinx.ext.ifconfig", "sphinx.ext.mathjax", "sphinx.ext.napoleon", + "sphinx_autodoc_typehints", ] # Add any paths that contain templates here, relative to this directory. @@ -166,12 +167,22 @@ # If this is True, todo emits a warning for each TODO entries. The default is False. todo_emit_warnings = True +autodoc_default_options = { + # 'members': 'var1, var2', + # 'member-order': 'bysource', + "special-members": True, + "undoc-members": True, + "exclude-members": "__weakref__, __dict__, __str__, __module__", +} + +autosummary_generate = True +autosummary_imported_members = True # -- Options for HTML output ------------------------------------------------- # The theme to use for HTML and HTML Help pages. See the documentation for # a list of builtin themes. -html_theme = "alabaster" +html_theme = "furo" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the diff --git a/docs/requirements.txt b/docs/requirements.txt index 0990c2a..21f47ae 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -4,3 +4,5 @@ # sphinx_rtd_theme myst-parser[linkify] sphinx>=3.2.1 +furo +sphinx-autodoc-typehints diff --git a/src/gypsum_client/__init__.py b/src/gypsum_client/__init__.py index 42bd389..0e086f2 100644 --- a/src/gypsum_client/__init__.py +++ b/src/gypsum_client/__init__.py @@ -18,6 +18,7 @@ from .auth import access_token, set_access_token from .clone_operations import clone_version +from .config import REQUESTS_MOD from .create_operations import create_project from .fetch_metadata_database import fetch_metadata_database from .fetch_metadata_schema import fetch_metadata_schema diff --git a/src/gypsum_client/auth.py b/src/gypsum_client/auth.py index e9001fc..51481b7 100644 --- a/src/gypsum_client/auth.py +++ b/src/gypsum_client/auth.py @@ -1,6 +1,6 @@ import os import time -from typing import Optional +from typing import Optional, Union import requests from filelock import FileLock @@ -25,13 +25,19 @@ def access_token( request: bool = True, cache_dir: Optional[str] = _cache_directory(), token_expiration_limit: int = 10, -) -> Optional[str]: +) -> Optional[Union[str, dict]]: """Get GitHub access token for authentication to the gypsum API's. + Example: + + .. code-block:: python + + token = access_token() + Args: full: Whether to return the full token details. - Defaults to False. + Defaults to False, only ``token`` is returned. request: Whether to request a new token if no token is found or the @@ -45,7 +51,10 @@ def access_token( Integer specifying the number of seconds until the token expires. Returns: - The GitHub token to access gypsum's resources. + If `full=False` A string specifying the GitHub token to + access gypsum's resources. + + If `full=True` retuns a dicionary containing the full token details. """ global TOKEN_CACHE @@ -123,7 +132,10 @@ def set_access_token( Defaults to None, indicating token is not cached to disk. Returns: - The GitHub token to access gypsum's resources. + Dictionary containing the following keys: + - ``token``, a string containing the token. + - ``name``, the name of the GitHub user authenticated by the token. + - ``expires``, the Unix time at which the token expires. """ global TOKEN_CACHE diff --git a/src/gypsum_client/clone_operations.py b/src/gypsum_client/clone_operations.py index a388557..9b36afc 100644 --- a/src/gypsum_client/clone_operations.py +++ b/src/gypsum_client/clone_operations.py @@ -1,3 +1,43 @@ +"""Clone a version's directory structure. + +Cloning of a versioned asset involves creating a directory at the destination +that has the same contents as the corresponding project-asset-version directory. +All files in the specified version are represented as symlinks from the +destination to the corresponding file in the cache. +The idea is that, when the destination is used in +:py:func:`~gypsum_client.prepare_directory_for_upload.prepare_directory_upload`, +the symlinks are converted into upload links, i.e., ``links=`` in +:py:func:`~gypsum_client.upload_api_operations.start_upload`. +This allows users to create new versions very cheaply as duplicate files +are not uploaded to/stored in the backend. + +Users can more-or-less do whatever they want inside the cloned destination, +but they should treat the symlink targets as read-only. +That is, they should not modify the contents of the linked-to file, as these +refer to assumed-immutable files in the cache. +If a file in the destination needs to be modified, the symlink should be +deleted and replaced with an actual file; +this avoids mutating the cache and it ensures that +:py:func:`~gypsum_client.prepare_directory_for_upload.prepare_directory_upload` +recognizes that a new file actually needs to be uploaded. + +Advanced users can set ``download=False``, in which case symlinks are created +even if their targets are not present in the cache. +In such cases, the destination should be treated as write-only due to the +potential presence of dangling symlinks. +This mode is useful for uploading a new version of an asset without +downloading the files from the existing version, +assuming that the modifications associated with the former can be +achieved without reading any of the latter. + +On Windows, the user may not have permissions to create symbolic links, +so the function will transparently fall back to creating hard links or +copies instead. +This precludes any optimization by prepare_directory_upload as the hard +links/copies cannot be converted into upload links. +It also assumes that download=True as dangling links/copies cannot be created. +""" + import errno import os import shutil @@ -26,42 +66,20 @@ def clone_version( Clone the directory structure for a versioned asset into a separate location. This is typically used to prepare a new version for a lightweight upload. - Cloning of a versioned asset involves creating a directory at the destination - that has the same contents as the corresponding project-asset-version directory. - All files in the specified version are represented as symlinks from the - destination to the corresponding file in the cache. - The idea is that, when the destination is used in - :py:func:`~gypsum_client.prepare_directory_upload.prepare_directory_upload`, - the symlinks are converted into upload links, i.e., links= in - :py:func:`~gypsum_client.start_upload.start_upload`. - This allows users to create new versions very cheaply as duplicate files - are not uploaded to/stored in the backend. - - Users can more-or-less do whatever they want inside the cloned destination, - but they should treat the symlink targets as read-only. - That is, they should not modify the contents of the linked-to file, as these - refer to assumed-immutable files in the cache. - If a file in the destination needs to be modified, the symlink should be - deleted and replaced with an actual file; - this avoids mutating the cache and it ensures that - :py:func:`~gypsum_client.prepare_directory_upload.prepare_directory_upload` - recognizes that a new file actually needs to be uploaded. - - Advanced users can set download=False, in which case symlinks are created - even if their targets are not present in the cache. - In such cases, the destination should be treated as write-only due to the - potential presence of dangling symlinks. - This mode is useful for uploading a new version of an asset without - downloading the files from the existing version, - assuming that the modifications associated with the former can be - achieved without reading any of the latter. - - On Windows, the user may not have permissions to create symbolic links, - so the function will transparently fall back to creating hard links or - copies instead. - This precludes any optimization by prepare_directory_upload as the hard - links/copies cannot be converted into upload links. - It also assumes that download=True as dangling links/copies cannot be created. + See Also: + :py:func:`~gypsum_client.prepare_directory_for_upload.prepare_directory_upload`, + to prepare an upload based on the directory contents. + + Example: + + .. code-block:: python + + import tempfile + + cache = tempfile.mkdtemp() + dest = tempfile.mkdtemp() + + clone_version("test-R", "basic", "v1", destination=dest, cache_dir=cache) Args: project: diff --git a/src/gypsum_client/config.py b/src/gypsum_client/config.py index fc70ffb..84e7dda 100644 --- a/src/gypsum_client/config.py +++ b/src/gypsum_client/config.py @@ -1,8 +1,20 @@ +""" +Set this to False if SSL certificates are not properly setup on your machine. + +essentially sets ``verify=False`` on all requests to the gypsum REST API. + +Example: + + .. code-block::python + + from gypsum_client import REQUESTS_MOD + # to set verify to False + REQUESTS_MOD["verify"] = False + +""" + __author__ = "Jayaram Kancherla" __copyright__ = "Jayaram Kancherla" __license__ = "MIT" -## Set this to False if SSL certificates are not properly setup on your machine. -## essentially sets verify=False on all requests going out. - REQUESTS_MOD = {"verify": True} diff --git a/src/gypsum_client/create_operations.py b/src/gypsum_client/create_operations.py index a57e99a..5964896 100644 --- a/src/gypsum_client/create_operations.py +++ b/src/gypsum_client/create_operations.py @@ -1,4 +1,4 @@ -from typing import List +from typing import List, Union from urllib.parse import quote_plus import requests @@ -14,7 +14,7 @@ def create_project( project: str, - owners: List[str], + owners: Union[str, List[str]], uploaders: List[str] = [], baseline: int = None, growth_rate: int = None, @@ -24,6 +24,20 @@ def create_project( ): """Create a new project with the associated permissions. + See Also: + :py:func:`~gypsum_client.remove_operations.remove_project`, + to remove the project. + + Example: + + .. code-block:: python + + createProject( + "test-Py-create", + owners="jkanche", + uploaders=[{"id": "ArtifactDB-bot"}] + ) + Args: project: Project name. @@ -32,6 +46,8 @@ def create_project( List of GitHub users or organizations that are owners of this project. + May also be a string containing the Github user or organization. + uploaders: List of authorized uploaders for this project. Defaults to an empty list. @@ -53,9 +69,7 @@ def create_project( token: GitHub access token to authenticate with the gypsum REST API. - - Returns: - True if project is successfully created. + The token must refer to a gypsum administrator account. """ url = _remove_slash_url(url) uploaders = _sanitize_uploaders(uploaders) if uploaders is not None else [] @@ -73,6 +87,9 @@ def create_project( if year is not None: quota["year"] = year + if isinstance(owners, str): + owners = [owners] + body = {"permissions": {"owners": owners, "uploaders": uploaders}} if len(quota) > 0: body["quota"] = quota @@ -89,5 +106,3 @@ def create_project( raise Exception( f"Failed to create a project, {req.status_code} and reason: {req.text}" ) from e - - return True diff --git a/src/gypsum_client/fetch_metadata_database.py b/src/gypsum_client/fetch_metadata_database.py index d2a3ff6..8b92946 100644 --- a/src/gypsum_client/fetch_metadata_database.py +++ b/src/gypsum_client/fetch_metadata_database.py @@ -1,3 +1,11 @@ +"""Fetch the metadata database. + +This function will automatically check for updates to the SQLite files +and will download new versions accordingly. New checks are performed when one hour +or more has elapsed since the last check. If the check fails, a warning is raised +and the function returns the currently cached file. +""" + import os import tempfile import time @@ -21,16 +29,27 @@ def fetch_metadata_database( ) -> str: """Fetch the SQLite database containing metadata from the gypsum backend. - This function will automatically check for updates to the SQLite files - and will download new versions accordingly. New checks are performed when one hour - or more has elapsed since the last check. If the check fails, a warning is raised - and the function returns the currently cached file. + See `metadata index `_ + for more details. + + Each database is generated by aggregating metadata across multiple assets + and/or projects, and can be used to perform searches for interesting objects. + + See Also: + :py:func:`~gypsum_client.fetch_metadata_schema.fetch_metadata_schema`, to get + the JSON schema used to define the database tables. + + Example: + + .. code-block:: python + + sql_path = fetch_metadata_database() Args: name: Name of the database. - This can be the name of any SQLite file in - https://github.com/ArtifactDB/bioconductor-metadata-index/releases/tag/latest. + This can be the name of any SQLite file published + `here `_. Defaults to "bioconductor.sqlite3". diff --git a/src/gypsum_client/fetch_metadata_schema.py b/src/gypsum_client/fetch_metadata_schema.py index f306433..d3db0d4 100644 --- a/src/gypsum_client/fetch_metadata_schema.py +++ b/src/gypsum_client/fetch_metadata_schema.py @@ -19,6 +19,27 @@ def fetch_metadata_schema( ) -> str: """Fetch a JSON schema file for metadata to be inserted into a SQLite database. + Fetch a JSON schema file for metadata to be inserted into a SQLite database + See `metadata index `_ + for more details. + + Each SQLite database is created from metadata files uploaded to the gypsum backend, + so clients uploading objects to be incorporated into the database should + validate their metadata against the corresponding JSON schema. + + See Also: + :py:func:`~gypsum_client.validate_metadata.validate_metadata`, to + validate metadata against a chosen schema. + + :py:func:`~gypsum_client.fetch_metadata_database.fetch_metadata_database`, + to obtain the SQLite database of metadata. + + Example: + + .. code-block:: python + + schema_path = fetch_metadata_schema() + Args: name: Name of the schema. diff --git a/src/gypsum_client/fetch_operations.py b/src/gypsum_client/fetch_operations.py index 25e484b..fa2f0b4 100644 --- a/src/gypsum_client/fetch_operations.py +++ b/src/gypsum_client/fetch_operations.py @@ -17,6 +17,16 @@ def fetch_latest(project: str, asset: str, url: str = _rest_url()) -> str: """Fetch the latest version of a project's asset. + See Also: + :py:func:`~gypsum_client.refresh_operations.refresh_latest`, + to refresh the latest version. + + Example: + + .. code-block:: python + + ver = fetch_latest("test-R", "basic") + Args: project: Project name. @@ -44,6 +54,12 @@ def fetch_manifest( ) -> dict: """Fetch the manifest for a version of an asset of a project. + Example: + + .. code-block:: python + + manifest = fetch_manifest("test-R", "basic", "v1") + Args: project: Project name. @@ -86,9 +102,19 @@ def fetch_manifest( ) -def fetch_permissions(project: str, url: str = _rest_url()) -> list: +def fetch_permissions(project: str, url: str = _rest_url()) -> dict: """Fetch the permissions for a project. + See Also: + :py:func:`~gypsum_client.set_operations.set_permissions`, + to update or modify the permissions. + + Example: + + .. code-block:: python + + perms = fetch_permissions("test-R") + Args: project: Project name. @@ -102,19 +128,21 @@ def fetch_permissions(project: str, url: str = _rest_url()) -> list: organizations that are owners of this project. - ``uploaders``, a list of lists specifying the users or organizations who are authorzied to upload to this project. + Each entry is a list with the following fields: - - ``id``, a string containing the GitHub user or organization - that is authorized to upload. - - Optional ``asset``, a string containing the name of the asset - that the uploader is allowed to upload to. If not provided, there is no - restriction on the uploaded asset name. - - Optional ``version``, a string containing the name of the version - that the uploader is allowed to upload to.If not provided, there is - no restriction on the uploaded version name. - - Optional ``until``a POSIXct object containing the expiry date of this - authorization. If not provided, the authorization does not expire. - - Optional ``trusted``, whether the uploader is trusted. - If not provided, defaults to False. + - ``id``, a string containing the GitHub user or organization + that is authorized to upload. + - Optional ``asset``, a string containing the name of the asset + that the uploader is allowed to upload to. If not provided, there is no + restriction on the uploaded asset name. + - Optional ``version``, a string containing the name of the version + that the uploader is allowed to upload to.If not provided, there is + no restriction on the uploaded version name. + - Optional ``until``a POSIXct object containing the expiry date of this + authorization. If not provided, the authorization does not expire. + - Optional ``trusted``, whether the uploader is trusted. + If not provided, defaults to False. + """ perms = _fetch_json(f"{project}/..permissions", url=url) @@ -125,9 +153,19 @@ def fetch_permissions(project: str, url: str = _rest_url()) -> list: return perms -def fetch_quota(project: str, url: str = _rest_url()): +def fetch_quota(project: str, url: str = _rest_url()) -> dict: """Fetch the quota details for a project. + See Also: + :py:func:`~gypsum_client.set_operations.set_quota`, + to update or modify the quota. + + Example: + + .. code-block:: python + + quota = fetch_quota("test-R") + Args: project: Project name. @@ -150,9 +188,15 @@ def fetch_summary( cache_dir: str = _cache_directory(), overwrite: bool = False, url: str = _rest_url(), -): +) -> dict: """Fetch the summary for a version of an asset of a project. + Example: + + .. code-block:: python + + summa = fetch_summary("test-R", "basic", "v1") + Args: project: Project name. @@ -203,9 +247,19 @@ def fetch_summary( return _out -def fetch_usage(project: str, url: str = _rest_url()): +def fetch_usage(project: str, url: str = _rest_url()) -> int: """Fetch the quota usage for a project. + See Also: + :py:func:`~gypsum_client.refresh_operations.refresh_usage`, + to refresh usage details. + + Example: + + .. code-block:: python + + usage = fetch_usage("test-R") + Args: project: Project name. @@ -214,7 +268,7 @@ def fetch_usage(project: str, url: str = _rest_url()): URL to the gypsum compatible API. Returns: - Numeric scalar specifying the quota usage for the project, in bytes. + Quota usage for the project, in bytes. """ _usage = _fetch_json(f"{project}/..usage", url=url) return _usage["total"] diff --git a/src/gypsum_client/list_operations.py b/src/gypsum_client/list_operations.py index 3d6ffce..39bf325 100644 --- a/src/gypsum_client/list_operations.py +++ b/src/gypsum_client/list_operations.py @@ -11,11 +11,16 @@ def list_projects(url: str = _rest_url()) -> list: """List all projects in the gypsum backend. + Example: + + .. code-block:: python + + all_prjs = list_projects() + Args: url: URL to the gypsum compatible API. - Returns: List of project names. """ @@ -25,6 +30,12 @@ def list_projects(url: str = _rest_url()) -> list: def list_assets(project: str, url: str = _rest_url()) -> list: """List all assets in a project. + Example: + + .. code-block:: python + + all_assets = list_assets("test-R") + Args: project: Project name. @@ -41,6 +52,12 @@ def list_assets(project: str, url: str = _rest_url()) -> list: def list_versions(project: str, asset: str, url=_rest_url()) -> list: """List all versions for a project asset. + Example: + + .. code-block:: python + + all_vers = list_versions("test-R", "basic") + Args: project: Project name. @@ -67,6 +84,12 @@ def list_files( ) -> list: """List all files for a specified version of a project and asset. + Example: + + .. code-block:: python + + all_files = list_files("test-R", "basic", "v1") + Args: project: Project name. diff --git a/src/gypsum_client/prepare_directory_for_upload.py b/src/gypsum_client/prepare_directory_for_upload.py index f20ac8c..fa98486 100644 --- a/src/gypsum_client/prepare_directory_for_upload.py +++ b/src/gypsum_client/prepare_directory_for_upload.py @@ -1,3 +1,59 @@ +"""Prepare to upload a directory's contents. + +Files in `directory` (that are not symlinks) are used as +regular uploads, i.e., `files=` in +:py:func:`~gypsum_client.upload_api_operations.start_upload`. + +If `directory` contains a symlink to a file in `cache`, +we assume that it points to a file that was previously downloaded +by, e.g., :py:func:`~gypsum_client.upload_api_operations.save_file` or +:py:func:`~gypsum_client.upload_api_operations.save_version`. +Thus, instead of performing a regular upload, we attempt to +create an upload link, i.e., ``links=`` in +:py:func:`~gypsum_client.upload_api_operations.start_upload`. +This is achieved by examining the destination path of the +symlink and inferring the link destination in the backend. +Note that this still works if the symlinks are dangling. + +If a symlink cannot be converted into an upload link, it will +be used as a regular upload, i.e., the contents of the symlink +destination will be uploaded by +:py:func:`~gypsum_client.upload_api_operations.start_upload`. +In this case, an error will be raised if the symlink is dangling +as there is no file that can actually be uploaded. +If ``links="always"``, an error is raised instead upon symlink +conversion failure. + +This function is intended to be used with +:py:func:`~gypsum_client.clone_operations.clone_version`, +which creates symlinks to files in `cache`. + +See Also: + :py:func:`~gypsum_client.upload_api_operations.start_upload`, + to actually start the upload. + + :py:func:`~gypsum_client.clone_operations.clone_version`, + to prepare the symlinks. + +Example: + + .. code-block:: python + + import tempfile + cache = tempfile.mkdtemp() + dest = tempfile.mkdtemp() + + # Clone a project + clone_version("test-R", "basic", "v1", destination=dest, cache_dir=cache) + + # Make some modification + with open(os.path.join(dest, "heanna"), "w") as f: + f.write("sumire") + + # Prepare the directory for upload + prepped = prepare_directory_upload(dest, cache_dir=cache) +""" + import os from typing import Literal @@ -23,29 +79,6 @@ def prepare_directory_upload( This goes through the directory to list its contents and convert symlinks to upload links. - Files in `directory` (that are not symlinks) are used as - regular uploads, i.e., `files=` in `start_upload`. - - If `directory` contains a symlink to a file in `cache`, - we assume that it points to a file that was previously downloaded - by, e.g., `save_file` or `save_version`. - Thus, instead of performing a regular upload, we attempt to - create an upload link, i.e., `links=` in `start_upload`. - This is achieved by examining the destination path of the - symlink and inferring the link destination in the backend. - Note that this still works if the symlinks are dangling. - - If a symlink cannot be converted into an upload link, it will - be used as a regular upload, i.e., the contents of the symlink - destination will be uploaded by `start_upload`. - In this case, an error will be raised if the symlink is dangling - as there is no file that can actually be uploaded. - If `links="always"`, an error is raised instead upon symlink - conversion failure. - - This function is intended to be used with `clone_version`, - which creates symlinks to files in `cache`. - Args: directory: Path to a directory, the contents of which are to be @@ -55,11 +88,11 @@ def prepare_directory_upload( Indicate how to handle symlinks in `directory`. Must be one of the following: - "auto": Will attempt to convert symlinks into upload links. - If the conversion fails, a regular upload is performed. + If the conversion fails, a regular upload is performed. - "always": Will attempt to convert symlinks into upload links. - If the conversion fails, an error is raised. + If the conversion fails, an error is raised. - "never": Will never attempt to convert symlinks into upload - links. All symlinked files are treated as regular uploads. + links. All symlinked files are treated as regular uploads. cache_dir: Path to the cache directory, used to convert symlinks into upload links. @@ -67,9 +100,10 @@ def prepare_directory_upload( Returns: Dictionary containing: - `files`: list of strings to be used as `files=` - in :py:func:`~gypsum_client.start_upload.start_upload`. + in :py:func:`~gypsum_client.start_upload.start_upload`. - `links`: dictionary to be used as `links=` in - :py:func:`~gypsum_client.start_upload.start_upload`. + :py:func:`~gypsum_client.start_upload.start_upload`. + """ _links_options = ["auto", "always", "never"] if links not in _links_options: @@ -80,14 +114,7 @@ def prepare_directory_upload( cache_dir = _cache_directory(cache_dir) out_files = [] - - out_links = { - "from_path": [], - "to_project": [], - "to_asset": [], - "to_version": [], - "to_path": [], - } + out_links = [] cache_dir = _normalize_and_sanitize_path(cache_dir) if not cache_dir.endswith("/"): @@ -116,11 +143,15 @@ def prepare_directory_upload( dest = _normalize_and_sanitize_path(dest) dest_components = _match_path_to_cache(dest, cache_dir) if dest_components: - out_links["from_path"].append(rel_path) - out_links["to_project"].append(dest_components["project"]) - out_links["to_asset"].append(dest_components["asset"]) - out_links["to_version"].append(dest_components["version"]) - out_links["to_path"].append(dest_components["path"]) + out_links.append( + { + "from.path": rel_path, + "to.project": dest_components["project"], + "to.asset": dest_components["asset"], + "to.version": dest_components["version"], + "to.path": dest_components["path"], + } + ) continue if links == "always": diff --git a/src/gypsum_client/probation_operations.py b/src/gypsum_client/probation_operations.py index 7574418..b1077f5 100644 --- a/src/gypsum_client/probation_operations.py +++ b/src/gypsum_client/probation_operations.py @@ -20,6 +20,31 @@ def approve_probation( This removes the ``on_probation`` tag from the uploaded version. + See Also: + :py:func:`~gypsum_client.upload_api_operations.start_upload`, + to specify probational upload. + + :py:func:`~.reject_probation`, + to reject the probational upload.. + + Example: + + .. code-block:: python + + init = start_upload( + project="test-Py", + asset="probation", + version="v1", + files=[], + probation=True + ) + + complete_upload(init) + approve_probation("test-Py", "probation", "v1") + + # Cleanup if this is just for testing + remove_asset("test-Py", "probation") + Args: project: Project name. @@ -62,6 +87,28 @@ def reject_probation( This removes all files associated with that version. + See Also: + :py:func:`~gypsum_client.upload_api_operations.start_upload`, + to specify probational upload. + + :py:func:`~.approve_probation`, + to approve the probational upload.. + + Example: + + .. code-block:: python + + init = start_upload( + project="test-Py", + asset="probation", + version="v1", + files=[], + probation=True + ) + + complete_upload(init) + reject_probation("test-Py", "probation", "v1") + Args: project: Project name. diff --git a/src/gypsum_client/refresh_operations.py b/src/gypsum_client/refresh_operations.py index 92bd397..6d82e8c 100644 --- a/src/gypsum_client/refresh_operations.py +++ b/src/gypsum_client/refresh_operations.py @@ -22,6 +22,16 @@ def refresh_latest( This is useful on rare occasions where multiple simultaneous uploads cause the latest version to be slightly out of sync. + See Also: + :py:func:`~gypsum_client.fetch_operations.fetch_latest`, + to fetch the latest version without recomputing. + + Example: + + .. code-block:: python + + ver = refresh_latest("test-R", "basic") + Args: project: Project name. @@ -66,6 +76,16 @@ def refresh_usage(project: str, url: str = _rest_url(), token: str = None) -> in This is useful on rare occasions where multiple simultaneous uploads cause the usage calculations to be out of sync. + See Also: + :py:func:`~gypsum_client.fetch_operations.fetch_usage`, + to fetch the usage without recomputing. + + Example: + + .. code-block:: python + + ver = refresh_usage("test-R", "basic") + Args: project: Project name. diff --git a/src/gypsum_client/remove_operations.py b/src/gypsum_client/remove_operations.py index 4a0d78f..4b1c052 100644 --- a/src/gypsum_client/remove_operations.py +++ b/src/gypsum_client/remove_operations.py @@ -13,6 +13,28 @@ def remove_asset(project: str, asset: str, url: str = _rest_url(), token: str = None): """Remove an asset of a project from the gypsum backend. + See Also: + :py:func:`~.remove_project`, + to remove a project. + + :py:func:`~.remove_version`, + to remove a specific version. + + Example: + + .. code-block:: python + + # Mock a project + init = start_upload( + project="test-Py-remove", + asset="mock-remove", + version="v1", + files=[], + ) + + complete_upload(init) + remove_asset("test-Py-remove", "mock-remove") + Args: project: Project name. @@ -25,6 +47,7 @@ def remove_asset(project: str, asset: str, url: str = _rest_url(), token: str = token: GitHub access token to authenticate to the gypsum REST API. + The token must refer to a gypsum administrator account. Returns: True if asset was successfully removed. @@ -42,6 +65,23 @@ def remove_asset(project: str, asset: str, url: str = _rest_url(), token: str = def remove_project(project: str, url: str = _rest_url(), token: str = None): """Remove a project from the gypsum backend. + See Also: + :py:func:`~gypsum_client.create_operations.create_project`, + to create a project. + + :py:func:`~.remove_asset`, + to remove a specific asset. + + :py:func:`~.remove_version`, + to remove a specific version. + + Example: + + .. code-block:: python + + create_project("test-Py-remove", owners=["jkanche"]) + remove_project("test-Py-remove") + Args: project: Project name. @@ -51,6 +91,7 @@ def remove_project(project: str, url: str = _rest_url(), token: str = None): token: GitHub access token to authenticate to the gypsum REST API. + The token must refer to a gypsum administrator account. Returns: True if the project was successfully removed. @@ -72,6 +113,29 @@ def remove_version( token: str = None, ): """Remove a project from the gypsum backend. + + See Also: + :py:func:`~.remove_asset`, + to remove a specific asset. + + :py:func:`~.remove_version`, + to remove a specific version. + + Example: + + .. code-block:: python + + # Mock a project + init = start_upload( + project="test-Py-remove", + asset="mock-remove", + version="v1", + files=[], + ) + + complete_upload(init) + + remove_version("test-Py-remove", "mock-remove", "v1") Args: project: diff --git a/src/gypsum_client/resolve_links.py b/src/gypsum_client/resolve_links.py index c637728..e044746 100644 --- a/src/gypsum_client/resolve_links.py +++ b/src/gypsum_client/resolve_links.py @@ -31,6 +31,17 @@ def resolve_links( are not supported) for linked-from files to their link destinations. + Example: + + .. code-block:: python + + cache = tempfile() + + save_version("test-R", "basic", "v3", relink=False, cache_dir=cache) + list_files(cache_dir, recursive=True, all_files=True) + + resolve_links("test-R", "basic", "v3", cache_dir=cache) + Args: project: Project name. diff --git a/src/gypsum_client/save_operations.py b/src/gypsum_client/save_operations.py index 358b15f..6dd22d9 100644 --- a/src/gypsum_client/save_operations.py +++ b/src/gypsum_client/save_operations.py @@ -44,6 +44,16 @@ def save_version( """Download all files associated with a version of an asset of a project from the gypsum bucket. + See Also: + + :py:func:`~.save_file`, to save a single file. + + Example: + + .. code-block:: python + + out <- save_version("test-R", "basic", "v1") + Args: project: Project name. @@ -203,6 +213,18 @@ def save_file( Download a file from the gypsum bucket, for a version of an asset of a project. + See Also: + + :py:func:`~.save_version`, to save all files associated + with a version. + + Example: + + .. code-block:: python + + out <- save_version("test-R", "basic", "v1", "blah.txt") + + Args: project: Project name. diff --git a/src/gypsum_client/search_metadata.py b/src/gypsum_client/search_metadata.py index d63c2ef..e308edb 100644 --- a/src/gypsum_client/search_metadata.py +++ b/src/gypsum_client/search_metadata.py @@ -19,7 +19,7 @@ :py:func:`~gypsum_client.fetch_metadata_database.fetch_metadata_database`, to download and cache the database files. - See `here `_, + See `metadata index `_, for details on the SQLite file contents and table structure. """ @@ -76,7 +76,7 @@ def search_metadata_text( Perform a text search on a SQLite database containing metadata from the gypsum backend. This is based on a precomputed tokenization of all string properties in each metadata document; - see `here `_ + see `metadata index `_ for details. Examples: @@ -102,10 +102,20 @@ def search_metadata_text( latest=False ) + # or use the ``&`` operation + query = define_text_query("sakugawa") & define_text_query("judgement") + result = search_metadata_text( + sqlite_path, + query, + include_metadata=False, + latest=False + ) + - Search for metadata container either of the keywords (`OR` operation): .. code-block:: python + # use the ``|`` operation query = define_text_query("uiharu") | define_text_query("rank") result = search_metadata_text( sqlite_path, diff --git a/src/gypsum_client/set_operations.py b/src/gypsum_client/set_operations.py index fd00473..ba7974a 100644 --- a/src/gypsum_client/set_operations.py +++ b/src/gypsum_client/set_operations.py @@ -22,6 +22,24 @@ def set_quota( """ Set the quota for a project. + See Also: + :py:func:`~gypsum_client.fetch_operations.fetch_quota`, + to fetch the usage details. + + Example: + + .. code-block:: python + + create_project("test-Py-quota", owners=["jkanche"]) + + set_quota( + "test-Py-quota", + baseline=1234, + growth_rate=5678, + year=2020 + ) + + Args: project: The project name. @@ -84,6 +102,27 @@ def set_permissions( """ Set the owner and uploader permissions for a project. + See Also: + :py:func:`~gypsum_client.fetch_operations.fetch_permission`, + to fetch the permissions for a project. + + Example: + + .. code-block:: python + + create_project("test-Py-perms", owners=["jkanche"]) + + until = ( + (datetime.now() + timedelta(seconds=1000000)) + .replace(microsecond=0) + ) + + set_permissions( + "test-Py-perms", + owners=["LTLA"], + uploaders=[{"id": "LTLA", "until": until}] + ) + Args: project: The project name. diff --git a/src/gypsum_client/upload_api_operations.py b/src/gypsum_client/upload_api_operations.py index 335e6fb..fb2e641 100644 --- a/src/gypsum_client/upload_api_operations.py +++ b/src/gypsum_client/upload_api_operations.py @@ -31,6 +31,50 @@ def start_upload( Start an upload of a new version of an asset, or a new asset of a project. + See Also: + :py:func:`~gypsum_client.upload_file_operations.upload_files`, + to actually upload the files. + + :py:func:`~.complete_upload`, + to indicate that the upload is completed. + + :py:func:`~.abort_upload`, + to abort an upload in progress. + + :py:func:`~gypsum_client.prepare_directory_for_upload.prepare_directory_upload`, + to create ``files`` and ``links`` from a directory. + + Example: + + .. code-block:: python + + import tempfile + tmp_dir = tempfile.mkdtemp() + + with open(f"{tmp_dir}/blah.txt", "w") as f: + f.write(blah_contents) + + os.makedirs(f"{tmp_dir}/foo", exist_ok=True) + + with open(f"{tmp_dir}/foo/blah.txt", "w") as f: + f.write(foobar_contents) + + files = [ + str(file.relative_to(tmp_dir)) + for file in Path(tmp_dir).rglob("*") + if not os.path.isdir(file) + ] + + init = start_upload( + project="test-Py-demo", + asset="upload", + version="1", + files=files, + directory=tmp_dir, + ) + + abort_upload(init) + Args: project: Project name. @@ -58,6 +102,7 @@ def start_upload( whether deduplication should be attempted for each file. If this is not available, the parameter ``deduplicate`` is used. + links: A List containing a dictionary with the following keys: - ``from.path``: a string containing the relative path of the @@ -194,18 +239,53 @@ def start_upload( return resp -def complete_upload(init: dict, url=_rest_url()) -> dict: + +def complete_upload(init: dict, url=_rest_url()): """Complete an upload session after all files have been uploaded. + See Also: + :py:func:`~gypsum_client.upload_api_operations.start_upload`, + to create the init. + + Example: + + .. code-block:: python + + import tempfile + tmp_dir = tempfile.mkdtemp() + + with open(f"{tmp_dir}/blah.txt", "w") as f: + f.write(blah_contents) + + os.makedirs(f"{tmp_dir}/foo", exist_ok=True) + + with open(f"{tmp_dir}/foo/blah.txt", "w") as f: + f.write(foobar_contents) + + files = [ + str(file.relative_to(tmp_dir)) + for file in Path(tmp_dir).rglob("*") + if not os.path.isdir(file) + ] + + init = start_upload( + project="test-Py-demo", + asset="upload", + version="1", + files=files, + directory=tmp_dir, + ) + + abort_upload(init) + Args: init: Dictionary containing ``complete_url`` and ``session_token``. + :py:func:`~.start_upload`, to create ``init``. + url: URL to the gypsum REST API. - - Returns: - True after completion. """ url = _remove_slash_url(url) req = requests.post( @@ -219,20 +299,53 @@ def complete_upload(init: dict, url=_rest_url()) -> dict: f"Failed to complete an upload session, {req.status_code} and reason: {req.text}" ) from e - return True -def abort_upload(init: dict, url=_rest_url()) -> dict: +def abort_upload(init: dict, url=_rest_url()): """Abort an upload session, usually after an irrecoverable error. + See Also: + :py:func:`~gypsum_client.upload_api_operations.start_upload`, + to create the init. + + Example: + + .. code-block:: python + + import tempfile + tmp_dir = tempfile.mkdtemp() + + with open(f"{tmp_dir}/blah.txt", "w") as f: + f.write(blah_contents) + + os.makedirs(f"{tmp_dir}/foo", exist_ok=True) + + with open(f"{tmp_dir}/foo/blah.txt", "w") as f: + f.write(foobar_contents) + + files = [ + str(file.relative_to(tmp_dir)) + for file in Path(tmp_dir).rglob("*") + if not os.path.isdir(file) + ] + + init = start_upload( + project="test-Py-demo", + asset="upload", + version="1", + files=files, + directory=tmp_dir, + ) + + complete_upload(init) + Args: init: Dictionary containing ``abort_url`` and ``session_token``. + :py:func:`~.start_upload`, to create ``init``. + url: URL to the gypsum REST API. - - Returns: - True after completion. """ url = _remove_slash_url(url) req = requests.post( @@ -246,5 +359,3 @@ def abort_upload(init: dict, url=_rest_url()) -> dict: raise Exception( f"Failed to abort the upload, {req.status_code} and reason: {req.text}" ) from e - - return True diff --git a/src/gypsum_client/upload_file_actions.py b/src/gypsum_client/upload_file_actions.py index 944e251..bcd50fa 100644 --- a/src/gypsum_client/upload_file_actions.py +++ b/src/gypsum_client/upload_file_actions.py @@ -4,11 +4,10 @@ import requests from ._utils import _cache_directory, _remove_slash_url, _rest_url -from .upload_api_operations import abort_upload, complete_upload from .auth import access_token from .config import REQUESTS_MOD from .prepare_directory_for_upload import prepare_directory_upload -from .upload_api_operations import start_upload +from .upload_api_operations import abort_upload, complete_upload, start_upload __author__ = "Jayaram Kancherla" __copyright__ = "Jayaram Kancherla" @@ -31,8 +30,8 @@ def upload_directory( """Upload a directory to the gypsum backend. This function is a wrapper around - :py:func:`~gypsum_client.prepare_directory_upload.prepare_directory_upload` - and :py:func:`~gypsum_client.start_upload.start_upload` and friends. + :py:func:`~gypsum_client.prepare_directory_for_upload.prepare_directory_upload` + and :py:func:`~gypsum_client.upload_api_operations.start_upload` and others. The aim is to streamline the upload of a directory's contents when no customization of the file listing is required. @@ -41,6 +40,24 @@ def upload_directory( as a versioned asset of a project. This requires uploader permissions to the relevant project. + Example: + + .. code-block:: python + + tmp_dir = tempfile.mkdtemp() + + with open(os.path.join(tmp, "blah.txt"), "w") as f: + f.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + os.makedirs(os.path.join(tmp, "foo")) + with open(os.path.join(tmp, "foo", "bar.txt"), "w") as f: + f.write("\n".join(map(str, range(1, 11)))) + + upload_directory( + tmp, "test-Py", + "upload-dir", version="1" + ) + Args: directory: Path to a directory containing the ``files`` to be uploaded. @@ -57,10 +74,10 @@ def upload_directory( cache_dir: Path to the cache for saving files, e.g., in - :py:func:`~gypsum_client.save_assets.save_version`. + :py:func:`~gypsum_client.save_operations.save_version`. Used to convert symbolic links to upload links,see - :py:func:`~gypsum_client.prepare_directory_upload.prepare_directory_upload`. + :py:func:`~gypsum_client.prepare_directory_for_upload.prepare_directory_upload`. deduplicate: Whether the backend should attempt deduplication of ``files`` diff --git a/src/gypsum_client/validate_metadata.py b/src/gypsum_client/validate_metadata.py index 33c674a..79a2bf4 100644 --- a/src/gypsum_client/validate_metadata.py +++ b/src/gypsum_client/validate_metadata.py @@ -13,6 +13,33 @@ def validate_metadata( ) -> bool: """Validate metadata against a JSON schema for a SQLite database. + See Also: + :py:func:`~gypsum_client.fetch_metadata_schema.fetch_metadata_schema`, to get + the JSON schema. + + :py:func:`~gypsum_client.fetch_metadata_database.fetch_metadata_database`, + to obtain the SQLite database of metadata. + + Example: + + .. code-block:: python + + _cache_dir = tempfile.mkdtemp() + + metadata = { + "title": "Fatherhood", + "description": "Luke ich bin dein Vater.", + "sources": [{"provider": "GEO", "id": "GSE12345"}], + "taxonomy_id": ["9606"], + "genome": ["GRCm38"], + "maintainer_name": "Darth Vader", + "maintainer_email": "vader@empire.gov", + "bioconductor_version": "3.10", + } + + schema = fetch_metadata_schema(cache_dir=_cache_dir) + validate_metadata(metadata, schema) + Args: metadata: Metadata to be checked. diff --git a/tests/test_latest.py b/tests/test_latest.py index 6c9ddb7..7a0007c 100644 --- a/tests/test_latest.py +++ b/tests/test_latest.py @@ -18,6 +18,6 @@ def test_refresh_latest(): if gh_token is None: raise ValueError("GitHub token not in environment") - ver = refresh_latest("test-Py", "upload", url=app_url, token=gh_token) - assert ver == "1" - assert fetch_latest("test-Py", "upload", url=app_url) == ver + ver = refresh_latest("test-R", "basic", url=app_url, token=gh_token) + assert ver == "v3" + assert fetch_latest("test-R", "basic", url=app_url) == ver diff --git a/tests/test_prepare_dir_upload.py b/tests/test_prepare_dir_upload.py index fcafca7..20effbd 100644 --- a/tests/test_prepare_dir_upload.py +++ b/tests/test_prepare_dir_upload.py @@ -20,18 +20,26 @@ def test_prepare_directory_upload_works_as_expected(): prepped = prepare_directory_upload(dest, cache_dir=cache) if os.name == "nt": # Windows assert prepped["files"] == ["blah.txt", "foo/bar.txt", "heanna"] - assert len(prepped["links"]["from_path"]) == 0 + assert len(prepped["links"]) == 0 else: # Unix assert prepped["files"] == ["heanna"] - assert prepped["links"]["from_path"] == ["blah.txt", "foo/bar.txt"] - assert prepped["links"]["to_project"] == ["test-R", "test-R"] - assert prepped["links"]["to_asset"] == ["basic", "basic"] - assert prepped["links"]["to_version"] == ["v1", "v1"] - assert prepped["links"]["to_path"] == ["blah.txt", "foo/bar.txt"] + assert sorted(x["from.path"] for x in prepped["links"]) == sorted( + ["blah.txt", "foo/bar.txt"] + ) + assert sorted(x["to.project"] for x in prepped["links"]) == sorted( + ["test-R", "test-R"] + ) + assert sorted(x["to.asset"] for x in prepped["links"]) == sorted( + ["basic", "basic"] + ) + assert sorted(x["to.version"] for x in prepped["links"]) == sorted(["v1", "v1"]) + assert sorted(x["to.path"] for x in prepped["links"]) == sorted( + ["blah.txt", "foo/bar.txt"] + ) prepped = prepare_directory_upload(dest, cache_dir=cache, links="never") assert sorted(prepped["files"]) == sorted(["blah.txt", "foo/bar.txt", "heanna"]) - assert len(prepped["links"]["from_path"]) == 0 + assert len(prepped["links"]) == 0 shutil.rmtree(cache) shutil.rmtree(dest) @@ -63,7 +71,9 @@ def test_prepare_directory_upload_with_some_odd_things(): prepped = prepare_directory_upload(dest, cache_dir=cache) assert sorted(prepped["files"]) == sorted(["..check", "arkansas"]) - assert sorted(prepped["links"]["from_path"]) == sorted(["blah.txt", "foo/bar.txt"]) + assert sorted(x["from.path"] for x in prepped["links"]) == sorted( + ["blah.txt", "foo/bar.txt"] + ) shutil.rmtree(cache) shutil.rmtree(dest) @@ -79,7 +89,7 @@ def test_prepare_directory_upload_handles_dangling_links_correctly(): prepped = prepare_directory_upload(dest, cache_dir=cache) assert prepped["files"] == [] - assert len(prepped["links"]["from_path"]) == 2 + assert len(prepped["links"]) == 2 with pytest.raises(Exception): prepare_directory_upload(dest, cache_dir=cache, links="never") diff --git a/tests/test_upload.py b/tests/test_upload.py index 318261e..6c7aea7 100644 --- a/tests/test_upload.py +++ b/tests/test_upload.py @@ -1,13 +1,18 @@ +import hashlib import os +import shutil import tempfile from pathlib import Path import pytest from gypsum_client import ( + abort_upload, complete_upload, fetch_manifest, + list_assets, remove_asset, start_upload, + upload_directory, upload_files, ) @@ -69,3 +74,123 @@ def test_upload_regular(): man = fetch_manifest("test-Py", "upload", "1", cache_dir=None, url=app_url) assert sorted(man.keys()) == ["blah.txt", "foo/blah.txt"] assert all(man[file].get("link") is None for file in man.keys()) + + +@pytest.mark.skipif( + "gh_token" not in os.environ, reason="GitHub token not in environment" +) +def test_upload_sequence_links(): + gh_token = os.environ.get("gh_token", None) + if gh_token is None: + raise ValueError("GitHub token not in environment") + + remove_asset("test-Py", asset="upload", token=gh_token, url=app_url) + + tmp = tempfile.mkdtemp() + try: + fpath = os.path.join(tmp, "test.json") + with open(fpath, "w") as f: + f.write('[ "Aaron" ]') + + link_df = [ + { + "from.path": "whee/stuff.txt", + "to.project": "test-R", + "to.asset": "basic", + "to.version": "v1", + "to.path": "blah.txt", + }, + { + "from.path": "michaela", + "to.project": "test-R", + "to.asset": "basic", + "to.version": "v1", + "to.path": "foo/bar.txt", + }, + ] + + init = start_upload( + project="test-Py", + asset="upload", + version="1", + files=[ + { + "path": "test.json", + "size": os.path.getsize(fpath), + "md5sum": hashlib.md5(open(fpath, "rb").read()).hexdigest(), + } + ], + links=link_df, + directory=tmp, + token=gh_token, + url=app_url, + ) + + assert len(init["file_urls"]) == 1 + upload_files(init, directory=tmp, url=app_url) + complete_upload(init, url=app_url) + + man = fetch_manifest("test-Py", "upload", "1", cache_dir=None, url=app_url) + + assert sorted(man.keys()) == ["michaela", "test.json", "whee/stuff.txt"] + assert man["michaela"]["link"] is not None + assert man["whee/stuff.txt"]["link"] is not None + assert "link" not in man["test.json"] + finally: + shutil.rmtree(tmp) + + +@pytest.mark.skipif( + "gh_token" not in os.environ, reason="GitHub token not in environment" +) +def test_aborting_upload(): + gh_token = os.environ.get("gh_token", None) + if gh_token is None: + raise ValueError("GitHub token not in environment") + + remove_asset("test-Py", asset="upload", token=gh_token, url=app_url) + + init = start_upload( + project="test-Py", + asset="upload", + version="1", + files=[], + token=gh_token, + url=app_url, + ) + + assert len(init["file_urls"]) == 0 + assert "upload" in list_assets("test-Py", url=app_url) + + abort_upload(init, url=app_url) + assert "upload" not in list_assets("test-Py", url=app_url) + + +@pytest.mark.skipif( + "gh_token" not in os.environ, reason="GitHub token not in environment" +) +def test_upload_directory(): + gh_token = os.environ.get("gh_token", None) + if gh_token is None: + raise ValueError("GitHub token not in environment") + + remove_asset("test-Py", asset="upload-dir", token=gh_token, url=app_url) + + tmp = tempfile.mkdtemp() + try: + with open(os.path.join(tmp, "blah.txt"), "w") as f: + f.write("ABCDEFGHIJKLMNOPQRSTUVWXYZ") + + os.makedirs(os.path.join(tmp, "foo")) + with open(os.path.join(tmp, "foo", "bar.txt"), "w") as f: + f.write("\n".join(map(str, range(1, 11)))) + + upload_directory( + tmp, "test-Py", "upload-dir", version="1", token=gh_token, url=app_url + ) + + man = fetch_manifest("test-Py", "upload-dir", "1", cache_dir=None, url=app_url) + assert sorted(man.keys()) == ["blah.txt", "foo/bar.txt"] + assert all("link" not in x for x in man.values()) + finally: + shutil.rmtree(tmp)