Major rework

.github/workflows/check.yml

@@ -2,7 +2,9 @@ name: check
 
 on:
   push:
+    branches: [main]
   pull_request:
+    branches: [main]
 
 jobs:
   tox:
@@ -16,7 +18,6 @@ jobs:
     steps:
       - uses: actions/checkout@v3
       - name: Set up system
-        shell: bash
         run: |
           sudo apt-get update -qq
           sudo apt-get install git-annex

.github/workflows/docbuild.yml

@@ -1,6 +1,10 @@
 name: docs
 
-on: [push, pull_request]
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
 
 jobs:
   build:

.gitignore

@@ -1,13 +1,14 @@
-.pybuild/
-.coverage
-/.tox
 *.egg-info
+*.grib
 *.py[coe]
 .#*
 .*.swp
-pip-wheel-metadata
-docs/build
-docs/source/generated
+.coverage
+.hypothesis
+.pybuild/
+/.tox
 build/
 dist/
-*.grib
+docs/build
+docs/source/generated
+pip-wheel-metadata

MANIFEST.in

@@ -1,4 +1,4 @@
-include CONTRIBUTORS LICENSE versioneer.py
+include CONTRIBUTORS LICENSE
 graft _datalad_buildsupport
 graft src/datalad_cds
 graft tests

docs/source/cli_reference.rst

@@ -0,0 +1,7 @@
+Command line reference
+======================
+
+.. toctree::
+   :maxdepth: 1
+
+   generated/man/datalad-download-cds

docs/source/conf.py

@@ -117,7 +117,7 @@
 todo_include_todos = True
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {"https://docs.python.org/": None}
+intersphinx_mapping = {"python": ("https://docs.python.org/3", None)}
 
 # -- Options for HTML output ----------------------------------------------
 

docs/source/index.rst

@@ -11,20 +11,19 @@ API
 High-level API commands
 -----------------------
 
-.. currentmodule:: datalad.api
-.. autosummary::
-   :toctree: generated
+.. toctree::
+   :maxdepth: 2
 
-   download_cds
+   python_reference.rst
 
 
 Command line reference
 ----------------------
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
-   generated/man/datalad-download-cds
+   cli_reference.rst
 
 
 Indices and tables

docs/source/python_reference.rst

@@ -0,0 +1,8 @@
+High-level API commands
+=======================
+
+.. currentmodule:: datalad.api
+.. autosummary::
+   :toctree: generated
+
+   download_cds

pyproject.toml

@@ -29,14 +29,12 @@ dependencies = [
 
 [project.optional-dependencies]
 devel = [
-  "black",
   "coverage",
-  "flake8",
-  "flake8-bugbear",
-  "isort",
+  "hypothesis",
   "mypy",
   "pytest",
   "pytest-cov",
+  "ruff",
   "sphinx",
   "sphinx_rtd_theme",
 ]
@@ -45,7 +43,7 @@ devel = [
 repository = "https://github.com/m.risse/datalad-cds"
 
 [project.scripts]
-git-annex-remote-cdsrequest = "datalad_cds.cdsrequest:main"
+git-annex-remote-cds = "datalad_cds.cds_remote:main"
 
 [project.entry-points."datalad.extensions"]
 download-cds = "datalad_cds:command_suite"
@@ -65,52 +63,25 @@ tag_prefix = "v"
 show_missing = true
 omit = ["src/datalad_cds/_version.py"]
 
-[tool.black]
+[tool.ruff]
 line-length = 88
-extend-exclude = '''
-(
-  ^/_datalad_buildsupport
-  | ^/versioneer.py
-  | ^/src/datalad_cds/_version.py
-)
-'''
-
-[tool.isort]
-profile = "black"
-skip_gitignore = true
-extend_skip = [
+extend-exclude = [
   "_datalad_buildsupport/",
-  "versioneer.py",
   "src/datalad_cds/_version.py",
+  "versioneer.py",
 ]
 
+[tool.ruff.lint]
+extend-select = ["I"]
+
 [tool.mypy]
 ignore_missing_imports = true
 disallow_untyped_calls = true
 disallow_untyped_defs = true
 disallow_incomplete_defs = true
 exclude = [
-  '.tox/',
-  'build/',
-  '_datalad_buildsupport/',
-  'datalad_cds/_version.py',
-  'versioneer.py',
-]
-
-[[tool.mypy.overrides]]
-module = [
-  "setup",
-  "datalad_cds",
-]
-disallow_untyped_calls = false
-
-[[tool.mypy.overrides]]
-module = [
-  "datalad_cds._version",
-  "versioneer",
-  "_datalad_buildsupport.*",
+  ".tox/",
+  "_datalad_buildsupport/",
+  "build/",
+  "venv/",
 ]
-follow_imports = "skip"
-
-# [tool.flake8]
-# flake8 still does not support pyproject.toml, so it's configured in .flake8

src/datalad_cds/__init__.py

@@ -16,7 +16,7 @@
         # specification of a command, any number of commands can be defined
         (
             # importable module that contains the command implementation
-            "datalad_cds.downloadcds",
+            "datalad_cds.download_cds",
             # name of the command class implementation in above module
             "DownloadCDS",
             # optional name of the command in the cmdline API

src/datalad_cds/cds_remote.py

@@ -0,0 +1,76 @@
+import inspect
+import logging
+
+import cdsapi
+from annexremote import Master, RemoteError, SpecialRemote
+
+import datalad_cds.spec
+
+logger = logging.getLogger("datalad.cds.cds_remote")
+
+CDS_REMOTE_UUID = "923e2755-e747-42f4-890a-9c921068fb82"
+
+
+class HandleUrlError(Exception):
+    pass
+
+
+class CDSRemote(SpecialRemote):
+    transfer_store = None
+    remove = None
+
+    def initremote(self) -> None:
+        pass
+
+    def prepare(self) -> None:
+        pass
+
+    def _retrieve_cds(self, spec: datalad_cds.spec.Spec, filename: str) -> None:
+        logger.debug(
+            "%s called with spec %s and filename %s",
+            inspect.stack()[0][3],
+            spec,
+            filename,
+        )
+        c = cdsapi.Client()
+        c.retrieve(spec.name, spec.request, filename)
+
+    def transfer_retrieve(self, key: str, filename: str) -> None:
+        logger.debug(
+            "%s called with key %s and filename %s",
+            inspect.stack()[0][3],
+            key,
+            filename,
+        )
+        urls = self.annex.geturls(key, "datalad-cds:")
+        logger.debug("urls for this key: %s", urls)
+        for url in urls:
+            try:
+                self._retrieve_cds(datalad_cds.spec.Spec.from_url(url), filename)
+                break
+            except:  # noqa: E722
+                pass
+        else:
+            raise RemoteError("Failed to handle key {}".format(key))
+
+    def checkpresent(self, key: str) -> bool:
+        # We just assume that we can always handle the key
+        return True
+
+    def claimurl(self, url: str) -> bool:
+        return url.startswith("datalad-cds:")
+
+    def checkurl(self, url: str) -> bool:
+        return url.startswith("datalad-cds:")
+
+
+def main() -> None:
+    master = Master()
+    remote = CDSRemote(master)
+    master.LinkRemote(remote)
+    logger.addHandler(master.LoggingHandler())
+    master.Listen()
+
+
+if __name__ == "__main__":
+    main()

src/datalad_cds/compat.py

@@ -0,0 +1,10 @@
+import sys
+
+
+def removeprefix(s: str, prefix: str) -> str:
+    if sys.version_info >= (3, 9):  # pragma: py-lt-39
+        return s.removeprefix(prefix)
+    else:  # pragma: py-gte-39
+        if s.startswith(prefix):
+            return s[len(prefix) :]
+        return s