Skip to content

Commit

Permalink
Remove usage of features unimplemented in CDS beta
Browse files Browse the repository at this point in the history
  • Loading branch information
matrss committed Aug 19, 2024
1 parent c200591 commit aba20f1
Show file tree
Hide file tree
Showing 3 changed files with 5 additions and 36 deletions.
37 changes: 2 additions & 35 deletions src/datalad_cds/cds_remote.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import pathlib
import subprocess
import time

import cdsapi
from annexremote import Master, ProtocolError, RemoteError, SpecialRemote
Expand Down Expand Up @@ -39,44 +38,12 @@ def _is_dry_run(self) -> bool:
)
return result.returncode == 0 and result.stdout.strip() == "true"

def _info(self, message: str) -> None:
try:
self.annex.info(message)
except ProtocolError:
pass

def _retrieve_cds(self, spec: datalad_cds.spec.Spec, filename: str) -> None:
if self._is_dry_run():
pathlib.Path(filename).write_text(spec.to_json())
return
c = cdsapi.Client(quiet=True, wait_until_complete=False)
r = c.retrieve(spec.dataset, spec.sub_selection)
self._info("CDS request is submitted")
previous_state = "submitted"
sleep_duration = 1.0
sleep_max = 60.0
while True:
r.update()
current_state = r.reply["state"]
if current_state != previous_state:
self._info("CDS request is {}".format(current_state))
if current_state == "completed":
break
if current_state == "failed":
error = r.reply["error"]
raise RemoteError(
"CDS request failed, message: {}, reason: {}".format(
error.get("message"), error.get("reason")
)
)
# Exponential backoff
sleep_duration *= 1.5
if sleep_duration > sleep_max:
sleep_duration = sleep_max
previous_state = current_state
time.sleep(sleep_duration)
self._info("Starting download from CDS")
r.download(filename)
c = cdsapi.Client()
c.retrieve(spec.dataset, spec.sub_selection, filename)

def transfer_retrieve(self, key: str, filename: str) -> None:
urls = self.annex.geturls(key, "cds:")
Expand Down
3 changes: 2 additions & 1 deletion tests/test_download_cds.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,10 @@
from typing import Union

import datalad.api as da
import datalad_cds
import pytest

import datalad_cds

request_dict = {
"dataset": "reanalysis-era5-pressure-levels",
"sub-selection": {
Expand Down
1 change: 1 addition & 0 deletions tests/test_spec.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import hypothesis as h
import pytest

from datalad_cds.spec import Spec


Expand Down

0 comments on commit aba20f1

Please sign in to comment.