Skip to content

Commit

Permalink
simplify the implementation a bit
Browse files Browse the repository at this point in the history
  • Loading branch information
isidentical committed Jan 28, 2021
1 parent 4e0b803 commit 82ab6c7
Show file tree
Hide file tree
Showing 3 changed files with 19 additions and 40 deletions.
23 changes: 4 additions & 19 deletions dvc/cache/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -302,22 +302,15 @@ def create_tasks(executor, amount):
yield rel_path_infos.pop(task), task.result()

def _transfer_directory(
self, from_tree, from_info, out_info, jobs, no_progress_bar=False
self, from_tree, from_info, jobs, no_progress_bar=False
):
if out_info:
initials = out_info.parts[-1:]
else:
initials = ()

strip_first = bool(initials)
dir_info = DirInfo()

with Tqdm(total=1, unit="Files", disable=no_progress_bar) as pbar:
for entry_info, entry_hash in self._transfer_directory_contents(
from_tree, from_info, jobs, pbar
):
dir_info.trie[
initials + entry_info.parts[strip_first:]
] = entry_hash
dir_info.trie[entry_info.parts] = entry_hash

local_cache = self.repo.cache.local
(
Expand All @@ -330,21 +323,13 @@ def _transfer_directory(
self.tree.upload(to_info, self.tree.hash_to_path_info(hash_info.value))
return hash_info

def transfer(
self,
from_tree,
from_info,
out_info=None,
jobs=None,
no_progress_bar=False,
):
def transfer(self, from_tree, from_info, jobs=None, no_progress_bar=False):
jobs = jobs or min((from_tree.jobs, self.tree.jobs))

if from_tree.isdir(from_info):
return self._transfer_directory(
from_tree,
from_info,
out_info=out_info,
jobs=jobs,
no_progress_bar=no_progress_bar,
)
Expand Down
18 changes: 6 additions & 12 deletions dvc/repo/add.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,12 +153,9 @@ def _process_stages(

if to_remote or to_cache:
# Already verified in the add()
assert len(stages) == 1
assert len(sub_targets) == 1
[stage], [target] = stages, sub_targets

assert len(stage.outs) == 1
[out] = stage.outs
(stage,) = stages
(target,) = sub_targets
(out,) = stage.outs

if to_remote:
out.hash_info = repo.cloud.transfer(
Expand All @@ -171,13 +168,10 @@ def _process_stages(
from dvc.tree import get_cloud_tree

from_tree = get_cloud_tree(repo, url=target)
out.hash_info = repo.cache.local.transfer(
from_tree,
from_tree.path_info,
out_info=out.path_info,
jobs=kwargs.get("jobs"),
out.hash_info = out.cache.transfer(
from_tree, from_tree.path_info, jobs=kwargs.get("jobs"),
)
repo.cache.local.checkout(out.path_info, out.hash_info)
out.checkout()

Dvcfile(repo, stage.path).dump(stage)
return link_failures
Expand Down
18 changes: 9 additions & 9 deletions tests/func/test_add.py
Original file line number Diff line number Diff line change
Expand Up @@ -1027,18 +1027,18 @@ def test_add_to_remote_invalid_combinations(dvc, invalid_opt, kwargs):
def test_add_to_cache_dir(tmp_dir, dvc, local_cloud):
local_cloud.gen({"data": {"foo": "foo", "bar": "bar"}})

[stage] = dvc.add(local_cloud.url, out="data")
[stage] = dvc.add(local_cloud.url + "/data", out="data")
assert len(stage.deps) == 0
assert len(stage.outs) == 1

data = tmp_dir / "data"
assert data.read_text() == {"data": {"foo": "foo", "bar": "bar"}}
assert data.read_text() == {"foo": "foo", "bar": "bar"}
assert (tmp_dir / "data.dvc").exists()

shutil.rmtree(data)
status = dvc.checkout(str(data))
assert status["added"] == ["data" + os.sep]
assert data.read_text() == {"data": {"foo": "foo", "bar": "bar"}}
assert data.read_text() == {"foo": "foo", "bar": "bar"}


def test_add_to_cache_file(tmp_dir, dvc, local_cloud):
Expand All @@ -1061,18 +1061,18 @@ def test_add_to_cache_file(tmp_dir, dvc, local_cloud):
def test_add_to_cache_different_name(tmp_dir, dvc, local_cloud):
local_cloud.gen({"data": {"foo": "foo", "bar": "bar"}})

dvc.add(local_cloud.url, out="not_data")
dvc.add(local_cloud.url + "/data", out="not_data")

not_data = tmp_dir / "not_data"
assert not_data.read_text() == {"not_data": {"foo": "foo", "bar": "bar"}}
assert not_data.read_text() == {"foo": "foo", "bar": "bar"}
assert (tmp_dir / "not_data.dvc").exists()

assert not (tmp_dir / "data").exists()
assert not (tmp_dir / "data.dvc").exists()

shutil.rmtree(not_data)
dvc.checkout(str(not_data))
assert not_data.read_text() == {"not_data": {"foo": "foo", "bar": "bar"}}
assert not_data.read_text() == {"foo": "foo", "bar": "bar"}
assert not (tmp_dir / "data").exists()


Expand All @@ -1081,12 +1081,12 @@ def test_add_to_cache_not_exists(tmp_dir, dvc, local_cloud):

dest_dir = tmp_dir / "dir" / "that" / "does" / "not" / "exist"
with pytest.raises(StagePathNotFoundError):
dvc.add(local_cloud.url, out=str(dest_dir))
dvc.add(local_cloud.url + "/data", out=str(dest_dir))

dest_dir.parent.mkdir(parents=True)
dvc.add(local_cloud.url, out=str(dest_dir))
dvc.add(local_cloud.url + "/data", out=str(dest_dir))

assert dest_dir.read_text() == {"exist": {"foo": "foo", "bar": "bar"}}
assert dest_dir.read_text() == {"foo": "foo", "bar": "bar"}
assert dest_dir.with_suffix(".dvc").exists()


Expand Down

0 comments on commit 82ab6c7

Please sign in to comment.