Skip to content

Commit

Permalink
Increase DBFS copy buffer size (#185)
Browse files Browse the repository at this point in the history
Fix #131
  • Loading branch information
nfx authored Jun 19, 2023
1 parent 452c5c8 commit 990ab59
Showing 1 changed file with 3 additions and 5 deletions.
8 changes: 3 additions & 5 deletions databricks/sdk/mixins/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -316,7 +316,7 @@ def open(self, path: str, *, read: bool = False, write: bool = False, overwrite:

def upload(self, path: str, src: BinaryIO, *, overwrite: bool = False):
with self.open(path, write=True, overwrite=overwrite) as dst:
shutil.copyfileobj(src, dst)
shutil.copyfileobj(src, dst, length=_DbfsIO.MAX_CHUNK_SIZE)

def download(self, path: str) -> BinaryIO:
return self.open(path, read=True)
Expand Down Expand Up @@ -370,14 +370,12 @@ def copy(self, src: str, dst: str, *, recursive=False, overwrite=False):
# copy single file
with src.open(read=True) as reader:
with dst.open(write=True, overwrite=overwrite) as writer:
for chunk in reader:
writer.write(chunk)
shutil.copyfileobj(reader, writer, length=_DbfsIO.MAX_CHUNK_SIZE)
return
# iterate through files
for child, reader in src.list_opened_handles(recursive=recursive):
with dst.child(child).open(write=True, overwrite=overwrite) as writer:
for chunk in reader:
writer.write(chunk)
shutil.copyfileobj(reader, writer, length=_DbfsIO.MAX_CHUNK_SIZE)

def move_(self, src: str, dst: str, *, recursive=False, overwrite=False):
source = self._path(src)
Expand Down

0 comments on commit 990ab59

Please sign in to comment.