From ba295467694f960322cea7b5e25c548b6c857975 Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Tue, 30 Apr 2024 23:50:35 +0200 Subject: [PATCH] allows fsspec until 2023.1.0 --- dlt/common/storages/fsspec_filesystem.py | 12 +++++------- tests/common/storages/test_local_filesystem.py | 2 +- tests/common/storages/utils.py | 2 +- 3 files changed, 7 insertions(+), 9 deletions(-) diff --git a/dlt/common/storages/fsspec_filesystem.py b/dlt/common/storages/fsspec_filesystem.py index 0c108e67d4..a21f0f2c0c 100644 --- a/dlt/common/storages/fsspec_filesystem.py +++ b/dlt/common/storages/fsspec_filesystem.py @@ -199,7 +199,7 @@ def open( # noqa: A003 mode: str = "rb", compression: Literal["auto", "disable", "enable"] = "auto", **kwargs: Any, - ) -> Union[GzipFile, IO[Any]]: + ) -> IO[Any]: """Open the file as a fsspec file. This method opens the file represented by this dictionary as a file-like object using @@ -226,7 +226,6 @@ def open( # noqa: A003 raise ValueError("""The argument `compression` must have one of the following values: "auto", "enable", "disable".""") - opened_file: Union[IO[Any], GzipFile] # if the user has already extracted the content, we use it so there is no need to # download the file again. if "file_content" in self: @@ -247,15 +246,14 @@ def open( # noqa: A003 **text_kwargs, ) else: - if "local" in self.fsspec.protocol: + if "file" in self.fsspec.protocol: # use native local file path to open file:// uris file_url = self.local_file_path else: file_url = self["file_url"] - opened_file = self.fsspec.open( + return self.fsspec.open( # type: ignore[no-any-return] file_url, mode=mode, compression=compression_arg, **kwargs ) - return opened_file def read_bytes(self) -> bytes: """Read the file content. @@ -267,7 +265,7 @@ def read_bytes(self) -> bytes: return self["file_content"] # type: ignore else: with self.open(mode="rb", compression="disable") as f: - return f.read() + return f.read() # type: ignore[no-any-return] def guess_mime_type(file_name: str) -> Sequence[str]: @@ -292,7 +290,7 @@ def glob_files( Returns: Iterable[FileItem]: The list of files. """ - is_local_fs = "local" in fs_client.protocol + is_local_fs = "file" in fs_client.protocol if is_local_fs and FilesystemConfiguration.is_local_path(bucket_url): bucket_url = FilesystemConfiguration.make_file_uri(bucket_url) bucket_url_parsed = urlparse(bucket_url) diff --git a/tests/common/storages/test_local_filesystem.py b/tests/common/storages/test_local_filesystem.py index 6bcf8234e6..7b16fa8d57 100644 --- a/tests/common/storages/test_local_filesystem.py +++ b/tests/common/storages/test_local_filesystem.py @@ -223,7 +223,7 @@ def test_filesystem_decompress() -> None: with file_dict.open(mode="tr") as f: lines = f.readlines() assert len(lines) > 1 - assert lines[0].startswith('"1200864931","2015-07-01 00:00:13"') # type: ignore + assert lines[0].startswith('"1200864931","2015-07-01 00:00:13"') # read as uncompressed binary with file_dict.open(compression="enable") as f: assert f.read().startswith(b'"1200864931","2015-07-01 00:00:13"') diff --git a/tests/common/storages/utils.py b/tests/common/storages/utils.py index 0737136622..3bfc3374a4 100644 --- a/tests/common/storages/utils.py +++ b/tests/common/storages/utils.py @@ -113,7 +113,7 @@ def assert_sample_files( # fieldnames below are not really correct but allow to load first 3 columns # even if first row does not have header names - elements = list(DictReader(f, fieldnames=["A", "B", "C"])) # type: ignore + elements = list(DictReader(f, fieldnames=["A", "B", "C"])) assert len(elements) > 0 if item["mime_type"] == "application/parquet": # verify it is a real parquet