Merge pull request #1305 from dlt-hub/rfix/fixes-fsspec-version

allows fsspec until 2023.1.0

dlt/common/storages/fsspec_filesystem.py

@@ -199,7 +199,7 @@ def open(  # noqa: A003
         mode: str = "rb",
         compression: Literal["auto", "disable", "enable"] = "auto",
         **kwargs: Any,
-    ) -> Union[GzipFile, IO[Any]]:
+    ) -> IO[Any]:
         """Open the file as a fsspec file.
 
         This method opens the file represented by this dictionary as a file-like object using
@@ -226,7 +226,6 @@ def open(  # noqa: A003
             raise ValueError("""The argument `compression` must have one of the following values:
                 "auto", "enable", "disable".""")
 
-        opened_file: Union[IO[Any], GzipFile]
         # if the user has already extracted the content, we use it so there is no need to
         # download the file again.
         if "file_content" in self:
@@ -247,15 +246,14 @@ def open(  # noqa: A003
                 **text_kwargs,
             )
         else:
-            if "local" in self.fsspec.protocol:
+            if "file" in self.fsspec.protocol:
                 # use native local file path to open file:// uris
                 file_url = self.local_file_path
             else:
                 file_url = self["file_url"]
-            opened_file = self.fsspec.open(
+            return self.fsspec.open(  # type: ignore[no-any-return]
                 file_url, mode=mode, compression=compression_arg, **kwargs
             )
-        return opened_file
 
     def read_bytes(self) -> bytes:
         """Read the file content.
@@ -267,7 +265,7 @@ def read_bytes(self) -> bytes:
             return self["file_content"]  # type: ignore
         else:
             with self.open(mode="rb", compression="disable") as f:
-                return f.read()
+                return f.read()  # type: ignore[no-any-return]
 
 
 def guess_mime_type(file_name: str) -> Sequence[str]:
@@ -292,7 +290,7 @@ def glob_files(
     Returns:
         Iterable[FileItem]: The list of files.
     """
-    is_local_fs = "local" in fs_client.protocol
+    is_local_fs = "file" in fs_client.protocol
     if is_local_fs and FilesystemConfiguration.is_local_path(bucket_url):
         bucket_url = FilesystemConfiguration.make_file_uri(bucket_url)
         bucket_url_parsed = urlparse(bucket_url)

tests/common/storages/test_local_filesystem.py

@@ -223,7 +223,7 @@ def test_filesystem_decompress() -> None:
         with file_dict.open(mode="tr") as f:
             lines = f.readlines()
             assert len(lines) > 1
-            assert lines[0].startswith('"1200864931","2015-07-01 00:00:13"')  # type: ignore
+            assert lines[0].startswith('"1200864931","2015-07-01 00:00:13"')
         # read as uncompressed binary
         with file_dict.open(compression="enable") as f:
             assert f.read().startswith(b'"1200864931","2015-07-01 00:00:13"')

tests/common/storages/utils.py

@@ -113,7 +113,7 @@ def assert_sample_files(
 
                 # fieldnames below are not really correct but allow to load first 3 columns
                 # even if first row does not have header names
-                elements = list(DictReader(f, fieldnames=["A", "B", "C"]))  # type: ignore
+                elements = list(DictReader(f, fieldnames=["A", "B", "C"]))
                 assert len(elements) > 0
         if item["mime_type"] == "application/parquet":
             # verify it is a real parquet