fix httpfile serialization bug (#973)

* fix httpfile serialization bug * Fix pickle of in-context local file Co-authored-by: Martin Durant <[email protected]>
fsspec · May 30, 2022 · 06c4003 · 06c4003
1 parent 73de504
commit 06c4003
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 2 deletions.
diff --git a/fsspec/implementations/http.py b/fsspec/implementations/http.py
@@ -642,7 +642,7 @@ def __reduce__(self):
                 self.url,
                 self.mode,
                 self.blocksize,
-                self.cache.name,
+                self.cache.name if self.cache else "none",
                 self.size,
             ),
         )

diff --git a/fsspec/implementations/local.py b/fsspec/implementations/local.py
@@ -354,7 +354,7 @@ def __getattr__(self, item):
 
     def __enter__(self):
         self._incontext = True
-        return self.f.__enter__()
+        return self
 
     def __exit__(self, exc_type, exc_value, traceback):
         self._incontext = False

diff --git a/fsspec/implementations/tests/test_http.py b/fsspec/implementations/tests/test_http.py
@@ -163,6 +163,11 @@ def test_file_pickle(server):
     # via HTTPFile
     h = fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"})
     out = server + "/index/realfile"
+
+    with fsspec.open(out, headers={"give_length": "true", "head_ok": "true"}) as f:
+        pic = pickle.loads(pickle.dumps(f))
+        assert pic.read() == data
+
     with h.open(out, "rb") as f:
         pic = pickle.dumps(f)
         assert f.read() == data

diff --git a/fsspec/implementations/tests/test_local.py b/fsspec/implementations/tests/test_local.py
@@ -587,6 +587,20 @@ def test_pickle(tmpdir):
     with pytest.raises(ValueError):
         pickle.dumps(f)
 
+    # with context
+    with fs.open(fn0, "rb") as f:
+        f.seek(1)
+        f2 = pickle.loads(pickle.dumps(f))
+        assert f2.tell() == 1
+        assert f2.read() == f.read()
+
+    # with fsspec.open https://github.com/fsspec/filesystem_spec/issues/579
+    with fsspec.open(fn0, "rb") as f:
+        f.seek(1)
+        f2 = pickle.loads(pickle.dumps(f))
+        assert f2.tell() == 1
+        assert f2.read() == f.read()
+
 
 def test_strip_protocol_expanduser():
     path = "file://~\\foo\\bar" if WIN else "file://~/foo/bar"