From 06c400335cb7d382cd57d94f93a4b30157dad9b2 Mon Sep 17 00:00:00 2001 From: Ryan Abernathey Date: Mon, 30 May 2022 12:44:09 -0400 Subject: [PATCH] fix httpfile serialization bug (#973) * fix httpfile serialization bug * Fix pickle of in-context local file Co-authored-by: Martin Durant --- fsspec/implementations/http.py | 2 +- fsspec/implementations/local.py | 2 +- fsspec/implementations/tests/test_http.py | 5 +++++ fsspec/implementations/tests/test_local.py | 14 ++++++++++++++ 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/fsspec/implementations/http.py b/fsspec/implementations/http.py index 126b4106e..8d65c277b 100644 --- a/fsspec/implementations/http.py +++ b/fsspec/implementations/http.py @@ -642,7 +642,7 @@ def __reduce__(self): self.url, self.mode, self.blocksize, - self.cache.name, + self.cache.name if self.cache else "none", self.size, ), ) diff --git a/fsspec/implementations/local.py b/fsspec/implementations/local.py index de4a41d44..d1062858c 100644 --- a/fsspec/implementations/local.py +++ b/fsspec/implementations/local.py @@ -354,7 +354,7 @@ def __getattr__(self, item): def __enter__(self): self._incontext = True - return self.f.__enter__() + return self def __exit__(self, exc_type, exc_value, traceback): self._incontext = False diff --git a/fsspec/implementations/tests/test_http.py b/fsspec/implementations/tests/test_http.py index a71838cf7..6bb5145d1 100644 --- a/fsspec/implementations/tests/test_http.py +++ b/fsspec/implementations/tests/test_http.py @@ -163,6 +163,11 @@ def test_file_pickle(server): # via HTTPFile h = fsspec.filesystem("http", headers={"give_length": "true", "head_ok": "true"}) out = server + "/index/realfile" + + with fsspec.open(out, headers={"give_length": "true", "head_ok": "true"}) as f: + pic = pickle.loads(pickle.dumps(f)) + assert pic.read() == data + with h.open(out, "rb") as f: pic = pickle.dumps(f) assert f.read() == data diff --git a/fsspec/implementations/tests/test_local.py b/fsspec/implementations/tests/test_local.py index 9c413920e..b32ad344f 100644 --- a/fsspec/implementations/tests/test_local.py +++ b/fsspec/implementations/tests/test_local.py @@ -587,6 +587,20 @@ def test_pickle(tmpdir): with pytest.raises(ValueError): pickle.dumps(f) + # with context + with fs.open(fn0, "rb") as f: + f.seek(1) + f2 = pickle.loads(pickle.dumps(f)) + assert f2.tell() == 1 + assert f2.read() == f.read() + + # with fsspec.open https://github.com/fsspec/filesystem_spec/issues/579 + with fsspec.open(fn0, "rb") as f: + f.seek(1) + f2 = pickle.loads(pickle.dumps(f)) + assert f2.tell() == 1 + assert f2.read() == f.read() + def test_strip_protocol_expanduser(): path = "file://~\\foo\\bar" if WIN else "file://~/foo/bar"