From 4fe210dd417626a0fe3857caabd469ef0476ebc7 Mon Sep 17 00:00:00 2001 From: JoFrhwld Date: Mon, 3 Jun 2024 17:05:27 -0400 Subject: [PATCH 1/4] cloudpickling --- pyproject.toml | 3 ++- src/fasttrackpy/processors/outputs.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 98e6479..db52a02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,10 +30,11 @@ python-magic-bin = {version = "^0.4.14", markers = "sys_platform == 'win32'"} click = "^8.1.7" cloup = "^3.0.3" matplotlib = "^3.8.2" -aligned-textgrid = "^0.6.2" +aligned-textgrid = "^0.6.7" tqdm = "^4.66.1" joblib = "^1.3.2" pyyaml = "^6.0.1" +cloudpickle = "^3.0.0" [tool.poetry.group.dev.dependencies] jupyter = "^1.0.0" diff --git a/src/fasttrackpy/processors/outputs.py b/src/fasttrackpy/processors/outputs.py index ac55f26..1e3f6a1 100644 --- a/src/fasttrackpy/processors/outputs.py +++ b/src/fasttrackpy/processors/outputs.py @@ -1,5 +1,5 @@ import numpy as np -import pickle +import cloudpickle import parselmouth as pm import polars as pl from aligned_textgrid import SequenceInterval @@ -429,7 +429,7 @@ def pickle_candidates( tmp_candidates = copy.deepcopy (candidates) with file.open('wb') as f: - pickle.dump(tmp_candidates, f) + cloudpickle.dump(tmp_candidates, f) def unpickle_candidates( @@ -452,6 +452,6 @@ def unpickle_candidates( file = Path(file) with file.open('rb') as f: - candidates = pickle.load(f) + candidates = cloudpickle.load(f) return candidates From 02d3c30d791a80dea4b32969d5443f11d0d9dde6 Mon Sep 17 00:00:00 2001 From: JoFrhwld Date: Mon, 3 Jun 2024 17:47:47 -0400 Subject: [PATCH 2/4] cloudpickling --- src/fasttrackpy/processors/outputs.py | 4 ++-- tests/test_outputs.py | 16 +++++++++++++++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/fasttrackpy/processors/outputs.py b/src/fasttrackpy/processors/outputs.py index 1e3f6a1..ce256c1 100644 --- a/src/fasttrackpy/processors/outputs.py +++ b/src/fasttrackpy/processors/outputs.py @@ -426,10 +426,10 @@ def pickle_candidates( if type(file) is str: file = Path(file) - tmp_candidates = copy.deepcopy (candidates) + #tmp_candidates = copy.deepcopy (candidates) with file.open('wb') as f: - cloudpickle.dump(tmp_candidates, f) + cloudpickle.dump(candidates, f) def unpickle_candidates( diff --git a/tests/test_outputs.py b/tests/test_outputs.py index 697f8ee..41383e3 100644 --- a/tests/test_outputs.py +++ b/tests/test_outputs.py @@ -9,6 +9,7 @@ pickle_candidates,\ unpickle_candidates from fasttrackpy.patterns.just_audio import process_audio_file +from fasttrackpy import process_audio_textgrid import parselmouth as pm import polars as pl import numpy as np @@ -142,6 +143,7 @@ def test_save_cand_spectrogram(self, tmp_path): class TestPickle: cands = CandidateTracks(SOUND) + def test_pickle_unpickle(self, tmp_path): pickle_file = tmp_path / "cand.pkl" @@ -154,4 +156,16 @@ def test_pickle_unpickle(self, tmp_path): re_read.winner.maximum_formant, self.cands.winner.maximum_formant ) - assert len(re_read.candidates) == len(self.cands.candidates) \ No newline at end of file + assert len(re_read.candidates) == len(self.cands.candidates) + + def test_big_pickle_unpickle(self, tmp_path): + cands2 = process_audio_textgrid( + audio_path=Path("tests", "test_data", "corpus", "josef-fruehwald_speaker.wav"), + textgrid_path=Path("tests", "test_data", "corpus", "josef-fruehwald_speaker.TextGrid"), + ) + pickle_file = tmp_path / "cand2.pkl" + pickle_candidates(cands2[0], file = pickle_file) + assert pickle_file.exists() + + #re_read = unpickle_candidates(file = pickle_file) + #assert len(re_read.candidates) == len(self.cands2[0].candidates) \ No newline at end of file From cd350f6e8d26361b92a6c6fd421a1dd0ca48b128 Mon Sep 17 00:00:00 2001 From: JoFrhwld Date: Tue, 4 Jun 2024 13:28:45 -0400 Subject: [PATCH 3/4] increasing recursion limit for pickling --- src/fasttrackpy/processors/outputs.py | 5 +++-- tests/test_outputs.py | 4 ++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/fasttrackpy/processors/outputs.py b/src/fasttrackpy/processors/outputs.py index ce256c1..06ee2c4 100644 --- a/src/fasttrackpy/processors/outputs.py +++ b/src/fasttrackpy/processors/outputs.py @@ -7,6 +7,7 @@ import matplotlib.pyplot as mp import copy import logging +import sys ptolmap = {"F1" :"#4477AA", "F1_s": "#4477AA", @@ -425,7 +426,7 @@ def pickle_candidates( """ if type(file) is str: file = Path(file) - + sys.setrecursionlimit(3000) #tmp_candidates = copy.deepcopy (candidates) with file.open('wb') as f: @@ -450,7 +451,7 @@ def unpickle_candidates( """ if type(file) is str: file = Path(file) - + sys.setrecursionlimit(3000) with file.open('rb') as f: candidates = cloudpickle.load(f) diff --git a/tests/test_outputs.py b/tests/test_outputs.py index 41383e3..46d398d 100644 --- a/tests/test_outputs.py +++ b/tests/test_outputs.py @@ -167,5 +167,5 @@ def test_big_pickle_unpickle(self, tmp_path): pickle_candidates(cands2[0], file = pickle_file) assert pickle_file.exists() - #re_read = unpickle_candidates(file = pickle_file) - #assert len(re_read.candidates) == len(self.cands2[0].candidates) \ No newline at end of file + re_read = unpickle_candidates(file = pickle_file) + assert len(re_read.candidates) == len(cands2[0].candidates) \ No newline at end of file From af8ae002f90f48894f17dfd414589aec76d3b675 Mon Sep 17 00:00:00 2001 From: JoFrhwld Date: Tue, 4 Jun 2024 14:28:13 -0400 Subject: [PATCH 4/4] version bump --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index db52a02..d514e47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "fasttrackpy" -version = "0.4.6" +version = "0.4.7" description = "A python implementation of FastTrack" authors = [ "JoFrhwld ",