DEBUG 0

.github/workflows/tests.yml

@@ -74,7 +74,7 @@ jobs:
     - name: Setup PROJECT_DATA dir
       run: echo -e "\nexport PROJECT_DATA=/tmp/project_data\n" >> ./scripts/project.env
     - name: Run tests
-      run: sudo -H -E env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" "HOME=$HOME" ./scripts/run.sh --maxfail=25 --durations=0 --junitxml=./test.xml
+      run: sudo -H -E env "PATH=$PATH" "LD_LIBRARY_PATH=$LD_LIBRARY_PATH" "HOME=$HOME" ./scripts/run.sh -k "test_in_memory" -s -vvvv --maxfail=25 --durations=0 --junitxml=./test.xml
       env:
         HUGGING_FACE_HUB_TOKEN: ${{ secrets.HUGGING_FACE_HUB_TOKEN }}
         OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

src/steps/step_output.py

@@ -755,6 +755,7 @@ def _output_to_dataset(  # noqa: C901
 ) -> OutputDataset | OutputIterableDataset:
     # Check whether we are running in the background or not
     if pipe:  # pragma: no cover
+        print("IN CHILD")
         # We are running in the background
         output = __output_to_dataset(
             step=step,
@@ -824,7 +825,9 @@ def dummy_empty_generator():
 
                 # Send the data card (sometimes this gets updated in .run())
                 # + send the pickled version of the OutputDataset to the parent process
+                print("PIPE PUT BEFORE")
                 pipe.put(dill.dumps((step._data_card, return_val)))
+                print("PIPE PUT AFTER")
 
                 return return_val  # Meaningless, see: pipe.put()
             except dill.PicklingError as e:

src/utils/background_utils.py

@@ -182,7 +182,9 @@ def _run_in_background_process_no_block(
     ):
         p, pipe = run_in_background_process(func, *args, **kwargs)
         result_process_func(p)
+        print("PIPE GET BEFORE")
         result_func(pipe.get())
+        print("PIPE GET AFTER")
         p.terminate()
 
     run_in_background_thread(