add single process property

Signed-off-by: Carla Vieira <[email protected]>
kedro-org · Oct 5, 2022 · 83f784e · 83f784e
1 parent 38de49d
commit 83f784e
Showing 1 changed file with 6 additions and 0 deletions.
diff --git a/kedro/io/cached_dataset.py b/kedro/io/cached_dataset.py
@@ -27,6 +27,12 @@ class CachedDataSet(AbstractDataSet):
     class as shown above.
     """
 
+    # this dataset cannot be used with ``ParallelRunner``,
+    # therefore it has the attribute ``_SINGLE_PROCESS = True``
+    # for parallelism within a Spark pipeline please consider
+    # ``ThreadRunner`` instead
+    _SINGLE_PROCESS = True
+
     def __init__(
         self,
         dataset: Union[AbstractDataSet, Dict],