Merge pull request #5144 from voxel51/doc/estore

Documentation for the ExecutionStore

docs/source/plugins/developing_plugins.rst

@@ -2428,6 +2428,91 @@ loaded only when the `brain_key` property is modified.
     Panel data is never readable in Python; it is only implicitly used by
     the types you define when they are rendered clientside.
 
+.. _panel-execution-store
+
+Execution store
+---------------
+
+Panels can store data in the execution store, which is a key-value store that
+is persisted beyond the lifetime of the panel. This is useful for storing
+information that should persist across panel instances and App sessions, such
+as cached data, long-lived panel state, or user preferences.
+
+You can create/retrieve execution stores scoped to the current ``ctx.dataset``
+via :meth:`ctx.store <fiftyone.operators.executor.ExecutionContext.store>`:
+
+.. code-block:: python
+    :linenos:
+
+    def on_load(ctx):
+        # Retrieve a store scoped to the current `ctx.dataset`
+        # The store is automatically created if necessary
+        store = ctx.store("my_store")
+
+        # Load a pre-existing value from the store
+        user_choice = store.get("user_choice")
+
+        # Store data with a TTL to ensure it is evicted after `ttl` seconds
+        store.set("my_key", {"foo": "bar"}, ttl=60)
+
+        # List all keys in the store
+        print(store.list_keys())  # ["user_choice", "my_key"]
+
+        # Retrieve data from the store
+        print(store.get("my_key"))  # {"foo": "bar"}
+
+        # Retrieve metadata about a key
+        print(store.get_metadata("my_key"))
+        # {"created_at": ..., "updated_at": ..., "expires_at": ...}
+
+        # Delete a key from the store
+        store.delete("my_key")
+
+        # Clear all data in the store
+        store.clear()
+
+.. note::
+
+    Did you know? Any execution stores associated with a dataset are
+    automatically  deleted when the dataset is deleted.
+
+For advanced use cases, it is also possible to create and use global stores
+that are available to all datasets via the
+:class:`ExecutionStore <fiftyone.operators.store.ExecutionStore>` class:
+
+.. code-block:: python
+    :linenos:
+
+    from fiftyone.operators import ExecutionStore
+
+    # Retrieve a global store
+    # The store is automatically created if necessary
+    store = ExecutionStore.create("my_store")
+
+    # Store data with a TTL to ensure it is evicted after `ttl` seconds
+    store.set("my_key", {"foo": "bar"}, ttl=60)
+
+    # List all keys in the global store
+    print(store.list_keys())  # ["my_key"]
+
+    # Retrieve data from the global store
+    print(store.get("my_key"))  # {"foo": "bar"}
+
+    # Retrieve metadata about a key
+    print(store.get_metadata("my_key"))
+    # {"created_at": ..., "updated_at": ..., "expires_at": ...}
+
+    # Delete a key from the global store
+    store.delete("my_key")
+
+    # Clear all data in the global store
+    store.clear()
+
+.. warning::
+
+    Global stores have no automatic garbage collection, so take care when
+    creating and using global stores whose keys do not utilize TTLs.
+
 .. _panel-saved-workspaces
 
 Saved workspaces

fiftyone/factory/repos/execution_store.py

@@ -78,28 +78,17 @@ def has_store(self, store_name) -> bool:
 
     def list_stores(self) -> list[str]:
         """Lists the stores associated with the current context."""
-        result = self._collection.find(
-            dict(key="__store__", dataset_id=self._dataset_id),
-            {"store_name": 1},
-        )
-        return [d["store_name"] for d in result]
+        pipeline = [
+            {"$match": {"dataset_id": self._dataset_id}},
+            {"$group": {"_id": "$store_name"}},
+        ]
+        return [d["_id"] for d in self._collection.aggregate(pipeline)]
 
     def count_stores(self) -> int:
         """Counts the stores associated with the current context."""
         pipeline = [
-            {
-                "$match": {
-                    "dataset_id": self._dataset_id,
-                }
-            },
-            {
-                "$group": {
-                    "_id": {
-                        "store_name": "$store_name",
-                        "dataset_id": "$dataset_id",
-                    }
-                }
-            },
+            {"$match": {"dataset_id": self._dataset_id}},
+            {"$group": {"_id": "$store_name"}},
             {"$count": "total_stores"},
         ]
 
@@ -233,9 +222,7 @@ def has_store_global(self, store_name):
         """Determines whether a store with the given name exists across all
         datasets and the global context.
         """
-        result = self._collection.find_one(
-            dict(store_name=store_name, key="__store__"), {}
-        )
+        result = self._collection.find_one(dict(store_name=store_name), {})
         return bool(result)
 
     def list_stores_global(self) -> list[StoreDocument]:

fiftyone/operators/__init__.py

@@ -19,6 +19,7 @@
 )
 from .utils import ProgressHandler, is_new
 from .panel import Panel, PanelConfig
+from .store import ExecutionStore
 from .categories import Categories
 
 # This enables Sphinx refs to directly use paths imported here

fiftyone/operators/executor.py

@@ -26,6 +26,7 @@
 from fiftyone.operators.operations import Operations
 from fiftyone.operators.panel import PanelRef
 from fiftyone.operators.registry import OperatorRegistry
+from fiftyone.operators.store import ExecutionStore
 import fiftyone.operators.types as types
 from fiftyone.plugins.secrets import PluginSecretsResolver, SecretsDictionary
 import fiftyone.server.view as fosv
@@ -876,17 +877,16 @@ def set_progress(self, progress=None, label=None):
             self.log(f"Progress: {progress} - {label}")
 
     def store(self, store_name):
-        """
-        Create (if not previously created) and use a store with the specified name.
+        """Retrieves the execution store with the given name.
+
+        The store is automatically created if necessary.
 
         Args:
             store_name: the name of the store
 
         Returns:
             a :class:`fiftyone.operators.store.ExecutionStore`
         """
-        from fiftyone.operators.store import ExecutionStore
-
         dataset_id = self.dataset._doc.id
         return ExecutionStore.create(store_name, dataset_id)
 

fiftyone/operators/store/service.py

@@ -63,7 +63,7 @@ def create_store(
         Returns:
             a :class:`fiftyone.store.models.StoreDocument`
         """
-        return self._repo.create_store(store_name, metadata)
+        return self._repo.create_store(store_name, metadata=metadata)
 
     def get_store(self, store_name: str) -> StoreDocument:
         """Gets the specified store for the current context.

fiftyone/operators/store/store.py

@@ -6,6 +6,7 @@
 |
 """
 
+from datetime import datetime
 from typing import Any, Optional
 
 from bson import ObjectId
@@ -101,19 +102,24 @@ def update_ttl(self, key: str, new_ttl: int) -> None:
         """
         self._store_service.update_ttl(self.store_name, key, new_ttl)
 
-    def get_ttl(self, key: str) -> Optional[int]:
-        """Retrieves the TTL for a specific key.
+    def get_metadata(self, key: str) -> Optional[dict]:
+        """Retrieves the metadata for the given key.
 
         Args:
-            key: the key to get the TTL for
+            key: the key to check
 
         Returns:
-            the TTL in seconds, or None if the key does not have a TTL
+            a dict of metadata about the key
         """
         key_doc = self._store_service.get_key(self.store_name, key)
         if key_doc is None:
             return None
-        return key_doc.ttl
+
+        return dict(
+            created_at=key_doc.created_at,
+            updated_at=key_doc.updated_at,
+            expires_at=key_doc.expires_at,
+        )
 
     def list_keys(self) -> list[str]:
         """Lists all keys in the store.

tests/unittests/execution_store_unit_tests.py

@@ -166,19 +166,6 @@ def test_list_keys(self):
             {"key": 1},
         )
 
-    def test_list_stores(self):
-        self.mock_collection.find.return_value = [
-            {"store_name": "widgets"},
-            {"store_name": "gadgets"},
-        ]
-        stores = self.store_repo.list_stores()
-        assert stores == ["widgets", "gadgets"]
-        self.mock_collection.find.assert_called_once()
-        self.mock_collection.find.assert_called_with(
-            {"key": "__store__", "dataset_id": None},
-            {"store_name": 1},
-        )
-
 
 class TestExecutionStoreIntegration(unittest.TestCase):
     def setUp(self) -> None: