add expire cache entries (#44)

* add expire cache entries

* cleanup

* add docs

cads_worker/entry_points.py

@@ -1,8 +1,11 @@
+import datetime
 import os
+from typing import Annotated, Optional
 
 import cacholote
 import structlog
 import typer
+from typer import Option
 
 from . import config
 
@@ -29,5 +32,38 @@ def _cache_cleaner() -> None:
         raise
 
 
+def _add_tzinfo(timestamp: datetime.datetime | None) -> datetime.datetime | None:
+    if timestamp is not None and timestamp.tzinfo is None:
+        timestamp = timestamp.replace(tzinfo=datetime.timezone.utc)
+    return timestamp
+
+
+def _expire_cache_entries(
+    collection_id: Annotated[
+        Optional[list[str]], Option(help="Collection ID to expire")
+    ] = None,
+    before: Annotated[
+        Optional[datetime.datetime],
+        Option(help="Expire entries created before this date"),
+    ] = None,
+    after: Annotated[
+        Optional[datetime.datetime],
+        Option(help="Expire entries created after this date"),
+    ] = None,
+) -> int:
+    """Expire cache entries."""
+    count = cacholote.expire_cache_entries(
+        tags=collection_id,
+        before=_add_tzinfo(before),
+        after=_add_tzinfo(after),
+    )
+    typer.echo(f"Number of entries expired: {count}")
+    return count
+
+
 def cache_cleaner() -> None:
     typer.run(_cache_cleaner)
+
+
+def expire_cache_entries() -> None:
+    typer.run(_expire_cache_entries)

cads_worker/worker.py

@@ -207,7 +207,8 @@ def submit_workflow(
         os.chdir(tmpdir)
         try:
             request = {k: request[k] for k in sorted(request.keys())}
-            result = cacholote.cacheable(adaptor.retrieve)(request=request)
+            with cacholote.config.set(tag=config.get("collection_id")):
+                result = cacholote.cacheable(adaptor.retrieve)(request=request)
         except Exception as err:
             logger.exception(job_id=job_id, event_type="EXCEPTION")
             context.add_user_visible_error(

pyproject.toml

@@ -27,6 +27,7 @@ readme = "README.md"
 
 [project.scripts]
 cache-cleaner = "cads_worker.entry_points:cache_cleaner"
+expire-cache-entries = "cads_worker.entry_points:expire_cache_entries"
 
 [tool.coverage.run]
 branch = true

tests/test_20_expire_cache_entries.py

@@ -0,0 +1,47 @@
+import datetime
+import pathlib
+
+import cacholote
+import pytest
+
+from cads_worker import entry_points
+
+TODAY = datetime.datetime.now(tz=datetime.timezone.utc)
+TOMORROW = TODAY + datetime.timedelta(days=1)
+YESTERDAY = TODAY - datetime.timedelta(days=1)
+
+
+@cacholote.cacheable
+def cached_now() -> datetime.datetime:
+    return datetime.datetime.now()
+
+
+@pytest.mark.parametrize(
+    "collection_id,before,after",
+    [
+        (["foo"], None, None),
+        (None, TOMORROW, None),
+        (None, None, YESTERDAY),
+        (["foo"], TOMORROW, YESTERDAY),
+    ],
+)
+def test_cache_entries(
+    tmp_path: pathlib.Path,
+    collection_id: list[str] | None,
+    before: datetime.datetime | None,
+    after: datetime.datetime | None,
+) -> None:
+    with cacholote.config.set(
+        cache_db_urlpath=f"sqlite:///{tmp_path / 'cacholote.db'}",
+        tag="foo",
+    ):
+        now = cached_now()
+        assert now == cached_now()
+
+        count = entry_points._expire_cache_entries(
+            collection_id=collection_id,
+            before=before,
+            after=after,
+        )
+        assert count == 1
+        assert now != cached_now()