Better handling of missing or bad package stats (#1829)

quiltdata · Oct 6, 2020 · 52e881b · 52e881b
1 parent b2c30b0
commit 52e881b
Show file tree

Hide file tree

Showing 3 changed files with 33 additions and 3 deletions.
diff --git a/lambdas/es/indexer/document_queue.py b/lambdas/es/indexer/document_queue.py
@@ -73,7 +73,7 @@ def append(
             metadata: str = '',
             pointer_file: str = '',
             package_hash: str = '',
-            package_stats: Dict[str, int] = {'total_files': 0, 'total_bytes': 0}.copy(),
+            package_stats: Dict[str, int] = None,
             tags: List[str] = (),
             text: str = '',
             version_id=None,
@@ -118,15 +118,25 @@ def append(
         if doc_type == DocTypes.PACKAGE:
             if not handle or not package_hash or not pointer_file:
                 raise ValueError("missing required argument for package document")
+            if (
+                package_stats
+                and not isinstance(package_stats, dict)
+                or isinstance(package_stats, dict)
+                and any(k not in package_stats for k in ['total_files', 'total_bytes'])
+            ):
+                raise ValueError("Malformed package_stats")
             body.update({
                 "_id": f"{handle}:{package_hash}",
                 "handle": handle,
                 "hash": package_hash,
                 "metadata": metadata,
                 "pointer_file": pointer_file,
-                "package_stats": package_stats,
                 "tags": ",".join(tags)
             })
+            if package_stats:
+                body.update({
+                    "package_stats": package_stats,
+                })
         elif doc_type == DocTypes.OBJECT:
             body.update({
                 # Elastic native keys

diff --git a/lambdas/es/indexer/index.py b/lambdas/es/indexer/index.py
@@ -244,7 +244,7 @@ def select_package_stats(s3_client, bucket, manifest_key) -> str:
         return stats
 
     except (botocore.exceptions.ClientError, AssertionError, KeyError) as err:
-        logger_.error("Unable to compute package status via S3 select: %s", err)
+        logger_.error("Unable to compute package stats via S3 select: %s", err)
 
     return None
 

diff --git a/lambdas/es/indexer/test/test_index.py b/lambdas/es/indexer/test/test_index.py
@@ -234,6 +234,7 @@ def _make_event(
                 "last_modified": datetime.datetime(2019, 5, 30, 23, 27, 29, tzinfo=tzutc()),
                 "pointer_file": "1598026253",
                 "package_hash": "abc",
+                "package_stats": None,
             }
         ),
         (
@@ -276,6 +277,25 @@ def _make_event(
                 "version_id": "abc",
             }
         ),
+        pytest.param(
+            "ObjectCreated:Put",
+            DocTypes.PACKAGE,
+            {
+                "bucket": "test",
+                "etag": "123",
+                "ext": "",
+                "handle": "pkg/usr",
+                "key": "foo",
+                "last_modified": datetime.datetime(2019, 5, 30, 23, 27, 29, tzinfo=tzutc()),
+                "pointer_file": "1598026253",
+                "package_hash": "abc",
+                "package_stats": {"bad": "data"},
+            },
+            marks=pytest.mark.xfail(
+                raises=ValueError,
+                reason="Malformed package_stats",
+            )
+        ),
         pytest.param(
             "ObjectCreated:Put",
             DocTypes.PACKAGE,