Make S3 bucket expiry deletion handling robust. (#11156) (#11160)

We now handle any number of deletions and unit test are added for this.

build-support/bin/BUILD

@@ -21,7 +21,7 @@ python_binary(
      ':common',
    ],
    tags = {'type_checked'},
- )
+)
 
 python_binary(
   name = 'check_banned_imports',
@@ -113,3 +113,9 @@ python_binary(
   ],
   tags = {"type_checked"},
 )
+
+python_tests(
+  dependencies = [
+    ":bootstrap_and_deploy_ci_pants_pex",
+  ],
+)

build-support/bin/bootstrap_and_deploy_ci_pants_pex.py

@@ -6,6 +6,7 @@
 import json
 import os
 import subprocess
+from collections import Counter
 from pathlib import Path
 
 from common import banner, die
@@ -105,6 +106,44 @@ def calculate_native_engine_so_hash() -> str:
     )
 
 
+class ListingError(Exception):
+    pass
+
+
+class NonUniqueVersionError(Exception):
+    pass
+
+
+def _s3_listing_has_unique_version(listing_prefix: str, listing_result: str) -> bool:
+    if not listing_result:
+        return False
+    result = json.loads(listing_result)
+    versions = result.get("Versions")
+    if not versions:
+        return False
+    tally = Counter(version["Key"] for version in versions)
+    if len(tally) > 1:
+        keys = "\n".join(f"{index}.) {key}" for index, key in enumerate(tally.keys(), start=1))
+        raise ListingError(
+            f"Multiple keys returned listing {listing_prefix} in AWS S3:\n{keys}\n"
+            "This is unexpected. Please raise this failure in the #infra channel in Slack so we "
+            "can investigate."
+        )
+    delete_markers = result.get("DeleteMarkers", [])
+    for delete_marker in delete_markers:
+        key = delete_marker["Key"]
+        if key in tally:
+            tally[key] -= 1
+    _, count = tally.popitem()
+    if count > 1:
+        raise NonUniqueVersionError(
+            f"Multiple copies found of {listing_prefix} in AWS S3. This is not allowed as a "
+            "security precaution. Please raise this failure in the #infra channel in Slack so that "
+            "we may investigate how this happened and delete the duplicate copy from S3."
+        )
+    return count == 1
+
+
 def native_engine_so_in_s3_cache(*, aws_bucket: str, native_engine_so_aws_key: str) -> bool:
     ls_output = subprocess.run(
         [
@@ -121,19 +160,10 @@ def native_engine_so_in_s3_cache(*, aws_bucket: str, native_engine_so_aws_key: s
         stdout=subprocess.PIPE,
         check=True,
     ).stdout.decode()
-    if not ls_output:
-        return False
-    versions = json.loads(ls_output).get("Versions")
-    if versions is None:
-        return False
-    if len(versions) > 1:
-        die(
-            f"Multiple copies found of {native_engine_so_aws_key} in AWS S3. This is not allowed "
-            "as a security precaution. Please raise this failure in the #infra channel "
-            "in Slack so that we may investigate how this happened and delete the duplicate "
-            "copy from S3."
-        )
-    return True
+    try:
+        return _s3_listing_has_unique_version(native_engine_so_aws_key, ls_output)
+    except NonUniqueVersionError as e:
+        die(str(e))
 
 
 def bootstrap_pants_pex(python_version: float) -> None:

build-support/bin/bootstrap_and_deploy_ci_pants_pex_test.py

@@ -0,0 +1,157 @@
+# Copyright 2020 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+import pytest
+from bootstrap_and_deploy_ci_pants_pex import (
+    ListingError,
+    NonUniqueVersionError,
+    _s3_listing_has_unique_version,
+)
+
+
+def test_listing_has_unique_version_no_results() -> None:
+    assert not _s3_listing_has_unique_version("prefix", "")
+    assert not _s3_listing_has_unique_version("prefix", "{}")
+
+
+def test_listing_has_unique_version_nominal() -> None:
+    assert _s3_listing_has_unique_version(
+        "prefix",
+        """
+        {
+            "Versions": [
+                {
+                    "ETag": "\\"42e2a84a76d7377e5c5a2c5c3e2e2afe\\"",
+                    "Size": 207011168,
+                    "StorageClass": "STANDARD",
+                    "Key": "prefix1",
+                    "VersionId": "0_x4c45bLKFS_zOETFrBB7u0HtZWD_j3",
+                    "IsLatest": false,
+                    "LastModified": "2020-10-11T11:23:32+00:00",
+                    "Owner": {
+                        "ID": "65a011a29cdf8ec533ec3d1ccaae921c"
+                    }
+                }
+            ]
+        }
+        """,
+    )
+
+
+def test_listing_has_unique_version_deleted() -> None:
+    assert not _s3_listing_has_unique_version(
+        "prefix",
+        """
+        {
+            "Versions": [
+                {
+                    "ETag": "\\"42e2a84a76d7377e5c5a2c5c3e2e2afe\\"",
+                    "Size": 207011168,
+                    "StorageClass": "STANDARD",
+                    "Key": "prefix1",
+                    "VersionId": "0_x4c45bLKFS_zOETFrBB7u0HtZWD_j3",
+                    "IsLatest": false,
+                    "LastModified": "2020-10-11T11:23:32+00:00",
+                    "Owner": {
+                        "ID": "65a011a29cdf8ec533ec3d1ccaae921c"
+                    }
+                }
+            ],
+            "DeleteMarkers": [
+                {
+                    "Key": "prefix1",
+                    "VersionId": "7h5go2iDSRrWPVX8hvDITmWNb0SrHnD_",
+                    "IsLatest": true,
+                    "LastModified": "2020-11-11T00:00:00+00:00"
+                }
+            ]
+        }
+        """,
+    )
+
+    assert _s3_listing_has_unique_version(
+        "prefix",
+        """
+        {
+            "Versions": [
+                {
+                    "Key": "prefix1"
+                },
+                {
+                    "Key": "prefix1"
+                }
+            ],
+            "DeleteMarkers": [
+                {
+                    "Key": "prefix1"
+                }
+            ]
+        }
+        """,
+    )
+
+    assert not _s3_listing_has_unique_version(
+        "prefix",
+        """
+        {
+            "Versions": [
+                {
+                    "Key": "prefix1"
+                },
+                {
+                    "Key": "prefix1"
+                }
+            ],
+            "DeleteMarkers": [
+                {
+                    "Key": "prefix1"
+                },
+                {
+                    "Key": "prefix1"
+                }
+            ]
+        }
+        """,
+    )
+
+
+def test_listing_has_unique_version_multiple() -> None:
+    with pytest.raises(ListingError):
+        assert _s3_listing_has_unique_version(
+            "prefix",
+            """
+            {
+                "Versions": [
+                    {
+                        "Key": "prefix1"
+                    },
+                    {
+                        "Key": "prefix2"
+                    }
+                ]
+            }
+            """,
+        )
+
+
+def test_listing_has_unique_version_non_unique() -> None:
+    with pytest.raises(NonUniqueVersionError):
+        assert _s3_listing_has_unique_version(
+            "prefix",
+            """
+            {
+                "Versions": [
+                    {
+                        "Key": "prefix1",
+                        "VersionId": "1_x4c45bLKFS_zOETFrBB7u0HtZWD_j3",
+                        "LastModified": "2020-10-11T11:23:32+00:00"
+                    },
+                    {
+                        "Key": "prefix1",
+                        "VersionId": "0_x4c45bLKFS_zOETFrBB7u0HtZWD_j3",
+                        "LastModified": "2020-9-11T11:23:32+00:00"
+                    }
+                ]
+            }
+            """,
+        )