Troubleshoot tails file rollover

aries_cloudagent/indy/tests/test_indy.py

@@ -1,5 +1,6 @@
 import pytest
 from os import makedirs
+from os.path import join
 from pathlib import Path
 from shutil import rmtree
 
@@ -8,19 +9,19 @@
 import indy.blob_storage
 
 from .. import create_tails_reader, create_tails_writer
-from .. import util as test_module_util
+from .. import util as test_module
 
 
 @pytest.mark.indy
 class TestIndyUtils(AsyncTestCase):
     TAILS_HASH = "8UW1Sz5cqoUnK9hqQk7nvtKK65t7Chu3ui866J23sFyJ"
 
     def tearDown(self):
-        tails_dir = test_module_util.indy_client_dir("tails", create=False)
+        tails_dir = test_module.indy_client_dir("tails", create=False)
         rmtree(tails_dir, ignore_errors=True)
 
     async def test_tails_reader(self):
-        tails_dir = test_module_util.indy_client_dir("tails", create=True)
+        tails_dir = test_module.indy_client_dir("tails", create=True)
         tails_local = f"{tails_dir}/{TestIndyUtils.TAILS_HASH}"
 
         with open(tails_local, "a") as f:
@@ -37,10 +38,28 @@ async def test_tails_reader(self):
             await create_tails_reader(tails_local)
 
     async def test_tails_writer(self):
-        tails_dir = test_module_util.indy_client_dir("tails", create=True)
+        tails_dir = test_module.indy_client_dir("tails", create=True)
         assert await create_tails_writer(tails_dir)
 
         rmtree(tails_dir, ignore_errors=True)
 
     async def test_nonce(self):
-        assert await test_module_util.generate_pr_nonce()
+        assert await test_module.generate_pr_nonce()
+
+    async def test_tails_path(self):
+        tails_dir = test_module.indy_client_dir("tails", create=False)
+        rmtree(tails_dir, ignore_errors=True)
+
+        tails_local_path = test_module.tails_path("rev-reg-id")
+        assert tails_local_path is None
+
+        tails_rr_dir = test_module.indy_client_dir(
+            join("tails", "rev-reg-id"), create=True
+        )
+        tails_local_path = test_module.tails_path("rev-reg-id")
+        assert tails_local_path is None
+
+        with open(join(tails_rr_dir, "tails-hash"), "w") as f:
+            f.write("content")
+        tails_local_path = test_module.tails_path("rev-reg-id")
+        assert tails_local_path

aries_cloudagent/indy/util.py

@@ -1,6 +1,6 @@
 """Libindy utility functions."""
 
-from os import getenv, makedirs
+from os import getenv, listdir, makedirs
 from os.path import isdir, join
 from pathlib import Path
 from platform import system
@@ -38,3 +38,17 @@ def indy_client_dir(subpath: str = None, create: bool = False) -> str:
         makedirs(target_dir, exist_ok=True)
 
     return target_dir
+
+
+def tails_path(rev_reg_id: str) -> str:
+    """Return path to indy tails file for input rev reg id."""
+
+    tails_dir = indy_client_dir(join("tails", rev_reg_id), create=False)
+    if not isdir(tails_dir):
+        return None
+
+    content = listdir(tails_dir)
+    if len(content) != 1:
+        return None
+
+    return join(tails_dir, content[0])

aries_cloudagent/messaging/credential_definitions/routes.py

@@ -129,7 +129,7 @@ async def credential_definitions_send_credential_definition(request: web.BaseReq
     schema_id = body.get("schema_id")
     support_revocation = bool(body.get("support_revocation"))
     tag = body.get("tag")
-    revocation_registry_size = body.get("revocation_registry_size")
+    rev_reg_size = body.get("revocation_registry_size")
 
     ledger: BaseLedger = await context.inject(BaseLedger, required=False)
     if not ledger:
@@ -160,12 +160,10 @@ async def credential_definitions_send_credential_definition(request: web.BaseReq
             raise web.HTTPBadRequest(reason="tails_server_base_url not configured")
         try:
             # Create registry
-            issuer_did = cred_def_id.split(":")[0]
             revoc = IndyRevocation(context)
             registry_record = await revoc.init_issuer_registry(
                 cred_def_id,
-                issuer_did,
-                max_cred_num=revocation_registry_size,
+                max_cred_num=rev_reg_size,
             )
 
         except RevocationNotSupportedError as e:
@@ -175,27 +173,35 @@ async def credential_definitions_send_credential_definition(request: web.BaseReq
             await registry_record.set_tails_file_public_uri(
                 context, f"{tails_base_url}/{registry_record.revoc_reg_id}"
             )
-            await registry_record.publish_registry_definition(context)
-            await registry_record.publish_registry_entry(context)
-
-            tails_server: BaseTailsServer = await context.inject(BaseTailsServer)
-            upload_success, reason = await tails_server.upload_tails_file(
-                context, registry_record.revoc_reg_id, registry_record.tails_local_path
-            )
-            if not upload_success:
-                raise web.HTTPInternalServerError(
-                    reason=f"Tails file failed to upload: {reason}"
-                )
+            await registry_record.send_def(context)
+            await registry_record.send_entry(context)
 
+            # stage pending registry independent of whether tails server is OK
             pending_registry_record = await revoc.init_issuer_registry(
                 registry_record.cred_def_id,
-                registry_record.issuer_did,
                 max_cred_num=registry_record.max_cred_num,
             )
             ensure_future(
-                pending_registry_record.stage_pending_registry_definition(context)
+                pending_registry_record.stage_pending_registry(context, max_attempts=16)
             )
 
+            tails_server: BaseTailsServer = await context.inject(BaseTailsServer)
+            (upload_success, reason) = await tails_server.upload_tails_file(
+                context,
+                registry_record.revoc_reg_id,
+                registry_record.tails_local_path,
+                interval=0.8,
+                backoff=-0.5,
+                max_attempts=5,  # heuristic: respect HTTP timeout
+            )
+            if not upload_success:
+                raise web.HTTPInternalServerError(
+                    reason=(
+                        f"Tails file for rev reg {registry_record.revoc_reg_id} "
+                        f"failed to upload: {reason}"
+                    )
+                )
+
         except RevocationError as e:
             raise web.HTTPBadRequest(reason=e.message) from e
 

aries_cloudagent/messaging/credential_definitions/tests/test_routes.py

@@ -99,9 +99,9 @@ async def test_send_credential_definition_revoc(self):
                     return_value=async_mock.MagicMock(
                         set_tails_file_public_uri=async_mock.CoroutineMock(),
                         generate_registry=async_mock.CoroutineMock(),
-                        publish_registry_definition=async_mock.CoroutineMock(),
-                        publish_registry_entry=async_mock.CoroutineMock(),
-                        stage_pending_registry_definition=async_mock.CoroutineMock(),
+                        send_def=async_mock.CoroutineMock(),
+                        send_entry=async_mock.CoroutineMock(),
+                        stage_pending_registry=async_mock.CoroutineMock(),
                     )
                 )
             )
@@ -184,8 +184,9 @@ async def test_send_credential_definition_revoc_upload_x(self):
                     return_value=async_mock.MagicMock(
                         set_tails_file_public_uri=async_mock.CoroutineMock(),
                         generate_registry=async_mock.CoroutineMock(),
-                        publish_registry_definition=async_mock.CoroutineMock(),
-                        publish_registry_entry=async_mock.CoroutineMock(),
+                        send_def=async_mock.CoroutineMock(),
+                        send_entry=async_mock.CoroutineMock(),
+                        stage_pending_registry=async_mock.CoroutineMock(),
                     )
                 )
             )
@@ -221,8 +222,8 @@ async def test_send_credential_definition_revoc_init_issuer_rev_reg_x(self):
                         async_mock.MagicMock(
                             set_tails_file_public_uri=async_mock.CoroutineMock(),
                             generate_registry=async_mock.CoroutineMock(),
-                            publish_registry_definition=async_mock.CoroutineMock(),
-                            publish_registry_entry=async_mock.CoroutineMock(),
+                            send_def=async_mock.CoroutineMock(),
+                            send_entry=async_mock.CoroutineMock(),
                         ),
                         test_module.RevocationError("Error on pending rev reg init"),
                     ]