Use a mutex for registry CLI txs in webapp deployment commands (#957)

Part of [Service provider auctions for web deployments](https://www.notion.so/Service-provider-auctions-for-web-deployments-104a6b22d47280dbad51d28aa3a91d75) and https://git.vdb.to/cerc-io/stack-orchestrator/issues/948 - Add a registry mutex decorator over tx methods in `LaconicRegistryClient` wrapper - Required to allow multiple process to run webapp deployment tooling without running into account sequence errors when sending laconicd txs Reviewed-on: https://git.vdb.to/cerc-io/stack-orchestrator/pulls/957 Reviewed-by: ashwin <[email protected]> Co-authored-by: Prathamesh Musale <[email protected]> Co-committed-by: Prathamesh Musale <[email protected]>
cerc-io · Oct 25, 2024 · 4a7df2d · 4a7df2d
1 parent 0c47da4
commit 4a7df2d
Show file tree

Hide file tree

Showing 6 changed files with 82 additions and 54 deletions.
diff --git a/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/deploy_webapp_from_registry.py
@@ -360,6 +360,9 @@ def dump_known_requests(filename, requests, status="SEEN"):
 @click.option(
     "--private-key-file", help="The private key for decrypting config.", required=True
 )
+@click.option(
+    "--registry-lock-file", help="File path to use for registry mutex lock", default=None
+)
 @click.option(
     "--private-key-passphrase",
     help="The passphrase for the private key.",
@@ -393,6 +396,7 @@ def command(  # noqa: C901
     private_key_passphrase,
     all_requests,
     auction_requests,
+    registry_lock_file,
 ):
     if request_id and discover:
         print("Cannot specify both --request-id and --discover", file=sys.stderr)
@@ -444,7 +448,7 @@ def command(  # noqa: C901
         include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
         exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
 
-        laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr)
+        laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
         webapp_deployer_record = laconic.get_record(lrn, require=True)
         payment_address = webapp_deployer_record.attributes.paymentAddress
         main_logger.log(f"Payment address: {payment_address}")
@@ -649,7 +653,7 @@ def command(  # noqa: C901
                         )
                         run_log_file = open(run_log_file_path, "wt")
                         run_reg_client = LaconicRegistryClient(
-                            laconic_config, log_file=run_log_file
+                            laconic_config, log_file=run_log_file, mutex_lock_file=registry_lock_file
                         )
 
                     build_logger = TimedLogger(run_id, run_log_file)

diff --git a/stack_orchestrator/deploy/webapp/handle_deployment_auction.py b/stack_orchestrator/deploy/webapp/handle_deployment_auction.py
@@ -120,6 +120,9 @@ def dump_known_auction_requests(filename, requests, status="SEEN"):
     help="Bid to place on application deployment auctions (in alnt)",
     required=True,
 )
+@click.option(
+    "--registry-lock-file", help="File path to use for registry mutex lock", default=None
+)
 @click.option(
     "--dry-run", help="Don't do anything, just report what would be done.", is_flag=True
 )
@@ -129,6 +132,7 @@ def command(
     laconic_config,
     state_file,
     bid_amount,
+    registry_lock_file,
     dry_run,
 ):
     if int(bid_amount) < 0:
@@ -138,7 +142,7 @@ def command(
     logger = TimedLogger(file=sys.stderr)
 
     try:
-        laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr)
+        laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
         auctions_requests = laconic.app_deployment_auctions()
 
         previous_requests = {}

diff --git a/stack_orchestrator/deploy/webapp/publish_deployment_auction.py b/stack_orchestrator/deploy/webapp/publish_deployment_auction.py
@@ -102,7 +102,7 @@ def command(
         "max_price": max_price,
         "num_providers": num_providers,
     }
-    auction_id = laconic.create_auction(provider_auction_params)
+    auction_id = laconic.create_deployment_auction(provider_auction_params)
     print("Deployment auction created:", auction_id)
 
     if not auction_id:

diff --git a/stack_orchestrator/deploy/webapp/registry_mutex.py b/stack_orchestrator/deploy/webapp/registry_mutex.py
@@ -0,0 +1,31 @@
+import fcntl
+from functools import wraps
+
+# Define default file path for the lock
+DEFAULT_LOCK_FILE_PATH = "/tmp/registry_mutex_lock_file"
+
+
+def registry_mutex():
+    def decorator(func):
+        @wraps(func)
+        def wrapper(self, *args, **kwargs):
+            lock_file_path = DEFAULT_LOCK_FILE_PATH
+            if self.mutex_lock_file is not None:
+                lock_file_path = self.mutex_lock_file
+
+            with open(lock_file_path, 'w') as lock_file:
+                try:
+                    # Try to acquire the lock
+                    fcntl.flock(lock_file, fcntl.LOCK_EX)
+
+                    # Call the actual function
+                    result = func(self, *args, **kwargs)
+                finally:
+                    # Always release the lock
+                    fcntl.flock(lock_file, fcntl.LOCK_UN)
+
+            return result
+
+        return wrapper
+
+    return decorator
diff --git a/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py b/stack_orchestrator/deploy/webapp/undeploy_webapp_from_registry.py
@@ -178,6 +178,9 @@ def dump_known_requests(filename, requests):
     "my payment address are examined).",
     is_flag=True,
 )
+@click.option(
+    "--registry-lock-file", help="File path to use for registry mutex lock", default=None
+)
 @click.pass_context
 def command(  # noqa: C901
     ctx,
@@ -195,6 +198,7 @@ def command(  # noqa: C901
     min_required_payment,
     lrn,
     all_requests,
+    registry_lock_file,
 ):
     if request_id and discover:
         print("Cannot specify both --request-id and --discover", file=sys.stderr)
@@ -212,7 +216,7 @@ def command(  # noqa: C901
     include_tags = [tag.strip() for tag in include_tags.split(",") if tag]
     exclude_tags = [tag.strip() for tag in exclude_tags.split(",") if tag]
 
-    laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr)
+    laconic = LaconicRegistryClient(laconic_config, log_file=sys.stderr, mutex_lock_file=registry_lock_file)
     deployer_record = laconic.get_record(lrn, require=True)
     payment_address = deployer_record.attributes.paymentAddress
     main_logger.log(f"Payment address: {payment_address}")

diff --git a/stack_orchestrator/deploy/webapp/util.py b/stack_orchestrator/deploy/webapp/util.py
@@ -26,6 +26,8 @@
 
 from enum import Enum
 
+from stack_orchestrator.deploy.webapp.registry_mutex import registry_mutex
+
 
 class AuctionStatus(str, Enum):
     COMMIT = "commit"
@@ -391,6 +393,7 @@ def app_deployment_auctions(self, criteria=None, all=True):
         criteria["type"] = "ApplicationDeploymentAuction"
         return self.list_records(criteria, all)
 
+    @registry_mutex()
     def publish(self, record, names=None):
         if names is None:
             names = []
@@ -421,6 +424,7 @@ def publish(self, record, names=None):
         finally:
             logged_cmd(self.log_file, "rm", "-rf", tmpdir)
 
+    @registry_mutex()
     def set_name(self, name, record_id):
         logged_cmd(
             self.log_file,
@@ -434,6 +438,7 @@ def set_name(self, name, record_id):
             record_id,
         )
 
+    @registry_mutex()
     def delete_name(self, name):
         logged_cmd(
             self.log_file,
@@ -446,6 +451,7 @@ def delete_name(self, name):
             name,
         )
 
+    @registry_mutex()
     def send_tokens(self, address, amount, type="alnt"):
         args = [
             "laconic",
@@ -464,58 +470,36 @@ def send_tokens(self, address, amount, type="alnt"):
 
         return AttrDict(json.loads(logged_cmd(self.log_file, *args)))
 
-    def create_auction(self, auction):
-        if auction["kind"] == AUCTION_KIND_PROVIDER:
-            args = [
-                "laconic",
-                "-c",
-                self.config_file,
-                "registry",
-                "auction",
-                "create",
-                "--kind",
-                auction["kind"],
-                "--commits-duration",
-                str(auction["commits_duration"]),
-                "--reveals-duration",
-                str(auction["reveals_duration"]),
-                "--denom",
-                auction["denom"],
-                "--commit-fee",
-                str(auction["commit_fee"]),
-                "--reveal-fee",
-                str(auction["reveal_fee"]),
-                "--max-price",
-                str(auction["max_price"]),
-                "--num-providers",
-                str(auction["num_providers"])
-            ]
-        else:
-            args = [
-                "laconic",
-                "-c",
-                self.config_file,
-                "registry",
-                "auction",
-                "create",
-                "--kind",
-                auction["kind"],
-                "--commits-duration",
-                str(auction["commits_duration"]),
-                "--reveals-duration",
-                str(auction["reveals_duration"]),
-                "--denom",
-                auction["denom"],
-                "--commit-fee",
-                str(auction["commit_fee"]),
-                "--reveal-fee",
-                str(auction["reveal_fee"]),
-                "--minimum-bid",
-                str(auction["minimum_bid"])
-            ]
+    @registry_mutex()
+    def create_deployment_auction(self, auction):
+        args = [
+            "laconic",
+            "-c",
+            self.config_file,
+            "registry",
+            "auction",
+            "create",
+            "--kind",
+            auction["kind"],
+            "--commits-duration",
+            str(auction["commits_duration"]),
+            "--reveals-duration",
+            str(auction["reveals_duration"]),
+            "--denom",
+            auction["denom"],
+            "--commit-fee",
+            str(auction["commit_fee"]),
+            "--reveal-fee",
+            str(auction["reveal_fee"]),
+            "--max-price",
+            str(auction["max_price"]),
+            "--num-providers",
+            str(auction["num_providers"])
+        ]
 
         return json.loads(logged_cmd(self.log_file, *args))["auctionId"]
 
+    @registry_mutex()
     def commit_bid(self, auction_id, amount, type="alnt"):
         args = [
             "laconic",
@@ -532,6 +516,7 @@ def commit_bid(self, auction_id, amount, type="alnt"):
 
         return json.loads(logged_cmd(self.log_file, *args))["reveal_file"]
 
+    @registry_mutex()
     def reveal_bid(self, auction_id, reveal_file_path):
         logged_cmd(
             self.log_file,