Merge pull request #154 from basecamp/lock-deploys

Deploy locks
basecamp · Mar 24, 2023 · 01a2b67 · 01a2b67
2 parents 5bbb4ae + 84540ce
commit 01a2b67
Show file tree

Hide file tree

Showing 18 changed files with 525 additions and 228 deletions.
diff --git a/README.md b/README.md
@@ -463,11 +463,11 @@ traefik:
 
 ### Configure docker options for traefik
 
-We allow users to pass additional docker options to the trafik container like 
+We allow users to pass additional docker options to the trafik container like
 
 ```yaml
 traefik:
-  options: 
+  options:
     publish:
     - 8080:8080
     volumes:
@@ -718,6 +718,30 @@ Note that by default old containers are pruned after 3 days when you run `mrsk d
 
 If you wish to remove the entire application, including Traefik, containers, images, and registry session, you can run `mrsk remove`. This will leave the servers clean.
 
+## Locking
+
+Commands that are unsafe to run concurrently will take a deploy lock while they run. The lock is the `mrsk_lock` directory on the primary server.
+
+You can check the lock status with:
+
+```
+mrsk lock status
+
+Locked by: AN Other at 2023-03-24 09:49:03 UTC
+Version: 77f45c0686811c68989d6576748475a60bf53fc2
+Message: Automatic deploy lock
+```
+
+You can also manually acquire and release the lock
+
+```
+mrsk lock acquire -m "Doing maintanence"
+```
+
+```
+mrsk lock release
+```
+
 ## Stage of development
 
 This is beta software. Commands may still move around. But we're live in production at [37signals](https://37signals.com).

diff --git a/lib/mrsk/cli/accessory.rb b/lib/mrsk/cli/accessory.rb
@@ -1,84 +1,98 @@
 class Mrsk::Cli::Accessory < Mrsk::Cli::Base
   desc "boot [NAME]", "Boot new accessory service on host (use NAME=all to boot all accessories)"
   def boot(name)
-    if name == "all"
-      MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
-    else
-      with_accessory(name) do |accessory|
-        directories(name)
-        upload(name)
+    with_lock do
+      if name == "all"
+        MRSK.accessory_names.each { |accessory_name| boot(accessory_name) }
+      else
+        with_accessory(name) do |accessory|
+          directories(name)
+          upload(name)
 
-        on(accessory.host) do
-          execute *MRSK.registry.login
-          execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
-          execute *accessory.run
-        end
+          on(accessory.host) do
+            execute *MRSK.registry.login
+            execute *MRSK.auditor.record("Booted #{name} accessory"), verbosity: :debug
+            execute *accessory.run
+          end
 
-        audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
+          audit_broadcast "Booted accessory #{name}" unless options[:skip_broadcast]
+        end
       end
     end
   end
 
   desc "upload [NAME]", "Upload accessory files to host", hide: true
   def upload(name)
-    with_accessory(name) do |accessory|
-      on(accessory.host) do
-        accessory.files.each do |(local, remote)|
-          accessory.ensure_local_file_present(local)
+    with_lock do
+      with_accessory(name) do |accessory|
+        on(accessory.host) do
+          accessory.files.each do |(local, remote)|
+            accessory.ensure_local_file_present(local)
 
-          execute *accessory.make_directory_for(remote)
-          upload! local, remote
-          execute :chmod, "755", remote
+            execute *accessory.make_directory_for(remote)
+            upload! local, remote
+            execute :chmod, "755", remote
+          end
         end
       end
     end
   end
 
   desc "directories [NAME]", "Create accessory directories on host", hide: true
   def directories(name)
-    with_accessory(name) do |accessory|
-      on(accessory.host) do
-        accessory.directories.keys.each do |host_path|
-          execute *accessory.make_directory(host_path)
+    with_lock do
+      with_accessory(name) do |accessory|
+        on(accessory.host) do
+          accessory.directories.keys.each do |host_path|
+            execute *accessory.make_directory(host_path)
+          end
         end
       end
     end
   end
 
   desc "reboot [NAME]", "Reboot existing accessory on host (stop container, remove container, start new container)"
   def reboot(name)
-    with_accessory(name) do |accessory|
-      stop(name)
-      remove_container(name)
-      boot(name)
+    with_lock do
+      with_accessory(name) do |accessory|
+        stop(name)
+        remove_container(name)
+        boot(name)
+      end
     end
   end
 
   desc "start [NAME]", "Start existing accessory container on host"
   def start(name)
-    with_accessory(name) do |accessory|
-      on(accessory.host) do
-        execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug
-        execute *accessory.start
+    with_lock do
+      with_accessory(name) do |accessory|
+        on(accessory.host) do
+          execute *MRSK.auditor.record("Started #{name} accessory"), verbosity: :debug
+          execute *accessory.start
+        end
       end
     end
   end
 
   desc "stop [NAME]", "Stop existing accessory container on host"
   def stop(name)
-    with_accessory(name) do |accessory|
-      on(accessory.host) do
-        execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug
-        execute *accessory.stop, raise_on_non_zero_exit: false
+    with_lock do
+      with_accessory(name) do |accessory|
+        on(accessory.host) do
+          execute *MRSK.auditor.record("Stopped #{name} accessory"), verbosity: :debug
+          execute *accessory.stop, raise_on_non_zero_exit: false
+        end
       end
     end
   end
 
   desc "restart [NAME]", "Restart existing accessory container on host"
   def restart(name)
-    with_accessory(name) do
-      stop(name)
-      start(name)
+    with_lock do
+      with_accessory(name) do
+        stop(name)
+        start(name)
+      end
     end
   end
 

diff --git a/lib/mrsk/cli/app.rb b/lib/mrsk/cli/app.rb
@@ -1,34 +1,36 @@
 class Mrsk::Cli::App < Mrsk::Cli::Base
   desc "boot", "Boot app on servers (or reboot app if already running)"
   def boot
-    say "Get most recent version available as an image...", :magenta unless options[:version]
-    using_version(version_or_latest) do |version|
-      say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
-
-      cli = self
-
-      on(MRSK.hosts) do |host|
-        roles = MRSK.roles_on(host)
+    with_lock do
+      say "Get most recent version available as an image...", :magenta unless options[:version]
+      using_version(version_or_latest) do |version|
+        say "Start container with version #{version} using a #{MRSK.config.readiness_delay}s readiness delay (or reboot if already running)...", :magenta
 
-        roles.each do |role|
-          execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
+        cli = self
 
-          begin
-            old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
-            execute *MRSK.app(role: role).run
-            sleep MRSK.config.readiness_delay
-            execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
+        on(MRSK.hosts) do |host|
+          roles = MRSK.roles_on(host)
 
-          rescue SSHKit::Command::Failed => e
-            if e.message =~ /already in use/
-              error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)"
-              execute *MRSK.auditor(role: role).record("Rebooted app version #{version}"), verbosity: :debug
+          roles.each do |role|
+            execute *MRSK.auditor(role: role).record("Booted app version #{version}"), verbosity: :debug
 
-              execute *MRSK.app(role: role).stop(version: version)
-              execute *MRSK.app(role: role).remove_container(version: version)
+            begin
+              old_version = capture_with_info(*MRSK.app(role: role).current_running_version).strip
               execute *MRSK.app(role: role).run
-            else
-              raise
+              sleep MRSK.config.readiness_delay
+              execute *MRSK.app(role: role).stop(version: old_version), raise_on_non_zero_exit: false if old_version.present?
+
+            rescue SSHKit::Command::Failed => e
+              if e.message =~ /already in use/
+                error "Rebooting container with same version #{version} already deployed on #{host} (may cause gap in zero-downtime promise!)"
+                execute *MRSK.auditor(role: role).record("Rebooted app version #{version}"), verbosity: :debug
+
+                execute *MRSK.app(role: role).stop(version: version)
+                execute *MRSK.app(role: role).remove_container(version: version)
+                execute *MRSK.app(role: role).run
+              else
+                raise
+              end
             end
           end
         end
@@ -38,24 +40,28 @@ def boot
 
   desc "start", "Start existing app container on servers"
   def start
-    on(MRSK.hosts) do |host|
-      roles = MRSK.roles_on(host)
+    with_lock do
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
 
-      roles.each do |role|
-        execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug
-        execute *MRSK.app(role: role).start, raise_on_non_zero_exit: false
+        roles.each do |role|
+          execute *MRSK.auditor.record("Started app version #{MRSK.config.version}"), verbosity: :debug
+          execute *MRSK.app(role: role).start, raise_on_non_zero_exit: false
+        end
       end
     end
   end
 
   desc "stop", "Stop app container on servers"
   def stop
-    on(MRSK.hosts) do |host|
-      roles = MRSK.roles_on(host)
+    with_lock do
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
 
-      roles.each do |role|
-        execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug
-        execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
+        roles.each do |role|
+          execute *MRSK.auditor(role: role).record("Stopped app"), verbosity: :debug
+          execute *MRSK.app(role: role).stop, raise_on_non_zero_exit: false
+        end
       end
     end
   end
@@ -160,40 +166,48 @@ def logs
 
   desc "remove", "Remove app containers and images from servers"
   def remove
-    stop
-    remove_containers
-    remove_images
+    with_lock do
+      stop
+      remove_containers
+      remove_images
+    end
   end
 
   desc "remove_container [VERSION]", "Remove app container with given version from servers", hide: true
   def remove_container(version)
-    on(MRSK.hosts) do |host|
-      roles = MRSK.roles_on(host)
+    with_lock do
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
 
-      roles.each do |role|
-        execute *MRSK.auditor(role: role).record("Removed app container with version #{version}"), verbosity: :debug
-        execute *MRSK.app(role: role).remove_container(version: version)
+        roles.each do |role|
+          execute *MRSK.auditor(role: role).record("Removed app container with version #{version}"), verbosity: :debug
+          execute *MRSK.app(role: role).remove_container(version: version)
+        end
       end
     end
   end
 
   desc "remove_containers", "Remove all app containers from servers", hide: true
   def remove_containers
-    on(MRSK.hosts) do |host|
-      roles = MRSK.roles_on(host)
+    with_lock do
+      on(MRSK.hosts) do |host|
+        roles = MRSK.roles_on(host)
 
-      roles.each do |role|
-        execute *MRSK.auditor(role: role).record("Removed all app containers"), verbosity: :debug
-        execute *MRSK.app(role: role).remove_containers
+        roles.each do |role|
+          execute *MRSK.auditor(role: role).record("Removed all app containers"), verbosity: :debug
+          execute *MRSK.app(role: role).remove_containers
+        end
       end
     end
   end
 
   desc "remove_images", "Remove all app images from servers", hide: true
   def remove_images
-    on(MRSK.hosts) do
-      execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
-      execute *MRSK.app.remove_images
+    with_lock do
+      on(MRSK.hosts) do
+        execute *MRSK.auditor.record("Removed all app images"), verbosity: :debug
+        execute *MRSK.app.remove_images
+      end
     end
   end
 

diff --git a/lib/mrsk/cli/base.rb b/lib/mrsk/cli/base.rb
@@ -6,6 +6,8 @@ module Mrsk::Cli
   class Base < Thor
     include SSHKit::DSL
 
+    class LockError < StandardError; end
+
     def self.exit_on_failure?() true end
 
     class_option :verbose, type: :boolean, aliases: "-v", desc: "Detailed logging"
@@ -71,5 +73,35 @@ def print_runtime
       def audit_broadcast(line)
         run_locally { execute *MRSK.auditor.broadcast(line), verbosity: :debug }
       end
+
+      def with_lock
+        acquire_lock
+
+        yield
+      ensure
+        release_lock
+      end
+
+      def acquire_lock
+        if MRSK.lock_count == 0
+          say "Acquiring the deploy lock"
+          on(MRSK.primary_host) { execute *MRSK.lock.acquire("Automatic deploy lock", MRSK.config.version) }
+        end
+        MRSK.lock_count += 1
+      rescue SSHKit::Runner::ExecuteError => e
+        if e.message =~ /cannot create directory/
+          invoke "mrsk:cli:lock:status", []
+        end
+
+        raise LockError, "Deploy lock found"
+      end
+
+      def release_lock
+        MRSK.lock_count -= 1
+        if MRSK.lock_count == 0
+          say "Releasing the deploy lock"
+          on(MRSK.primary_host) { execute *MRSK.lock.release }
+        end
+      end
   end
 end