Add stoplight for object storage failures, return HTTP 503

app/controllers/api/base_controller.rb

@@ -40,7 +40,7 @@ class Api::BaseController < ApplicationController
     render json: { error: 'This action is not allowed' }, status: 403
   end
 
-  rescue_from Mastodon::RaceConditionError do
+  rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight do
     render json: { error: 'There was a temporary problem serving your request, please try again' }, status: 503
   end
 

app/controllers/application_controller.rb

@@ -28,7 +28,7 @@ class ApplicationController < ActionController::Base
   rescue_from ActiveRecord::RecordNotFound, with: :not_found
   rescue_from Mastodon::NotPermittedError, with: :forbidden
   rescue_from HTTP::Error, OpenSSL::SSL::SSLError, with: :internal_server_error
-  rescue_from Mastodon::RaceConditionError, with: :service_unavailable
+  rescue_from Mastodon::RaceConditionError, Seahorse::Client::NetworkingError, Stoplight::Error::RedLight, with: :service_unavailable
 
   before_action :store_current_location, except: :raise_not_found, unless: :devise_controller?
   before_action :require_functional!, if: :user_signed_in?

config/initializers/paperclip.rb

@@ -95,3 +95,45 @@
     url: ENV.fetch('PAPERCLIP_ROOT_URL', '/system') + '/:class/:attachment/:id_partition/:style/:filename',
   )
 end
+
+# In some places in the code, we rescue this exception, but we don't always
+# load the S3 library, so it may be an undefined constant:
+
+unless defined?(Seahorse)
+  module Seahorse
+    module Client
+      class NetworkingError < StandardError; end
+    end
+  end
+end
+
+class Paperclip::Attachment
+  def save
+    circuit_break! do
+      flush_deletes unless @options[:keep_old_files]
+
+      process = only_process
+      @queued_for_write.except!(:original) if process.any? && !process.include?(:original)
+
+      flush_writes
+    end
+
+    @dirty = false
+    true
+  end
+
+  private
+
+  STOPLIGHT_THRESHOLD = 10
+  STOPLIGHT_COOLDOWN  = 30
+
+  def circuit_break!(&block)
+    Stoplight('object-storage', &block).with_threshold(STOPLIGHT_THRESHOLD).with_cool_off_time(STOPLIGHT_COOLDOWN).with_error_handler do |error, handle|
+      if error.is_a?(Seahorse::Client::NetworkingError)
+        handle.call(error)
+      else
+        raise error
+      end
+    end.run
+  end
+end