diff --git a/plugins/kubernetes/app/controllers/kubernetes/clusters_controller.rb b/plugins/kubernetes/app/controllers/kubernetes/clusters_controller.rb index 9e4eaa9acb..7d348fda99 100644 --- a/plugins/kubernetes/app/controllers/kubernetes/clusters_controller.rb +++ b/plugins/kubernetes/app/controllers/kubernetes/clusters_controller.rb @@ -110,7 +110,7 @@ def update_secret(namespace) def secret_exist?(secret) @cluster.client.get_secret(secret.fetch(:metadata).fetch(:name), secret.fetch(:metadata).fetch(:namespace)) true - rescue KubeException + rescue *SamsonKubernetes.connection_errors false end end diff --git a/plugins/kubernetes/app/models/kubernetes/api/pod.rb b/plugins/kubernetes/app/models/kubernetes/api/pod.rb index ef9f23ebe1..4502d8b7db 100644 --- a/plugins/kubernetes/app/models/kubernetes/api/pod.rb +++ b/plugins/kubernetes/app/models/kubernetes/api/pod.rb @@ -61,10 +61,10 @@ def containers # tries to get logs from current or previous pod depending on if it restarted def logs(container, end_time) fetch_logs(container, end_time, previous: restarted?) - rescue KubeException # not found or pod is initializing + rescue *SamsonKubernetes.connection_errors # not found or pod is initializing begin fetch_logs(container, end_time, previous: !restarted?) - rescue KubeException + rescue *SamsonKubernetes.connection_errors nil end end diff --git a/plugins/kubernetes/app/models/kubernetes/cluster.rb b/plugins/kubernetes/app/models/kubernetes/cluster.rb index 4bd9353bee..7736337eba 100644 --- a/plugins/kubernetes/app/models/kubernetes/cluster.rb +++ b/plugins/kubernetes/app/models/kubernetes/cluster.rb @@ -45,7 +45,7 @@ def namespaces def namespace_exists?(namespace) connection_valid? && namespaces.include?(namespace) - rescue KubeException + rescue *SamsonKubernetes.connection_errors false end @@ -63,7 +63,7 @@ def schedulable_nodes def connection_valid? client.api_valid? - rescue KubeException, Errno::ECONNREFUSED + rescue *SamsonKubernetes.connection_errors false end diff --git a/plugins/kubernetes/app/models/kubernetes/deploy_executor.rb b/plugins/kubernetes/app/models/kubernetes/deploy_executor.rb index ef6b837dec..9779806db7 100644 --- a/plugins/kubernetes/app/models/kubernetes/deploy_executor.rb +++ b/plugins/kubernetes/app/models/kubernetes/deploy_executor.rb @@ -124,7 +124,7 @@ def pod_statuses(release, release_docs) # efficient pod fetching by querying once per cluster instead of once per deploy group def fetch_pods(release) release.clients.flat_map do |client, query| - pods = Vault.with_retries(KubeException, attempts: 3) { client.get_pods(query) } + pods = Vault.with_retries(*SamsonKubernetes.connection_errors, attempts: 3) { client.get_pods(query) } pods.map! { |p| Kubernetes::Api::Pod.new(p, client: client) } end end diff --git a/plugins/kubernetes/app/models/kubernetes/resource.rb b/plugins/kubernetes/app/models/kubernetes/resource.rb index d8d6c625c4..34d22ec1c7 100644 --- a/plugins/kubernetes/app/models/kubernetes/resource.rb +++ b/plugins/kubernetes/app/models/kubernetes/resource.rb @@ -99,7 +99,7 @@ def update def fetch_resource reply = request(:get, name, namespace, as: :raw) JSON.parse(reply, symbolize_names: true) - rescue KubeException => e + rescue *SamsonKubernetes.connection_errors => e raise e unless e.error_code == 404 nil end diff --git a/plugins/kubernetes/lib/samson_kubernetes/samson_plugin.rb b/plugins/kubernetes/lib/samson_kubernetes/samson_plugin.rb index 8d4bd35f4c..62d8c3495c 100644 --- a/plugins/kubernetes/lib/samson_kubernetes/samson_plugin.rb +++ b/plugins/kubernetes/lib/samson_kubernetes/samson_plugin.rb @@ -6,6 +6,13 @@ class Engine < Rails::Engine app.config.assets.precompile.append %w[kubernetes/icon.png] end end + + # http errors and ssl errors are not handled uniformly, but we want to ignore/retry on both + # see https://github.com/abonas/kubeclient/issues/240 + # using a method to avoid loading kubeclient on every boot ~0.1s + def self.connection_errors + [OpenSSL::SSL::SSLError, KubeException, Errno::ECONNREFUSED].freeze + end end Samson::Hooks.view :project_tabs_view, 'samson_kubernetes/project_tab' diff --git a/plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb b/plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb index 0b66aebdd3..3ce44ac1a4 100644 --- a/plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb +++ b/plugins/kubernetes/test/models/kubernetes/deploy_executor_test.rb @@ -547,5 +547,12 @@ def worker_is_unstable executor.send(:fetch_pods, kubernetes_releases(:test_release)) end end + + it "retries on ssl failure" do + Kubeclient::Client.any_instance.expects(:get_pods).times(4).raises(OpenSSL::SSL::SSLError.new) + assert_raises OpenSSL::SSL::SSLError do + executor.send(:fetch_pods, kubernetes_releases(:test_release)) + end + end end end diff --git a/plugins/kubernetes/test/samson_kubernetes/samson_plugin_test.rb b/plugins/kubernetes/test/samson_kubernetes/samson_plugin_test.rb index c33fc51766..6d884c1db7 100644 --- a/plugins/kubernetes/test/samson_kubernetes/samson_plugin_test.rb +++ b/plugins/kubernetes/test/samson_kubernetes/samson_plugin_test.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true require_relative "../test_helper" +require "kubeclient" SingleCov.covered! @@ -50,4 +51,10 @@ def link_parts(resource) link_parts(cluster).must_equal ["test", cluster] end end + + describe ".connection_errors" do + it "works" do + SamsonKubernetes.connection_errors + end + end end