Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Kubernetes-ify the container orchestrator #19500

Merged
merged 1 commit into from
Nov 13, 2019
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions app/models/miq_worker/deployment_per_worker.rb
Original file line number Diff line number Diff line change
@@ -3,14 +3,14 @@ module DeploymentPerWorker
extend ActiveSupport::Concern

def create_container_objects
ContainerOrchestrator.new.create_deployment_config(worker_deployment_name) do |definition|
ContainerOrchestrator.new.create_deployment(worker_deployment_name) do |definition|
configure_worker_deployment(definition, 1)
definition[:spec][:template][:spec][:containers].first[:env] << {:name => "EMS_IDS", :value => Array.wrap(self.class.ems_id_from_queue_name(queue_name)).join(",")}
end
end

def delete_container_objects
ContainerOrchestrator.new.delete_deployment_config(worker_deployment_name)
ContainerOrchestrator.new.delete_deployment(worker_deployment_name)
end

def stop_container
4 changes: 2 additions & 2 deletions app/models/miq_worker/replica_per_worker.rb
Original file line number Diff line number Diff line change
@@ -3,14 +3,14 @@ module ReplicaPerWorker
extend ActiveSupport::Concern

def create_container_objects
ContainerOrchestrator.new.create_deployment_config(worker_deployment_name) do |definition|
ContainerOrchestrator.new.create_deployment(worker_deployment_name) do |definition|
configure_worker_deployment(definition)
end
scale_deployment
end

def delete_container_objects
ContainerOrchestrator.new.delete_deployment_config(worker_deployment_name)
ContainerOrchestrator.new.delete_deployment(worker_deployment_name)
end

def stop_container
4 changes: 2 additions & 2 deletions app/models/miq_worker/service_worker.rb
Original file line number Diff line number Diff line change
@@ -8,7 +8,7 @@ def create_container_objects
orchestrator = ContainerOrchestrator.new

orchestrator.create_service(worker_deployment_name, SERVICE_PORT)
orchestrator.create_deployment_config(worker_deployment_name) do |definition|
orchestrator.create_deployment(worker_deployment_name) do |definition|
configure_worker_deployment(definition)

definition[:spec][:serviceName] = worker_deployment_name
@@ -24,7 +24,7 @@ def create_container_objects

def delete_container_objects
orch = ContainerOrchestrator.new
orch.delete_deployment_config(worker_deployment_name)
orch.delete_deployment(worker_deployment_name)
orch.delete_service(worker_deployment_name)
end

34 changes: 11 additions & 23 deletions lib/container_orchestrator.rb
Original file line number Diff line number Diff line change
@@ -10,14 +10,14 @@ def self.available?
File.exist?(TOKEN_FILE) && File.exist?(CA_CERT_FILE)
end

def scale(deployment_config_name, replicas)
connection.patch_deployment_config(deployment_config_name, { :spec => { :replicas => replicas } }, my_namespace)
def scale(deployment_name, replicas)
kube_apps_connection.patch_deployment(deployment_name, { :spec => { :replicas => replicas } }, my_namespace)
end

def create_deployment_config(name)
definition = deployment_config_definition(name)
def create_deployment(name)
definition = deployment_definition(name)
yield(definition) if block_given?
connection.create_deployment_config(definition)
kube_apps_connection.create_deployment(definition)
rescue KubeException => e
raise unless e.message =~ /already exists/
end
@@ -38,21 +38,9 @@ def create_secret(name, data)
raise unless e.message =~ /already exists/
end

def delete_deployment_config(name)
rc = kube_connection.get_replication_controllers(
:label_selector => "openshift.io/deployment-config.name=#{name}",
:namespace => my_namespace
).first

def delete_deployment(name)
scale(name, 0)
connection.delete_deployment_config(name, my_namespace)
delete_replication_controller(rc.metadata.name) if rc
rescue KubeException => e
raise unless e.message =~ /not found/
end

def delete_replication_controller(name)
kube_connection.delete_replication_controller(name, my_namespace)
kube_apps_connection.delete_deployment(name, my_namespace)
rescue KubeException => e
raise unless e.message =~ /not found/
end
@@ -71,14 +59,14 @@ def delete_secret(name)

private

def connection
@connection ||= raw_connect(manager_uri("/oapi"))
end

def kube_connection
@kube_connection ||= raw_connect(manager_uri("/api"))
end

def kube_apps_connection
@kube_apps_connection ||= raw_connect(manager_uri("/apis/apps"))
end

def raw_connect(uri)
ssl_options = {
:verify_ssl => OpenSSL::SSL::VERIFY_PEER,
5 changes: 2 additions & 3 deletions lib/container_orchestrator/object_definition.rb
Original file line number Diff line number Diff line change
@@ -2,19 +2,18 @@ class ContainerOrchestrator
module ObjectDefinition
private

def deployment_config_definition(name)
def deployment_definition(name)
{
:metadata => {
:name => name,
:labels => {:app => app_name},
:namespace => my_namespace,
},
:spec => {
:selector => {:name => name, :app => app_name},
:selector => {:matchLabels => {:name => name}},
:template => {
:metadata => {:name => name, :labels => {:name => name, :app => app_name}},
:spec => {
:serviceAccount => "miq-anyuid",
:serviceAccountName => "miq-anyuid",
:containers => [{
:name => name,
83 changes: 31 additions & 52 deletions spec/lib/container_orchestrator_spec.rb
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
describe ContainerOrchestrator do
let(:connection) { subject.send(:connection) }
let(:kube_connection) { subject.send(:kube_connection) }
let(:cert) { Tempfile.new("cert") }
let(:token) { Tempfile.new("token") }
let(:cert_path) { cert.path }
let(:token_path) { token.path }
let(:kube_host) { "kube.example.com" }
let(:kube_port) { "8443" }
let(:namespace) { "manageiq" }
let(:kube_apps_connection) { subject.send(:kube_apps_connection) }
let(:kube_connection) { subject.send(:kube_connection) }
let(:cert) { Tempfile.new("cert") }
let(:token) { Tempfile.new("token") }
let(:cert_path) { cert.path }
let(:token_path) { token.path }
let(:kube_host) { "kube.example.com" }
let(:kube_port) { "8443" }
let(:namespace) { "manageiq" }

before do
stub_const("ContainerOrchestrator::CA_CERT_FILE", cert_path)
@@ -37,14 +37,6 @@
end
end

describe "#connection (private)" do
it "connects to the correct uri" do
expect(connection.api_endpoint.to_s).to eq("https://kube.example.com:8443/oapi")
expect(connection.auth_options[:bearer_token_file]).to eq(token_path)
expect(connection.ssl_options[:verify_ssl]).to eq(1)
end
end

describe "#kube_connection (private)" do
it "connects to the correct uri" do
expect(kube_connection.api_endpoint.to_s).to eq("https://kube.example.com:8443/api")
@@ -53,38 +45,44 @@
end
end

describe "#kube_apps_connection (private)" do
it "connects to the correct uri" do
expect(kube_apps_connection.api_endpoint.to_s).to eq("https://kube.example.com:8443/apis/apps")
expect(kube_apps_connection.auth_options[:bearer_token_file]).to eq(token_path)
expect(kube_apps_connection.ssl_options[:verify_ssl]).to eq(1)
end
end

context "with stub connections" do
let(:connection_stub) { double("OpenShiftConnection") }
let(:apps_connection_stub) { double("AppsConnection") }
let(:kube_connection_stub) { double("KubeConnection") }

before do
allow(subject).to receive(:connection).and_return(connection_stub)
allow(subject).to receive(:kube_apps_connection).and_return(apps_connection_stub)
allow(subject).to receive(:kube_connection).and_return(kube_connection_stub)
end

describe "#scale" do
it "patches the deployment config with the specified number of replicas" do
it "patches the deployment with the specified number of replicas" do
deployment_patch = {:spec => {:replicas => 4}}
expect(connection_stub).to receive(:patch_deployment_config).with("deployment", deployment_patch, namespace)
expect(apps_connection_stub).to receive(:patch_deployment).with("deployment", deployment_patch, namespace)

subject.scale("deployment", 4)
end
end

describe "#create_deployment_config" do
it "creates a deployment config with the given name and edits" do
expect(connection_stub).to receive(:create_deployment_config) do |definition|
describe "#create_deployment" do
it "creates a deployment with the given name and edits" do
expect(apps_connection_stub).to receive(:create_deployment) do |definition|
expect(definition[:metadata][:name]).to eq("test")
expect(definition[:metadata][:namespace]).to eq("manageiq")

expect(definition[:spec][:template][:spec][:serviceAccount]).to eq("test-account")
expect(definition[:spec][:template][:spec][:serviceAccountName]).to eq("test-account")

expect(definition[:spec][:template][:spec][:containers].first[:image]).to eq("test-image")
end

subject.create_deployment_config("test") do |spec|
spec[:spec][:template][:spec][:serviceAccount] = "test-account"
subject.create_deployment("test") do |spec|
spec[:spec][:template][:spec][:serviceAccountName] = "test-account"

spec[:spec][:template][:spec][:containers].first[:image] = "test-image"
@@ -93,9 +91,9 @@

it "doesn't raise an exception for an existing object" do
error = KubeException.new(500, "deployment config already exists", "")
expect(connection_stub).to receive(:create_deployment_config).and_raise(error)
expect(apps_connection_stub).to receive(:create_deployment).and_raise(error)

expect { subject.create_deployment_config("test") }.not_to raise_error
expect { subject.create_deployment("test") }.not_to raise_error
end
end

@@ -148,31 +146,12 @@
end
end

describe "#delete_deployment_config" do
describe "#delete_deployment" do
it "deletes the replication controller if it exists" do
rc = double("ReplicationController", :metadata => double(:name => "rc_name"))

expect(kube_connection_stub).to receive(:get_replication_controllers)
.with(:label_selector => "openshift.io/deployment-config.name=dc_name", :namespace => "manageiq")
.and_return([rc])

expect(connection_stub).to receive(:delete_deployment_config).with("dc_name", "manageiq")
expect(kube_connection_stub).to receive(:delete_replication_controller).with("rc_name", "manageiq")
expect(subject).to receive(:scale).with("dc_name", 0)

subject.delete_deployment_config("dc_name")
end

it "doesn't try to delete the replication controler if it doesn't exist" do
expect(kube_connection_stub).to receive(:get_replication_controllers)
.with(:label_selector => "openshift.io/deployment-config.name=dc_name", :namespace => "manageiq")
.and_return([])

expect(connection_stub).to receive(:delete_deployment_config).with("dc_name", "manageiq")
expect(kube_connection_stub).not_to receive(:delete_replication_controller)
expect(subject).to receive(:scale).with("dc_name", 0)
expect(apps_connection_stub).to receive(:delete_deployment).with("deploy_name", "manageiq")
expect(subject).to receive(:scale).with("deploy_name", 0)

subject.delete_deployment_config("dc_name")
subject.delete_deployment("deploy_name")
end
end
end