Introduce FirmwareRegistry and FirmwareBinary

One can update physical server's firmware, but the firmware has
to be accessible somewhere, usually on some HTTP/FTP/SAMBA link.
Furthermore, there is usually some metadata associated with the
firmware to tell what kind of hardware (manufacturer, model type)
is it meant for.

With this commit we introduce a new model, FirmwareRegistry, that
supports various kinds of metadata parsing by leveraging STI. User
is supposed to create FirmwareRegistry instance via UI feeding it
url+authentication and then click "Refresh Relationships" on it.
ManageIQ will then invoke

```
registry.sync_fw_binaries_queue
```

which in turn will connect to the metadata repository to list
available firmwares and store them as FirmwareBinary (no binary
data is really stored, only HTTP/FTP/SAMBA link and some metadata).

In followup PRs we will then be able to list available firmware
binaries to update physical server's firmware.

Signed-off-by: Miha Pleško <[email protected]>

app/models/firmware_binary.rb

@@ -0,0 +1,15 @@
+class FirmwareBinary < ApplicationRecord
+  validates :name, :presence => true
+  has_many :endpoints, :dependent => :destroy, :as => :resource, :inverse_of => :resource
+  has_many :firmware_binary_firmware_constraints, :dependent => :destroy
+  has_many :firmware_constraints, :through => :firmware_binary_firmware_constraints
+  belongs_to :firmware_registry
+
+  def allow_duplicate_endpoint_url?
+    true
+  end
+
+  def urls
+    endpoints.map(&:url)
+  end
+end

app/models/firmware_binary_firmware_constraint.rb

@@ -0,0 +1,6 @@
+class FirmwareBinaryFirmwareConstraint < ApplicationRecord
+  self.table_name = 'firmware_binaries_firmware_constraints'
+
+  belongs_to :firmware_binary
+  belongs_to :firmware_constraint
+end

app/models/firmware_constraint.rb

@@ -0,0 +1,9 @@
+class FirmwareConstraint < ApplicationRecord
+  has_many :firmware_binary_firmware_constraints, :dependent => :destroy
+  has_many :firmware_binaries, :through => :firmware_binary_firmware_constraints
+
+  # Firmware constraints are nameless, but we need to show something on the UI.
+  def name
+    _('Firmware Constraint')
+  end
+end

app/models/firmware_registry.rb

@@ -0,0 +1,37 @@
+class FirmwareRegistry < ApplicationRecord
+  include NewWithTypeStiMixin
+
+  has_many :firmware_binaries, :dependent => :destroy
+  has_one :endpoint, :as => :resource, :dependent => :destroy, :inverse_of => :resource
+  has_one :authentication, :as => :resource, :dependent => :destroy, :inverse_of => :resource
+
+  validates :name, :presence => true, :uniqueness => true
+
+  def sync_fw_binaries_queue
+    MiqQueue.put_unless_exists(
+      :class_name  => self.class.name,
+      :instance_id => id,
+      :method_name => 'sync_fw_binaries'
+    )
+  end
+
+  def sync_fw_binaries
+    _log.info("Synchronizing FirmwareBinaries from #{self.class.name} [#{id}|#{name}]...")
+    sync_fw_binaries_raw
+    self.last_refresh_error = nil
+    _log.info("Synchronizing FirmwareBinaries from #{self.class.name} [#{id}|#{name}]... Complete")
+  rescue MiqException::Error => e
+    self.last_refresh_error = e
+  ensure
+    self.last_refresh_on = Time.now.utc
+    save!
+  end
+
+  def sync_fw_binaries_raw
+    raise NotImplementedError, 'Must be implemented in subclass'
+  end
+
+  def self.display_name(number = 1)
+    n_('Firmware Registry', 'Firmware Registries', number)
+  end
+end

app/models/firmware_registry_simple.rb

@@ -0,0 +1,61 @@
+class FirmwareRegistrySimple < FirmwareRegistry
+  CONSTRAINTS = %i[manufacturer model].freeze
+
+  def sync_fw_binaries_raw
+    remote_binaries.each do |binary_hash|
+      binary = FirmwareBinary.find_or_create_by(:firmware_registry => self, :external_id => binary_hash['id'])
+      _log.info("Updating FirmwareBinary [#{binary.id} | #{binary.name}]...")
+
+      binary.name = binary_hash['filename'] || binary_hash['id']
+      binary.description = binary_hash['description']
+      binary.version = binary_hash['version']
+      binary.save!
+
+      unless binary.urls.sort == binary_hash['urls'].sort
+        _log.info("Updating FirmwareBinary [#{binary.id} | #{binary.name}] endpoints...")
+        binary.endpoints.destroy_all
+        binary.endpoints = binary_hash['urls'].map { |url| Endpoint.create(:url => url) }
+      end
+
+      currents = binary.firmware_constraints.map { |c| c.attributes.slice(*CONSTRAINTS) }
+      remotes = binary_hash['compatible_server_models'].map { |r| r.symbolize_keys.slice(*CONSTRAINTS) }
+      unless currents.map(&:to_a).sort == remotes.map(&:to_a).sort
+        _log.info("Updating FirmwareBinary [#{binary.id} | #{binary.name}] constraints...")
+        binary.firmware_constraints = remotes.map do |remote|
+          FirmwareConstraint.find_or_create_by(remote)
+        end
+      end
+
+      _log.info("Updating FirmwareBinary [#{binary.id} | #{binary.name}]... completed.")
+    end
+  end
+
+  def remote_binaries
+    self.class.fetch_from_remote(
+      endpoint.url,
+      authentication.userid,
+      authentication.password,
+      :verify_ssl => endpoint.security_protocol
+    )
+  end
+
+  def self.fetch_from_remote(url, username, password, verify_ssl: OpenSSL::SSL::VERIFY_PEER)
+    uri = URI.parse(url)
+    request = Net::HTTP::Get.new(uri.request_uri)
+    request['Content-Type'] = 'application/json'
+    request.basic_auth(username, password)
+    response = Net::HTTP.new(uri.host, uri.port).tap do |http|
+      http.open_timeout = 5.seconds
+      http.use_ssl = url.start_with?('https')
+      http.verify_mode = verify_ssl
+    end.request(request)
+
+    raise MiqException::Error, "Bad status returned: #{response.code}" if response.code.to_i != 200
+
+    JSON.parse(response.body)
+  rescue SocketError => e
+    raise MiqException::Error, e
+  rescue JSON::ParserError => e
+    raise MiqException::Error, e
+  end
+end

spec/factories/firmware_binary.rb

@@ -0,0 +1,14 @@
+FactoryBot.define do
+  factory :firmware_binary do
+    sequence(:name)        { |n| "firmware_binary_#{seq_padded_for_sorting(n)}" }
+    sequence(:description) { 'Firmware Binary Description' }
+    sequence(:version)     { 'v1.2.3' }
+
+    trait :with_endpoints do
+      after(:create) do |binary|
+        binary.endpoints << FactoryBot.create(:endpoint, :url => 'http://test.binary.1', :resource => binary)
+        binary.endpoints << FactoryBot.create(:endpoint, :url => 'http://test.binary.2', :resource => binary)
+      end
+    end
+  end
+end

spec/factories/firmware_constraint.rb

@@ -0,0 +1,6 @@
+FactoryBot.define do
+  factory :firmware_constraint do
+    sequence(:manufacturer)  { |n| "manufacturer_#{seq_padded_for_sorting(n)}" }
+    sequence(:model)         { |n| "model_#{seq_padded_for_sorting(n)}" }
+  end
+end

spec/factories/firmware_registry.rb

@@ -0,0 +1,9 @@
+FactoryBot.define do
+  factory :firmware_registry do
+    sequence(:name) { |n| "firmware_registry_#{seq_padded_for_sorting(n)}" }
+    endpoint { |n| FactoryBot.create(:endpoint, :url => "http://test.registry.#{n}") }
+    authentication { FactoryBot.create(:authentication) }
+  end
+
+  factory :firmware_registry_simple, :parent => :firmware_registry, :class => 'FirmwareRegistrySimple'
+end

spec/models/firmware_binary_spec.rb

@@ -0,0 +1,49 @@
+describe FirmwareBinary do
+  subject { FactoryBot.create(:firmware_binary) }
+
+  describe '#allow_duplicate_endpoint_url?' do
+    let(:binary1) { FactoryBot.create(:firmware_binary) }
+    let(:binary2) { FactoryBot.create(:firmware_binary) }
+
+    it 'has the flag set' do
+      expect(subject.allow_duplicate_endpoint_url?).to eq(true)
+    end
+
+    it 'two different binaries are allowed to have same url' do
+      expect do
+        binary1.endpoints << FactoryBot.create(:endpoint, :url => 'same-url', :resource => binary1)
+        binary2.endpoints << FactoryBot.create(:endpoint, :url => 'same-url', :resource => binary2)
+      end.not_to raise_error
+    end
+  end
+
+  describe '#urls' do
+    it 'lists from all endpoints' do
+      subject.endpoints << FactoryBot.create(:endpoint, :url => 'url1')
+      subject.endpoints << FactoryBot.create(:endpoint, :url => 'url2')
+      expect(subject.urls).to eq(%w[url1 url2])
+    end
+  end
+
+  describe '#has_many endpoints' do
+    before { subject.endpoints = [endpoint1, endpoint2] }
+    let(:endpoint1) { FactoryBot.create(:endpoint) }
+    let(:endpoint2) { FactoryBot.create(:endpoint) }
+
+    it 'one to many connection exists' do
+      expect(subject.endpoints).to match_array([endpoint1, endpoint2])
+    end
+  end
+
+  describe '#has_many firmware_constraints' do
+    before { subject.firmware_constraints = [constraint1, constraint2] }
+    let(:constraint1) { FactoryBot.create(:firmware_constraint) }
+    let(:constraint2) { FactoryBot.create(:firmware_constraint) }
+
+    it 'many to many connection exists' do
+      expect(subject.firmware_constraints).to match_array([constraint1, constraint2])
+      expect(constraint1.firmware_binaries).to match_array([subject])
+      expect(constraint2.firmware_binaries).to match_array([subject])
+    end
+  end
+end

spec/models/firmware_registry_simple/data/sample_firmware_binaries.json

@@ -0,0 +1,54 @@
+[
+  {
+    "description": "DELL BIOS update",
+    "urls": [
+      "http://172.16.93.126:8080/files/dell-bios-2019-03-23.bin",
+      "https://172.16.93.126:8443/files/dell-bios-2019-03-23.bin"
+    ],
+    "compatible_server_models": [
+      {
+        "model": "Common Model",
+        "manufacturer": "Common Manufacturer"
+      }
+    ],
+    "id": 1,
+    "version": "10.4.3",
+    "filename": "dell-bios-2019-03-23.bin"
+  },
+  {
+    "description": "DELL BIOS update",
+    "urls": [
+      "http://172.16.93.126:8080/files/dell-bios-2019-04-13.bin",
+      "https://172.16.93.126:8443/files/dell-bios-2019-04-13.bin"
+    ],
+    "compatible_server_models": [
+      {
+        "model": "Common Model",
+        "manufacturer": "Common Manufacturer"
+      }
+    ],
+    "id": 2,
+    "version": "10.4.4",
+    "filename": "dell-bios-2019-04-13.bin"
+  },
+  {
+    "description": "SuperMicro NIC firmware",
+    "urls": [
+      "http://172.16.93.126:8080/files/nic-update-3.exe",
+      "https://172.16.93.126:8443/files/nic-update-3.exe"
+    ],
+    "compatible_server_models": [
+      {
+        "model": "SYS-6033A-GHJ7Z",
+        "manufacturer": "SuperMicro"
+      },
+      {
+        "model": "SYS-5019D-FN8TP",
+        "manufacturer": "SuperMicro"
+      }
+    ],
+    "id": 3,
+    "version": "4.3.2-101-z",
+    "filename": "nic-update-3.exe"
+  }
+]