Pluggable ScanItems

app/models/scan_item.rb

@@ -1,81 +1,10 @@
 class ScanItem < ApplicationRecord
+  include_concern "Seeding"
+
   serialize :definition
   acts_as_miq_set_member
   include UuidMixin
 
-  YAML_DIR = File.expand_path(File.join(Rails.root, "product/scan_items"))
-  Dir.mkdir(YAML_DIR) unless File.exist?(YAML_DIR)
-
-  SAMPLE_VM_PROFILE    = {:name => "sample",       :description => "VM Sample",    :mode => 'Vm',   :read_only => true}.freeze
-  SAMPLE_HOST_PROFILE  = {:name => "host sample",  :description => "Host Sample",  :mode => 'Host', :read_only => true}.freeze
-  DEFAULT_HOST_PROFILE = {:name => "host default", :description => "Host Default", :mode => 'Host'}.freeze
-
-  def self.sync_from_dir
-    where(:prod_default => 'Default').where.not(:filename => nil).each do |f|
-      next unless f.filename
-      unless File.exist?(File.join(YAML_DIR, f.filename))
-        $log.info("Scan Item: file [#{f.filename}] has been deleted from disk, deleting from model")
-        f.destroy
-      end
-    end
-
-    Dir.glob(File.join(YAML_DIR, "*.yaml")).sort.each do |f|
-      sync_from_file(f)
-    end
-  end
-
-  def self.sync_from_file(filename)
-    fd = File.open(filename)
-    item = YAML.load(fd.read)
-    fd.close
-
-    item[:filename] = filename.sub(YAML_DIR + "/", "")
-    item[:file_mtime] = File.mtime(filename).utc
-    item[:prod_default] = "Default"
-
-    rec = find_by(:name => item[:name], :filename => item[:filename])
-
-    if rec
-      if rec.filename && (rec.file_mtime.nil? || rec.file_mtime.utc < item[:file_mtime])
-        $log.info("Scan Item: [#{rec.name}] file has been updated on disk, synchronizing with model")
-        rec.update!(item)
-      end
-    else
-      $log.info("Scan Item: [#{item[:name]}] file has been added to disk, adding to model")
-      create(item)
-    end
-  end
-
-  def self.seed
-    sync_from_dir
-    preload_default_profile
-  end
-
-  def self.preload_default_profile
-    # Create sample VM scan profiles
-    vm_profile = ScanItemSet.find_or_initialize_by(:name => SAMPLE_VM_PROFILE[:name])
-    vm_profile.update(SAMPLE_VM_PROFILE)
-
-    # Create sample Host scan profiles
-    host_profile = ScanItemSet.find_or_initialize_by(:name => SAMPLE_HOST_PROFILE[:name])
-    host_profile.update(SAMPLE_HOST_PROFILE)
-
-    # Create default Host scan profiles
-    host_default = ScanItemSet.find_or_initialize_by(:name => DEFAULT_HOST_PROFILE[:name])
-    load_host_default = host_default.new_record?
-    host_default.update(DEFAULT_HOST_PROFILE)
-
-    where(:prod_default => 'Default').each do |s|
-      case s.mode
-      when "Host"
-        host_profile.add_member(s) unless host_profile.members.include?(s)
-        host_default.add_member(s) if load_host_default && !host_default.members.include?(s)
-      when "Vm"
-        vm_profile.add_member(s) unless vm_profile.members.include?(s)
-      end
-    end
-  end
-
   def self.get_default_profiles
     get_profile('default')
   end

app/models/scan_item/seeding.rb

@@ -0,0 +1,121 @@
+module ScanItem::Seeding
+  extend ActiveSupport::Concern
+
+  SCAN_ITEMS_DIR = Rails.root.join("product", "scan_items")
+
+  # Default ScanItemSets
+  SAMPLE_VM_PROFILE    = {:name => "sample",       :description => "VM Sample",    :mode => 'Vm',   :read_only => true}.freeze
+  SAMPLE_HOST_PROFILE  = {:name => "host sample",  :description => "Host Sample",  :mode => 'Host', :read_only => true}.freeze
+  DEFAULT_HOST_PROFILE = {:name => "host default", :description => "Host Default", :mode => 'Host'}.freeze
+
+  module ClassMethods
+    def seed
+      transaction do
+        seed_all_scan_items
+        seed_scan_item_sets
+      end
+    end
+
+    # Used for seeding a specific scan_item for test purposes
+    def seed_scan_item(name)
+      path = seed_files.detect { |f| File.basename(f).include?(name) }
+      raise "scan item #{name.inspect} not found" if path.nil?
+
+      seed_record(path, ScanItem.find_by(:filename => seed_filename(path)))
+    end
+
+    private
+
+    def seed_all_scan_items
+      scan_items = where(:prod_default => 'Default').index_by do |f|
+        seed_filename(f.filename)
+      end
+      seed_files.each do |f|
+        seed_record(f, scan_items.delete(seed_filename(f)))
+      end
+
+      if scan_items.any?
+        _log.info("Deleting the following ScanItems(s) as they no longer exist: #{scan_items.keys.sort.collect(&:inspect).join(", ")}")
+        ScanItem.destroy(scan_items.values.map(&:id))
+      end
+    end
+
+    def seed_scan_item_sets
+      vm_profile = ScanItemSet.find_or_initialize_by(:name => SAMPLE_VM_PROFILE[:name])
+      vm_profile.update!(SAMPLE_VM_PROFILE)
+
+      # Create sample Host scan profiles
+      host_profile = ScanItemSet.find_or_initialize_by(:name => SAMPLE_HOST_PROFILE[:name])
+      host_profile.update!(SAMPLE_HOST_PROFILE)
+
+      # Create default Host scan profiles
+      host_default = ScanItemSet.find_or_initialize_by(:name => DEFAULT_HOST_PROFILE[:name])
+      load_host_default = host_default.new_record?
+      host_default.update!(DEFAULT_HOST_PROFILE)
+
+      where(:prod_default => 'Default').each do |s|
+        case s.mode
+        when "Host"
+          host_profile.add_member(s)
+          host_default.add_member(s) if load_host_default
+        when "Vm"
+          vm_profile.add_member(s)
+        end
+      end
+    end
+
+    def seed_record(path, scan_item)
+      scan_item ||= ScanItem.new
+
+      # DB and filesystem have different precision so calling round is done in
+      # order to eliminate the second fractions diff otherwise the comparison
+      # of the file time and the scan_item time from db will always be different.
+      mtime = File.mtime(path).utc.round
+      scan_item.file_mtime = mtime
+
+      if scan_item.new_record? || scan_item.changed?
+        filename = seed_filename(path)
+
+        _log.info("#{scan_item.new_record? ? "Creating" : "Updating"} ScanItem #{filename.inspect}")
+
+        yml  = YAML.load_file(path).symbolize_keys
+        name = yml[:name].strip
+
+        attrs = yml.slice(*column_names_symbols)
+        attrs.delete(:id)
+        attrs[:filename]     = filename
+        attrs[:file_mtime]   = mtime
+        attrs[:prod_default] = "Default"
+        attrs[:name]         = name
+
+        begin
+          scan_item.update!(attrs)
+        rescue ActiveRecord::RecordInvalid
+          duplicate = find_by(:name => name)
+          if duplicate&.prod_default == "Custom"
+            _log.warn("A custom scan_item already exists with the name #{duplicate.name.inspect}.  Skipping...")
+          elsif duplicate
+            _log.warn("A default scan_item named '#{duplicate.name.inspect}' loaded from '#{duplicate.filename}' already exists. Updating attributes of existing report...")
+            duplicate.update!(attrs)
+          else
+            raise
+          end
+        end
+      end
+    end
+
+    def seed_files
+      Dir.glob(SCAN_ITEMS_DIR.join("*.{yml,yaml}")).sort + seed_plugin_files
+    end
+
+    def seed_plugin_files
+      Vmdb::Plugins.flat_map do |plugin|
+        Dir.glob(plugin.root.join("content/scan_items/*.{yml,yaml}")).sort
+      end
+    end
+
+    def seed_filename(path)
+      File.basename(path)
+    end
+  end
+end

spec/models/scan_item/data/product/scan_items/testing_scan_item.yaml

@@ -0,0 +1,13 @@
+---
+:item_type: category
+:definition:
+  content:
+  - target: vmconfig
+  - target: vmevents
+  - target: accounts
+  - target: software
+  - target: services
+  - target: system
+:name: testing_scan_item
+:description: Testing ScanItem
+:mode: Vm

spec/models/scan_item/seeding_spec.rb

@@ -0,0 +1,87 @@
+require 'fileutils'
+
+describe ScanItem do
+  describe "::Seeding" do
+    include_examples(".seed called multiple times", 6)
+
+    describe ".seed" do
+      let(:tmpdir)         { Pathname.new(Dir.mktmpdir) }
+      let(:scan_item_dir)  { tmpdir.join("product/scan_items") }
+      let(:scan_item_yml)  { scan_item_dir.join("testing_scan_item.yaml") }
+      let(:data_dir)       { Pathname.new(__dir__).join("data/product") }
+
+      before do
+        FileUtils.mkdir_p(scan_item_dir)
+        FileUtils.cp_r(Rails.root.join("product", "scan_items", "scan_item_cat.yaml"), scan_item_dir, :preserve => true)
+
+        stub_const("ScanItem::Seeding::SCAN_ITEMS_DIR", scan_item_dir)
+        expect(Vmdb::Plugins).to receive(:flat_map).at_least(:once) { [] }
+      end
+
+      after do
+        FileUtils.rm_rf(tmpdir)
+      end
+
+      it "creates, updates, and changes records" do
+        described_class.seed
+
+        orig_scan_item = ScanItem.find_by(:name => "sample_category")
+        expect(orig_scan_item).to_not be_nil
+
+        expect(ScanItem.where(:name => "testing_scan_item")).to_not exist
+
+        # Add new records
+        FileUtils.cp_r(data_dir, tmpdir, :preserve => true)
+
+        described_class.seed
+
+        scan_item = ScanItem.find_by(:name => "testing_scan_item")
+
+        expect(scan_item).to have_attributes(
+          :name         => "testing_scan_item",
+          :description  => "Testing ScanItem",
+          :filename     => "testing_scan_item.yaml",
+          :file_mtime   => File.mtime(scan_item_yml).utc.round,
+          :prod_default => "Default",
+          :mode         => "Vm",
+          :definition   => a_hash_including("content" => include("target" => "vmconfig"))
+        )
+
+        # Update reports
+        orig_scan_item_mtime = orig_scan_item.file_mtime
+        scan_item_mtime = scan_item.file_mtime
+
+        # The mtime rounding is granular to the second, so need to be higher
+        # than that for test purposes
+        FileUtils.touch(scan_item_yml, :mtime => 1.second.from_now.to_time)
+
+        described_class.seed
+
+        expect(orig_scan_item.reload.file_mtime).to eq(orig_scan_item_mtime)
+        expect(scan_item.reload.file_mtime).to_not eq(scan_item_mtime)
+
+        # Delete reports
+        FileUtils.rm_f(scan_item_yml)
+
+        described_class.seed
+
+        expect { scan_item.reload }.to raise_error(ActiveRecord::RecordNotFound)
+      end
+    end
+  end
+
+  describe ".seed_files (private)" do
+    it "will include files from core" do
+      expect(described_class.send(:seed_files)).to include(
+        a_string_starting_with(Rails.root.join("product", "scan_items").to_s)
+      )
+    end
+
+    it "will include files from plugins" do
+      skip "Until a plugin brings a scan item"
+      expect(described_class.send(:seed_files)).to include(
+        a_string_matching(%r{#{Bundler.install_path}/.+/content/scan_items/})
+      )
+    end
+  end
+end