Graph refresh refactoring internal indexes

app/models/manager_refresh/inventory_collection/data_storage.rb

@@ -0,0 +1,128 @@
+module ManagerRefresh
+  class InventoryCollection
+    class DataStorage
+      include Vmdb::Logging
+
+      # @return [Array<InventoryObject>] objects of the InventoryCollection in an Array
+      attr_accessor :data
+
+      attr_reader :index_proxy, :inventory_collection
+
+      delegate :each, :size, :to => :data
+
+      delegate :find,
+               :primary_index,
+               :build_primary_index_for,
+               :build_secondary_indexes_for,
+               :to => :index_proxy
+
+      delegate :builder_params,
+               :inventory_object?,
+               :inventory_object_lazy?,
+               :manager_ref,
+               :new_inventory_object,
+               :to => :inventory_collection
+
+      def initialize(inventory_collection, secondary_refs)
+        @inventory_collection = inventory_collection
+        @data                 = []
+
+        @index_proxy = ManagerRefresh::InventoryCollection::Index::Proxy.new(inventory_collection, secondary_refs)
+      end
+
+      def <<(inventory_object)
+        unless primary_index.find(inventory_object.manager_uuid)
+          data << inventory_object
+
+          # TODO(lsmola) Maybe we do not need the secondary indexes here?
+          # Maybe we should index it like LocalDb indexes, on demand, and storing what was
+          # indexed? Maybe we should allow only lazy access and no direct find from a parser. Since for streaming
+          # refresh, things won't be parsed together and no full state will be taken.
+          build_primary_index_for(inventory_object)
+          build_secondary_indexes_for(inventory_object)
+        end
+        inventory_collection
+      end
+
+      alias push <<
+
+      def find_or_build(manager_uuid)
+        raise "The uuid consists of #{manager_ref.size} attributes, please find_or_build_by method" if manager_ref.size > 1
+
+        find_or_build_by(manager_ref.first => manager_uuid)
+      end
+
+      def find_or_build_by(manager_uuid_hash)
+        if !manager_uuid_hash.keys.all? { |x| manager_ref.include?(x) } || manager_uuid_hash.keys.size != manager_ref.size
+          raise "Allowed find_or_build_by keys are #{manager_ref}"
+        end
+
+        # Not using find by since if could take record from db, then any changes would be ignored, since such record will
+        # not be stored to DB, maybe we should rethink this?
+        primary_index.find(manager_uuid_hash) || build(manager_uuid_hash)
+      end
+
+      def build(hash)
+        hash             = builder_params.merge(hash)
+        inventory_object = new_inventory_object(hash)
+
+        uuid = inventory_object.manager_uuid
+        # Each InventoryObject must be able to build an UUID, return nil if it can't
+        return nil if uuid.blank?
+        # Return existing InventoryObject if we have it
+        return primary_index.find(uuid) if primary_index.find(uuid)
+        # Store new InventoryObject and return it
+        push(inventory_object)
+        inventory_object
+      end
+
+      def to_a
+        data
+      end
+
+      # Import/export methods
+      def from_raw_data(inventory_objects_data, available_inventory_collections)
+        inventory_objects_data.each do |inventory_object_data|
+          hash = inventory_object_data.each_with_object({}) do |(key, value), result|
+            result[key.to_sym] = if value.kind_of?(Array)
+                                   value.map { |x| from_raw_value(x, available_inventory_collections) }
+                                 else
+                                   from_raw_value(value, available_inventory_collections)
+                                 end
+          end
+          build(hash)
+        end
+      end
+
+      def from_raw_value(value, available_inventory_collections)
+        if value.kind_of?(Hash) && (value['type'] || value[:type]) == "ManagerRefresh::InventoryObjectLazy"
+          value.transform_keys!(&:to_s)
+        end
+
+        if value.kind_of?(Hash) && value['type'] == "ManagerRefresh::InventoryObjectLazy"
+          inventory_collection = available_inventory_collections[value['inventory_collection_name'].try(:to_sym)]
+          raise "Couldn't build lazy_link #{value} the inventory_collection_name was not found" if inventory_collection.blank?
+          inventory_collection.lazy_find(value['ems_ref'], :key => value['key'], :default => value['default'])
+        else
+          value
+        end
+      end
+
+      def to_raw_data
+        data.map do |inventory_object|
+          inventory_object.data.transform_values do |value|
+            if inventory_object_lazy?(value)
+              value.to_raw_lazy_relation
+            elsif value.kind_of?(Array) && (inventory_object_lazy?(value.compact.first) || inventory_object?(value.compact.first))
+              value.compact.map(&:to_raw_lazy_relation)
+            elsif inventory_object?(value)
+              value.to_raw_lazy_relation
+            else
+              value
+            end
+          end
+        end
+      end
+    end
+  end
+end

app/models/manager_refresh/inventory_collection/index/proxy.rb

@@ -0,0 +1,164 @@
+module ManagerRefresh
+  class InventoryCollection
+    module Index
+      class Proxy
+        include Vmdb::Logging
+
+        def initialize(inventory_collection, secondary_refs = {})
+          @inventory_collection = inventory_collection
+
+          @primary_ref    = {:manager_ref => @inventory_collection.manager_ref}
+          @secondary_refs = secondary_refs
+          @all_refs       = @primary_ref.merge(@secondary_refs)
+
+          @data_indexes     = {}
+          @local_db_indexes = {}
+
+          @all_refs.each do |index_name, attribute_names|
+            @data_indexes[index_name] = ManagerRefresh::InventoryCollection::Index::Type::Data.new(
+              inventory_collection,
+              attribute_names
+            )
+
+            @local_db_indexes[index_name] = ManagerRefresh::InventoryCollection::Index::Type::LocalDb.new(
+              inventory_collection,
+              attribute_names,
+              @data_indexes[index_name]
+            )
+          end
+        end
+
+        def build_primary_index_for(inventory_object)
+          # Building the object, we need to provide all keys of a primary index
+          assert_index(inventory_object.data, primary_index_ref)
+          primary_index.store_index_for(inventory_object)
+        end
+
+        def build_secondary_indexes_for(inventory_object)
+          secondary_refs.keys.each do |ref|
+            data_index(ref).store_index_for(inventory_object)
+          end
+        end
+
+        def reindex_secondary_indexes!
+          data_indexes.each do |ref, index|
+            next if ref == primary_index_ref
+
+            index.reindex!
+          end
+        end
+
+        def primary_index_ref
+          :manager_ref
+        end
+
+        def primary_index
+          data_index(primary_index_ref)
+        end
+
+        def find(manager_uuid, ref: :manager_ref)
+          # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
+          return if manager_uuid.nil?
+
+          manager_uuid = stringify_index_value(manager_uuid, ref)
+
+          return unless assert_index(manager_uuid, ref)
+
+          case strategy
+          when :local_db_find_references, :local_db_cache_all
+            local_db_index(ref).find(manager_uuid)
+          when :local_db_find_missing_references
+            data_index(ref).find(manager_uuid) || local_db_index(ref).find(manager_uuid)
+          else
+            data_index(ref).find(manager_uuid)
+          end
+        end
+
+        def find_by(manager_uuid_hash, ref: :manager_ref)
+          # TODO(lsmola) deprecate this, it's enough to have find method
+          find(manager_uuid_hash, :ref => ref)
+        end
+
+        def lazy_find_by(manager_uuid_hash, ref: :manager_ref, key: nil, default: nil)
+          # TODO(lsmola) deprecate this, it's enough to have lazy_find method
+
+          lazy_find(manager_uuid_hash, :ref => ref, :key => key, :default => default)
+        end
+
+        def lazy_find(manager_uuid, ref: :manager_ref, key: nil, default: nil)
+          # TODO(lsmola) also, it should be enough to have only 1 find method, everything can be lazy, until we try to
+          # access the data
+          # TODO(lsmola) lazy_find will support only hash, then we can remove the _by variant
+          return if manager_uuid.nil?
+          return unless assert_index(manager_uuid, ref)
+
+          ::ManagerRefresh::InventoryObjectLazy.new(inventory_collection,
+                                                    stringify_index_value(manager_uuid, ref),
+                                                    manager_uuid,
+                                                    :ref => ref, :key => key, :default => default)
+        end
+
+        private
+
+        delegate :strategy, :hash_index_with_keys, :to => :inventory_collection
+
+        attr_reader :all_refs, :data_indexes, :inventory_collection, :primary_ref, :local_db_indexes, :secondary_refs
+
+        def stringify_index_value(index_value, ref)
+          # TODO(lsmola) !!!!!!!!!! Important, move this inside of the index. We should be passing around a full hash
+          # index. Then all references should be turned into {stringified_index => full_index} hash. So that way, we can
+          # keep fast indexing using string, but we can use references to write queries autmatically (targeted,
+          # db_based, etc.)
+          # We can also save {stringified_index => full_index}, so we don't have to compute it twice.
+          if index_value.kind_of?(Hash)
+            hash_index_with_keys(named_ref(ref), index_value)
+          else
+            # TODO(lsmola) raise deprecation warning, we want to use only hash indexes
+            index_value
+          end
+        end
+
+        def data_index(name)
+          data_indexes[name] || raise("Index #{name} not defined for #{inventory_collection}")
+        end
+
+        def local_db_index(name)
+          local_db_indexes[name] || raise("Index #{name} not defined for #{inventory_collection}")
+        end
+
+        def named_ref(ref)
+          all_refs[ref]
+        end
+
+        def missing_keys(data_keys, ref)
+          named_ref(ref) - data_keys
+        end
+
+        def required_index_keys_present?(data_keys, ref)
+          missing_keys(data_keys, ref).empty?
+        end
+
+        def assert_index(manager_uuid, ref)
+          if manager_uuid.kind_of?(Hash)
+            # Test we are sending all keys required for the index
+            unless required_index_keys_present?(manager_uuid.keys, ref)
+              missing_keys = missing_keys(manager_uuid.keys, ref)
+
+              if !Rails.env.production?
+                raise "Invalid index for '#{inventory_collection}' using #{manager_uuid}. Missing keys for index #{ref} are #{missing_keys}"
+              else
+                _log.error("Invalid index for '#{inventory_collection}' using #{manager_uuid}. Missing keys for index #{ref} are #{missing_keys}")
+                return false
+              end
+            end
+          end
+
+          true
+        rescue => e
+          _log.error("Error when asserting index: #{manager_uuid}, with ref: #{ref} of #{inventory_collection}")
+          raise e
+        end
+      end
+    end
+  end
+end

app/models/manager_refresh/inventory_collection/index/type/base.rb

@@ -0,0 +1,48 @@
+module ManagerRefresh
+  class InventoryCollection
+    module Index
+      module Type
+        class Base
+          include Vmdb::Logging
+
+          def initialize(inventory_collection, attribute_names, *_args)
+            @index = {}
+
+            @inventory_collection = inventory_collection
+            @attribute_names      = attribute_names
+          end
+
+          delegate :keys, :to => :index
+
+          def store_index_for(inventory_object)
+            index[inventory_object.manager_uuid(attribute_names)] = inventory_object
+          end
+
+          def reindex!
+            self.index = {}
+            data.each do |inventory_object|
+              store_index_for(inventory_object)
+            end
+          end
+
+          # Find value based on index_value
+          #
+          # @param _index_value [String] a index_value of the InventoryObject we search for
+          def find(_index_value)
+            raise "Implement in subclass"
+          end
+
+          protected
+
+          attr_reader :attribute_names, :index, :inventory_collection
+
+          private
+
+          attr_writer :index
+
+          delegate :data, :to => :inventory_collection
+        end
+      end
+    end
+  end
+end

app/models/manager_refresh/inventory_collection/index/type/data.rb

@@ -0,0 +1,16 @@
+module ManagerRefresh
+  class InventoryCollection
+    module Index
+      module Type
+        class Data < ManagerRefresh::InventoryCollection::Index::Type::Base
+          # Find value based on index_value
+          #
+          # @param index_value [String] a index_value of the InventoryObject we search in data
+          def find(index_value)
+            index[index_value]
+          end
+        end
+      end
+    end
+  end
+end