Merge pull request #283 from betadots/issue-264

Allow doing lookups

.rubocop_todo.yml

@@ -256,11 +256,6 @@ Style/BlockDelimiters:
 Style/ClassAndModuleChildren:
   Exclude:
     - 'test/channels/application_cable/connection_test.rb'
-    - 'test/models/hiera_data/config_test.rb'
-    - 'test/models/hiera_data/data_file_test.rb'
-    - 'test/models/hiera_data/git_repo_test.rb'
-    - 'test/models/hiera_data/interpolation_test.rb'
-    - 'test/models/hiera_data/yaml_file_test.rb'
 
 # Offense count: 1
 # This cop supports safe autocorrection (--autocorrect).

Gemfile

@@ -33,6 +33,7 @@ gem 'net-ldap', require: "net/ldap"
 gem 'puppet'
 gem 'puppetdb-ruby', require: 'puppetdb'
 gem 'ruby-saml'
+gem 'deep_merge', require: "deep_merge/core"
 
 # To use retry middleware with Faraday v2.0+
 gem 'faraday-retry'

Gemfile.lock

@@ -191,10 +191,10 @@ GEM
       rake (>= 10.0)
     globalid (1.2.1)
       activesupport (>= 6.1)
-    google-protobuf (3.25.2)
-    google-protobuf (3.25.2-arm64-darwin)
-    google-protobuf (3.25.2-x86_64-darwin)
-    google-protobuf (3.25.2-x86_64-linux)
+    google-protobuf (3.25.3)
+    google-protobuf (3.25.3-arm64-darwin)
+    google-protobuf (3.25.3-x86_64-darwin)
+    google-protobuf (3.25.3-x86_64-linux)
     hiera-eyaml (3.4.0)
       highline
       optimist
@@ -479,6 +479,7 @@ DEPENDENCIES
   capybara (>= 2.15)
   dartsass-rails
   dartsass-sprockets
+  deep_merge
   diffy
   factory_bot_rails
   faker

app/controllers/application_controller.rb

@@ -40,6 +40,13 @@ def load_environments
     authorize! :show, @environment
   end
 
+  def load_nodes
+    @nodes = Node.all
+    @nodes.select! { |n| current_user.may_access?(n) }
+    @node = Node.find(params[:node_id])
+    authorize! :show, @node
+  end
+
   def display_error_page(error)
     @error = error
     render template: "page/error", status: :internal_server_error

app/controllers/keys_controller.rb

@@ -42,13 +42,4 @@ def destroy
     redirect_to environment_node_key_path(@environment, @node, @key),
       notice: "Value was removed successfully"
   end
-
-  private
-
-  def load_nodes
-    @nodes = Node.all
-    @nodes.select! { |n| current_user.may_access?(n) }
-    @node = Node.find(params[:node_id])
-    authorize! :show, @node
-  end
 end

app/controllers/lookups_controller.rb

@@ -0,0 +1,22 @@
+class LookupsController < ApplicationController
+  before_action :load_environments
+  before_action :load_nodes
+  before_action :load_keys
+
+  def show
+    @result = @key.lookup(@node)
+  rescue Hdm::Error => e
+    @error = e
+
+    render action: "error"
+  end
+
+  private
+
+  def load_keys
+    @keys = Key.all_for(@node, environment: @environment)
+    @keys.select! { |k| current_user.may_access?(k) }
+    @key = Key.new(environment: @environment, name: params[:key_id])
+    authorize! :show, @key
+  end
+end

app/models/hiera_data.rb

@@ -107,32 +107,35 @@ def remove_key(hierarchy_name, path, key, facts: {})
 
   def decrypt_value(hierarchy_name, value)
     hierarchy = find_hierarchy(hierarchy_name)
-    Hiera::Backend::Eyaml::Options["pkcs7_private_key"] = hierarchy.private_key
-    Hiera::Backend::Eyaml::Options["pkcs7_public_key"] = hierarchy.public_key
-    parser = Hiera::Backend::Eyaml::Parser::Parser.new([Hiera::Backend::Eyaml::Parser::EncHieraTokenType.new, Hiera::Backend::Eyaml::Parser::EncBlockTokenType.new])
-    tokens = parser.parse(value)
-    tokens.map(&:to_plain_text).join
+    public_key = hierarchy.public_key
+    private_key = hierarchy.private_key
+    EYamlFile.decrypt_value(value, public_key:, private_key:)
   end
 
   def encrypt_value(hierarchy_name, value)
     hierarchy = find_hierarchy(hierarchy_name)
-    Hiera::Backend::Eyaml::Options["pkcs7_private_key"] = hierarchy.private_key
-    Hiera::Backend::Eyaml::Options["pkcs7_public_key"] = hierarchy.public_key
-    encryptor = Hiera::Backend::Eyaml::Encryptor.find
-    ciphertext = encryptor.encode(encryptor.encrypt(value))
-    token = Hiera::Backend::Eyaml::Parser::EncToken.new(:block, value, encryptor, ciphertext, nil, '  ')
-    token.to_encrypted format: :string
+    public_key = hierarchy.public_key
+    private_key = hierarchy.private_key
+    EYamlFile.encrypt_value(value, public_key:, private_key:)
+  end
+
+  def lookup_options_for(key, facts: {}, decrypt: false)
+    candidates = lookup_for(facts, decrypt:)
+                 .lookup("lookup_options", merge_strategy: :hash)
+    merge = extract_merge_value(key, candidates)
+    case merge
+    when String
+      merge
+    when Hash
+      merge["strategy"]
+    else
+      "first"
+    end
   end
 
-  def lookup_options(facts)
-    result = {}
-    config.hierarchies.each do |hierarchy|
-      hierarchy.resolved_paths(facts:).each do |path|
-        file = DataFile.new(path: hierarchy.datadir(facts:).join(path))
-        result = (file["lookup_options"] || {}).merge(result)
-      end
-    end
-    result
+  def lookup(key, facts: {}, decrypt: false)
+    merge_strategy = lookup_options_for(key, facts:, decrypt:).to_sym
+    lookup_for(facts).lookup(key, merge_strategy:)
   end
 
   private
@@ -145,5 +148,29 @@ def config
   def find_hierarchy(name)
     config.hierarchies.find { |h| h.name == name }
   end
+
+  def lookup_for(facts, decrypt: false)
+    @cached_lookups ||= {}
+    @cached_lookups[facts] ||= begin
+      hashes = config.hierarchies.flat_map do |hierarchy|
+        hierarchy.resolved_paths(facts:).map do |path|
+          DataFile.new(
+            path: hierarchy.datadir(facts:).join(path),
+            type: hierarchy.backend,
+            options: hierarchy.file_options.merge({ decrypt: })
+          ).content
+        end
+      end
+      Lookup.new(hashes.compact)
+    end
+  end
+
+  def extract_merge_value(key, candidates)
+    result = candidates&.dig(key)
+    result ||= candidates&.select { |k, _v| k[0] == "^" }
+                         &.find { |k, _v| key.match(Regexp.new(k)) }
+                         &.last
+    result&.dig("merge")
+  end
 end
 # rubocop:enable Metrics/ClassLength

app/models/hiera_data/data_file.rb

@@ -2,12 +2,14 @@ class HieraData
   class DataFile
     attr_reader :path, :file
 
-    delegate :exist?, :writable?, :keys, :content_for_key, :[],
+    delegate :content, :exist?, :writable?, :keys,
+      :content_for_key, :[],
       to: :file
 
-    def initialize(path:, facts: {}, type: :yaml)
+    def initialize(path:, facts: {}, options: {}, type: :yaml)
       @path = path
       @facts = facts
+      @options = options
       @replaced_from_git = false
       setup_git_location
       @file = create_file(type)
@@ -58,6 +60,8 @@ def matching_git_location
 
     def create_file(type)
       case type
+      when :eyaml
+        EYamlFile.new(path: @path, options: @options)
       when :yaml
         YamlFile.new(path: @path)
       else

app/models/hiera_data/e_yaml_file.rb

@@ -0,0 +1,55 @@
+class HieraData
+  class EYamlFile < YamlFile
+    ENCRYPTED_PATTERN = /.*ENC\[.*\]/
+
+    def self.encrypted?(value)
+      value.is_a?(String) && !!value.match(ENCRYPTED_PATTERN)
+    end
+
+    def self.decrypt_value(value, public_key:, private_key:)
+      Hiera::Backend::Eyaml::Options["pkcs7_private_key"] = private_key
+      Hiera::Backend::Eyaml::Options["pkcs7_public_key"] = public_key
+      parser = Hiera::Backend::Eyaml::Parser::Parser.new([Hiera::Backend::Eyaml::Parser::EncHieraTokenType.new, Hiera::Backend::Eyaml::Parser::EncBlockTokenType.new])
+      tokens = parser.parse(value)
+      tokens.map(&:to_plain_text).join
+    end
+
+    def self.encrypt_value(value, public_key:, private_key:)
+      Hiera::Backend::Eyaml::Options["pkcs7_private_key"] = private_key
+      Hiera::Backend::Eyaml::Options["pkcs7_public_key"] = public_key
+      encryptor = Hiera::Backend::Eyaml::Encryptor.find
+      ciphertext = encryptor.encode(encryptor.encrypt(value))
+      token = Hiera::Backend::Eyaml::Parser::EncToken.new(:block, value, encryptor, ciphertext, nil, '  ')
+      token.to_encrypted format: :string
+    end
+
+    def content
+      @content ||=
+        begin
+          super
+          decrypt_content if @content && decrypt?
+          @content
+        end
+    end
+
+    private
+
+    def decrypt_content
+      public_key = @options[:public_key]
+      private_key = @options[:private_key]
+      @content.transform_values! do |value|
+        if self.class.encrypted?(value)
+          self.class.decrypt_value(value, public_key:, private_key:)
+        else
+          value
+        end
+      end
+    end
+
+    def decrypt?
+      @options[:decrypt] &&
+        @options[:public_key] &&
+        @options[:private_key]
+    end
+  end
+end

app/models/hiera_data/hierarchy.rb

@@ -55,6 +55,13 @@ def public_key
       @base_path.join(raw_hash.dig("options", "pkcs7_public_key"))
     end
 
+    def file_options
+      {
+        public_key:,
+        private_key:
+      }.compact
+    end
+
     def encryptable?
       backend == :eyaml &&
         File.readable?(private_key) &&

app/models/hiera_data/lookup.rb

@@ -0,0 +1,57 @@
+class HieraData
+  class Lookup
+    def initialize(hashes)
+      @hashes = hashes
+    end
+
+    def lookup(key, merge_strategy: :first)
+      applicable_values = extract_key_from_hashes(key)
+      merge_values(applicable_values, merge_strategy)
+    end
+
+    private
+
+    def extract_key_from_hashes(key)
+      @hashes
+        .select { |h| h.key?(key) }
+        .pluck(key)
+    end
+
+    def merge_values(values, strategy)
+      case strategy
+      when :first
+        values.first
+      when :unique
+        unique_array_merge(values)
+      when :hash
+        flat_hash_merge(values)
+      when :deep
+        deep_merge(values)
+      end
+    end
+
+    def unique_array_merge(values)
+      values.inject([]) do |memo, value|
+        raise Hdm::Error, "Merge strategy `unique` is not applicable to a hash." if value.is_a?(Hash)
+
+        memo + Array(value)
+      end.uniq
+    end
+
+    def flat_hash_merge(values)
+      values.inject({}) do |memo, value|
+        raise Hdm::Error, "Merge strategy `hash` can only be used with hashes." unless value.is_a?(Hash)
+
+        value.merge(memo)
+      end
+    end
+
+    def deep_merge(values)
+      values.inject do |memo, value|
+        memo ||= {}
+
+        DeepMerge.deep_merge!(memo, value, merge_hash_arrays: true)
+      end
+    end
+  end
+end

app/models/hiera_data/util.rb

@@ -0,0 +1,9 @@
+class HieraData
+  module Util
+    module_function
+
+    def yaml_format(value)
+      value.to_yaml.sub(/\A---(\n| )/, '').chomp
+    end
+  end
+end

app/models/hiera_data/yaml_file.rb

@@ -2,8 +2,9 @@ class HieraData
   class YamlFile
     attr_reader :path
 
-    def initialize(path:)
+    def initialize(path:, options: {})
       @path = path
+      @options = options
     end
 
     def exist?
@@ -46,8 +47,7 @@ def content_for_key(key)
       return "false" if value == false
       return value if [String, Integer, Float].include?(value.class)
 
-      value_string = value.to_yaml
-      value_string.sub(/\A---(\n| )/, '').gsub(/^$\n/, '')
+      Util.yaml_format(value)
     end
 
     def write_key(key, value)